odania-static-pages 0.1.0

data/lib/odania_static_pages/config.rb

@@ -0,0 +1,127 @@
+require_relative 'config/docker_compose'
+require_relative 'config/generator/jekyll'
+require_relative 'config/rsync'
+require_relative 'config/s3'
+
+module OdaniaStaticPages
+	class Config
+		attr_accessor :environment
+		attr_reader :project_dir, :base_dir, :output_sites_path, :output_nginx_path, :environments,
+								:generator_type, :generator
+
+		def initialize(path=nil)
+			@base_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..'))
+			@project_dir = path.nil? ? Dir.getwd : path
+			@output_sites_path = 'sites'
+			@output_nginx_path = 'nginx'
+			@generator_type = 'Jekyll'
+			@environments = {
+				develop: Environment.new('_develop', 'DockerCompose'),
+				live: Environment.new('_live', 'Rsync')
+			}
+			@generator = Generator::Jekyll.new(@project_dir, {})
+		end
+
+		def load!
+			if File.exist? config_file
+				config = YAML.load_file(config_file).stringify_keys!
+				@output_sites_path = config['output_sites_path'] unless config['output_sites_path'].nil?
+				@output_nginx_path = config['output_nginx_path'] unless config['output_nginx_path'].nil?
+				@generator_type = config['generator_type'] unless config['generator_type'].nil?
+
+				unless config['environments'].nil?
+					@environments = {}
+					config['environments'].each_pair do |name, data|
+						@environments[name] = Environment.from_hash data
+					end
+				end
+
+				@deploy = Deploy.from_hash(config['deploy']) unless config['deploy'].nil?
+
+				unless config['generator'].nil?
+					generator_config_module = "OdaniaStaticPages::Config::Generator::#{@generator_type}".constantize
+					@generator = generator_config_module.from_hash @project_dir, config['generator']
+				end
+			end
+
+			self
+		end
+
+		def config_file
+			File.join(@project_dir, '_config.yml')
+		end
+
+		def output_site_path
+			File.join @project_dir, current_environment.output_path, @output_sites_path
+		end
+
+		def output_path
+			File.join @project_dir, current_environment.output_path
+		end
+
+		def pages_path
+			File.join @project_dir, 'pages'
+		end
+
+		def current_environment
+			@environments[@environment]
+		end
+
+		def save!
+			puts "Writing config file #{config_file}"
+			File.write config_file, YAML.dump(to_h)
+		end
+
+		private
+
+		def to_h
+			environment_hash = {}
+			@environments.each_pair do |name, environment|
+				environment_hash[name.to_s] = environment.to_h
+			end
+
+			config = {
+				output_sites_path: @output_sites_path,
+				output_nginx_path: @output_nginx_path,
+				generator_type: @generator_type,
+				generator: @generator.to_h,
+				environments: environment_hash
+			}
+
+			config.stringify_keys!
+		end
+
+		class Environment
+			attr_reader :output_path, :deploy_type, :deploy_module, :notify
+
+			def initialize(output_path, deploy_type, deploy_module_config={}, notify='echo Switched from OLD-COLOR to NEW-COLOR!')
+				@output_path = output_path
+				@deploy_type = deploy_type
+				@notify = notify
+				clazz_name = "OdaniaStaticPages::Config::Deploy::#{deploy_type}"
+				@deploy_module = clazz_name.constantize.new deploy_module_config.symbolize_keys!
+			end
+
+			def to_h
+				{
+					output_path: @output_path,
+					deploy_type: @deploy_type,
+					notify: @notify,
+					deploy_config: @deploy_module.to_h
+				}.stringify_keys!
+			end
+
+			def self.from_hash(data)
+				Environment.new data['output_path'], data['deploy_type'], data['deploy_config'], data['notify']
+			end
+
+			def do_notify(new_color, old_color)
+				unless @notify.nil?
+					replaced_notify = @notify.gsub('NEW-COLOR', new_color).gsub('OLD-COLOR', old_color)
+					puts "Ececuting notify: #{replaced_notify}"
+					puts `#{replaced_notify}`
+				end
+			end
+		end
+	end
+end

data/lib/odania_static_pages/deploy/docker_compose.rb

@@ -0,0 +1,148 @@
+module OdaniaStaticPages
+	module Deploy
+		class DockerCompose
+			def initialize
+				@config = OdaniaStaticPages.config
+				@environment = @config.current_environment
+				@deploy_config = @environment.deploy_module
+				@generator_config = @config.generator
+				@nginx_dir = File.join(@config.output_path, 'nginx')
+				@nginx_conf_dir = File.join(@nginx_dir, 'conf.d')
+			end
+
+			def prepare
+				puts 'Preparing docker-compose state'
+				load_state
+				state_path = File.dirname(@deploy_config.state_file)
+				FileUtils.mkdir_p state_path unless File.exist? state_path
+				save_state
+			end
+
+			def publish(color, do_rebuild)
+				puts 'docker-compose'
+				load_state
+				color = color.nil? ? @state[:color] : color
+				new_color = 'green'.eql?(color) ? 'blue' : 'green'
+				puts " -> Current color: #{color}"
+				@site_path = @config.output_site_path
+				puts " -> Deploying to color: #{new_color} [Path: #{@site_path}]"
+
+				generate_compose_config
+				generate_nginx_config(do_rebuild)
+				prepare_varnish
+
+				@config.current_environment.do_notify new_color, color
+				puts
+				puts "Finished deploying color #{new_color}"
+			end
+
+			private
+
+			def load_state
+				@state = {color: 'blue'}
+				@state = YAML.load_file(@deploy_config.state_file).symbolize_keys! if File.exist? @deploy_config.state_file
+
+				@state
+			end
+
+			def save_state
+				File.write @deploy_config.state_file, YAML.dump(@state)
+			end
+
+			def generate_compose_config
+				puts 'Generating docker-compose.yml'
+				environment = @config.current_environment
+				compose_file = File.join(@config.project_dir, environment.output_path, 'docker-compose.yml')
+
+				puts "Writing docker compose to #{compose_file}"
+				docker_compose_generator = DockerComposeGenerator.new @config, environment
+				docker_compose_generator.write compose_file
+			end
+
+			def generate_nginx_config(do_rebuild)
+				vhost_renderer = ERB.new(File.read(File.join(@config.base_dir, 'templates', 'nginx', 'vhost.conf.erb')))
+				sites = {}
+				grouped_domains.each_pair do |site_name, page_config|
+					full_site_name = "#{site_name}.lvh.me"
+					puts "Writing vhost for: #{full_site_name}"
+					expires = @deploy_config.expires
+					File.write File.join(@nginx_conf_dir, "#{site_name}.conf"), vhost_renderer.result(binding)
+
+					sites[site_name] = "#{site_name}.lvh.me:8080"
+
+					page_config.each do |config|
+						sites["#{site_name}#{config[:baseurl]}"] = "#{site_name}.lvh.me:8080#{config[:baseurl]}"
+					end if page_config.count > 1
+				end
+
+				puts 'Generating index.html'
+				renderer = ERB.new(File.read(File.join(@config.base_dir, 'templates', 'nginx', 'index.html.erb')))
+				File.write File.join(@config.output_site_path, 'index.html'), renderer.result(binding)
+
+				puts 'Copy default vhost'
+				FileUtils.cp File.join(@config.base_dir, 'templates', 'nginx', 'default-vhost.conf'), File.join(@nginx_conf_dir, 'default.conf')
+			end
+
+			def prepare_varnish
+				docker_folder = File.join(@config.project_dir, 'docker')
+				FileUtils.mkdir_p docker_folder
+				puts "Checkout repositories in #{docker_folder}"
+
+				if File.exist? "#{docker_folder}/varnish"
+					cmd = "cd #{docker_folder}/varnish && git pull"
+				else
+					cmd = "git clone -q -b develop https://github.com/Odania-IT/odania-varnish.git #{docker_folder}/varnish"
+				end
+				puts " -> Executing #{cmd}"
+				puts `#{cmd}`
+				exit 1 unless $?.success?
+
+				if File.exist? "#{docker_folder}/varnish-generator"
+					cmd = "cd #{docker_folder}/varnish-generator && git pull"
+				else
+					cmd = "git clone -q https://github.com/Odania-IT/odania-varnish-generator.git #{docker_folder}/varnish-generator"
+				end
+				puts " -> Executing #{cmd}"
+				puts `#{cmd}`
+				exit 1 unless $?.success?
+
+				puts 'Prepare varnish secret'
+				File.write File.join(docker_folder, 'varnish', 'varnish-secret'), @deploy_config.varnish_secret
+				File.write File.join(docker_folder, 'varnish-generator', 'varnish-secret'), @deploy_config.varnish_secret
+			end
+
+			def grouped_domains
+				result = Hash.new { |k, v| k[v] = [] }
+
+				@generator_config.jekyll_config['pages'].each do |page|
+					uri = URI.parse(page['url'])
+					host = uri.host
+					result[host] << {baseurl: page['baseurl'], relative_path: @generator_config.page_path(page)}
+				end
+
+				puts result.inspect
+				result
+			end
+
+			class DockerComposeGenerator
+				attr_reader :nginx_volume_html, :nginx_volume_conf_d, :nginx_volume_nginx_conf, :compose_images
+
+				def initialize(config, environment)
+					@nginx_volume_html = "#{config.output_site_path}:/srv:ro"
+					@nginx_volume_conf_d = "#{@nginx_conf_dir}:/etc/nginx/conf.d:ro"
+					@nginx_volume_nginx_conf = "#{File.join(config.output_path, 'nginx', 'nginx.conf')}:/etc/nginx/nginx.conf"
+					@compose_images = environment.deploy_module.compose_images
+					@erb_template = File.join(config.base_dir, 'templates', 'docker-compose', 'docker-compose.yml.erb')
+				end
+
+				def render
+					ERB.new(File.read(@erb_template)).result(binding)
+				end
+
+				def write(out_dir)
+					File.write out_dir, render
+				end
+			end
+		end
+	end
+end

data/lib/odania_static_pages/deploy/rsync.rb

@@ -0,0 +1,73 @@
+module OdaniaStaticPages
+	module Deploy
+		class Rsync
+			def initialize
+				@config = OdaniaStaticPages.config
+				@deploy_config = @config.current_environment.deploy_module
+				@local_state = '_local_state.yml'
+			end
+
+			def prepare
+				puts 'Preparing rsync state'
+				load_state
+				save_state
+			end
+
+			def publish(color, do_rebuild)
+				puts 'Rsync website'
+				load_state
+				color = color.nil? ? @state[:color] : color
+				new_color = 'green'.eql?(color) ? 'blue' : 'green'
+				puts " -> Current color: #{color}"
+				@site_path = @config.output_site_path
+				puts " -> Deploying to color: #{new_color} [Path: #{@site_path}]"
+
+				@deploy_config.targets[new_color].each do |target|
+					puts
+					puts
+					puts "Syncing target #{target} " + '-' * 50
+
+					cmd = "cd #{@site_path} && rsync #{@deploy_config.rsync_options} . #{target}"
+					puts "Executing: #{cmd}"
+					puts `#{cmd}`.split("\n").join("\n    ")
+
+					unless $?.success?
+						puts 'Error during rsync!!'
+						exit 1
+					end
+				end
+
+				@state[:color] = new_color
+				save_state
+
+				@config.current_environment.do_notify new_color, color
+				puts
+				puts "Finished deploying color #{new_color}"
+			end
+
+			private
+
+			def load_state
+				@state = {color: 'blue'}
+				cmd = "rsync #{@deploy_config.state_file} #{@local_state}"
+				puts "Syncing state: #{cmd}"
+				puts `#{cmd}`
+
+				@state = YAML.load_file(@local_state).symbolize_keys! if $?.success?
+				@state
+			end
+
+			def save_state
+				File.write @local_state, YAML.dump(@state)
+				cmd = "rsync #{@local_state} #{@deploy_config.state_file}"
+				puts "Syncing state: #{cmd}"
+				puts `#{cmd}`
+
+				unless $?.success?
+					puts 'Error saving state!!'
+					exit 1
+				end
+			end
+		end
+	end
+end

data/lib/odania_static_pages/deploy/s3.rb

@@ -0,0 +1,165 @@
+module OdaniaStaticPages
+	module Deploy
+		class S3
+			def initialize
+				@config = OdaniaStaticPages.config
+				@s3_config = @config.current_environment.deploy_module
+				params = {
+					region: @s3_config.region
+				}
+				params[:profile] = @s3_config.profile unless @s3_config.profile.nil?
+
+				puts "Configuring AWS with params: #{params}"
+				Aws.config.update(params)
+			end
+
+			def prepare
+				tags = @s3_config.tags
+				logging_stack_name = 'odania-logging'
+				params = {
+					BucketName: @s3_config.logging_bucket
+				}
+				Stacker.create_or_update_stack(logging_stack_name, File.join(@config.base_dir, 'cf-templates', 's3-logging.yml'), params, nil, nil, tags)
+
+				params = {
+					BucketName: @s3_config.state_bucket
+				}
+				Stacker.create_or_update_stack('odania-state', File.join(@config.base_dir, 'cf-templates', 's3-state.yml'), params, nil, nil, tags)
+
+				params = {
+					BucketNameGreen: @s3_config.bucket_name_green,
+					BucketNameBlue: @s3_config.bucket_name_blue,
+					LoggingStackName: logging_stack_name
+				}
+				Stacker.create_or_update_stack('odania-static-http', File.join(@config.base_dir, 'cf-templates', 's3-http.yml'), params, logging_stack_name, nil, tags)
+			end
+
+			def publish(color)
+				puts 'Publishing website to s3'
+				load_state
+				color = color.nil? ? @state[:color] : color
+				new_color = 'green'.eql?(color) ? 'blue' : 'green'
+				puts " -> Current color: #{color}"
+				@site_path = @config.output_site_path
+				puts " -> Deploying to color: #{new_color} [Path: #{@site_path}]"
+
+				@uploaded_files = []
+				bucket_name = 'green'.eql?(new_color) ? @s3_config.bucket_name_green : @s3_config.bucket_name_blue
+				recursive_file_upload bucket_name, @site_path
+				delete_not_uploaded_files bucket_name
+
+				@state[:color] = new_color
+				save_state
+
+				url = bucket_url bucket_name
+
+				@config.current_environment.do_notify new_color, color
+				puts
+				puts "Finished deploying color #{new_color} to bucket #{bucket_name}"
+				puts "Public url: #{url}"
+			end
+
+			private
+
+			def client
+				@s3_client = Aws::S3::Client.new if @s3_client.nil?
+				@s3_client
+			end
+
+			def bucket_url(bucket_name)
+				bucket = Aws::S3::Bucket.new bucket_name
+				bucket.url
+			end
+
+			def s3_exists?(bucket, key)
+				result = client.list_objects({bucket: bucket, prefix: key})
+
+				result.contents.each do |content|
+					return true if key.eql? content.key
+				end
+
+				false
+			end
+
+			def s3_files_in_bucket(bucket)
+				is_truncated = true
+				next_marker = nil
+				result = []
+
+				while is_truncated
+					params = {bucket: bucket}
+					params[:next_marker] = next_marker unless next_marker.nil?
+					response = client.list_objects(params)
+
+					response.contents.each do |content|
+						result << content.key
+					end
+
+					next_marker = response.next_marker
+					is_truncated = response.is_truncated
+				end
+
+				result
+			end
+
+			def recursive_file_upload(bucket, path)
+				Dir.glob(File.join(path, '**')).each do |file|
+					if File.directory? file
+						recursive_file_upload bucket, file
+					else
+						target_file = file.gsub("#{@site_path}/", '')
+						content_type = MimeMagic.by_extension File.extname file
+						@uploaded_files << target_file
+						puts "  *> #{file} => #{target_file} [Content-Type: #{content_type}]"
+
+						File.open(file, 'rb') do |opened_file|
+							client.put_object({
+								acl: 'public-read',
+								bucket: bucket,
+								key: target_file,
+								body: opened_file,
+								content_type: content_type.to_s,
+								server_side_encryption: 'AES256'
+							})
+						end
+					end
+				end
+			end
+
+			def delete_not_uploaded_files(bucket)
+				s3_files = s3_files_in_bucket bucket
+
+				new_files = @uploaded_files - s3_files
+				files_to_delete = s3_files - @uploaded_files
+
+				puts '*'*100
+				puts @uploaded_files
+				puts '*'*100
+				puts s3_files
+				puts '*'*100
+				puts "New Files: #{new_files}"
+				puts '*'*100
+				puts "Files to delete: #{files_to_delete}"
+			end
+
+			def load_state
+				@state = {color: 'blue'}
+				if s3_exists?(@s3_config.state_bucket, 'state.yml')
+					response = client.get_object({bucket: @s3_config.state_bucket, key: 'state.yml'})
+					@state = YAML.load response.body.read
+				end
+
+				@state
+			end
+
+			def save_state
+				client.put_object({
+						bucket: @s3_config.state_bucket,
+						key: 'state.yml',
+						body: YAML.dump(@state),
+						server_side_encryption: 'AES256'
+				})
+			end
+		end
+	end
+end

data/lib/odania_static_pages/generator/jekyll.rb

@@ -0,0 +1,112 @@
+module OdaniaStaticPages
+	module Generator
+		class Jekyll
+			def init
+				puts 'Initliazing jekyll pages'
+				puts
+				setup_generator
+
+				FileUtils.mkdir_p @pages_path unless File.exist? @pages_path
+
+				unless File.exist? @jekyll_config_file
+					FileUtils.cp File.join(@config.base_dir, 'templates', 'jekyll', 'example_config.yml'), @jekyll_config_file
+					setup_generator
+				end
+
+				FileUtils.mkdir_p @generator_config.full_common_folder unless File.exist? @generator_config.full_common_folder
+
+				puts 'Initialize Gemfile'
+				gemfile_template = File.join(@config.base_dir, 'templates', 'jekyll', 'Gemfile.erb')
+				gem_extra = @generator_config.gem_extra
+				File.write File.join(@generator_config.full_common_folder, 'Gemfile'), ERB.new(File.read(gemfile_template)).result(binding)
+
+				puts 'Install gems'
+				puts `cd #{@generator_config.full_common_folder} && bundle check`
+				puts `cd #{@generator_config.full_common_folder} && bundle install --path ~/.gems` unless $?.success?
+
+				@jekyll_config['pages'].each do |page|
+					relative_page_path = @generator_config.page_path page
+					relative_page_path += page['baseurl'] unless page['baseurl'].empty?
+					page_path = File.join @pages_path, relative_page_path
+
+					puts '*' * 100
+					puts "Processing #{relative_page_path} => #{page_path}"
+
+					unless File.directory? page_path
+						puts "Creating page #{page['name']}"
+						`cd #{@pages_path} && octopress new #{relative_page_path}`
+					end
+
+					page_config = @generator_config.prepare_config page
+					current_config_file = File.join page_path, '_config.yml'
+					current_config = YAML.load_file current_config_file
+
+					unless current_config.eql? page_config
+						puts ' -> Updating config'
+						File.write current_config_file, YAML.dump(page_config)
+					end
+
+					@generator_config.link @config.base_dir, page_path, page
+
+					puts
+				end
+
+			end
+
+			def build(env)
+				puts 'Building all jekyll websites'
+				setup_generator
+
+				grouped_domains.each_pair do |site_name, page_config|
+					build_for_configs(page_config, site_name, @config.output_site_path, env)
+				end
+			end
+
+			private
+
+			def setup_generator
+				@config = OdaniaStaticPages.config
+				@current_environment = @config.current_environment
+				@generator_config = @config.generator
+				@pages_path = @config.pages_path
+				@jekyll_config_file = @generator_config.jekyll_config_file
+				@jekyll_config = @generator_config.jekyll_config
+			end
+
+			def grouped_domains
+				result = Hash.new { |k, v| k[v] = [] }
+
+				@generator_config.jekyll_config['pages'].each do |page|
+					uri = URI.parse(page['url'])
+					host = uri.host
+					result[host] << {baseurl: page['baseurl'], relative_path: @generator_config.page_path(page)}
+				end
+
+				puts result.inspect
+				result
+			end
+
+			def build_site(site_path, options, jekyll_env='development')
+				puts 'Install gems'
+				puts `cd #{File.join(@generator_config.pages_dir, site_path)} && bundle check`
+				puts `cd #{File.join(@generator_config.pages_dir, site_path)} && bundle install --path ~/.gems` unless $?.success?
+
+				cmd = "cd #{ File.join(@generator_config.pages_dir, site_path) } && JEKYLL_ENV=#{jekyll_env} bundle exec jekyll build #{options}"
+				puts " -> Building site [cmd: #{cmd}]"
+				unless system(cmd)
+					puts "Error building site: #{site_path}"
+					exit 1
+				end
+			end
+
+			def build_for_configs(page_config, site_name, target_site_path, jekyll_env='development')
+				page_config.each do |config|
+					site_path = site_name
+					site_path = File.join(site_path, config[:baseurl]) unless config[:baseurl].nil?
+					options = "-d #{File.join(target_site_path, site_path)}"
+					build_site site_path, options, jekyll_env
+				end
+			end
+		end
+	end
+end

data/lib/odania_static_pages/server/nginx.rb

@@ -0,0 +1,7 @@
+module OdaniaStaticPages
+	module Server
+		class Nginx
+
+		end
+	end
+end

data/lib/odania_static_pages/version.rb

@@ -0,0 +1,3 @@
+module OdaniaStaticPages
+	VERSION = '0.1.0'
+end

data/lib/odania_static_pages.rb

@@ -0,0 +1,28 @@
+require 'active_support/all'
+require 'autostacker24'
+require 'aws-sdk'
+require 'erb'
+require 'mimemagic'
+require 'thor'
+require 'uri'
+require 'yaml'
+
+require_relative 'odania_static_pages/cli'
+require_relative 'odania_static_pages/config'
+require_relative 'odania_static_pages/deploy/docker_compose'
+require_relative 'odania_static_pages/deploy/rsync'
+require_relative 'odania_static_pages/deploy/s3'
+require_relative 'odania_static_pages/generator/jekyll'
+require_relative 'odania_static_pages/server/nginx'
+require_relative 'odania_static_pages/version'
+
+module OdaniaStaticPages
+	def self.config(path=nil)
+		if @config.nil? or !path.nil?
+			@config = OdaniaStaticPages::Config.new(path)
+			@config.load!
+		end
+
+		@config
+	end
+end

data/odania-static-pages.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require_relative 'lib/odania_static_pages/version'
+
+Gem::Specification.new do |spec|
+	spec.name = 'odania-static-pages'
+	spec.version = OdaniaStaticPages::VERSION
+	spec.authors = ['Mike Petersen']
+	spec.email = ['mike@odania-it.de']
+
+	spec.summary = %q{Helper For creating static pages}
+	spec.description = %q{Helper for creating static pages}
+	spec.homepage = 'http://www.odania.com'
+	spec.license = 'MIT'
+
+	spec.files = `git ls-files -z`.split("\x0").reject do |f|
+		f.match(%r{^(test|spec|features)/})
+	end
+	spec.bindir = 'exe'
+	spec.executables = spec.files.grep(%r{^exe/}) {|f| File.basename(f)}
+	spec.require_paths = ['lib']
+
+	spec.add_runtime_dependency 'activesupport'
+	spec.add_runtime_dependency 'autostacker24', '>= 2.8.0'
+	spec.add_runtime_dependency 'aws-sdk', '>= 3'
+	spec.add_runtime_dependency 'mimemagic'
+	spec.add_runtime_dependency 'thor'
+
+	spec.add_development_dependency 'bundler', '~> 1.15'
+	spec.add_development_dependency 'rake', '~> 10.0'
+	spec.add_development_dependency 'minitest', '~> 5.0'
+end