capistrano-cluster 0.0.10

data/lib/capistrano/cluster/files.rb

@@ -0,0 +1,57 @@
+require 'nokogiri'
+module Capistrano
+  module Cluster
+    module Files
+      module DSL
+
+        def remote_file(url, file, checksum: nil)
+          file = File.expand_path(file)
+
+          if test "[ -f '#{file}' ]"
+            if checksum
+              return if checksum == capture("sha256sum '#{file}' | awk '{ print $1}'")
+              execute :rm, "-f", file
+            else
+              return
+            end
+          end
+          execute :mkdir, "-p", File.dirname(file)
+          execute :wget, "-q", "-o", "/dev/null" , "-O", file, url
+        end
+
+        def upload_as(user, file, remote_file, options={})
+          group = options.fetch(:group, user)
+
+          tmp_name = "/tmp/#{SecureRandom.uuid}"
+          upload! file, tmp_name
+
+          sudo :mv, tmp_name, remote_file
+
+          unless test "[[ -d #{File.dirname(remote_file)} ]]"
+            sudo :mkdir, "-p", File.dirname(remote_file)
+            sudo :chown, "-R", "#{user}:#{group}", File.dirname(remote_file)
+
+          end
+
+          sudo :chown, "#{user}:#{group}", remote_file
+          sudo :chmod, options.fetch(:mode, 644), remote_file
+
+        end
+
+        def item(identifier)
+          consume = false
+          xml = File.read(caller.first.split(":").first).lines.select { |l| consume ||= l =~/^__END__$/ || consume}[1..-1].join()
+          doc = Nokogiri::HTML(xml)
+          doc.css("##{identifier}").inner_html
+        end
+
+      end
+    end
+  end
+end
+
+
+
+include Capistrano::Cluster::Files::DSL
+
+

data/lib/capistrano/cluster/packages.rb

@@ -0,0 +1,62 @@
+require 'erb'
+require 'ostruct'
+require 'delegate'
+
+module Capistrano
+  module Cluster
+    module Packages
+      module DSL
+
+        def install(*packages)
+          update_apt_source_list
+          sudo "apt-get", "-q", "-y", "--force-yes","install", *packages.flatten
+        end
+
+        class TemplateContext < SimpleDelegator
+          def initialize(obj, variables={})
+            __setobj__(obj)
+            @vars = variables
+            variables.each_pair do |name, value|
+              class <<self; self;end.send(:define_method,name, &->(){ value })
+            end
+          end
+
+        end
+
+        def file(source, locals={})
+
+          if file = file_path(source)
+            content = if file =~ /\.erb$/
+              ERB.new(File.read(file)).result(TemplateContext.new(self, locals).instance_eval { binding })
+            else
+              File.read(file)
+            end
+            StringIO.new(content)
+          else
+            raise "File #{source} not found"
+          end
+
+        end
+
+        def file_path(source)
+          lookup_paths = [fetch(:files_path), fetch(:default_templates_path)].compact
+          lookup_paths.map do |path|
+            [File.join(path, "#{source}"), File.join(path, "#{source}.erb")]
+          end.flatten.select do |file|
+            File.exists? file
+          end.first
+        end
+
+        def update_apt_source_list(forced: false)
+          sudo "apt-get", "-q", "-y", "update" if forced or test "[ ! -z $(find /var/lib/apt/periodic/update-success-stamp -mmin +120) ]"
+        end
+
+      end
+    end
+  end
+end
+
+
+include Capistrano::Cluster::Packages::DSL
+
+set :files_path, fetch(:files_path, Pathname(fetch(:deploy_config_path)).dirname.join("files"))

data/lib/capistrano/cluster/paths.rb

@@ -0,0 +1,18 @@
+module Capistrano
+  module Cluster
+    module Paths
+
+      def home_path
+        fetch(:home_path, "/home/#{fetch(:user)}")
+      end
+
+    end
+  end
+end
+
+
+
+include Capistrano::Cluster::Paths
+
+
+

data/lib/capistrano/cluster/service.rb

@@ -0,0 +1,70 @@
+module Capistrano
+  module Cluster
+    class Service
+      attr :name, :cmd
+      def initialize(name, &block)
+        @name = name
+        instance_eval(&block)
+      end
+
+      def start(*args)
+        @start = args.first if args.length > 0
+        @start || fail("no start command defined for #{@name}")
+      end
+
+      def working_dir(*args)
+        @working_dir = args.first if args.length > 0
+        @working_dir || "/tmp"
+      end
+
+      def pid_file(*args)
+        @pid_file = args.first if args.length > 0
+        @pid_file || "/var/run/#{@name}.pid"
+      end
+
+      def reload(*args)
+        @reload = args.first if args.length > 0
+        @reload || "stop;start"
+      end
+
+      def stop(*args)
+        @stop = args.first if args.length > 0
+        @stop || "kill -TERM $(< #{pid_file})"
+      end
+
+      def user(*args)
+        @user = args.first if args.length > 0
+        @user
+      end
+
+      def log_file(*args)
+        @log_file = args.first if args.length > 0
+        @log_file || "/var/log/service.#{@name}.log"
+      end
+
+      def script
+        file("service", name: name, start_cmd: start, pid_file: pid_file, working_dir: working_dir, reload_cmd: reload, stop_cmd: stop, user: user)
+      end
+
+      module DSL
+
+        def service(name, &block)
+          service = Capistrano::Cluster::Service.new(name, &block)
+          upload_as :root, service.script, "/etc/init.d/#{name}"
+          sudo :chmod, "0777", "/etc/init.d/#{name}"
+          sudo "update-rc.d", name, :defaults
+        end
+
+      end
+
+    end
+
+  end
+
+end
+
+
+
+include Capistrano::Cluster::Service::DSL
+
+

data/lib/capistrano/cluster/version.rb

@@ -0,0 +1,5 @@
+module Capistrano
+  module Cluster
+    VERSION = "0.0.10"
+  end
+end

data/lib/capistrano/cluster.rb

@@ -0,0 +1,35 @@
+
+require 'capistrano/setup'
+require 'capistrano/deploy'
+require 'capistrano/bundler'
+require 'capistrano/rails/assets'
+require 'capistrano/rails/migrations'
+require 'capistrano/rails'
+
+require "capistrano/cluster/version"
+require "capistrano/cluster/core_ext/object"
+require "capistrano/cluster/application"
+require "capistrano/cluster/packages"
+require "capistrano/cluster/files"
+require "capistrano/cluster/paths"
+require "capistrano/cluster/service"
+
+set :default_templates_path, File.expand_path('../../../files',__FILE__)
+
+module Capistrano
+  module Cluster
+  end
+end
+
+tasks_path = Pathname(File.expand_path('../../../tasks/',__FILE__))
+
+import tasks_path.join("setup.rake")
+import tasks_path.join("setup/firewall.rake")
+
+Dir.glob(tasks_path.join("*/*.rake")).each do |r|
+  import r
+end
+
+import tasks_path.join("deploy.rake")
+import tasks_path.join("deploy/application.rake")
+

data/tasks/deploy/application.rake

@@ -0,0 +1,48 @@
+namespace :deploy do
+
+  namespace :check do
+
+    task :application do
+
+      on roles(:app) do
+        sudo :mkdir, "-p", deploy_to, shared_path.join("config")
+        sudo :chown, "-R", fetch(:user), deploy_to, shared_path.join("config")
+      end
+
+      invoke "deploy:application:secrets"
+      invoke "deploy:application:web"
+      invoke "deploy:application:indexer"
+      invoke "deploy:application:db"
+      invoke "deploy:application:proxy"
+    end
+
+  end
+
+  namespace :application do
+
+    task :secrets do
+
+      on roles(:app) do
+        env = StringIO.new
+        gc_settings = {
+          "RUBY_HEAP_MIN_SLOTS" => 800000,
+          "RUBY_FREE_MIN" => 100000,
+          "RUBY_GC_MALLOC_LIMIT" => 59000000
+        }
+        fetch(:secrets, {}).merge(gc_settings).each_pair do |key,value|
+          env.puts "export #{"#{key}".upcase}=#{value}"
+        end
+        env.rewind
+        upload! env, shared_path.join(".env")
+      end
+
+    end
+
+
+  end
+
+end
+
+before "deploy:symlink:linked_files", "check:application"
+before "deploy:check:directories", "check:application"
+

data/tasks/deploy.rake

@@ -0,0 +1,71 @@
+require 'shellwords'
+
+namespace :deploy do
+
+  task :restart do
+
+    on roles(:web) do
+      sudo "/etc/init.d/#{fetch(:application)}-web", :reload
+    end
+
+  end
+
+end
+
+
+class Rake::Task
+  def overwrite(&block)
+    @actions.clear
+    prerequisites.clear
+    enhance(&block)
+  end
+  def abandon
+    prerequisites.clear
+    @actions.clear
+  end
+end
+
+namespace :deploy do
+  after :publishing, :restart
+end
+
+Rake::Task["deploy:check:linked_files"].overwrite do
+  next unless any? :linked_files
+  on release_roles :app do |host|
+    linked_files(shared_path).each do |file|
+      unless test "[ -f #{file} ]"
+        error t(:linked_file_does_not_exist, file: file, host: host)
+        exit 1
+      end
+    end
+  end
+end
+
+
+Rake::Task["deploy"].clear
+
+desc 'Deploy a new release.'
+
+task :deploy, :app do |t,options|
+
+  applications options[:app] do
+    %w{ starting started
+        updating updated
+        publishing published
+        finishing finished }.each do |task|
+
+      invoke "deploy:#{task}"
+    end
+
+  end
+
+end
+
+
+
+desc "On app"
+task :application, :app do |t,options|
+  if application = applications(options[:app]).first
+    configure_application application
+  end
+end

data/tasks/roles/app.rake

@@ -0,0 +1,71 @@
+namespace :setup do
+
+  namespace :app do
+    task :packages do
+      on roles(:app) do
+        install fetch(:app_packages)
+        if test "[ ! -f /etc/gemrc ]"
+          upload_as :root, StringIO.new("install: --no-rdoc --no-ri\nupdate:  --no-rdoc --no-ri"), "/etc/gemrc"
+          sudo :chmod, '0644', "/etc/gemrc"
+        end
+
+        if test "[ -z $(which bundler) ]"
+          sudo :gem, 'install', 'bundler'
+        end
+
+      end
+    end
+
+    task :package_sources do
+      on roles(:app) do
+        sudo "apt-add-repository", "-y", "ppa:brightbox/ruby-ng"
+      end
+    end
+
+  end
+end
+
+before "setup:packages", "setup:app:package_sources"
+after "setup:system", "setup:app:packages"
+
+set :app_packages, %w[
+  autoconf
+  bind9-host
+  bison
+  build-essential
+  curl
+  daemontools
+  dnsutils
+  ed
+  git
+  imagemagick
+  iputils-tracepath
+  libcurl4-openssl-dev
+  libevent-dev
+  libglib2.0-dev
+  libjpeg-dev
+  libjpeg62
+  libpng12-0
+  libpng12-dev
+  libmagickcore-dev
+  libmagickwand-dev
+  libmcrypt-dev
+  libmysqlclient-dev
+  libpq-dev
+  libsqlite3-dev
+  libssl-dev
+  libssl0.9.8
+  libxml2-dev
+  libxslt-dev
+  mercurial
+  netcat-openbsd
+  ruby2.1-dev
+  ruby2.1
+  socat
+  sqlite3
+  telnet
+  zlib1g-dev
+  postgresql-client-9.3
+  jpegoptim
+  libv8-dev
+]

data/tasks/roles/cache.rake

@@ -0,0 +1,16 @@
+namespace :setup do
+
+  task :cache  do
+    on roles(:cache) do
+      unless test "[ -f /etc/redis/redis.conf ]"
+        install "redis-server"
+        upload_as :root, file("redis/redis.conf"), "/etc/redis/redis.conf"
+        sudo 'nohup /etc/init.d/redis-server restart'
+      end
+    end
+  end
+
+end
+
+
+after "setup:system", "setup:cache"

data/tasks/roles/db.rake

@@ -0,0 +1,66 @@
+#TODO (vvlad): tune using pgtune
+namespace :setup do
+
+  task :db do
+
+    on roles(:db) do
+
+      install "postgresql-9.3", "postgresql-9.3-postgis-2.1"
+      allowed_hosts = roles(:app).map(&:to_s)
+      upload_as :postgres, file("pg_hba.conf", allowed_hosts: allowed_hosts), "/etc/postgresql/9.3/main/pg_hba.conf"
+      upload_as :postgres, file("postgresql.conf", allowed_hosts: allowed_hosts), "/etc/postgresql/9.3/main/postgresql.conf"
+
+      if test "[ ! -f /var/lib/postgresql/.first-time ]"
+        sudo :service, "postgresql", :restart
+        sudo :touch, '/var/lib/postgresql/.first-time'
+      else
+        sudo :service, "postgresql", :reload
+      end
+    end
+
+  end
+
+end
+
+after "setup:system", "setup:db"
+
+namespace :deploy do
+  namespace :application do
+
+    task :db do
+
+      db_user = fetch(:database)[:username]
+      db_pass = fetch(:database)[:password]
+      db_name = fetch(:database).fetch(:name, fetch(:application))
+      enconding = fetch(:database)[:enconding] || "utf-8"
+
+      on roles(:db) do
+
+        if capture(:psql, "-U", "postgres", "template1", "-t", "-c", "SELECT 1 FROM pg_catalog.pg_user WHERE usename = '#{db_user}'".shellescape).empty?
+          execute :psql, "-U", "postgres", "-c", "CREATE ROLE #{db_user} LOGIN PASSWORD '#{db_pass}';".shellescape
+        end
+
+        if capture(:psql, "-U", "postgres", "template1", "-t", "-c", "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '#{db_name}'".shellescape).empty?
+          execute :psql, "-U", "postgres", "-c", "CREATE DATABASE #{db_name} ENCODING '#{enconding}' OWNER #{db_user};".shellescape
+          execute :psql, "-U", "postgres", "#{db_name}", "-c", "CREATE EXTENSION postgis;".shellescape
+        end
+
+      end
+
+      on roles(:app) do
+        defaults = {
+          environment: fetch(:framework_env),
+          hostname: primary(:db),
+          name: db_name,
+          adapter: 'postgresql',
+          encoding: 'utf-8',
+          options:{}
+        }
+        settings = defaults.merge(fetch(:database))
+        upload! file("database.yml", settings), "#{shared_path}/config/database.yml"
+      end
+
+    end
+
+  end
+end

data/tasks/roles/indexer.rake

@@ -0,0 +1,132 @@
+solr_version = fetch(:solr_version, "4.9.0")
+
+set :solr_version, solr_version
+set :solr_url, fetch(:solr_url, "http://www.eu.apache.org/dist/lucene/solr/#{solr_version}/solr-#{solr_version}.tgz")
+set :solr_checksum, 'ae47a89f35b5e2a6a4e55732cccc64fb10ed9779'
+set :solr_data, "/var/lib/solr"
+set :solr_dist, "/opt/solr"
+set :solr_user, "solr"
+
+
+namespace :setup do
+
+  desc "Boostraps solr nodes"
+  task :indexer do
+
+    on roles(:indexer) do
+
+      solr_dist = fetch(:solr_dist)
+      solr_data = fetch(:solr_data)
+
+
+      if test "[ ! -e '/etc/init.d/solr' ]"
+
+
+        install "openjdk-7-jre-headless"
+
+        sudo :mkdir, "-p", solr_dist
+        sudo :chown, "-R", "#{fetch(:user)}:#{fetch(:user)}", solr_dist
+        archive = "/tmp/solr-#{fetch(:solr_version)}.tgz"
+        remote_file fetch(:solr_url), archive, checksum: fetch(:solr_checksum)
+
+        execute :tar, "-xzC", solr_dist, "-f", archive, "--strip-components", "1"
+
+        sudo :mkdir, "-p", solr_data
+        sudo :chown, "-R", "#{fetch(:user)}:#{fetch(:user)}", solr_data
+
+        upload! file("solr/solr.xml"), "#{solr_data}/solr.xml"
+        service "solr" do
+          pid_file "/tmp/solr.pid"
+          user fetch(:user)
+          working_dir "#{solr_dist}/example"
+          start "/usr/bin/java -server -Dsolr.solr.home='#{solr_data}' -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -DSTOP.PORT=8079 -DSTOP.KEY=stopkey -Djetty.port=8983 -jar start.jar"
+          stop "pidof java && kill -TERM $(pidof java)"
+        end
+
+      end
+
+      sudo "nohup /etc/init.d/solr restart"
+
+    end
+
+
+  end
+
+end
+
+after "setup:system", "setup:indexer"
+
+namespace :deploy do
+  namespace :application do
+
+    task :indexer do
+
+      config = { application: fetch(:application), name: fetch(:solr_core) }
+
+      on roles(:indexer) do
+        solr_data = fetch(:solr_data)
+        if test "[ ! -d '#{solr_data}/#{config[:name]}/data' ]"
+          execute :mkdir, "-p", "#{solr_data}/#{config[:name]}/data"
+
+          fetch(:solr_core_files).each do |core_file|
+            execute :mkdir, "-p", File.dirname("#{solr_data}/#{config[:name]}/#{core_file}")
+            upload! file("solr/#{core_file}"), "#{solr_data}/#{config[:name]}/#{core_file}"
+          end
+          execute :touch, "#{solr_data}/#{config[:name]}/core.properties"
+        end
+
+      end if config[:name]
+
+    end
+
+  end
+end
+
+
+set :solr_core_files , %w[
+  conf/mapping-FoldToASCII.txt
+  conf/schema.xml
+  conf/solrconfig.xml
+  conf/stopwords.txt
+  conf/lang/contractions_ca.txt
+  conf/lang/contractions_fr.txt
+  conf/lang/contractions_ga.txt
+  conf/lang/contractions_it.txt
+  conf/lang/hyphenations_ga.txt
+  conf/lang/stemdict_nl.txt
+  conf/lang/stoptags_ja.txt
+  conf/lang/stopwords_ar.txt
+  conf/lang/stopwords_bg.txt
+  conf/lang/stopwords_ca.txt
+  conf/lang/stopwords_cz.txt
+  conf/lang/stopwords_da.txt
+  conf/lang/stopwords_de.txt
+  conf/lang/stopwords_el.txt
+  conf/lang/stopwords_en.txt
+  conf/lang/stopwords_es.txt
+  conf/lang/stopwords_eu.txt
+  conf/lang/stopwords_fa.txt
+  conf/lang/stopwords_fi.txt
+  conf/lang/stopwords_fr.txt
+  conf/lang/stopwords_ga.txt
+  conf/lang/stopwords_gl.txt
+  conf/lang/stopwords_hi.txt
+  conf/lang/stopwords_hu.txt
+  conf/lang/stopwords_hy.txt
+  conf/lang/stopwords_id.txt
+  conf/lang/stopwords_it.txt
+  conf/lang/stopwords_ja.txt
+  conf/lang/stopwords_lv.txt
+  conf/lang/stopwords_nl.txt
+  conf/lang/stopwords_no.txt
+  conf/lang/stopwords_pt.txt
+  conf/lang/stopwords_ro.txt
+  conf/lang/stopwords_ru.txt
+  conf/lang/stopwords_sv.txt
+  conf/lang/stopwords_th.txt
+  conf/lang/stopwords_tr.txt
+  conf/lang/userdict_ja.txt
+]
+
+
+