mamiya 0.0.1.alpha2

data/lib/mamiya/agent/actions.rb

@@ -0,0 +1,12 @@
+module Mamiya
+  class Agent
+    module Actions
+      def distribute(application, package)
+        trigger('fetch',
+          application: application,
+          package: package,
+        )
+      end
+    end
+  end
+end

data/lib/mamiya/agent/fetcher.rb

@@ -0,0 +1,137 @@
+require 'thread'
+require 'mamiya/steps/fetch'
+
+require 'mamiya/storages/abstract'
+
+module Mamiya
+  class Agent
+    ##
+    # This class has a queue for fetching packages.
+    class Fetcher
+      GRACEFUL_TIMEOUT = 60
+
+      def initialize(config, logger: Mamiya::Logger.new)
+        @thread = nil
+        @queue = Queue.new
+
+        @config = config
+        @destination = config[:packages_dir]
+        @keep_packages = config[:keep_packages]
+        @current_job = nil
+
+        @logger = logger['fetcher']
+        @working = nil
+      end
+
+      attr_reader :thread
+      attr_reader :current_job
+      attr_writer :cleanup_hook
+
+      def enqueue(app, package, before: nil, &callback)
+        @queue << [app, package, before, callback]
+      end
+
+      def queue_size
+        @queue.size
+      end
+
+      def start!
+        @logger.info 'Starting...'
+
+        @thread = Thread.new(&method(:main_loop))
+        @thread.abort_on_exception = true
+      end
+
+      def stop!(graceful = false)
+        return unless @thread
+
+        if graceful
+          @queue << :suicide
+          @thread.join(GRACEFUL_TIMEOUT)
+        end
+
+        @thread.kill if @thread.alive?
+      ensure
+        @thread = nil
+      end
+
+      def running?
+        @thread && @thread.alive?
+      end
+
+      def working?
+        !!@current_job
+      end
+
+      def cleanup
+        Dir[File.join(@destination, '*')].each do |app|
+          packages = Dir[File.join(app, "*.tar.gz")]
+          packages.sort_by! { |_| [File.mtime(_), _] }
+          packages[0...-@keep_packages].each do |victim|
+            @logger.info "Cleaning up: remove #{victim}"
+            File.unlink(victim) if File.exist?(victim)
+
+            meta_victim = victim.sub(/\.tar\.gz\z/, '.json')
+            if File.exist?(meta_victim)
+              @logger.info "Cleaning up: remove #{meta_victim}"
+              File.unlink(meta_victim)
+            end
+
+            package_name = File.basename(victim, '.tar.gz')
+            if @cleanup_hook
+              @cleanup_hook.call(File.basename(app), package_name)
+            end
+          end
+        end
+      end
+
+      private
+
+      def main_loop
+        while order = @queue.pop
+          break if order == :suicide
+          handle_order(*order)
+        end
+      end
+
+      def handle_order(app, package, before_hook = nil, callback = nil)
+        @current_job = [app, package]
+        @logger.info "fetching #{app}:#{package}"
+        # TODO: Limit apps by configuration
+
+        destination = File.join(@destination, app)
+
+        Dir.mkdir(destination) unless File.exist?(destination)
+
+        before_hook.call if before_hook
+
+        # TODO: before run hook for agent.update_tags!
+        Mamiya::Steps::Fetch.new(
+          application: app,
+          package: package,
+          destination: destination,
+          config: @config,
+        ).run!
+
+        callback.call if callback
+
+        @logger.info "fetched #{app}:#{package}"
+
+        cleanup
+
+      rescue Mamiya::Storages::Abstract::AlreadyFetched => e
+        @logger.info "skipped #{app}:#{package} (already fetched)"
+        callback.call(e) if callback
+      rescue Exception => e
+        @logger.fatal "fetch failed (#{app}:#{package}): #{e.inspect}"
+        e.backtrace.each do |line|
+          @logger.fatal "\t#{line}"
+        end
+
+        callback.call(e) if callback
+      ensure
+        @current_job = nil
+      end
+    end
+  end
+end

data/lib/mamiya/agent/handlers/abstract.rb

@@ -0,0 +1,20 @@
+require 'json'
+
+module Mamiya
+  class Agent
+    module Handlers
+      class Abstract
+        def initialize(agent, event)
+          @agent = agent
+          @event = event
+          @payload = (event.payload && !event.payload.empty?) ? JSON.parse(event.payload) : {}
+        end
+
+        attr_reader :agent, :event, :payload
+
+        def run!
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/agent/handlers/fetch.rb

@@ -0,0 +1,68 @@
+require 'mamiya/agent/handlers/abstract'
+require 'mamiya/storages/abstract'
+
+module Mamiya
+  class Agent
+    module Handlers
+      class Fetch < Abstract
+        FETCH_ACK_EVENT = 'mamiya:fetch-result:ack'
+        FETCH_START_EVENT = 'mamiya:fetch-result:start'
+        FETCH_SUCCESS_EVENT = 'mamiya:fetch-result:success'
+        FETCH_ERROR_EVENT = 'mamiya:fetch-result:error'
+
+        IGNORED_ERRORS = [
+          Mamiya::Storages::Abstract::AlreadyFetched.new(''),
+        ].freeze
+
+        def run!
+          agent.serf.event(FETCH_ACK_EVENT,
+            {
+              name: agent.serf.name,
+              application: payload['application'],
+              package: payload['package'],
+              pending: agent.fetcher.queue_size.succ,
+            }.to_json
+          )
+
+          agent.fetcher.enqueue(
+            payload['application'], payload['package'],
+            before: proc {
+              agent.serf.event(FETCH_START_EVENT,
+                {
+                  name: agent.serf.name,
+                  application: payload['application'],
+                  package: payload['package'],
+                  pending: agent.fetcher.queue_size.succ,
+                }.to_json
+              )
+              agent.update_tags!
+            }
+          ) do |error|
+            if error && IGNORED_ERRORS.lazy.grep(error.class).none?
+              agent.serf.event(FETCH_ERROR_EVENT,
+                {
+                  name: agent.serf.name,
+                  application: payload['application'],
+                  package: payload['package'],
+                  error: error.inspect,
+                  pending: agent.fetcher.queue_size,
+                }.to_json
+              )
+            else
+              agent.serf.event(FETCH_SUCCESS_EVENT,
+                {
+                  name: agent.serf.name,
+                  application: payload['application'],
+                  package: payload['package'],
+                  pending: agent.fetcher.queue_size,
+                }.to_json
+              )
+            end
+
+            agent.update_tags!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/cli.rb

@@ -0,0 +1,322 @@
+require 'mamiya'
+
+require 'mamiya/config'
+require 'mamiya/script'
+require 'mamiya/logger'
+
+require 'mamiya/steps/build'
+require 'mamiya/steps/push'
+require 'mamiya/steps/fetch'
+require 'mamiya/steps/extract'
+
+require 'mamiya/agent'
+require 'mamiya/master'
+
+require 'thor'
+require 'thread'
+
+module Mamiya
+  class CLI < Thor
+    class_option :config, aliases: '-C', type: :string
+    class_option :script, aliases: '-S', type: :string
+    class_option :application, aliases: %w(-a --app), type: :string
+    class_option :debug, aliases: %w(-d), type: :boolean
+    class_option :color, type: :boolean
+    class_option :no_color, type: :boolean
+    # TODO: class_option :set, aliases: '-s', type: :array
+
+    no_commands do
+      def invoke_command(*)
+        super
+      rescue SystemExit
+      rescue Exception => e
+        logger.fatal "#{e.class}: #{e.message}"
+
+        e.backtrace.map{ |_| _.prepend("\t") }.each do |line|
+          logger.debug line
+        end
+      end
+    end
+
+    desc "status", "Show status of servers"
+    def status
+      # TODO:
+    end
+
+    desc "list-packages", "List packages in storage"
+    method_option :name_only, aliases: '-n'
+    def list_packages
+      unless options[:name_only]
+        puts "Available packages in #{application}:"
+        puts ""
+      end
+
+      puts storage.packages.sort
+    end
+
+    desc "list-applications", "List applications in storage"
+    def list_applications
+      puts _applications.keys
+    end
+
+    desc "show PACKAGE", "Show package"
+    method_option :format, aliases: %w(-f), type: :string, default: 'pp'
+    def show(package)
+      meta = storage.meta(package)
+
+      case options[:format]
+      when 'pp'
+        require 'pp'
+        pp meta
+      when 'json'
+        require 'json'
+        puts meta.to_json
+      when 'yaml'
+        require 'yaml'
+        puts meta.to_yaml
+      end
+    end
+
+    # ---
+
+    desc "deploy PACKAGE", "Run build->push->distribute->prepare->finalize"
+    def deploy
+    end
+
+    desc "rollback", "Switch back to previous release then finalize"
+    def rollback
+    end
+
+    desc "build", "Build package."
+    method_option :build_from, aliases: %w(--source -f), type: :string
+    method_option :build_to, aliases: %w(--destination -t), type: :string
+    method_option :skip_prepare_build, aliases: %w(--no-prepare-build -P), type: :boolean
+    method_option :force_prepare_build, aliases: %w(--prepare-build -p), type: :boolean
+    def build
+      # TODO: overriding name
+      %i(build_from build_to).each { |k| script.set(k, File.expand_path(options[k])) if options[k] }
+
+      if options[:force_prepare_build] && options[:skip_prepare_build]
+        logger.warn 'Both force_prepare_build and skip_prepare_build are enabled. ' \
+          'This results skipping prepare_build...'
+      end
+
+      if options[:force_prepare_build]
+        script.set :skip_prepare_build, false
+      end
+
+      if options[:skip_prepare_build]
+        script.set :skip_prepare_build, true
+      end
+
+      Mamiya::Steps::Build.new(script: script, logger: logger).run!
+    end
+
+    desc "push PACKAGE", "Upload built packages to storage."
+    def push(package_atom)
+      package_path = package_path_from_atom(package_atom)
+
+      if options[:application]
+        logger.warn "Overriding package's application name with given one: #{options[:application]}"
+        sleep 2
+      end
+
+      Mamiya::Steps::Push.new(
+        config: config,
+        package: package_path,
+        application: options[:application],
+      ).run!
+    end
+
+    desc "fetch PACKAGE DESTINATION", "Retrieve package from storage"
+    def fetch(package_atom, destination)
+      Mamiya::Steps::Fetch.new(
+        script: script(:no_error),
+        config: config,
+        package: package_atom,
+        application: application,
+        destination: destination,
+      ).run!
+    end
+
+    desc "extract PACKAGE DESTINATION", "Unpack package to DESTINATION"
+    def extract(package_atom, destination)
+      package_path = package_path_from_atom(package_atom)
+
+      Mamiya::Steps::Extract.new(
+        package: package_path,
+        destination: destination
+      ).run!
+    end
+
+    desc "distribute PACKAGE", "Order clients to download specified package."
+    def distribute
+    end
+
+    desc "prepare", "Prepare package on clients."
+    def prepare
+    end
+
+    desc "finalize", "Finalize (start) prepared package on clients."
+    def finalize
+    end
+
+
+    # ---
+
+    desc "agent", "Start agent."
+    method_option :serf, type: :array
+    method_option :daemonize, aliases: '-D', type: :boolean, default: false
+    method_option :log, aliases: '-l', type: :string
+    method_option :pidfile, aliases: '-p', type: :string
+    def agent
+      prepare_agent_behavior!
+      merge_serf_option!
+
+      agent = Agent.new(config, logger: logger)
+      agent.run!
+    end
+
+    desc "master", "Start master"
+    method_option :serf, type: :array
+    method_option :daemonize, aliases: '-D', type: :boolean, default: false
+    method_option :log, aliases: '-l', type: :string
+    method_option :pidfile, aliases: '-p', type: :string
+    def master
+      prepare_agent_behavior!
+      merge_serf_option!
+
+      agent = Master.new(config, logger: logger)
+      agent.run!
+    end
+
+    # def worker
+    # end
+
+    # def event_handler
+    # end
+
+    private
+
+    def prepare_agent_behavior!
+      pidfile = File.expand_path(options[:pidfile]) if options[:pidfile]
+      logger # insitantiate
+
+      Process.daemon(:nochdir) if options[:daemonize]
+
+      if pidfile
+        open(pidfile, 'w') { |io| io.puts $$ }
+        at_exit { File.unlink(pidfile) if File.exist?(pidfile) }
+      end
+
+      reload_queue = Queue.new
+      reload_thread = Thread.new do
+        while reload_queue.pop
+          logger.info "Reopening"
+          logger.reopen
+          logger.info "Log reopened"
+        end
+      end
+      reload_thread.abort_on_exception = true
+
+      trap(:HUP) do
+        reload_queue << true
+      end
+    end
+
+    def config(dont_raise_error = false)
+      return @config if @config
+      path = [options[:config], './mamiya.yml', './config.yml'].compact.find { |_| File.exists?(_) }
+
+      if path
+        logger.debug "Using configuration: #{path}"
+        @config = Mamiya::Config.load(File.expand_path(path))
+      else
+        logger.debug "Couldn't find configuration file"
+        return nil if dont_raise_error
+        fatal! "Configuration File not found (try --config(-C) option or place it at ./mamiya.yml or ./config.yml)"
+      end
+    end
+
+    def script(dont_raise_error = false)
+      return @script if @script
+      path = [options[:script], './mamiya.rb', './deploy.rb'].compact.find { |_| File.exists?(_) }
+
+      if path
+        logger.debug "Using deploy script: #{path}"
+        @script = Mamiya::Script.new.load!(File.expand_path(path)).tap do |s|
+          s.set :application, options[:application] if options[:application]
+          s.set :logger, logger
+        end
+      else
+        logger.debug "Couldn't find deploy script."
+        return nil if dont_raise_error
+        fatal! "Deploy Script File not found (try --script(-S) option or place it at ./mamiya.rb or ./deploy.rb)"
+      end
+    end
+
+    def fatal!(message)
+      logger.fatal message
+      exit 1
+    end
+
+    def merge_serf_option!
+      (config[:serf] ||= {})[:agent] ||= {}
+
+      options[:serf].flat_map{ |_| _.split(/,/) }.each do |conf|
+        k,v = conf.split(/=/,2)
+        config[:serf][:agent][k.to_sym] = v
+      end
+    end
+
+    def application
+      options[:application] || config[:application] || script.application
+    end
+
+    def storage
+      config.storage_class.new(
+        config[:storage].merge(
+          application: application
+        )
+      )
+    end
+
+    def _applications
+      config.storage_class.find(config[:storage])
+    end
+
+    def logger
+      @logger ||= begin
+        $stdout.sync = ENV["MAMIYA_SYNC_OUT"] == '1'
+        outs = [$stdout]
+        outs << File.expand_path(options[:log]) if options[:log]
+        Mamiya::Logger.new(
+          color: options[:no_color] ? false : (options[:color] ? true : nil),
+          outputs: outs,
+          level: options[:debug] ? Mamiya::Logger::DEBUG : Mamiya::Logger.defaults[:level],
+        )
+      end
+    end
+
+    def package_path_from_atom(package_atom)
+      candidates = [
+        package_atom,
+        options[:build_to] && File.join(options[:build_to], package_atom),
+        options[:build_to] && File.join(options[:build_to], "#{package_atom}.tar.gz"),
+        script(:no_error) && script.build_to && File.join(script.build_to, package_atom),
+        script(:no_error) && script.build_to && File.join(script.build_to, "#{package_atom}.tar.gz"),
+      ]
+      logger.debug "Candidates: #{candidates.inspect}"
+
+      package_path = candidates.select { |_| _ }.find { |_| File.exists?(_) }
+
+      unless package_path
+        fatal! "Package (#{package_atom}) couldn't find at #{candidates.join(', ')}"
+      end
+
+      package_path
+    end
+  end
+end
+
+require 'mamiya/cli/client'