mamiya 0.0.1.alpha2

data/lib/mamiya/master/agent_monitor_handlers.rb

@@ -0,0 +1,44 @@
+require 'mamiya/master'
+
+module Mamiya
+  class Master
+    module AgentMonitorHandlers
+      def fetch_result__ack(status, payload, event)
+        status['fetcher'] ||= {}
+        status['fetcher']['pending'] = payload['pending']
+      end
+
+      def fetch_result__start(status, payload, event)
+        status['fetcher'] ||= {}
+        status['fetcher']['fetching'] = [payload['application'], payload['package']]
+      end
+
+      def fetch_result__error(status, payload, event)
+        status['fetcher'] ||= {}
+
+        if status['fetcher']['fetching'] == [payload['application'], payload['package']]
+          status['fetcher']['fetching'] = nil
+        end
+      end
+
+      def fetch_result__success(status, payload, event)
+        status['fetcher'] ||= {}
+
+        if status['fetcher']['fetching'] == [payload['application'], payload['package']]
+          status['fetcher']['fetching'] = nil
+        end
+
+        status['packages'] ||= {}
+        status['packages'][payload['application']] ||= []
+        status['packages'][payload['application']] << payload['package']
+      end
+
+      def fetch_result__remove(status, payload, event)
+        status['packages'] ||= {}
+        packages = status['packages'][payload['application']]
+        packages.delete(payload['package']) if packages
+      end
+
+    end
+  end
+end

data/lib/mamiya/master/web.rb

@@ -0,0 +1,148 @@
+require 'mamiya/version'
+require 'mamiya/agent'
+require 'sinatra/base'
+require 'json'
+
+module Mamiya
+  class Master < Agent
+    class Web < Sinatra::Base
+      helpers do
+        def master
+          env['mamiya.master']
+        end
+
+        def agent_monitor
+          master.agent_monitor
+        end
+
+        def storage(app)
+          master.storage(app)
+        end
+      end
+
+      get '/' do
+        "mamiya v#{Mamiya::VERSION}"
+      end
+
+      get '/packages/:application' do
+        content_type :json
+        packages = storage(params[:application]).packages
+
+        status 404 if packages.empty?
+        {packages: packages}.to_json
+      end
+
+      get '/packages/:application/:package' do
+        content_type :json
+
+        meta = storage(params[:application]).meta(params[:package])
+
+        if meta
+          {application: params[:application], name: params[:package],
+           meta: meta}.to_json
+        else
+          status 404
+          {}.to_json
+        end
+      end
+
+      get '/packages' do
+        content_type :json
+        applications = master.applications
+
+        {applications: applications}.to_json
+      end
+
+      post '/packages/:application/:package/distribute' do
+        # TODO: filter with label
+        if storage(params[:application]).meta(params[:package])
+          status 204
+          master.distribute(params[:application], params[:package])
+        else
+          status 404
+          content_type :json
+          {error: 'not found'}.to_json
+        end
+      end
+
+      get '/packages/:application/:package/distribution' do
+        # TODO: filter with label
+        content_type :json
+        meta = storage(params[:application]).meta(params[:package])
+        unless meta
+          status 404
+          next {error: 'not found'}.to_json
+        end
+
+        result = {application: params[:application], package: params[:package], distributed: [], not_distributed: []}
+        statuses = agent_monitor.statuses
+
+        statuses.each do |name, status|
+          next if status["master"]
+          if status["packages"] && status["packages"][params[:application]] &&
+            status["packages"][params[:application]].include?(params[:package])
+
+            result[:distributed] << name
+          else
+            result[:not_distributed] << name
+          end
+        end
+
+        result[:distributed_count] = result[:distributed].size
+        result[:not_distributed_count] = result[:not_distributed].size
+
+        result.to_json
+      end
+
+
+      get '/agents' do
+        statuses = agent_monitor.statuses
+        members = agent_monitor.agents
+        failed_agents = agent_monitor.failed_agents
+
+        agents = {}
+        members.each do |name, status|
+          agents[name] ||= {}
+          agents[name]["membership"] = status
+        end
+
+        statuses.each do |name, status|
+          if status["master"]
+            agents.delete name
+            next
+          end
+
+          agents[name] ||= {}
+          agents[name]["status"] = status
+        end
+
+
+        content_type :json
+
+        {
+          agents: agents,
+          failed_agents: failed_agents,
+        }.to_json
+      end
+
+      get '/agents/:name' do
+        content_type :json
+
+        status = agent_monitor.statuses[params[:name]]
+        membership = agent_monitor.agents[params[:name]]
+
+        if status || membership
+          {name: params[:name], status: status, membership: membership}.to_json
+        else
+          status 404
+          {error: 'not found'}.to_json
+        end
+      end
+
+      post '/agents/refresh' do
+        agent_monitor.refresh
+        status 204
+      end
+    end
+  end
+end

data/lib/mamiya/package.rb

@@ -0,0 +1,122 @@
+require 'mamiya/logger'
+
+require 'pathname'
+require 'digest/sha2'
+require 'json'
+
+module Mamiya
+  class Package
+    class NotExists < Exception; end
+    class InternalError < Exception; end
+    PATH_SUFFIXES = /\.(?:tar\.gz|json)\z/
+
+    def initialize(path)
+      @path_without_ext = Pathname.new(path.sub(PATH_SUFFIXES, ''))
+      @meta_loaded_from_file = false
+      @loaded_meta = nil
+      meta # load
+    end
+
+    attr_reader :path
+    attr_writer :meta
+
+    def name
+      meta['name'] || @path_without_ext.basename.to_s
+    end
+
+    def path
+      Pathname.new(@path_without_ext.to_s + '.tar.gz')
+    end
+
+    def meta_path
+      Pathname.new(@path_without_ext.to_s + '.json')
+    end
+
+    def meta
+      if !@meta_loaded_from_file && meta_path.exist?
+        @meta_loaded_from_file = true
+        loaded_meta = load_meta()
+        if @loaded_meta == @meta
+          @loaded_meta = loaded_meta
+          @meta = load_meta()
+        end
+      end
+      @meta ||= {}
+    end
+
+    def application
+      meta['application'] || meta[:application]
+    end
+
+    def checksum
+      return nil unless exist?
+      Digest::SHA2.file(path).hexdigest
+    end
+
+    def valid?
+      raise NotExists, 'package not exist' unless exist?
+      raise NotExists, 'meta not exist' unless meta_path.exist?
+      !meta['checksum'] || checksum == meta['checksum']
+    end
+
+    def exists?
+      path.exist?
+    end
+    alias exist? exists?
+
+    def build!(build_dir, exclude_from_package: [], dereference_symlinks: false, package_under: nil, logger: Mamiya::Logger.new)
+      logger = logger['Package']
+
+      exclude_from_package.push('.svn', '.git').uniq!
+
+      build_dir = Pathname.new(build_dir)
+      build_dir += package_under if package_under
+      meta_in_build = build_dir.join('.mamiya.meta.json')
+
+      meta['name'] = self.name
+      File.write meta_in_build, self.meta.to_json
+
+      Dir.chdir(build_dir) do
+        excludes = exclude_from_package.flat_map { |exclude| ['--exclude', exclude] }
+        dereference = dereference_symlinks ? ['-h'] : []
+
+        cmd = ["tar", "czf", self.path.to_s,
+               *dereference,
+               *excludes,
+               "."]
+
+        logger.debug "$ #{cmd.join(' ')}"
+        result = system(*cmd)
+        raise InternalError, "failed to run: #{cmd.inspect}" unless result
+      end
+
+      checksum = self.checksum()
+      raise InternalError, 'checksum should not be nil after package built' unless checksum
+      meta['checksum'] = checksum
+
+      File.write meta_path, self.meta.to_json
+      nil
+    ensure
+      if meta_in_build && meta_in_build.exist?
+        meta_in_build.delete()
+      end
+    end
+
+    def extract_onto!(destination)
+      raise NotExists unless exist?
+      Dir.mkdir(destination) unless File.directory?(destination)
+
+      cmd = ["tar", "xf", path.to_s, "-C", destination.to_s]
+      result = system(*cmd)
+      raise InternalError, "Failed to run: #{cmd.inspect}" unless result
+
+      nil
+    end
+
+    private
+
+    def load_meta
+      meta_path.exist? && JSON.parse(meta_path.read)
+    end
+  end
+end

data/lib/mamiya/script.rb

@@ -0,0 +1,117 @@
+require 'mamiya/dsl'
+require 'mamiya/logger'
+
+require 'shellwords'
+
+module Mamiya
+  class Script < DSL
+    class CommandFailed < Exception; end
+
+    add_hook :before_build
+    add_hook :prepare_build
+    add_hook :build
+    add_hook :after_build
+
+    add_hook :before_distribute
+    add_hook :after_distribute
+
+    add_hook :before_prepare
+    add_hook :prepare
+    add_hook :after_prepare
+
+    add_hook :before_finalize
+    add_hook :finalize
+    add_hook :after_finalize
+
+    add_hook :before_rollback
+    add_hook :rollback
+    add_hook :after_rollback
+
+    add_hook :package_name, chain: true
+    add_hook :package_meta, chain: true
+
+    set_default :application, nil
+    set_default :repository, nil
+    set_default :ref, nil
+
+    set_default :discover_servers, true
+    set_default :on_client_failure, :error # error, warn, ignore
+
+    set_default :build_from, nil
+    set_default :build_to, nil
+    set_default :package_under, nil
+    set_default :exclude_from_package, []
+    set_default :dereference_symlinks, true
+
+    set_default :package_to, nil
+
+    # TODO: use variable in config.yml
+    set_default :deploy_to, nil
+    set_default :prepare_to, nil
+
+    set_default :logger, Mamiya::Logger.new(outputs: [])
+
+    set_default :skip_prepare_build, false
+
+    def run(*args, allow_failure: false)
+      # TODO: Stop when fail
+      actual = -> do
+        logger = self.logger['RUN']
+
+        logger.info("$ #{args.shelljoin}")
+
+        err_r, err_w = IO.pipe
+        out_r, out_w = IO.pipe
+
+        pid = spawn(*args, out: out_w, err: err_w)
+
+        [out_w, err_w].each(&:close)
+
+        buf = ""
+
+        ths = {:debug => out_r, :warn => err_r}.map do |severity, io|
+          Thread.new {
+            until io.eof?
+              str = io.gets
+              logger.__send__(severity, str.chomp)
+              buf << str
+            end
+          }.tap { |_| _.abort_on_exception = true }
+        end
+
+        pid, status = Process.waitpid2(pid)
+
+        begin
+          timeout(3) { ths.each(&:join) }
+        rescue Timeout::Error
+        end
+        ths.each { |_| _.alive? && _.kill }
+
+        [out_r, err_r].each(&:close)
+
+        unless allow_failure || status.success?
+          raise CommandFailed,
+            "Excecution failed (" \
+            "status=#{status.exitstatus}" \
+            " pid=#{status.pid}" \
+            "#{status.signaled? ? "termsig=#{status.termsig.inspect} stopsig=#{status.stopsig.inspect}" : nil}" \
+            "#{status.stopped? ? " stopped" : nil}" \
+            "): #{args.inspect}"
+        end
+
+        buf
+      end
+
+      if defined? Bundler
+        Bundler.with_clean_env(&actual)
+      else
+        actual.call
+      end
+    end
+
+    def cd(*args)
+      logger.info "$ cd #{args[0]}"
+      Dir.chdir *args
+    end
+  end
+end

data/lib/mamiya/steps/abstract.rb

@@ -0,0 +1,19 @@
+require 'mamiya/config'
+require 'mamiya/script'
+require 'mamiya/logger'
+
+module Mamiya
+  module Steps
+    class Abstract
+      def initialize(script: Mamiya::Script.new, config: Mamiya::Config.new, logger: Mamiya::Logger.new, **options)
+        @script, @config, @options = script, config, options
+        @logger = logger[self.class.name.sub(/^Mamiya::Steps::/,'')]
+      end
+
+      attr_reader :script, :config, :options, :logger
+
+      def run!
+      end
+    end
+  end
+end