mamiya 0.0.1.alpha21 → 0.0.1.alpha22

data/lib/mamiya/steps/build.rb

@@ -1,24 +1,73 @@
 require 'mamiya/steps/abstract'
 require 'mamiya/package'
 
+require 'fileutils'
+
 module Mamiya
   module Steps
     class Build < Abstract
+      class ScriptFileNotSpecified < Exception; end
+      class ApplicationNotSpecified < Exception; end
+
       def run!
         @exception = nil
 
+        unless script_file
+          raise ScriptFileNotSpecified, "Set script files to :script_file"
+        end
+
+        unless script.application
+          raise ApplicationNotSpecified, ":application should be specified in your script file"
+        end
+
         logger.info "Initiating package build"
 
+        run_before_build
+        run_prepare_build
+        run_build
+
+        # XXX: Is this really suitable here? Package class should do?
+        copy_deploy_scripts
+
+        set_metadata
+
+        build_package
+
+        logger.info "Packed."
+
+      rescue Exception => e
+        @exception = e
+        raise
+      ensure
+        logger.warn "Exception occured, cleaning up..." if @exception
+
+        if script_dest.exist?
+          FileUtils.remove_entry_secure script_dest
+        end
+
+        logger.info "Running script.after_build"
+        script.after_build[@exception]
+
+        logger.info "DONE!" unless @exception
+      end
+
+      private
+
+      def run_before_build
         logger.info "Running script.before_build"
         script.before_build[]
+      end
 
+      def run_prepare_build
         unless script.skip_prepare_build
           logger.info "Running script.prepare_build"
           script.prepare_build[File.exists?(script.build_from)]
         else
           logger.debug "prepare_build skipped due to script.skip_prepare_build"
         end
+      end
 
+      def run_build
         old_pwd = Dir.pwd
         begin
           # Using without block because chdir in block shows warning
@@ -29,24 +78,42 @@ module Mamiya
         ensure
           Dir.chdir old_pwd
         end
+      end
 
+      def copy_deploy_scripts
+        # XXX: TODO: move to another class?
+        logger.info "Copying script files..."
 
-        logger.debug "Determining package name..."
-        package_name = Dir.chdir(script.build_from) {
-          script.package_name[
-            [Time.now.strftime("%Y-%m-%d_%H.%M.%S"), script.application]
-          ].join('-')
-        }
-        logger.info "Package name determined: #{package_name}"
+        if script_dest.exist?
+          logger.warn "Removing existing .mamiya.script"
+          FileUtils.remove_entry_secure script_dest
+        end
+        script_dest.mkdir
 
-        package_path = File.join(script.build_to, package_name)
-        package = Mamiya::Package.new(package_path)
-        package.meta[:application] = script.application
+        logger.info "- #{script_file} -> #{script_dest}"
+        FileUtils.cp script_file, script_dest
+
+        if script.script_additionals
+          script_dir = Pathname.new(File.dirname(script_file))
+          script.script_additionals.each do |additional|
+            src = script_dir.join(additional)
+            dst = script_dest.join(additional)
+            logger.info "- #{src} -> #{dst}"
+            FileUtils.mkdir_p dst.dirname
+            FileUtils.cp_r src, dst
+          end
+        end
+      end
 
+      def set_metadata
+        package.meta[:application] = script.application
+        package.meta[:script] = File.basename(script_file)
         Dir.chdir(script.build_from) do
           package.meta.replace script.package_meta[package.meta]
         end
+      end
 
+      def build_package
         logger.info "Packaging to: #{package.path}"
         logger.debug "meta=#{package.meta.inspect}"
         package.build!(script.build_from,
@@ -54,18 +121,39 @@ module Mamiya
            dereference_symlinks: script.dereference_symlinks || false,
            package_under: script.package_under || nil,
            logger: logger)
-        logger.info "Packed."
+      end
 
-      rescue Exception => e
-        @exception = e
-        raise
-      ensure
-        logger.warn "Exception occured, cleaning up..." if @exception
+      def package_name
+        @package_name ||= begin
+          logger.debug "Determining package name..."
+          name = Dir.chdir(script.build_from) {
+            script.package_name[
+              [Time.now.strftime("%Y-%m-%d_%H.%M.%S"), script.application]
+            ].join('-')
+          }
+          logger.info "Package name determined: #{name}"
+          name
+        end
+      end
 
-        logger.info "Running script.after_build"
-        script.after_build[@exception]
+      def package_path
+        @package_path ||= File.join(script.build_to, package_name)
+      end
 
-        logger.info "DONE!" unless @exception
+      def package
+        @package ||= Mamiya::Package.new(package_path)
+      end
+
+      def script_file
+        @script_file ||= script.script_file || script._file
+      end
+
+      def script_dest
+        @script_dest ||= if script.package_under
+          Pathname.new File.join(script.build_from, script.package_under, '.mamiya.script')
+        else
+          Pathname.new File.join(script.build_from, '.mamiya.script')
+        end
       end
     end
   end

data/lib/mamiya/steps/extract.rb

@@ -4,6 +4,7 @@ require 'mamiya/steps/abstract'
 module Mamiya
   module Steps
     class Extract < Abstract
+      # XXX: extract step is really needed? doing this in prepare step for consistency with agent/tasks/prepare, is better, I guess.
       def run!
         package = case options[:package]
                   when Mamiya::Package

data/lib/mamiya/steps/prepare.rb

@@ -0,0 +1,60 @@
+require 'mamiya/steps/abstract'
+
+require 'pathname'
+require 'json'
+
+module Mamiya
+  module Steps
+    class Prepare < Abstract
+      def run!
+        @exception = nil
+        old_pwd = Dir.pwd
+        Dir.chdir(target)
+
+        logger.info "Preparing #{target}..."
+
+        script.before_prepare(labels)[]
+        script.prepare(labels)[]
+
+        File.write target.join('.mamiya.prepared'), "#{Time.now.to_i}\n"
+      rescue Exception => e
+        @exception = e
+        raise e
+      ensure
+        Dir.chdir old_pwd if old_pwd
+        logger.warn "Exception occured, cleaning up..." if @exception
+
+        script.after_prepare(labels)[@exception]
+
+        logger.info "DONE!" unless @exception
+      end
+
+      # This class see target_dir's script
+      alias given_script script 
+
+      def script
+        @target_script ||= Mamiya::Script.new.load!(
+          target.join('.mamiya.script', target_meta['script'])).tap do |script|
+          # XXX: release_path is set by options[:target] but deploy_to is set by script?
+          script.set(:release_path, target)
+          script.set(:logger, logger)
+        end
+      end
+
+      private
+
+      def target
+        @target ||= Pathname.new(options[:target]).realpath
+      end
+
+      def target_meta
+        @target_meta ||= JSON.parse target.join('.mamiya.meta.json').read
+      end
+
+      def labels
+        # XXX: TODO: is it sure that passing labels via options of step?
+        options[:labels]
+      end
+    end
+  end
+end

data/lib/mamiya/steps/switch.rb

@@ -0,0 +1,76 @@
+require 'mamiya/steps/abstract'
+
+module Mamiya
+  module Steps
+    class Switch < Abstract
+      def run!
+        @exception = nil
+        logger.info "Switching to #{target}"
+
+        script.before_switch(labels)[]
+
+        # TODO: link with relative if available?
+        File.unlink script.current_path if script.current_path.symlink?
+        script.current_path.make_symlink(target.realpath)
+
+        if do_release?
+          begin
+            old_pwd = Dir.pwd
+            Dir.chdir(target)
+
+            logger.info "Releasing..."
+
+            script.release(labels)[@exception]
+          ensure
+            Dir.chdir old_pwd if old_pwd
+          end
+        else
+          logger.warn "Skipping release (:no_release is set)"
+        end
+
+      rescue Exception => e
+        @exception = e
+        raise e
+      ensure
+        logger.warn "Exception occured, cleaning up..." if @exception
+
+        script.after_switch(labels)[@exception]
+
+        logger.info "DONE!" unless @exception
+      end
+
+      # XXX: dupe with prepare step. modulize?
+
+      # This class see target_dir's script
+      alias given_script script 
+
+      def script
+        @target_script ||= Mamiya::Script.new.load!(
+          target.join('.mamiya.script', target_meta['script'])).tap do |script|
+          # XXX: release_path is set by options[:target] but deploy_to is set by script?
+          script.set(:release_path, target)
+          script.set(:logger, logger)
+        end
+      end
+
+      private
+
+      def do_release?
+        !options[:no_release]
+      end
+
+      def target
+        @target ||= Pathname.new(options[:target]).realpath
+      end
+
+      def target_meta
+        @target_meta ||= JSON.parse target.join('.mamiya.meta.json').read
+      end
+
+      def labels
+        # XXX: TODO: is it sure that passing labels via options of step?
+        options[:labels]
+      end
+    end
+  end
+end

data/lib/mamiya/storages/filesystem.rb

@@ -0,0 +1,92 @@
+require 'mamiya/package'
+require 'mamiya/storages/abstract'
+require 'fileutils'
+
+module Mamiya
+  module Storages
+    class Filesystem < Mamiya::Storages::Abstract
+      def self.find(config={})
+        Hash[Dir[File.join(config[:path], '*')].map do |app_path|
+          app = File.basename(app_path)
+          [app, self.new(config.merge(application: app))]
+        end]
+      end
+
+      def packages
+        storage_path.children(false).group_by { |child|
+          child.to_s.sub(Package::PATH_SUFFIXES,'')
+        }.select { |key, files|
+          files.find { |file| file.to_s.end_with?('.tar.gz') } &&
+          files.find { |file| file.to_s.end_with?('.json') }
+        }.keys.sort
+      end
+
+      def push(package)
+        raise TypeError, "package should be a kind of Mamiya::Package" unless package.kind_of?(Mamiya::Package)
+        raise NotBuilt, "package not built" unless package.exists?
+
+        if package_exist?(package.name)
+          raise AlreadyExists 
+        end
+
+        storage_path.mkpath
+
+        FileUtils.cp package.path, storage_path.join("#{package.name}.tar.gz")
+        FileUtils.cp package.meta_path, storage_path.join("#{package.name}.json")
+      end
+
+      def fetch(package_name, destination)
+        package_name = normalize_package_name(package_name)
+        raise NotFound unless package_exist?(package_name)
+
+        package_path = File.join(destination, "#{package_name}.tar.gz")
+        meta_path = File.join(destination, "#{package_name}.json")
+
+        if File.exists?(package_path) || File.exists?(meta_path)
+          raise AlreadyFetched
+        end
+
+        FileUtils.cp storage_path.join("#{package_name}.tar.gz"), package_path
+        FileUtils.cp storage_path.join("#{package_name}.json"), meta_path
+        
+        return Mamiya::Package.new(package_path)
+      end
+
+      def meta(package_name)
+        package_name = normalize_package_name(package_name)
+        return unless package_exist?(package_name)
+
+        JSON.parse storage_path.join("#{package_name}.json").read
+      end
+
+      def remove(package_name)
+        package_name = normalize_package_name(package_name)
+
+        package_path = storage_path.join("#{package_name}.tar.gz")
+        meta_path = storage_path.join("#{package_name}.json")
+
+        if [package_path, meta_path].all? { |_| !_.exist? }
+          raise Mamiya::Storages::Abstract::NotFound
+        end
+
+        package_path.delete if package_path.exist?
+        meta_path.delete if meta_path.exist?
+      end
+
+      private
+
+      def storage_path
+        @storage_path ||= Pathname.new(@config[:path]).join(@config[:application])
+      end
+
+      def package_exist?(name)
+        storage_path.join("#{name}.tar.gz").exist? &&
+          storage_path.join("#{name}.json").exist?
+      end
+
+      def normalize_package_name(name)
+        name.sub(/\.(?:tar\.gz|json)\z/, '')
+      end
+    end
+  end
+end

data/lib/mamiya/storages/mock.rb

@@ -14,6 +14,7 @@ module Mamiya
       end
 
       def self.find(config={})
+        # TODO: FIXME: Should be an Hash
         storage.keys
       end
 

data/lib/mamiya/util/label_matcher.rb

@@ -1,8 +1,12 @@
 module Mamiya
   module Util
     module LabelMatcher
+      def self.parse_string_expr(str)
+        str.split(/\|/).map{ |_| _.split(/,/) }
+      end
+
       def match?(*expressions)
-        labels = self.labels()
+        labels = self.labels().map(&:to_s)
 
         if expressions.all? { |_| _.kind_of?(Symbol) || _.kind_of?(String) }
           return self.match?(expressions)
@@ -11,12 +15,12 @@ module Mamiya
         expressions.any? do |expression|
           case expression
           when Symbol, String
-            labels.include?(expression)
+            labels.include?(expression.to_s)
           when Array
             if expression.any? { |_| _.kind_of?(Array) }
               self.match?(*expression)
             else
-              expression.all? { |_| labels.include?(_) }
+              expression.all? { |_| labels.include?(_.to_s) }
             end
           end
         end

data/lib/mamiya/version.rb

@@ -1,3 +1,3 @@
 module Mamiya
-  VERSION = "0.0.1.alpha21"
+  VERSION = "0.0.1.alpha22"
 end

data/mamiya.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "aws-sdk-core", "2.0.0.rc15"
   spec.add_runtime_dependency "term-ansicolor", ">= 1.3.0"
   unless ENV["MAMIYA_VILLEIN_PATH"]
-    spec.add_runtime_dependency "villein", ">= 0.3.2"
+    spec.add_runtime_dependency "villein", ">= 0.5.0"
   end
 
   spec.add_runtime_dependency "sinatra", ">= 1.4.5"

data/spec/agent/actions_spec.rb

@@ -28,5 +28,30 @@ describe Mamiya::Agent::Actions do
 
       agent.distribute('myapp', 'mypkg')
     end
+
+    context "with labels" do
+      it "adds _labels on task" do
+        expect(agent).to receive(:trigger).with('task', task: 'fetch', app: 'myapp', pkg: 'mypkg', _labels: ['foo'], coalesce: false)
+
+        agent.distribute('myapp', 'mypkg', labels: ['foo'])
+      end
+    end
+
+  end
+
+  describe "#prepare" do
+    it "sends prepare request" do
+      expect(agent).to receive(:trigger).with('task', task: 'prepare', app: 'myapp', pkg: 'mypkg', coalesce: false)
+
+      agent.prepare('myapp', 'mypkg')
+    end
+
+    context "with labels" do
+      it "adds _labels on task" do
+        expect(agent).to receive(:trigger).with('task', task: 'prepare', app: 'myapp', pkg: 'mypkg', _labels: ['foo'], coalesce: false)
+
+        agent.prepare('myapp', 'mypkg', labels: ['foo'])
+      end
+    end
   end
 end

data/spec/agent/task_queue_spec.rb

@@ -103,6 +103,42 @@ describe Mamiya::Agent::TaskQueue do
       expect(task_class_a.runs[1]['foo']).to eq '2'
     end
 
+    describe "(task with _labels)" do
+      it "runs on matched agent" do
+        queue.start!
+        expect(agent).to receive(:match?).with(['foo', 'bar']).and_return(true)
+
+        queue.enqueue(:a, 'foo' => '1', '_labels' => ['foo', 'bar'])
+
+        100.times { break if task_class_a.runs.size == 1; sleep 0.01 }
+        expect(task_class_a.runs.size).to eq 1
+        expect(task_class_a.runs[0]['foo']).to eq '1'
+      end
+
+      it "doesn't run on not matched agent" do
+        queue.start!
+        expect(agent).to receive(:match?).with(['foo', 'bar']).and_return(false)
+
+        queue.enqueue(:a, 'foo' => '1', '_labels' => ['foo', 'bar'])
+        queue.enqueue(:a, 'foo' => '2')
+
+        100.times { break if task_class_a.runs.size == 1; sleep 0.01 }
+        expect(task_class_a.runs.size).to eq 1
+        expect(task_class_a.runs[0]['foo']).to eq '2'
+      end
+
+      it "removes _labels from task on run" do
+        queue.start!
+        expect(agent).to receive(:match?).with(['foo', 'bar']).and_return(true)
+
+        queue.enqueue(:a, 'foo' => '1', '_labels' => ['foo', 'bar'])
+
+        100.times { break if task_class_a.runs.size == 1; sleep 0.01 }
+        expect(task_class_a.runs.size).to eq 1
+        expect(task_class_a.runs[0].key?('_labels')).to be_false
+      end
+    end
+
     describe "#working?" do
       it "returns true if there're any working tasks" do
         queue.start!
@@ -134,12 +170,12 @@ describe Mamiya::Agent::TaskQueue do
 
         100.times { break unless task_class_a.locks.empty?; sleep 0.01 }
         expect(task_class_a.locks).not_to be_empty
-        expect(queue.status[:a][:working]).to eq('wait' => true, 'id' => 1)
+        expect(queue.status[:a][:working]).to eq('wait' => true, 'id' => 1, 'task' => :a)
 
         queue.enqueue(:a, 'id' => 2)
         100.times { break unless queue.status[:a][:queue].empty?; sleep 0.01 }
         expect(queue.status[:a][:queue].size).to eq 1
-        expect(queue.status[:a][:queue].first).to eq('id' => 2)
+        expect(queue.status[:a][:queue].first).to eq('id' => 2, 'task' => :a)
 
         task_class_a.locks.values.last << true
 
@@ -200,16 +236,16 @@ describe Mamiya::Agent::TaskQueue do
           expect(task_class_a.locks).not_to be_empty
           expect(task_class_b.locks).not_to be_empty
 
-          expect(queue.status[:a][:working]).to eq('wait' => true, 'id' => 1)
-          expect(queue.status[:b][:working]).to eq('wait' => true, 'id' => 2)
+          expect(queue.status[:a][:working]).to eq('wait' => true, 'id' => 1, 'task' => :a)
+          expect(queue.status[:b][:working]).to eq('wait' => true, 'id' => 2, 'task' => :b)
 
           queue.enqueue(:a, 'id' => 3)
           queue.enqueue(:b, 'id' => 4)
           100.times { break if !queue.status[:a][:queue].empty? && !queue.status[:b][:queue].empty?; sleep 0.01 }
           expect(queue.status[:a][:queue].size).to eq 1
-          expect(queue.status[:a][:queue].first).to eq('id' => 3)
+          expect(queue.status[:a][:queue].first).to eq('id' => 3, 'task' => :a)
           expect(queue.status[:b][:queue].size).to eq 1
-          expect(queue.status[:b][:queue].first).to eq('id' => 4)
+          expect(queue.status[:b][:queue].first).to eq('id' => 4, 'task' => :b)
 
           task_class_a.locks.values.last << true
           task_class_b.locks.values.last << true

data/spec/agent/tasks/abstract_spec.rb

@@ -41,6 +41,41 @@ describe Mamiya::Agent::Tasks::Abstract do
 
       expect(task.error).to eq err
     end
+
+    context "with _chain" do
+      let(:job) { {'foo' => 'bar', '_chain' => ['another']} }
+
+      it "enqueues next _chain, keeping same task specification" do
+        expect(queue).to receive(:enqueue).with(:another,
+          'foo' => 'bar',
+        )
+
+        task.execute
+      end
+    end
+
+    context "with multiple _chain" do
+      let(:job) { {'foo' => 'bar', '_chain' => ['b', 'c']} }
+
+      it "enqueues next _chain, keeping same task specification" do
+        expect(queue).to receive(:enqueue).with(:b,
+          'foo' => 'bar',
+          '_chain' => ['c'],
+        )
+
+        task.execute
+      end
+    end
+
+    context "with empty _chain" do
+      let(:job) { {'foo' => 'bar', '_chain' => []} }
+
+      it "doesn't enqueue nothing" do
+        expect(queue).not_to receive(:enqueue)
+        task.execute
+      end
+    end
+
   end
 
   describe ".identifier" do