mamiya 0.0.1.alpha21 → 0.0.1.alpha22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4a0a40c67ec9701a56d9d5454ed090bca4f8c7c4
-  data.tar.gz: c6989472920d7302838ca131b425b62715383809
+  metadata.gz: 132e823632be17c5738d084b0c0137d92846408f
+  data.tar.gz: 19a669e42c52946e603884bf9a27816e7b22bbf7
 SHA512:
-  metadata.gz: 60cfeb34db85ab130c792c28723e91518f2367a6bf29ff9221b7b969fa956c8fb2dfd12f69e0b44143081619ac346862270ab6c46b6afb4d343cb1fe2c7ac815
-  data.tar.gz: fc6c115392f4dca0acd05aa5c4708d796a29086b4c8863a65432a82206c397829b065d063bc909a98adb5bf09d70cb6b3bbf08cd5fc9218735d55f50dc0eb62c
+  metadata.gz: d1ad837469c4bf7ddbeb9bb2d740070e9705344df425c2047b8f9ccec4e3a0a75626533f19f7b7a189407f147be538d7b628d884633bf68c5f969f2773df20c2
+  data.tar.gz: 80e843665be039f845917ce97bc9c47bced11d2f147096868617d6c0d0306e407e600a95516dbab64a4e08ad3cd976358dd11df75939baa4a6297e8aa594113a

data/example/.gitignore

@@ -0,0 +1,5 @@
+target/
+packages/*
+source/built_at
+targets/
+builds/

data/example/Procfile

@@ -1,2 +1,2 @@
 master: bundle exec env MAMIYA_SYNC_OUT=1 mamiya master -d --serf bind=0.0.0.0:7760,rpc_addr=127.0.0.1:17760,node=${HOSTNAME}_${PORT}
-agent: bundle exec env MAMIYA_SYNC_OUT=1 sh -c 'sleep 2; mamiya agent -d --serf bind=0.0.0.0:$PORT,rpc_addr=127.0.0.1:1$PORT,join=127.0.0.1:7760,node=${HOSTNAME}_${PORT}'
+agent: bundle exec env MAMIYA_SYNC_OUT=1 sh -c 'sleep 2; mamiya agent -C config.agent.rb -d'

data/example/README.md

@@ -0,0 +1,83 @@
+## mamiya example configuration
+
+You can start both master and agent(s) using foreman, in local.
+
+- `deploy.rb`: mamiya script file (for package building, preparing)
+- `config.rb`: master node configuration
+- `config.agent.rb`: agent node configuration
+
+
+### Building and pushing packages
+
+- specify `$S3_BUCKET`, `$AWS_ACCESS_KEY_ID`, and `$AWS_SECRET_ACCESS_KEY` to use `S3` storage.
+  - if not all specified, defaults to `Filesystem` storage uses `./packages` directory.
+
+```
+$ bundle exec mamiya build
+09/09 09:41:00   INFO | [Build] Initiating package build
+09/09 09:41:00   INFO | [Build] Running script.before_build
+09/09 09:41:00   INFO | [Build] Running script.prepare_build
+09/09 09:41:00   INFO | [Build] Running script.build ...
+09/09 09:41:00   INFO | [Build] Copying script files...
+09/09 09:41:00   INFO | [Build] - /.../example/deploy.rb -> /.../example/source/.mamiya.script
+09/09 09:41:00   INFO | [Build] Package name determined: 2014-09-09_09.41.00-myapp
+09/09 09:41:00   INFO | [Build] Packaging to: /.../example/builds/2014-09-09_09.41.00-myapp.tar.gz
+09/09 09:41:00   INFO | [Build] Packed.
+09/09 09:41:00   INFO | [Build] Running script.after_build
+09/09 09:41:00   INFO | [Build] DONE!
+
+$ tree packages
+builds
+├── 2014-09-09_09.41.00-myapp.json
+└── 2014-09-09_09.41.00-myapp.tar.gz
+
+0 directories, 2 files
+```
+
+```
+$ bundle exec mamiya push builds/2014-09-09_09.41.00-myapp.tar.gz
+$ bundle exec mamiya list-packages
+$ bundle exec mamiya fetch 2014-09-09_09.41.00-myapp /tmp/
+```
+
+### Test steps individually (without agents)
+
+```
+$ bundle exec mamiya extract /tmp/2014-09-09_09.41.00-myapp.tar.gz /tmp/test
+09/09 09:41:00   INFO | [Extract] Extracting /tmp/2014-09-09_09.41.00-myapp.tar.gz onto /tmp/test/2014-09-09_09.41.00-myapp
+```
+
+```
+$ bundle exec mamiya prepare /tmp/test/2014-09-09_09.41.00-myapp
+09/09 09:41:00   INFO | [Prepare] Preparing /private/tmp/test/2014-09-09_09.41.00-myapp...
+```
+
+### master and agents
+
+#### Start cluster in local
+
+You have to install [Serf](https://www.serfdom.io/) to run.
+
+```
+$ foreman start
+
+(or)
+
+$ foreman start -m 'master=1,agent=2'
+```
+
+#### Test `mamiya client`
+
+```
+$ bundle exec mamiya client list-packages -a myapp
+2014-09-09_09.41.00-myapp
+
+$ bundle exec mamiya client show-package -a myapp 2014-09-09_09.41.00-myapp
+{"application"=>"myapp",
+ "name"=>"2014-09-09_09.41.00-myapp",
+ "meta"=>
+  {"application"=>"myapp",
+   "script"=>"deploy.rb",
+   "name"=>"2014-09-09_09.41.00-myapp",
+   "checksum"=>"..."}}
+```

data/example/config.agent.rb

@@ -0,0 +1,40 @@
+require 'pathname'
+
+if ENV['MAMIYA_S3_BUCKET'] || ENV['S3_BUCKET']
+  set :storage, {
+    type: :s3,
+    bucket: ENV["MAMIYA_S3_BUCKET"] || ENV["S3_BUCKET"],
+    region: ENV["AWS_REGION"] || 'ap-northeast-1',
+  }
+else
+  set :storage, {
+    type: :filesystem,
+    path: File.join(File.dirname(__FILE__), 'packages'),
+  }
+end
+
+set :serf, {
+  agent: {
+    bind: "0.0.0.0:#{ENV['PORT']}",
+    rpc_addr: "127.0.0.1:#{ENV['PORT']}",
+    join: "127.0.0.1:7760",
+    node: "#{ENV['HOSTNAME']}_#{ENV['PORT']}",
+  }
+}
+p serf
+
+agent_name = ENV['PORT'].to_s
+
+targets = Pathname.new(File.dirname(__FILE__)).join('targets')
+target = targets.join(agent_name).tap(&:mkpath)
+
+set :packages_dir, target.join('packages')
+set :prereleases_dir, target.join('prereleases')
+
+# To test
+Dir.mkdir packages_dir unless File.exist?(packages_dir)
+Dir.mkdir prereleases_dir unless File.exist?(prereleases_dir)
+
+set :keep_packages, 3
+set :keep_prereleases, 3
+set :fetch_sleep, 2

data/example/config.rb

@@ -1,9 +1,23 @@
-set :storage, {
-  type: :s3,
-  bucket: ENV["MAMIYA_S3_BUCKET"] || ENV["S3_BUCKET"] || raise("specify your S3 bucket via $MAMIYA_S3_BUCKET or $S3_BUCKET"),
-  region: ENV["AWS_REGION"] || 'ap-northeast-1',
-}
+if ENV['MAMIYA_S3_BUCKET'] || ENV['S3_BUCKET']
+  set :storage, {
+    type: :s3,
+    bucket: ENV["MAMIYA_S3_BUCKET"] || ENV["S3_BUCKET"],
+    region: ENV["AWS_REGION"] || 'ap-northeast-1',
+  }
+else
+  set :storage, {
+    type: :filesystem,
+    path: File.join(File.dirname(__FILE__), 'packages'),
+  }
+end
+
+set :packages_dir, "#{File.dirname(__FILE__)}/packages"
+set :prereleases_dir, "#{File.dirname(__FILE__)}/target/prereleases"
+
+# To test
+Dir.mkdir packages_dir unless File.exist?(packages_dir)
+prereleases_dir.mkpath unless File.exist?(prereleases_dir)
 
-set :packages_dir, "#{File.dirname(__FILE__)}/dst"
 set :keep_packages, 3
+set :keep_prereleases, 3
 set :fetch_sleep, 2

data/example/deploy.rb

@@ -10,25 +10,41 @@ set :exclude_from_package, ['tmp', 'log', 'spec', '.sass-cache']
 set :dereference_symlinks, true
 
 # http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?revision=45360&view=revision
-set :build_to, "#{File.dirname(__FILE__)}/pkg"
-set :deploy_to, "..."
+set :build_from, "#{File.dirname(__FILE__)}/source"
+set :build_to, "#{File.dirname(__FILE__)}/builds"
+
+dir = File.dirname(__FILE__)
+if File.basename(dir) == 'example'
+  set :deploy_to, File.join(dir, 'targets', 'default')
+else
+  # target/{releases,prereleases}/<name>/.mamiya.script/../../..
+  set :deploy_to, File.expand_path(File.join(dir, '..', '..', '..'))
+end
+
+# to test it
+Dir.mkdir build_to unless File.exist?(build_to)
+deploy_to.mkpath unless deploy_to.exist?
 
 set :bundle_without, [:development, :test]
 set :bundle_dir, "#{deploy_to}/shared/bundle"
 
 #use :git, exclude_git_clean_targets: true
 
-prepare_build("bundle install") do
-  run "bundle", "install"
-end
+#prepare_build("bundle install") do
+#  run "bundle", "install"
+#end
 
-build("include assets and .bundle") do
-  exclude_from_package.reject! { |_| _ == 'public/assets/' }
-  exclude_from_package.reject! { |_| _ == '.bundle/' }
+build("test") do
+  File.write('built_at', "#{Time.now}\n")
 end
 
-build("assets compile") do
-  run "bundle", "exec", "rake", "assets:precompile"
-end
+#build("include assets and .bundle") do
+#  exclude_from_package.reject! { |_| _ == 'public/assets/' }
+#  exclude_from_package.reject! { |_| _ == '.bundle/' }
+#end
+#
+#build("assets compile") do
+#  run "bundle", "exec", "rake", "assets:precompile"
+#end
 
 

data/example/source/README.md

@@ -0,0 +1 @@
+Hello

data/lib/mamiya/agent/actions.rb

@@ -1,7 +1,8 @@
 module Mamiya
   class Agent
     module Actions
-      def order_task(task, coalesce: false, **payload)
+      def order_task(task, coalesce: false, labels: nil, **payload)
+        payload[:_labels] = labels if labels
         trigger('task',
           coalesce: coalesce,
           task: task,
@@ -9,8 +10,12 @@ module Mamiya
         )
       end
 
-      def distribute(application, package)
-        order_task('fetch', app: application, pkg: package)
+      def distribute(application, package, labels: nil)
+        order_task('fetch', app: application, pkg: package, labels: labels)
+      end
+
+      def prepare(application, package, labels: nil)
+        order_task('prepare', app: application, pkg: package, labels: labels)
       end
     end
   end

data/lib/mamiya/agent/task_queue.rb

@@ -84,6 +84,15 @@ module Mamiya
       def enqueue(task_name, task)
         raise Stopped, 'this task queue is stopped' unless running?
 
+        if task['_labels'] && !agent.match?(task['_labels'])
+          @logger.debug "skipping enqueue #{task_name.inspect}, #{task.inspect}, because agent doesn't match"
+          return self
+        end
+
+        task.delete '_labels'
+        task.delete 'task'
+        task['task'] = task_name
+
         @logger.debug "enqueue #{task_name.inspect}, #{task.inspect}, #{@external_queue.inspect}"
         @external_queue << [task_name, task]
         self

data/lib/mamiya/agent/tasks/abstract.rb

@@ -37,6 +37,7 @@ module Mamiya
           errored
           @logger.error "Encountered error: #{error.inspect}\n\t#{error.backtrace.join("\n\t")}"
         ensure
+          enqueue_chained unless error
           after
           @logger.info "Task finished"
         end
@@ -55,6 +56,18 @@ module Mamiya
 
         private
 
+        def enqueue_chained
+          return if !task['_chain'] || task['_chain'].empty?
+
+          next_task = task.dup
+          next_task.delete('task')
+
+          next_task_name, *next_task['_chain'] = task['_chain']
+          next_task.delete('_chain') if next_task['_chain'].empty?
+
+          task_queue.enqueue(next_task_name.to_sym, next_task)
+        end
+
         def config
           @config ||= agent ? agent.config : nil
         end

data/lib/mamiya/agent/tasks/clean.rb

@@ -6,8 +6,14 @@ module Mamiya
       class Clean < Abstract
 
         def run
-          victims.each do |app, victim|
-            @logger.info "Cleaning up: remove #{victim}"
+          # XXX:
+          clean_packages
+          clean_prereleases
+        end
+
+        def clean_packages
+          package_victims.each do |app, victim|
+            @logger.info "Cleaning package: remove #{victim}"
             File.unlink(victim) if File.exist?(victim)
 
             meta_victim = victim.sub(/\.tar\.gz\z/, '.json')
@@ -16,9 +22,10 @@ module Mamiya
               File.unlink(meta_victim)
             end
 
+            # XXX: depends on FS structure
             package_name = File.basename(victim, '.tar.gz')
 
-            # XXX: depends on FS structure
+            # XXX: TODO: application->app, package->pkg
             agent.trigger('pkg', action: 'remove',
               application: app,
               package: package_name,
@@ -27,7 +34,7 @@ module Mamiya
           end
         end
 
-        def victims
+        def package_victims
           Dir[File.join(config[:packages_dir], '*')].flat_map do |app|
             packages = Dir[File.join(app, "*.tar.gz")].
               sort_by { |_| [File.mtime(_), _] }
@@ -38,6 +45,31 @@ module Mamiya
           end
         end
 
+        def clean_prereleases
+          prerelease_victims.each do |app, victim|
+            @logger.info "Cleaning prerelease: remove #{victim}"
+            package_name = File.basename(victim)
+            FileUtils.remove_entry_secure victim
+
+            agent.trigger('prerelease', action: 'remove',
+              app: app,
+              pkg: package_name,
+              coalesce: false,
+            )
+          end
+        end
+
+        def prerelease_victims
+          Dir[File.join(config[:prereleases_dir], '*')].flat_map do |app|
+            prereleases = Dir[File.join(app, "*")].
+              sort_by { |_| [File.mtime(_), _] }
+
+            prereleases[0...-(config[:keep_prereleases])].map do |victim|
+              [File.basename(app), victim]
+            end
+          end
+        end
+
       end
     end
   end

data/lib/mamiya/agent/tasks/fetch.rb

@@ -63,6 +63,7 @@ module Mamiya
             package: package,
             destination: destination,
             config: config,
+            logger: logger,
           )
         end
       end

data/lib/mamiya/agent/tasks/notifyable.rb

@@ -1,4 +1,3 @@
-require 'mamiya/agent'
 require 'mamiya/agent/tasks/abstract'
 
 module Mamiya

data/lib/mamiya/agent/tasks/prepare.rb

@@ -0,0 +1,103 @@
+require 'fileutils'
+
+require 'mamiya/agent/tasks/notifyable'
+require 'mamiya/steps/extract'
+require 'mamiya/steps/prepare'
+
+module Mamiya
+  class Agent
+    module Tasks
+      class Prepare < Notifyable
+        def execute
+          return unless check
+          super
+        end
+
+        def check
+          unless package_path.exist?
+            new_chain = ['prepare'] + (task['_chain'] || [])
+            logger.info "Package not fetched, enqueueing fetch task with #{new_chain.inspect}"
+            task_queue.enqueue(
+              :fetch,
+              task.merge('_chain' => new_chain)
+            )
+            return false
+          end
+
+          true
+        end
+
+        def run
+          if prerelease_path.exist? 
+            if prerelease_path.join('.mamiya.prepared').exist?
+              return
+            else
+              FileUtils.remove_entry_secure prerelease_path
+            end
+          end
+
+          packages_dir.join(application).mkpath
+          prereleases_dir.join(application).mkpath
+
+          extract_step.run!
+          prepare_step.run!
+        end
+
+        private
+
+        def application
+          task['app']
+        end
+
+        def package
+          task['pkg']
+        end
+
+        def release_name
+          task['release'] || package
+        end
+
+        def packages_dir
+          @packages_dir ||= config.packages_dir
+        end
+
+        def prereleases_dir
+          @prereleases_dir ||= config.prereleases_dir
+        end
+
+        def package_path
+          packages_dir.join(application, "#{package}.tar.gz")
+        end
+
+        def prerelease_path
+          prereleases_dir.join(application, release_name)
+        end
+
+
+        def labels
+          @labels ||= agent.labels
+        end
+
+
+        def extract_step
+          @extract_step ||= Mamiya::Steps::Extract.new(
+            package: package_path,
+            destination: prerelease_path,
+            config: config,
+            logger: logger,
+          )
+        end
+
+        def prepare_step
+          @prepare_step ||= Mamiya::Steps::Prepare.new(
+            script: nil,
+            target: prerelease_path,
+            config: config,
+            labels: labels,
+            logger: logger,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/agent.rb

@@ -2,12 +2,17 @@ require 'thread'
 require 'villein'
 require 'mamiya/version'
 
+require 'mamiya/util/label_matcher'
+
 require 'mamiya/logger'
 
 require 'mamiya/steps/fetch'
+require 'mamiya/steps/prepare'
+require 'mamiya/steps/extract'
 require 'mamiya/agent/task_queue'
 
 require 'mamiya/agent/tasks/fetch'
+require 'mamiya/agent/tasks/prepare'
 require 'mamiya/agent/tasks/clean'
 
 require 'mamiya/agent/handlers/task'
@@ -20,6 +25,7 @@ module Mamiya
     def initialize(config, logger: Mamiya::Logger.new, events_only: nil)
       @config = config
       @serf = init_serf
+      @trigger_lock = Mutex.new
       @events_only = events_only
 
       @terminate = false
@@ -32,6 +38,7 @@ module Mamiya
     def task_queue
       @task_queue ||= Mamiya::Agent::TaskQueue.new(self, logger: logger, task_classes: [
         Mamiya::Agent::Tasks::Fetch,
+        Mamiya::Agent::Tasks::Prepare,
         Mamiya::Agent::Tasks::Clean,
       ])
     end
@@ -66,14 +73,6 @@ module Mamiya
       @terminate = false
     end
 
-    def update_tags!
-      serf.tags['mamiya'] = ','.tap do |status|
-        status.concat('ready,') if status == ','
-      end
-
-      nil
-    end
-
     ##
     # Returns agent status. Used for HTTP API and `serf query` inspection.
     def status(packages: true)
@@ -81,19 +80,29 @@ module Mamiya
       {}.tap do |s|
         s[:name] = serf.name
         s[:version] = Mamiya::VERSION
+        s[:labels] = labels
 
         s[:queues] = task_queue.status
 
-        s[:packages] = self.existing_packages if packages
+        if packages
+          s[:packages] = self.existing_packages
+          s[:prereleases] = self.existing_prereleases
+        end
       end
     end
 
+    def labels
+      config.labels[[]]
+    end
+
+    include Mamiya::Util::LabelMatcher
+
     ##
     # Returns hash with existing packages (where valid) by app name.
     # Packages which has json and tarball is considered as valid.
     def existing_packages
       paths_by_app = Dir[File.join(config[:packages_dir], '*', '*.{tar.gz,json}')].group_by { |path|
-        path.split('/')[-2]
+        path.split(File::SEPARATOR)[-2]
       }
 
       Hash[
@@ -117,11 +126,34 @@ module Mamiya
       ]
     end
 
+    def existing_prereleases
+      paths_by_app = Dir[File.join(config[:prereleases_dir], '*', '*')].group_by { |path|
+        path.split(File::SEPARATOR)[-2]
+      }
+
+      Hash[
+        paths_by_app.map { |app, paths|
+          [
+            app,
+            paths.select { |path|
+              File.exist? File.join(path, '.mamiya.prepared')
+            }.map { |path|
+              File.basename(path)
+            }.sort
+          ]
+        }
+      ]
+    end
+
     def trigger(type, action: nil, coalesce: true, **payload)
       name = "mamiya:#{type}"
       name << ":#{action}" if action
 
-      serf.event(name, payload.merge(name: self.serf.name).to_json, coalesce: coalesce)
+      payload_str = payload.merge(name: self.serf.name).to_json
+
+      @trigger_lock.synchronize do
+        serf.event(name, payload_str, coalesce: coalesce)
+      end
     end
 
     private
@@ -139,7 +171,9 @@ module Mamiya
         end
 
         serf.respond('mamiya:packages') do |event|
-          self.existing_packages.to_json
+          {'packages' => self.existing_packages,
+           'prereleases' => self.existing_prereleases,
+          }.to_json
         end
       end
     end