mamiya 0.0.1.alpha19 → 0.0.1.alpha20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37666b908bc503abe07629a7308eeb762be79cf7
-  data.tar.gz: aef8d218df47dcdb03e5ff069d7e3a5c4a78a13b
+  metadata.gz: e7ed13e8778df9ab9ce99e885013120a638a2bb6
+  data.tar.gz: 4d85eca80055453cc3aac803a6dc5c0d232d2d38
 SHA512:
-  metadata.gz: ada7d2968785edd0b9b1f51dfa53d4d8da428ca87f3e85562d05a0505553fe16b303e60d07d6478e090ff7183996a4dcc16322ac21f70038f61e181dd7d31efa
-  data.tar.gz: 044f922700e0f5d873058564b263108ef3ee7350084a5561e17550c205f45b37526ef0e1b347ad3788bb51ba080db71e1b32814b2708310e33c9e9b23367f913
+  metadata.gz: efef0fb46485a44362ae151ad79eaf6baec6ade4877bf02588b802fe553e1b2115408fda4fbf5628c77ba2d141391bfc1742341d43bfea3d76b6a2a9ee51fa58
+  data.tar.gz: c8dc1424d06eda65ce392f5d8e3acd3fa7f667c0d29c1bb6993a3cad27e099c5da2ffa97b3b573fdab045e957c060edbabdc200d1cbb1ae60d3297404f669f2f

data/README.md

@@ -34,6 +34,24 @@ Or install it yourself as:
 
 TODO: Write usage instructions here
 
+## Upgrade Notes
+
+### 0.0.1.alpha20
+
+_tl;dr_ Don't mix alpha19 and alpha20.
+
+#### Internal component for distribution has been replaced completely
+
+alpha20 introduces new class `TaskQueue` and removes `Fetcher`. This changes way to distribute packages -- including internal serf events, job tracking that Distribution API does, etc.
+So basically there's no compatibility for distribution, between alpha19 and alpha20 and later. Distribute task from alpha20 doesn't effect to alpha19, and vice versa.
+
+Good new: There's no change in Distribution API.
+
+#### Agent status has changed
+
+- Due to removal of `Fetcher`, alpha20 removes `.fetcher` object from agent status.
+- Added `.queues`, represents task queues that managed by `TaskQueue` class.
+
 ## Contributing
 
 1. Fork it ( http://github.com/sorah/mamiya/fork )

data/lib/mamiya/agent.rb

@@ -5,15 +5,17 @@ require 'mamiya/version'
 require 'mamiya/logger'
 
 require 'mamiya/steps/fetch'
-require 'mamiya/agent/fetcher'
+require 'mamiya/agent/task_queue'
 
-require 'mamiya/agent/handlers/fetch'
+require 'mamiya/agent/tasks/fetch'
+require 'mamiya/agent/tasks/clean'
+
+require 'mamiya/agent/handlers/task'
 require 'mamiya/agent/actions'
 
 module Mamiya
   class Agent
     include Mamiya::Agent::Actions
-    FETCH_REMOVE_EVENT = 'mamiya:fetch-result:remove'
 
     def initialize(config, logger: Mamiya::Logger.new, events_only: nil)
       @config = config
@@ -27,10 +29,11 @@ module Mamiya
 
     attr_reader :config, :serf, :logger
 
-    def fetcher
-      @fetcher ||= Mamiya::Agent::Fetcher.new(config, logger: logger).tap do |f|
-        f.cleanup_hook = self.method(:cleanup_handler)
-      end
+    def task_queue
+      @task_queue ||= Mamiya::Agent::TaskQueue.new(self, logger: logger, task_classes: [
+        Mamiya::Agent::Tasks::Fetch,
+        Mamiya::Agent::Tasks::Clean,
+      ])
     end
 
     def run!
@@ -53,19 +56,18 @@ module Mamiya
 
     def start
       serf_start
-      fetcher_start
+      task_queue_start
     end
 
     def terminate
       serf.stop!
-      fetcher.stop!
+      task_queue.stop!
     ensure
       @terminate = false
     end
 
     def update_tags!
       serf.tags['mamiya'] = ','.tap do |status|
-        status.concat('fetching,') if fetcher.working?
         status.concat('ready,') if status == ','
       end
 
@@ -80,11 +82,7 @@ module Mamiya
         s[:name] = serf.name
         s[:version] = Mamiya::VERSION
 
-        s[:fetcher] = {
-          fetching: fetcher.current_job,
-          pending: fetcher.queue_size,
-          pending_jobs: fetcher.pending_jobs.map{ |_| _[0,2] },
-        }
+        s[:queues] = task_queue.status
 
         s[:packages] = self.existing_packages
       end
@@ -123,13 +121,14 @@ module Mamiya
       name = "mamiya:#{type}"
       name << ":#{action}" if action
 
-      serf.event(name, payload.to_json, coalesce: coalesce)
+      serf.event(name, payload.merge(name: self.serf.name).to_json, coalesce: coalesce)
     end
 
     private
 
     def init_serf
       agent_config = (config[:serf] && config[:serf][:agent]) || {}
+      # agent_config.merge!(log: $stderr)
       Villein::Agent.new(**agent_config).tap do |serf|
         serf.on_user_event do |event|
           user_event_handler(event)
@@ -151,10 +150,9 @@ module Mamiya
       logger.debug "Serf became ready"
     end
 
-    def fetcher_start
-      logger.debug "Starting fetcher"
-
-      fetcher.start!
+    def task_queue_start
+      logger.debug "Starting task_queue"
+      task_queue.start!
     end
 
     def user_event_handler(event)
@@ -174,9 +172,14 @@ module Mamiya
 
       if Handlers.const_defined?(class_name)
         handler = Handlers.const_get(class_name).new(self, event)
-        handler.send(action || :run!)
+        meth = action || :run!
+        if handler.respond_to?(meth)
+          handler.send meth
+        else
+          logger.debug "Handler #{class_name} doesn't respond to #{meth}, skipping"
+        end
       else
-        logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
+        #logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
       end
     rescue Exception => e
       logger.fatal("Error during handling event: #{e.inspect}")
@@ -188,13 +191,5 @@ module Mamiya
     rescue JSON::ParserError
       logger.warn("Discarded event[#{event.user_event}] with invalid payload (unable to parse as json)")
     end
-
-    def cleanup_handler(app, package)
-      trigger('fetch-result', action: 'remove', coalesce: false,
-        name: self.serf.name,
-        application: app,
-        package: package,
-      )
-    end
   end
 end

data/lib/mamiya/agent/actions.rb

@@ -1,14 +1,17 @@
 module Mamiya
   class Agent
     module Actions
-      # XXX: dupe?
-      def distribute(application, package)
-        trigger('fetch',
-          application: application,
-          package: package,
-          coalesce: false
+      def order_task(task, coalesce: false, **payload)
+        trigger('task',
+          coalesce: coalesce,
+          task: task,
+          **payload,
         )
       end
+
+      def distribute(application, package)
+        order_task('fetch', app: application, pkg: package)
+      end
     end
   end
 end

data/lib/mamiya/agent/handlers/task.rb

@@ -0,0 +1,13 @@
+require 'mamiya/agent/handlers/abstract'
+
+module Mamiya
+  class Agent
+    module Handlers
+      class Task < Abstract
+        def run!
+          agent.task_queue.enqueue(payload['task'].to_sym, payload)
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/agent/task_queue.rb

@@ -0,0 +1,151 @@
+require 'mamiya/agent'
+require 'thread'
+
+# XXX: TODO: have to refactor
+module Mamiya
+  class Agent
+    class TaskQueue
+      GRACEFUL_TIMEOUT = 30
+      JOIN_TIMEOUT = 30
+
+      def initialize(agent, task_classes: [], logger: Mamiya::Logger.new)
+        @agent = agent
+        @task_classes = task_classes
+        @external_queue = Queue.new
+        @queues = {}
+        @worker_threads = nil
+        @statuses = nil
+        @queueing_thread = nil
+        @lifecycle_mutex = Mutex.new
+        @terminate = false
+        @logger = logger['task_queue']
+      end
+
+      attr_reader :worker_threads, :task_classes, :agent
+
+      def start!
+        @lifecycle_mutex.synchronize do
+          return if running?
+
+          worker_threads = {}
+          queues = {}
+          statuses = {}
+
+          @task_classes.each { |klass|
+            name = klass.identifier.to_sym
+            queue = queues[name] = Queue.new
+            statuses[name] = {pending: [], lock: Mutex.new}
+            th = worker_threads[name] = Thread.new(
+              klass, queue,
+              statuses[name],
+              &method(:worker_loop)
+            )
+            th.abort_on_exception = true
+          }
+
+          @terminate = false
+          @statuses = statuses 
+          @queues = queues
+          exqueue = @external_queue = Queue.new
+          @queueing_thread = Thread.new(queues, exqueue, statuses, &method(:queueing_loop))
+          @queueing_thread.abort_on_exception = true
+          @worker_threads = worker_threads
+        end
+      end
+
+      def stop!(graceful = false)
+        @lifecycle_mutex.synchronize do
+          return unless running?
+          @terminate = true
+          @queueing_thread.kill if @queueing_thread.alive?
+          if graceful
+            @worker_threads.each do |th|
+              th.join(GRACEFUL_TIMEOUT)
+            end
+          end
+          @worker_threads.each do |name, th|
+            next unless th.alive?
+            th.kill
+            th.join(JOIN_TIMEOUT) 
+          end
+          @queues = nil
+          @worker_threads = nil
+        end
+      end
+
+      def running?
+        @worker_threads && !@terminate
+      end
+
+      def working?
+        running? && status.any? { |name, stat| stat[:working] }
+      end
+
+      def enqueue(task_name, task)
+        raise Stopped, 'this task queue is stopped' unless running?
+
+        @logger.debug "enqueue #{task_name.inspect}, #{task.inspect}, #{@external_queue.inspect}"
+        @external_queue << [task_name, task]
+        self
+      end
+
+      def status
+        return nil unless running?
+        Hash[@statuses.map do |name, st|
+          [name, {
+            queue: st[:pending].dup,
+            working: st[:working] ? st[:working].dup : nil,
+          }]
+        end]
+      end
+
+      private
+
+      def worker_loop(task_class, queue, status)
+        while task = queue.pop
+          break if @terminate
+          begin
+            status[:lock].synchronize do
+              status[:pending].delete task
+              status[:working] = task
+            end
+            task_class.new(self, task, agent: @agent, logger: @logger).execute
+          rescue Exception => e
+            @logger.error "#{task_class} worker catched error: #{e}\n\t#{e.backtrace.join("\n\t")}"
+          ensure
+            status[:lock].synchronize do
+              status[:working] = nil
+            end
+          end
+          break if @terminate
+        end
+      end
+
+      def queueing_loop(queues, external_queue, statuses)
+        @logger.debug "queueing thread started #{external_queue.inspect}"
+        while _ = external_queue.pop
+          task_name, task = _
+
+          break if @terminate
+
+          queue = queues[task_name]
+          unless queue
+            @logger.debug "Ignoring task #{task_name} (queue not defined)"
+            next
+          end
+
+          statuses[task_name][:lock].synchronize do
+            statuses[task_name][:pending] << task
+          end
+          @logger.info "Queueing task #{task_name}: #{task.inspect}"
+          queue << task
+          break if @terminate
+        end
+        @logger.debug "queueing thread finish"
+      rescue Exception => e
+        @logger.error "queueing thread error #{e.inspect}\n\t#{e.backtrace.join("\n\t")}"
+        raise e
+      end
+    end
+  end
+end

data/lib/mamiya/agent/tasks/abstract.rb

@@ -0,0 +1,61 @@
+require 'mamiya/logger'
+
+module Mamiya
+  class Agent
+    module Tasks
+      class Abstract
+        def initialize(task_queue, task, agent: nil, logger: Mamiya::Logger.new, raise_error: false)
+          @agent = agent
+          @task_queue = task_queue
+          @task = task.merge('task' => self.class.identifier)
+          @error = nil
+          @raise_error = raise_error
+          @logger = logger["#{self.class.identifier}:#{self.task_id}"]
+        end
+
+        def self.identifier
+          self.name.split(/::/).last.gsub(/(.)([A-Z])/, '\1_\2').downcase
+        end
+
+        attr_reader :task, :error, :logger, :agent, :task_queue
+
+        def raise_error?
+          !!@raise_error
+        end
+
+        def task_id
+          task['id'] || "0x#{self.__id__.to_s(16)}"
+        end
+
+        def execute
+          before
+          run
+        rescue Exception => error
+          @error = error
+          raise if raise_error?
+          errored
+        ensure
+          after
+        end
+
+        def before
+        end
+
+        def run
+        end
+
+        def after
+        end
+
+        def errored
+        end
+
+        private
+
+        def config
+          @config ||= agent ? agent.config : nil
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/agent/tasks/clean.rb

@@ -0,0 +1,44 @@
+require 'mamiya/agent/tasks/abstract'
+
+module Mamiya
+  class Agent
+    module Tasks
+      class Clean < Abstract
+
+        def run
+          victims.each do |app, victim|
+            @logger.info "Cleaning up: remove #{victim}"
+            File.unlink(victim) if File.exist?(victim)
+
+            meta_victim = victim.sub(/\.tar\.gz\z/, '.json')
+            if File.exist?(meta_victim)
+              @logger.info "Cleaning up: remove #{meta_victim}"
+              File.unlink(meta_victim)
+            end
+
+            package_name = File.basename(victim, '.tar.gz')
+
+            # XXX: depends on FS structure
+            agent.trigger('pkg', action: 'remove',
+              application: app,
+              package: package_name,
+              coalesce: false,
+            )
+          end
+        end
+
+        def victims
+          Dir[File.join(config[:packages_dir], '*')].flat_map do |app|
+            packages = Dir[File.join(app, "*.tar.gz")].
+              sort_by { |_| [File.mtime(_), _] }
+
+            packages[0...-(config[:keep_packages])].map do |victim|
+              [File.basename(app), victim]
+            end
+          end
+        end
+
+      end
+    end
+  end
+end