distribot 0.1.1

data/examples/controller

@@ -0,0 +1,168 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'distribot'
+require 'byebug'
+require 'pp'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/array'
+require 'active_support/json'
+require 'dotenv'
+
+Dotenv.load
+
+Distribot.configure do |config|
+  config.redis_url = ENV['DISTRIBOT_REDIS_URL']
+  config.rabbitmq_url = ENV['DISTRIBOT_RABBITMQ_URL']
+end
+
+module Example
+
+  def self.phase_groups
+    [
+      [
+        {
+          name: 'start',
+          is_initial: true,
+          transitions_to: 'get-latest-stories'
+        },
+        {
+          name: 'get-latest-stories',
+          transitions_to: 'download-news',
+          handlers: [
+            {
+              name: 'News::ArticleLister',
+              version: '~> 1.0'
+            }
+          ]
+        },
+        {
+          name: 'download-news',
+          transitions_to: 'finish',
+          handlers: [
+            {
+              name: 'News::ArticleDownloaderWorker',
+              version: '1.0.0'
+            }
+          ]
+        },
+        {
+          name: 'finish',
+          is_final: true
+        }
+      ]
+      # [
+      #   {
+      #     name: 'start',
+      #     is_initial: true,
+      #     transitions_to: 'part1'
+      #   },
+      #   {
+      #     name: 'part1',
+      #     handlers: [
+      #       {
+      #         name: 'HardWorker',
+      #         version: '~> 1.0'
+      #       }
+      #     ],
+      #     transitions_to: 'part2'
+      #   },
+      #   {
+      #     name: 'part2',
+      #     handlers: %w(
+      #       GoodWorker
+      #       FastWorker
+      #     ),
+      #     transitions_to: 'part3'
+      #   },
+      #   {
+      #     name: 'part3',
+      #     handlers: %w(
+      #       CheapWorker
+      #       ForeignWorker
+      #     ),
+      #     transitions_to: 'part4'
+      #   },
+      #   {
+      #     name: 'part4',
+      #     handlers: %w(
+      #       SlowWorker
+      #     ),
+      #     transitions_to: 'finish'
+      #   },
+      #   {
+      #     name: 'finish',
+      #     is_final: true
+      #   }
+      # ],
+      # [
+      #   {
+      #     name: 'start',
+      #     is_initial: true,
+      #     transitions_to: 'searching'
+      #   },
+      #   {
+      #     name: 'searching',
+      #     handlers: %w(
+      #       GoodWorker
+      #       FastWorker
+      #       CheapWorker
+      #     ),
+      #     transitions_to: 'finish'
+      #   },
+      #   {
+      #     name: 'finish',
+      #     is_final: true
+      #   }
+      # ]
+    ]
+  end
+
+  def self.make_flow(name)
+    name += "#1"
+    @flow = Distribot::Flow.new(
+      name: name,
+      phases: phase_groups.sample
+    )
+    @flow.save! do |info|
+      finished = Distribot.redis.incr('finished')
+      flow = Distribot::Flow.find(info[:flow_id])
+      first_transition = flow.transitions.sort_by(&:timestamp).first
+      last_transition = flow.transitions.sort_by(&:timestamp).last
+      duration = last_transition.timestamp - first_transition.timestamp
+      puts "WORKFLOW FINISHED: after #{duration} seconds -- #{info}: #{finished} total"
+    end
+    @flow
+  end
+end
+
+Distribot.redis.set("finished", 0)
+
+max = ARGV.shift.to_i
+max = 1 if max <= 0
+puts "<ENTER> when ready:"
+gets
+
+start_time = Time.now.to_f
+Distribot.redis.set('dificulty', 3)
+Distribot.redis.set('distribot.flows.running', 0)
+Distribot.redis.set('distribot.flows.max', max)
+Example.make_flow "Flow[1]"
+
+loop do
+  sleep 2
+  max = Distribot.redis.get('distribot.flows.max').to_i
+  current = Distribot.redis.get('distribot.flows.running').to_i
+  running_time = Time.now.to_f - start_time
+  finished = Distribot.redis.get('finished').to_i
+  rate = ( finished / running_time ).to_f.round(2)
+  puts "\tRUNNING: at #{Time.now} -- #{current} / #{max} (#{finished} finished in #{running_time.round(2)}sec = #{rate}/sec)"
+  diff = max - current
+  diff.times do |num|
+    new_index = Distribot.redis.get('finished')
+    puts "+++++++++ SPAWN #{num + 1}/#{diff} +++++++++++"
+    Example.make_flow("Flow: #{new_index} -> ")
+  end
+end
+
+

data/examples/distribot.eye

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+#^syntax detection
+
+ROOT = File.expand_path('../../', __FILE__)
+
+Eye.config do
+  logger "#{ROOT}/log/eye.log"
+end
+
+unless File.directory? "#{ROOT}/log"
+  `mkdir -p #{ROOT}/log`
+end
+unless File.directory? "#{ROOT}/tmp"
+  `mkdir -p #{ROOT}/tmp`
+end
+
+Eye.application :distribot do
+  working_dir ROOT
+  trigger :flapping, :times => 10, :within => 1.minute
+
+  # Usage:
+  # eye load eye/distribot.eye
+  # eye start all
+  # eye stop all
+  # eye restart distribot:phase-started
+  # tail -f log/*
+  # See https://github.com/kostya/eye for details.
+
+  things = %w(
+    phase-started
+    phase-finished
+    task-finished
+    handler-finished
+    flow-finished
+    flow-created
+  )
+
+  things.each do |thing|
+    process thing do
+      daemonize true
+      pid_file "tmp/#{thing}.pid"
+      stdall "log/#{thing}.log"
+      start_command "dotenv bundle exec bin/distribot.#{thing}"
+      stop_signals [:TERM, 5.seconds, :KILL]
+      restart_command "kill -USR2 {PID}"
+      restart_grace 10.seconds
+    end
+  end
+end

data/examples/status

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'distribot'
+require 'byebug'
+
+redis = Distribot.redis
+loop do
+  puts "\n"
+  redis
+    .smembers('distribot.flows.active')
+    .map{|id| Distribot::Flow.find(id) }
+    .sort_by(&:created_at)
+    .reverse
+    .each do |flow|
+      # Cover the odd case of fetching a flow that hasn't had its first transition yet:
+      latest_transition = flow.transitions.last or next
+
+      latest_phase = flow.phase(latest_transition.to) || Distribot::Phase.new(name: "UNKNOWN")
+      next if latest_phase.is_final
+      time_in_phase = Time.now.to_f - latest_transition.timestamp
+      total_age = Time.now.to_f - flow.created_at
+
+      if latest_phase.is_final
+        puts "Flow #{flow.id} is finished."
+      else
+        handler_counts = latest_phase.handlers.sort_by(&:name).map do |handler|
+          remaining_tasks = redis.get("distribot.flow.#{flow.id}.#{latest_phase.name}.#{handler}.finished").to_i
+          total_tasks = redis.get("distribot.flow.#{flow.id}.#{latest_phase.name}.#{handler}.finished.total").to_i
+          percent_finished = ((total_tasks - remaining_tasks) * 1.0) / (total_tasks * 1.0) * 100
+          finished_tasks = total_tasks - remaining_tasks
+          "#{handler}:\t#{['=' * (finished_tasks > 0 ? finished_tasks : 0), '-' * (remaining_tasks > 0 ? remaining_tasks : 0)].join('')} #{finished_tasks}/#{total_tasks} (#{percent_finished.round(2)}%)"
+        end
+        puts "Flow #{flow.id} (age:#{sprintf('%05.02f',total_age)} sec)\tphase:#{latest_phase.name}\t(#{time_in_phase.round(2)} seconds\t#{handler_counts.join("\t")}"
+      end
+  end
+  puts "-------------------------------------------------------------------------------------------------------------------------------------------------------"
+  sleep 0.5
+end

data/examples/worker

@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'distribot'
+require 'byebug'
+require 'pp'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/array'
+require 'active_support/json'
+require 'dotenv'
+
+Dotenv.load
+
+Distribot.configure do |config|
+  config.redis_url = ENV['DISTRIBOT_REDIS_URL']
+  config.rabbitmq_url = ENV['DISTRIBOT_RABBITMQ_URL']
+end
+
+class SimpleWorker
+  include Distribot::Worker
+  version '1.0.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+
+  def enumerate(_context, &callback)
+    jobs = [ ]
+    2.times do |chunk|
+      max = 5
+      jobs << (1..max).to_a.map{|n| {id: SecureRandom.uuid, args: [chunk, n] } }
+    end
+    callback.call(jobs.flatten)
+  end
+
+  def process(_context, job)
+    logger.info job.to_s
+    raise "Test Error!" if rand >= 0.9
+    job_time = Distribot.redis.get('difficulty').to_f
+    sleep job_time <= 0 ? 3 : job_time
+  end
+end
+
+class HardWorker < SimpleWorker;
+  version '1.1.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+end
+class GoodWorker < SimpleWorker;
+  version '1.0.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+end
+class FastWorker < SimpleWorker;
+  version '1.0.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+end
+class CheapWorker < SimpleWorker;
+  version '1.0.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+end
+class ForeignWorker < SimpleWorker;
+  version '1.0.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+end
+class SlowWorker < SimpleWorker;
+  version '1.0.0'
+  enumerate_with :enumerate
+  process_tasks_with :process
+end
+
+module News
+  class ArticleLister
+    include Distribot::Worker
+    version '1.0.1'
+    enumerate_with :enumerate
+    process_tasks_with :process
+    def enumerate(context, &callback)
+      @enumerations ||= 0
+      story_ids = (1..20).to_a
+      tasks = story_ids.map { |id| {story_id: "#{@enumerations}.#{id}"} }
+      @enumerations += 1
+      return tasks
+    end
+    def process(context, task)
+      json = { url: "https://infra/?story_id=#{task[:story_id]}" }.to_json
+      data_key = "flow.#{context.flow_id}.stories"
+      parsed = JSON.parse(json, symbolize_names: true) rescue {}
+      if parsed.key? :url
+        Distribot.redis.multi do |redis|
+          place = [data_key, task[:story_id]].join(':')
+          redis.set place, json
+          redis.sadd data_key, place
+        end
+      end
+    end
+  end
+
+  class ArticleDownloaderWorker
+    include Distribot::Worker
+    require 'wrest'
+    version '1.0.0'
+    enumerate_with :enumerate
+    process_tasks_with :process
+
+    def enumerate(context)
+      data_key = "flow.#{context.flow_id}.stories"
+      tasks = Distribot.redis.smembers(data_key).map { |location| {data_locator: location} }
+      Distribot.redis.del data_key
+      return tasks
+    end
+
+    def process(context, task)
+      data = JSON.parse(Distribot.redis.get(task[:data_locator]), symbolize_names: true)
+puts data
+#      data[:url].to_uri(verify_mode: OpenSSL::SSL::VERIFY_NONE).get
+      Distribot.redis.del(task[:data_locator])
+    end
+  end
+end
+
+puts News::ArticleLister.new.run
+puts News::ArticleDownloaderWorker.new.run
+
+# Distribot.logger.info HardWorker.new.run
+# Distribot.logger.info GoodWorker.new.run
+# Distribot.logger.info FastWorker.new.run
+# Distribot.logger.info CheapWorker.new.run
+# Distribot.logger.info ForeignWorker.new.run
+# Distribot.logger.info SlowWorker.new.run
+
+puts "Worker up and running!"
+
+sleep

data/lib/distribot/connector.rb

@@ -0,0 +1,162 @@
+
+require 'pp'
+require 'securerandom'
+require 'bunny'
+require 'byebug'
+require 'active_support/json'
+require 'uri'
+require 'wrest'
+
+module Distribot
+  class Connector
+    attr_accessor :amqp_url, :bunny, :channel
+    def initialize(amqp_url = 'amqp://localhost:5672')
+      self.amqp_url = amqp_url
+      setup
+    end
+
+    def queues
+      uri = URI.parse(amqp_url)
+      uri.scheme = 'http'
+      uri.port = '15672'.to_i
+      uri.path = '/api/queues'
+      uri.to_s.to_uri.get.deserialize.map { |x| x['name'] }
+    end
+
+    def logger
+      Distribot.logger
+    end
+
+    private
+
+    def setup
+      self.bunny = Bunny.new(amqp_url)
+      bunny.start
+      self.channel = bunny.create_channel
+      channel.prefetch(1)
+    end
+  end
+
+  class ConnectionSharer
+    attr_accessor :bunny, :channel
+    def initialize(bunny)
+      self.bunny = bunny
+      @channel = nil
+    end
+
+    def channel
+      @channel ||= bunny.create_channel
+    end
+
+    def logger
+      Distribot.logger
+    end
+  end
+
+  class Subscription < ConnectionSharer
+    attr_accessor :queue
+    def start(topic, options = {}, &block)
+      self.queue = channel.queue(topic, auto_delete: false, durable: true)
+      subscribe_args = options.merge(manual_ack: true)
+      queue.subscribe(subscribe_args) do |delivery_info, _properties, payload|
+        begin
+          parsed_message = JSON.parse(payload, symbolize_names: true)
+          block.call(parsed_message)
+          channel.acknowledge(delivery_info.delivery_tag, false)
+        rescue StandardError => e
+          logger.error "ERROR: #{e} -- #{e.backtrace.join("\n")}"
+          channel.basic_reject(delivery_info.delivery_tag, true)
+        end
+      end
+      self
+    end
+  end
+
+  class MultiSubscription < ConnectionSharer
+    attr_accessor :queue
+    def start(topic, options = {}, &block)
+      self.queue = channel.queue('', exclusive: true, auto_delete: true)
+      exchange = channel.fanout(topic)
+      queue.bind(exchange).subscribe(options) do |_delivery_info, _, payload|
+        begin
+          block.call(JSON.parse(payload, symbolize_names: true))
+        rescue StandardError => e
+          logger.error "Error #{e} - #{payload} --- #{e.backtrace.join("\n")}"
+        end
+      end
+      self
+    end
+  end
+
+  class BunnyConnector < Connector
+    attr_accessor :subscribers, :channel
+    def initialize(*args)
+      super(*args)
+      self.subscribers = []
+    end
+
+    def channel
+      @channel ||= bunny.create_channel
+    end
+
+    def queue_exists?(topic)
+      bunny.queue_exists?(topic)
+    end
+
+    def subscribe(topic, options = {}, &block)
+      if options[:solo]
+        options.delete :solo
+        setup
+      end
+      subscriber = Subscription.new(bunny)
+      subscribers << subscriber.start(topic, options) do |message|
+        logger.debug "received(#{topic} -> #{message})"
+        block.call(message)
+      end
+    end
+
+    def subscribe_multi(topic, options = {}, &block)
+      subscriber = MultiSubscription.new(bunny)
+      subscribers << subscriber.start(topic, options) do |message|
+        logger.debug "received-multi(#{topic} -> #{message})"
+        block.call(message)
+      end
+    end
+
+    def publish(topic, message)
+      queue = stubbornly :get_queue do
+        channel.queue(topic, auto_delete: false, durable: true)
+      end
+      logger.debug "publish(#{topic} -> #{message})"
+      channel.default_exchange.publish message.to_json, routing_key: queue.name
+    end
+
+    def broadcast(topic, message)
+      exchange = channel.fanout(topic)
+      logger.debug "broadcast(#{topic} -> #{message})"
+      exchange.publish(message.to_json, routing_key: topic)
+    end
+
+    private
+
+    def setup
+      self.bunny = Bunny.new(amqp_url)
+      bunny.start
+    end
+
+    def stubbornly(task, &block)
+      result = nil
+      loop do
+        begin
+          result = block.call
+          break
+        rescue Timeout::Error
+          logger.error "Connection timed out during '#{task}' :retrying in 1sec"
+          sleep 1
+          next
+        end
+      end
+      result
+    end
+  end
+end