worker_bee 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 067c96f441e33257c6f7c793659ad797cd74649b858508a8422f843c4c0a75e0
+  data.tar.gz: 308d20e8531c852715f84a5de31ddc5c0d7e34488f44b990b4dc25708e7b4f99
+SHA512:
+  metadata.gz: c011fa40377d36f291df13302bd356b1de078d4cbc5dae5ea06ae7b6cf1bc9e0045129a815d22f9297518e9b72e7b48833c63efa2a760168ead752b43c4b8900
+  data.tar.gz: 0f99f63c706a6ea5998df00ba6713e0e96c0ebc501534a56b28fa49d6952a00d6de8223d330cae74f6c4df9a256a4a6a2cba41e1542fa5f52b21c1a3ca9e382f

data/.autotest

@@ -0,0 +1,26 @@
+# -*- ruby -*-
+
+require "autotest/restart"
+
+Autotest.add_hook :initialize do |at|
+  at.testlib = "minitest/autorun"
+  at.add_exception "tmp"
+
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception "vendor"
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2011-03-10
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,7 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/worker_bee.rb
+test/test_worker_bee.rb

data/README.txt

@@ -0,0 +1,74 @@
+= worker_bee
+
+home :: http://seattlerb.org/
+
+== DESCRIPTION:
+
+WorkerBee encapsulates a simple pipeline of workers.
+
+== FEATURES/PROBLEMS:
+
+* Simple API to wrap up the usual Thread/Queue patterns.
+* Expressive shortcuts to make the pipeline really easy to read.
+
+== SYNOPSIS:
+
+    bee = WorkerBee.new
+    bee.input enum_of_work_to_do
+    bee.work(20) { |work| ... stuff with input ... }
+    bee.work(5)  { |work| ... stuff with results of previous ... }
+    bee.results # the final set of results
+
+Example of shortcuts:
+
+    bee = WorkerBee.new
+    bee.input(*urls)
+    bee.work     { |url| url_to_api    url } # -> api_url
+    bee.work(20) { |url| url_to_ary    url } # -> [ issues_or_pulls ]
+    bee.work     { |ary| issues_to_url ary } # -> url_to_check
+    bee.compact                              # -> url
+    bee.results.sort
+
+versus:
+
+    WorkerBee.new(self) # self = receiver of send for `then`s below
+      .input(*urls)
+      .then(:url_to_api)
+      .then(:url_to_ary, n:20)
+      .then(:issues_to_url)
+      .compact
+      .results
+      .sort
+
+== REQUIREMENTS:
+
+* ruby... awesome, no?
+
+== INSTALL:
+
+* sudo gem install worker_bee
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) Ryan Davis, seattle.rb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+
+require "rubygems"
+require "hoe"
+
+Hoe.plugin :isolate
+Hoe.plugin :seattlerb
+
+Hoe.spec "worker_bee" do
+  developer "Ryan Davis", "ryand-ruby@zenspider.com"
+
+  license "MIT"
+
+  require_ruby_version "> 3.1"
+end
+
+# vim: syntax=ruby

data/lib/worker_bee.rb

@@ -0,0 +1,478 @@
+require 'thread'
+
+##
+# Defines a WorkerBee instance which provides a simple means of
+# defining pipelines of parallel tasks working from one queue of work
+# to a queue of results (which can then be a queue of work for another
+# pipeline of workers).
+#
+#     WorkerBee.new(self) # the self is important for context in `then` calls
+#       .input(*urls)
+#       .then(:url_to_api)
+#       .then(:api_to_ary, n:20)
+#       .flatten
+#       .then(:issues_to_url)
+#       .compact
+#       .results
+#       .sort
+#
+# General Helper methods:
+#
+# * input *data
+# * trap_interrupt!
+# * periodic time = 5, &update
+# * show_progress([cols])
+# * catch_up!
+# * finish
+# * results
+#
+# Data Transformers / Filters:
+#
+# * work n:1, &block
+# * compact
+# * filter n:1, &work
+# * flatten
+# * grep arg, n:1
+# * grep_v_clear arg, n:1
+# * map arg, n:1
+# * match arg, n:1
+# * non_empty n:1
+# * then msg_name, n:1
+
+class WorkerBee
+  VERSION = "1.0.0"     # :nodoc:
+  SENTINAL = Object.new # :nodoc:
+
+  ##
+  # A slightly better Queue.
+
+  class BQ < Queue
+    ##
+    # Push multiple objects into the queue
+
+    def concat objs
+      objs.each do |obj|
+        self << obj
+      end
+    end
+
+    ##
+    # Close and return all data as an array.
+
+    def drain
+      close
+
+      result = []
+      result.push self.shift until self.empty?
+      result
+    end
+  end
+
+  ##
+  # The queue of queues of tasks to perform.
+
+  attr_accessor :tasks
+
+  ##
+  # Pipelines of workers. Each worker is wired up to task queues directly.
+
+  attr_accessor :workers
+
+  ##
+  # The context for the current run. This is only important for +then+.
+
+  attr_accessor :context
+
+  ##
+  # Threads that run parallel to everything else. Can be used to
+  # show progress or do periodic cleanup. Killed by #finish.
+
+  attr_accessor :updaters
+
+  ##
+  # Creates a new WorkerBee with one queue and no pipelines of workers.
+
+  def initialize context = nil
+    self.tasks    = [BQ.new]
+    self.workers  = []
+    self.context  = context || self
+    self.updaters = []
+  end
+
+  ##
+  # The front of the pipeline
+
+  def front
+    tasks.first
+  end
+
+  ##
+  # The current back of the pipeline
+
+  def back
+    tasks.last
+  end
+
+  ##
+  # Add another queue to back of the pipeline.
+
+  def add_to_pipeline
+    tasks << BQ.new
+    back
+  end
+
+  ##
+  # Add +data+ to the front of the pipeline of tasks.
+
+  def input *data
+    front.concat data
+    self
+  end
+
+  ##
+  # Meant to be used in an INT trap. Clears the remaining input and
+  # then finishes all work.
+
+  def interrupted!
+    non_empty = tasks.find { |pool| ! pool.empty? }
+    non_empty.clear if non_empty
+
+    trap "INT", "EXIT"
+
+    finish
+  end
+
+  ##
+  # Set up an interrupt handler that cleanly finishes.
+
+  def trap_interrupt!
+    trap("INT") { $stderr.puts "finishing..."; interrupted! }
+    self
+  end
+
+  ##
+  # Schedules a side task to run every +time+ seconds parallel to
+  # everything else. Shut down by #finish.
+
+  def periodic time = 5, &update
+    self.updaters << Thread.new do
+      loop do
+        update.call self
+        sleep time
+      end
+    end
+    self
+  end
+
+  ##
+  # Print out the +columns+, and then periodically print counts to
+  # show progress.
+
+  def show_progress(columns = nil)
+    puts columns.join "\t" if columns
+    periodic do
+      print_progress
+    end
+  end
+
+  ##
+  # Print 1 line of pipeline counts.
+
+  def print_progress
+    print "\r"
+    print counts.join "\t"
+  end
+
+  ##
+  # Returns the current counts of all tasks in the pipeline.
+
+  def counts
+    tasks.zip(workers).map { |t, w| t.size + (w || []).count(&:working?) }
+  end
+
+  ##
+  # A generic worker bee. Does work in the input queue and puts it
+  # into the output queue until it gets the +SENTINAL+ value.
+
+  class Worker < Thread
+    ##
+    # True if working on a task
+    attr_accessor :working
+    alias working? working
+
+    ##
+    # The input queue of work
+
+    attr_accessor :input
+
+    ##
+    # The output queue of work
+
+    attr_accessor :output
+
+    ##
+    # The actual work to do (a proc).
+
+    attr_accessor :work
+
+    ##
+    # Initialize and start a worker.
+
+    def initialize input, output, &work
+      self.input  = input
+      self.output = output
+      self.work   = work
+      self.working = false
+
+      self.abort_on_exception = true
+
+      super() do
+        loop do
+          task = input.shift
+
+          break if task == SENTINAL
+
+          self.working = true
+          call task
+          self.working = false
+        end
+      end
+    end
+
+    ##
+    # Do +work+ on +task+ and put the result into the +output+ queue.
+
+    def call task
+      output << work[task]
+    end
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers of a certain
+  # +type+ (defaulting to Worker) performing +block+ as the task for
+  # this pipeline.
+  #
+  #   bee.work(n:3) { |task| do_the_work task }
+
+  def work n:1, type:Worker, &block
+    input  = back
+    output = add_to_pipeline
+
+    workers << n.times.map { type.new input, output, &block }
+
+    self
+  end
+
+  alias :toil :work
+  alias :slog :work
+
+  ##
+  # A worker that filters non-truthy results of work from the next pipeline.
+
+  class CompactWorker < Worker
+    ##
+    # Do +work+ on +task+ and put the truthy results into the +output+
+    # queue.
+
+    def call task
+      out = work[task]
+      output << out if out
+    end
+  end
+
+  ##
+  # Add a pipeline of work that removes all non-truthy tasks.
+  #
+  #   bee
+  #     .input(*urls)
+  #     .then(:url_or_nil)
+  #     .compact
+
+  def compact
+    work type:CompactWorker, &:itself
+  end
+
+  ##
+  # A worker that flattens the results of work into the next pipeline.
+
+  class FlattenWorker < Worker
+    ##
+    # Do +work+ on +task+ and put the (multiple) results into the
+    # +output+ queue.
+
+    def call task
+      output.concat work[task]
+    end
+  end
+
+  ##
+  # Add a pipeline of work that flattens out all tasks. Lets you have
+  # one task create arrays of subtasks.
+  #
+  #     bee
+  #       .input(*urls)
+  #       .then(:fetch_jobs, n:20)
+  #       .flatten
+
+  def flatten
+    work type:FlattenWorker, &:itself
+  end
+
+  ##
+  # A worker that passes task through only if the work is truthy.
+
+  class Filter < Worker
+    def call task
+      output << task if work[task]
+    end
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers that filter
+  # tasks out if +work+ is doesn't evaluate to truthy. Eg:
+  #
+  #   bee
+  #     .input(*Dir["**/*"])
+  #     .filter      { |path| File.file? path }
+  #     .filter(n:4) { |path| `file -b #{path}` =~ /Ruby script/ }
+
+  def filter n:1, &work
+    work n:n, type:Filter, &work
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers that filter tasks
+  # that match +arg+ pattern (regexp or anything that responds to
+  # +===+).
+  #
+  #   bee
+  #     .input(*Dir["**/*"])
+  #     .grep(/\.rb$|Rakefile/)
+
+  def grep arg, n:1
+    work(n:n) { |data| data.grep arg }
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers that maps tasks
+  # by calling +arg+ on them and return the results for the next
+  # input.
+  #
+  #   bee.map(:to_s)
+
+  def map arg, n:1
+    work(n:n) { |data| data.send arg }
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers that calls clear
+  # on all data that does not match pattern +arg+. Usually followed by
+  # +non_empty+.
+  #
+  #   bee
+  #     .grep_v_clear(/ruby/)
+  #     .non_empty
+
+  def grep_v_clear arg, n:1
+    work(n:n) { |data| data.grep_v(arg).each(&:clear); data }
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers that removes all
+  # empty data.
+  #
+  #   bee
+  #     .grep_v_clear(/ruby/)
+  #     .non_empty
+
+  def non_empty n:1
+    work(n:n) { |data| data.reject(&:empty?) }
+  end
+
+  ##
+  # Add a pipeline of work with +n+ parallel workers that finds data
+  # matching +arg+ by calling +===+.
+  #
+  #   bee.match(matcher)
+
+  def match arg, n:1
+    work(n:n) { |data| arg === data }
+  end
+
+  ##
+  # Convenience function:
+  #
+  #   bee.then :msg_name, n:3
+  #
+  # is a shortcut equivalent to:
+  #
+  #   bee.work(n:3) { |task| msg_name task }
+  #
+  # Also helps prototype systems as the method doesn't need to exist
+  # yet. Prints that it doesn't exist and passes the data through
+  # as-is.
+
+  def then msg_name, n:1
+    m = context.method(msg_name) rescue nil
+    if m then
+      work(n:n) { |obj| context.send msg_name, obj }
+    else
+      warn "warning: prototyping #{msg_name}"
+      work(n:n) { |obj| obj }
+    end
+  end
+
+  ##
+  # Forces a choke point at this point in the pipeline, causing all
+  # data prior to this point to finish before moving on. Returns a new
+  # instance of the pipeline.
+  #
+  #   bee
+  #     .input(*urls)
+  #     .then(:fetch_tasks, n:10)
+  #     .catch_up!
+  #     .then(:do_heavy_tasks, n:3)
+
+  def catch_up!
+    updaters = self.updaters
+    self.updaters = nil
+
+    input = self.results
+    bee = self.class.new(self.context)
+    bee.input(*input)
+    bee.updaters = updaters
+    bee
+  end
+
+  ##
+  # Finish all work on all tasks, from front to back.
+
+  def finish
+    # TODO: zip workers and tasks?
+
+    workers.each do |pool|
+      input = pool.first.input
+
+      pool.size.times do
+        input << SENTINAL
+      end
+
+      pool.each do |thread|
+        thread.join
+      end
+    end
+
+    updaters.each(&:kill)
+
+    self
+  end
+
+  ##
+  # Finish and return the contents of all work.
+
+  def results
+    finish
+
+    back.drain - [SENTINAL]
+  end
+end

data/test/test_worker_bee.rb

@@ -0,0 +1,31 @@
+require "minitest/autorun"
+require "worker_bee"
+
+class TestWorkerBee < Minitest::Test
+  def test_sanity_manual
+    bee = WorkerBee.new
+
+    bee.input(*(1..25).to_a)
+
+    bee.work(n:20) { |n| n ** 2 }
+    bee.finish
+    bee.work(n:5)  { |n| Math.sqrt n }
+    bee.finish
+
+    expected = (1..25).to_a
+    assert_equal expected, bee.results.map(&:to_i).sort
+  end
+
+  def test_sanity_automatic
+    bee = WorkerBee.new
+
+    bee.input(*(1..25).to_a)
+
+    bee.work(n:20) { |n| n ** 2 }
+    # bee.finish # commented out on purpose
+    bee.work(n:5)  { |n| Math.sqrt n }
+
+    expected = (1..25).to_a
+    assert_equal expected, bee.results.map(&:to_i).sort
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: worker_bee
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Ryan Davis
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDPjCCAiagAwIBAgIBCTANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
+  GRYDY29tMB4XDTI1MDEwNjIzMjcwMVoXDTI2MDEwNjIzMjcwMVowRTETMBEGA1UE
+  AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
+  JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
+  b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
+  taCPaLmfYIaFcHHCSY4hYDJijRQkLxPeB3xbOfzfLoBDbjvx5JxgJxUjmGa7xhcT
+  oOvjtt5P8+GSK9zLzxQP0gVLS/D0FmoE44XuDr3iQkVS2ujU5zZL84mMNqNB1znh
+  GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
+  qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
+  gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
+  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
+  AQAC0WQJcPOWPFwkojhzweilRVjTJ19UiLhiBTw3C1wJO3LVdBkWDmnnhAmKuX4D
+  r7vjQvESlABGIPdutI1Yl7mrHQzTkfLfXvNN6MT0nLChPyIYauT6SZZxubwJrUfA
+  7R0c2CJTIboZ0XaGpLsXqHEF1c29H7TV1QvVuqKAN2mCjh4N82QVn+ZKtys28AwT
+  6GfQX2fqLoi4KSc7xIzHKaNzqxeOICmJofk9w5VZ2rRN6yes8jvFYwz9HR41wdj8
+  bwfinv7Yp5fA6AysuZLhCykyfDuZVRrUp0Vb68YCKsLjJly/Theak+euNTxvHsB+
+  al9oSgPPHICMEX65qvLywitx
+  -----END CERTIFICATE-----
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+description: WorkerBee encapsulates a simple pipeline of workers.
+email:
+- ryand-ruby@zenspider.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- History.txt
+- Manifest.txt
+- README.txt
+files:
+- ".autotest"
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/worker_bee.rb
+- test/test_worker_bee.rb
+homepage: http://seattlerb.org/
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://seattlerb.org/
+rdoc_options:
+- "--main"
+- README.txt
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.2
+specification_version: 4
+summary: WorkerBee encapsulates a simple pipeline of workers.
+test_files: []