threaded_pipeline 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5a894c4ca00f14b3a16899f18115ece8958520a36e1cc8ea2ede67abf19a1ed9
+  data.tar.gz: e9aa1280acbeb6bf5bccb3812e315efe9912134b850a39c8048f599d001badab
+SHA512:
+  metadata.gz: 6de0080d5b599d5aa40ef8d86649b0ad7af98c1962df936274e9d24b33105140f529c0333d041c6bc276ffc7175985c4e40bba170e200a0b05e4250e601e5928
+  data.tar.gz: e769f97578a2bf9a91dd761d21633e68eb0411a3af1daab5b4df1a52cd574e19b0799c3d47e435bc44ca2e6c84a14e16995bf6478152220aa743bcee6728de3e

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/lib/threaded_pipeline.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'threaded_pipeline/version'
+
+# Create a pipeline where each stage runs in its own thread.  Each stage must
+# accept a single argument and will pass its result to the next stage.  The
+# results of the last stage are then returned (unless opted out).
+#
+# = Example
+#
+#  threaded_pipeline = ThreadedPipeline.new
+#  threaded_pipeline.stages << -> (url) { fetch_large_csv(url) }
+#  threaded_pipeline.stages << -> (local_file) { process_local_file(local_file) }
+#  results = threaded_pipeline.process([list, of, large, csv, urls])
+#
+# = Example
+#
+#  another_pipeline = ThreadedPipeline.new(discard_results: true)
+#  another_pipeline.stages << -> (url) { api_query(url) }
+#  another_pipeline.stages << -> (returned_data) { process_returned_data(returned_data) }
+#  another_pipeline.stages << -> (processed_results) { record_results_in_database(processed_results) }
+#  while url = web_crawl_urls
+#    another_pipeline.feed(url)
+#  end
+#  another_pipeline.finish
+#
+class ThreadedPipeline
+  # Each stage will process the results of the previous one.
+  #
+  #   my_threaded_pipeline.stages << ->(arg) { process(arg) }
+  attr_accessor :stages
+  attr_reader   :started
+
+  def initialize(discard_results: false)
+    @stages = []
+    @started = false
+    @discard_results = discard_results
+  end
+
+  # The elements of enumerable will begin processing immediately.
+  def process(enumerable)
+    initialize_run
+    initialize_first_queue(enumerable)
+    finish
+  end
+
+  # Process the enumerale list without using threads.
+  # Maybe you have a bug you want to work on without threading.  Or you have a
+  # benchmark you want to run.
+  def process_unthreaded(enumerable)
+    initialize_run
+    @results = enumerable.map do |element|
+      stages.each do |stage|
+        element = stage[element]
+      end
+      element
+    end
+    finish
+  end
+
+  # Add another element to the list of work to be processed.  Work will start
+  # on the first element immediately (only feed once you have all your stages added).
+  # You could use .process if you already have the full list.
+  # This method is not thread safe (wrap access in a mutex if feeding from
+  # multiple threads).
+  def feed(element)
+    initialize_run unless @started
+    queue_hash[stages.first] << element
+  end
+
+  # Wait for all the threads to finish and return the results.
+  # @return results of last stage (unless discard_results was set to true)
+  def finish
+    raise "You never started pipeline #{inspect}" unless @started
+
+    queue_hash[stages.first] << finish_object
+    @threads.each(&:join)
+    @started = false
+    @queue_hash = nil
+    @finish_object = nil
+    @results unless @discard_results
+  end
+
+  private
+
+  def initialize_run(without_threads = false)
+    raise "Already started pipeline #{inspect}" if @started
+
+    @started = true
+    @queue_hash = nil
+    @threads = []
+    @results = []
+    return if without_threads
+
+    queue_hash # initialize outside of threads
+    @threads = stages.each_with_index.map do |stage, index|
+      Thread.new do
+        # Grab the next element off our queue
+        while (element = queue_hash[stage].pop) != finish_object
+          # The way you call a lambda is with []'s.  Who knew?
+          result = stage[element]
+          if index == stages.count - 1
+            # Only one thread is accessing @results
+            @results << result unless @discard_results
+          else
+            queue_hash[stages[index + 1]] << result
+          end
+        end
+        queue_hash[stages[index + 1]] << finish_object unless index == stages.count - 1
+      end
+    end
+  end
+
+  def initialize_first_queue(enumerable)
+    first_queue = queue_hash[stages.first]
+    enumerable.each do |element|
+      first_queue << element
+    end
+    first_queue << finish_object
+  end
+
+  # one queue after each stage but the last
+  def queue_hash
+    return @queue_hash unless @queue_hash.nil?
+
+    @queue_hash = stages.map { |stage| [stage, Queue.new] }.to_h
+  end
+
+  # How we know we're done?
+  def finish_object
+    @finish_object ||= Object.new
+  end
+end

data/lib/threaded_pipeline/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ThreadedPipeline
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: threaded_pipeline
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kurt Werle
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-color
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: If you are eg. downloading a large file, processing that file, then writing
+  the results to a database, ThreadedPipeline may be for you.  Download in one thread,
+  process in another, and write in a third, etc.
+email:
+- kurt@CircleW.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- lib/threaded_pipeline.rb
+- lib/threaded_pipeline/version.rb
+homepage: https://github.com/kwerle/threaded_pipeline
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/kwerle/threaded_pipeline
+  source_code_uri: https://github.com/kwerle/threaded_pipeline
+  changelog_uri: https://github.com/kwerle/threaded_pipeline
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: Run several stages of a pipeline in threads.
+test_files: []