peteonrails-threaded-collections 0.1.0

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2008 Peter Jackson
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,33 @@
+Threaded Collections is a package for iterating through collections 
+over multiple threads. With large collections, sometimes it can be 
+more efficient to process a collection in parallel, provided that the 
+collected items don't have a interdependencies, or need to be processed
+in a specific order. 
+
+Usage: 
+
+require "threaded_collections"
+
+threadcount = 2
+arr = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]
+tps = ThreadedCollectionProcessor.new(arr)
+tps.process(2) { |thread_id, item| puts "Thread #{thread_id} processed item: #{item}" }
+
+I abstracted this pattern from a web services client that posted items from a collection, but
+each request took a second to process. The remote service had plenty of threads available, so
+I parallelized the task with this pattern. 
+
+THIS IS AN ALPHA RELEASE. 
+
+I have no plans to break the interface but I do plan to make two major enhancements: 
+
+1. Make it possible to mix this functionality in to the Ruby iterators, so you don't have create the 
+ThreadedCollectionProcessor.
+2. Make it work with fibers and processes in addition to threads. 
+
+Note: Don't implement the Observer pattern inside the block! 
+
+Submit bugs, enhancement requests, and other issues through Lighthouse: 
+
+http://peteonrails.lighthouseapp.com/projects/11058-threaded-collections/overview
+

data/Rakefile

@@ -0,0 +1,36 @@
+require 'rubygems'
+require 'rake/testtask'
+Gem::manage_gems
+require 'rake/gempackagetask'
+
+spec = Gem::Specification.new do |s|
+    s.platform  =   Gem::Platform::RUBY
+    s.name      =   "threaded-collections"
+    s.version   =   "0.1.0"
+    s.author    =   "Peter Jackson"
+    s.email     =   "pete @nospam@ peteonrails.com"
+    s.summary   =   "A package for manipulating collections over many threads"
+    s.files     =   FileList['lib/*.rb', 'test/*'].to_a
+    s.require_path  =   "lib"
+    s.test_files = Dir.glob('tests/*.rb')
+    s.has_rdoc  =   true
+    s.extra_rdoc_files  =   ["README"]
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.need_tar = true
+end
+
+
+task :gem => "pkg/#{spec.name}-#{spec.version}.gem" do
+    puts "generated latest version"
+end
+
+task :default => [:test]
+
+desc "Test the Threaded Collection library"
+Rake::TestTask.new do |t|
+   t.libs << "test"
+   t.test_files = FileList['test/*test*.rb']
+   t.verbose = true
+end  

data/lib/threaded-collections.rb

@@ -0,0 +1 @@
+require 'threaded_collections/threaded_collections.rb'

data/lib/threaded_collections.rb

@@ -0,0 +1 @@
+require 'threaded_collections/threaded_collections.rb'

data/lib/threaded_collections/threaded_collections.rb

@@ -0,0 +1,41 @@
+require 'thread'
+
+module ThreadedCollections
+  # This class will build a threadsafe Queue from a file
+  # and process the file on a specified number of threads
+  # with a supplied block
+  class ThreadedCollectionProcessor
+  
+    # Take the collection passed in as an argument 
+    # and parse it into a threadsafe queue, returning 
+    # the number of items safely enqueued
+    def initialize(collection)
+      @queue = Queue.new()
+      collection.each() { |item| @queue.enq(item) }
+    end
+  
+    # The ThreadedCollectionProcessor#process method will spin up threads to process the queue.
+    # The queue will be processed until empty, calling out to the associated
+    # block to perform the actual work. 
+    def process(threadcount, &block)
+      threads = []
+    
+      # Spawn a set of threads to process the queue
+      1.upto(threadcount) do |thread_id|
+        threads << Thread.new(thread_id) do |i|   # Each thread 
+          until @queue.empty?() do                # grabs an item from the queue 
+            yield(thread_id, @queue.deq())        # and yields it to the block
+          end 
+        end  
+      end
+  
+      # Block until all threads are complete
+      threads.each() {|thr| thr.join() }
+    end
+
+    def queue_length
+      return @queue.length
+    end
+  
+  end # class ThreadedCollectionProcessor
+end # module

data/threadedcollections.gemspec

@@ -0,0 +1,13 @@
+spec = Gem::Specification.new do |s|
+    s.platform  =   Gem::Platform::RUBY
+    s.name      =   "threaded-collections"
+    s.version   =   "0.1.0"
+    s.author    =   "Peter Jackson"
+    s.email     =   "pete @nospam@ peteonrails.com"
+    s.summary   =   "A package for manipulating collections over many threads"
+    s.files     =   ['lib/threaded_collections.rb', 'lib/threaded-collections.rb', 'lib/threaded_collections/threaded_collections.rb', 'test/threaded_collections_test.rb', 'LICENSE', 'Rakefile', 'README', 'threadedcollections.gemspec']
+    s.require_path  =   "lib"
+    s.test_files = ['tests/threaded_collections_test.rb']
+    s.has_rdoc  =   true
+    s.extra_rdoc_files  =   ["README"]
+end   

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification 
+name: peteonrails-threaded-collections
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Peter Jackson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-05-14 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: pete @nospam@ peteonrails.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- lib/threaded_collections.rb
+- lib/threaded-collections.rb
+- lib/threaded_collections/threaded_collections.rb
+- test/threaded_collections_test.rb
+- LICENSE
+- Rakefile
+- README
+- threadedcollections.gemspec
+has_rdoc: true
+homepage: 
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: A package for manipulating collections over many threads
+test_files: 
+- tests/threaded_collections_test.rb