peteonrails-threaded-collections 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +22 -0
- data/README +33 -0
- data/Rakefile +36 -0
- data/lib/threaded-collections.rb +1 -0
- data/lib/threaded_collections.rb +1 -0
- data/lib/threaded_collections/threaded_collections.rb +41 -0
- data/threadedcollections.gemspec +13 -0
- metadata +60 -0
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2008 Peter Jackson
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person
|
4
|
+
obtaining a copy of this software and associated documentation
|
5
|
+
files (the "Software"), to deal in the Software without
|
6
|
+
restriction, including without limitation the rights to use,
|
7
|
+
copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8
|
+
copies of the Software, and to permit persons to whom the
|
9
|
+
Software is furnished to do so, subject to the following
|
10
|
+
conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be
|
13
|
+
included in all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
17
|
+
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
19
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
20
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
21
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
22
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
data/README
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
Threaded Collections is a package for iterating through collections
|
2
|
+
over multiple threads. With large collections, sometimes it can be
|
3
|
+
more efficient to process a collection in parallel, provided that the
|
4
|
+
collected items don't have a interdependencies, or need to be processed
|
5
|
+
in a specific order.
|
6
|
+
|
7
|
+
Usage:
|
8
|
+
|
9
|
+
require "threaded_collections"
|
10
|
+
|
11
|
+
threadcount = 2
|
12
|
+
arr = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]
|
13
|
+
tps = ThreadedCollectionProcessor.new(arr)
|
14
|
+
tps.process(2) { |thread_id, item| puts "Thread #{thread_id} processed item: #{item}" }
|
15
|
+
|
16
|
+
I abstracted this pattern from a web services client that posted items from a collection, but
|
17
|
+
each request took a second to process. The remote service had plenty of threads available, so
|
18
|
+
I parallelized the task with this pattern.
|
19
|
+
|
20
|
+
THIS IS AN ALPHA RELEASE.
|
21
|
+
|
22
|
+
I have no plans to break the interface but I do plan to make two major enhancements:
|
23
|
+
|
24
|
+
1. Make it possible to mix this functionality in to the Ruby iterators, so you don't have create the
|
25
|
+
ThreadedCollectionProcessor.
|
26
|
+
2. Make it work with fibers and processes in addition to threads.
|
27
|
+
|
28
|
+
Note: Don't implement the Observer pattern inside the block!
|
29
|
+
|
30
|
+
Submit bugs, enhancement requests, and other issues through Lighthouse:
|
31
|
+
|
32
|
+
http://peteonrails.lighthouseapp.com/projects/11058-threaded-collections/overview
|
33
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake/testtask'
|
3
|
+
Gem::manage_gems
|
4
|
+
require 'rake/gempackagetask'
|
5
|
+
|
6
|
+
spec = Gem::Specification.new do |s|
|
7
|
+
s.platform = Gem::Platform::RUBY
|
8
|
+
s.name = "threaded-collections"
|
9
|
+
s.version = "0.1.0"
|
10
|
+
s.author = "Peter Jackson"
|
11
|
+
s.email = "pete @nospam@ peteonrails.com"
|
12
|
+
s.summary = "A package for manipulating collections over many threads"
|
13
|
+
s.files = FileList['lib/*.rb', 'test/*'].to_a
|
14
|
+
s.require_path = "lib"
|
15
|
+
s.test_files = Dir.glob('tests/*.rb')
|
16
|
+
s.has_rdoc = true
|
17
|
+
s.extra_rdoc_files = ["README"]
|
18
|
+
end
|
19
|
+
|
20
|
+
Rake::GemPackageTask.new(spec) do |pkg|
|
21
|
+
pkg.need_tar = true
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
task :gem => "pkg/#{spec.name}-#{spec.version}.gem" do
|
26
|
+
puts "generated latest version"
|
27
|
+
end
|
28
|
+
|
29
|
+
task :default => [:test]
|
30
|
+
|
31
|
+
desc "Test the Threaded Collection library"
|
32
|
+
Rake::TestTask.new do |t|
|
33
|
+
t.libs << "test"
|
34
|
+
t.test_files = FileList['test/*test*.rb']
|
35
|
+
t.verbose = true
|
36
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'threaded_collections/threaded_collections.rb'
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'threaded_collections/threaded_collections.rb'
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module ThreadedCollections
|
4
|
+
# This class will build a threadsafe Queue from a file
|
5
|
+
# and process the file on a specified number of threads
|
6
|
+
# with a supplied block
|
7
|
+
class ThreadedCollectionProcessor
|
8
|
+
|
9
|
+
# Take the collection passed in as an argument
|
10
|
+
# and parse it into a threadsafe queue, returning
|
11
|
+
# the number of items safely enqueued
|
12
|
+
def initialize(collection)
|
13
|
+
@queue = Queue.new()
|
14
|
+
collection.each() { |item| @queue.enq(item) }
|
15
|
+
end
|
16
|
+
|
17
|
+
# The ThreadedCollectionProcessor#process method will spin up threads to process the queue.
|
18
|
+
# The queue will be processed until empty, calling out to the associated
|
19
|
+
# block to perform the actual work.
|
20
|
+
def process(threadcount, &block)
|
21
|
+
threads = []
|
22
|
+
|
23
|
+
# Spawn a set of threads to process the queue
|
24
|
+
1.upto(threadcount) do |thread_id|
|
25
|
+
threads << Thread.new(thread_id) do |i| # Each thread
|
26
|
+
until @queue.empty?() do # grabs an item from the queue
|
27
|
+
yield(thread_id, @queue.deq()) # and yields it to the block
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# Block until all threads are complete
|
33
|
+
threads.each() {|thr| thr.join() }
|
34
|
+
end
|
35
|
+
|
36
|
+
def queue_length
|
37
|
+
return @queue.length
|
38
|
+
end
|
39
|
+
|
40
|
+
end # class ThreadedCollectionProcessor
|
41
|
+
end # module
|
@@ -0,0 +1,13 @@
|
|
1
|
+
spec = Gem::Specification.new do |s|
|
2
|
+
s.platform = Gem::Platform::RUBY
|
3
|
+
s.name = "threaded-collections"
|
4
|
+
s.version = "0.1.0"
|
5
|
+
s.author = "Peter Jackson"
|
6
|
+
s.email = "pete @nospam@ peteonrails.com"
|
7
|
+
s.summary = "A package for manipulating collections over many threads"
|
8
|
+
s.files = ['lib/threaded_collections.rb', 'lib/threaded-collections.rb', 'lib/threaded_collections/threaded_collections.rb', 'test/threaded_collections_test.rb', 'LICENSE', 'Rakefile', 'README', 'threadedcollections.gemspec']
|
9
|
+
s.require_path = "lib"
|
10
|
+
s.test_files = ['tests/threaded_collections_test.rb']
|
11
|
+
s.has_rdoc = true
|
12
|
+
s.extra_rdoc_files = ["README"]
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: peteonrails-threaded-collections
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Peter Jackson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2008-05-14 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description:
|
17
|
+
email: pete @nospam@ peteonrails.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files:
|
23
|
+
- README
|
24
|
+
files:
|
25
|
+
- lib/threaded_collections.rb
|
26
|
+
- lib/threaded-collections.rb
|
27
|
+
- lib/threaded_collections/threaded_collections.rb
|
28
|
+
- test/threaded_collections_test.rb
|
29
|
+
- LICENSE
|
30
|
+
- Rakefile
|
31
|
+
- README
|
32
|
+
- threadedcollections.gemspec
|
33
|
+
has_rdoc: true
|
34
|
+
homepage:
|
35
|
+
post_install_message:
|
36
|
+
rdoc_options: []
|
37
|
+
|
38
|
+
require_paths:
|
39
|
+
- lib
|
40
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: "0"
|
45
|
+
version:
|
46
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
47
|
+
requirements:
|
48
|
+
- - ">="
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: "0"
|
51
|
+
version:
|
52
|
+
requirements: []
|
53
|
+
|
54
|
+
rubyforge_project:
|
55
|
+
rubygems_version: 1.0.1
|
56
|
+
signing_key:
|
57
|
+
specification_version: 2
|
58
|
+
summary: A package for manipulating collections over many threads
|
59
|
+
test_files:
|
60
|
+
- tests/threaded_collections_test.rb
|