thimble 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE +21 -0
- data/README.md +126 -0
- data/lib/{Manager.rb → manager.rb} +38 -20
- data/lib/{QueueItem.rb → queue_item.rb} +6 -4
- data/lib/thimble/version.rb +6 -0
- data/lib/{Thimble.rb → thimble.rb} +52 -40
- data/lib/{ThimbleQueue.rb → thimble_queue.rb} +47 -34
- metadata +46 -14
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 9d5dba5ce3a256f520a9c75924f886fe3d94fd5e64369528570ef57d7b2e226a
|
|
4
|
+
data.tar.gz: d99df41c042a2360e4f8c36f8f9b00d4923174abc71b78ce6c35d06fa966cfff
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a9d708451d9b0fe90a133401f4f999930067abd4af81856ad039851f8d1c0bfc44be8da8c99fcfc19eb121927490b0b1283512c8be2ff445c63fcdbd6d80f64d
|
|
7
|
+
data.tar.gz: e8026d10b4f8d7265c282d59acce8c5e8186aff83b76b75f93f54ad7cceba2f24d428941629d14a3210b467381b531ac31c8b82c2a3ee2f9c6f237ae422ceb15
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Andrew Kovanda
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Thimble
|
|
2
|
+
Thimble is a Ruby gem for parallelism and concurrency. It lets you choose threads (good for IO) or processes (good for CPU) and build pipelines using stages backed by a thread-safe queue.
|
|
3
|
+
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
gem 'thimble'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
bundle install
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Or install it yourself as:
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
gem install thimble
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Supported Ruby and platforms
|
|
26
|
+
- Ruby >= 3.0
|
|
27
|
+
- MRI: threads are limited by the GVL for CPU-bound work. Use `worker_type: :fork` for CPU-bound pipelines.
|
|
28
|
+
- JRuby/TruffleRuby: threads can run in parallel; `:thread` often suffices.
|
|
29
|
+
- Windows: `fork` is not available. Use `worker_type: :thread`.
|
|
30
|
+
|
|
31
|
+
## Quick start
|
|
32
|
+
|
|
33
|
+
Example 1: parallel map using forked processes (CPU-bound)
|
|
34
|
+
```
|
|
35
|
+
require 'thimble'
|
|
36
|
+
|
|
37
|
+
manager = Thimble::Manager.new(max_workers: 5, batch_size: 5, queue_size: 10, worker_type: :fork)
|
|
38
|
+
thimble = Thimble::Thimble.new((1..100).to_a, manager)
|
|
39
|
+
results = thimble.map { |x| x * 1000 }
|
|
40
|
+
# results is a Thimble::ThimbleQueue; consume it as needed
|
|
41
|
+
p results.to_a
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Example 2: feed an intermediate queue from a threaded stage (IO-bound)
|
|
45
|
+
```
|
|
46
|
+
require 'thimble'
|
|
47
|
+
# We create a queue to store intermediate work
|
|
48
|
+
queue = Thimble::ThimbleQueue.new(3, 'stage 2')
|
|
49
|
+
# Our array of data
|
|
50
|
+
ary = (1..10).to_a
|
|
51
|
+
# A separate thread worker who will be processing the intermediate queue
|
|
52
|
+
thread = Thimble::Thimble.async do
|
|
53
|
+
queue.each { |x| puts "I did work on #{x}!"; sleep 1 }
|
|
54
|
+
end
|
|
55
|
+
# Our Thimble, plus its manager. Note we are using Thread in this example.
|
|
56
|
+
thim = Thimble::Thimble.new(ary, Thimble::Manager.new(batch_size: 1, worker_type: :thread))
|
|
57
|
+
# We in parallel push data to the Thimble Queue
|
|
58
|
+
thim.map { |e| queue.push(e); sleep 0.1; puts "I pushed #{e} to the queue!" }
|
|
59
|
+
# The queue is closed (no more work can come in)
|
|
60
|
+
queue.close
|
|
61
|
+
# join the thread
|
|
62
|
+
thread.join
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Manager quick reference
|
|
66
|
+
```
|
|
67
|
+
m = Thimble::Manager.new(max_workers: 10, batch_size: 100, worker_type: :fork)
|
|
68
|
+
Thimble::Thimble.new(array, m)
|
|
69
|
+
```
|
|
70
|
+
- max_workers: how many workers can run at the same time
|
|
71
|
+
- batch_size: how many items to send to each worker (tune for workload)
|
|
72
|
+
- worker_type: :thread or :fork
|
|
73
|
+
|
|
74
|
+
The same Manager can be shared across Thimble instances to coordinate concurrency limits.
|
|
75
|
+
|
|
76
|
+
All thimbles require an explicit manager.
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## ThimbleQueue
|
|
81
|
+
ThimbleQueue is the queue underpinning Thimble. Taking from it is destructive. It is thread-safe for multi-thread producers/consumers.
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
q = Thimble::ThimbleQueue.new(10, 'name')
|
|
85
|
+
q.push(1)
|
|
86
|
+
q.close
|
|
87
|
+
q.each { |x| puts x }
|
|
88
|
+
# => 1
|
|
89
|
+
```
|
|
90
|
+
If you do not close the queue, consumers will wait for more data. Creating a Thimble creates a "closed" input queue; transformations create a new queue.
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Caveats and best practices
|
|
95
|
+
These are common pitfalls and how Thimble helps you avoid them:
|
|
96
|
+
|
|
97
|
+
- MRI GVL and workload choice
|
|
98
|
+
- Threads do not run CPU-bound Ruby in parallel on MRI. Use `worker_type: :fork` for CPU-bound tasks; `:thread` shines for IO-bound tasks.
|
|
99
|
+
- Platform differences
|
|
100
|
+
- `fork` is Unix-only. On Windows, use `:thread`.
|
|
101
|
+
- Forking and safety
|
|
102
|
+
- Thimble forks child workers before creating additional threads inside children. Children trap HUP and exit cleanly; the parent detaches workers to avoid zombies.
|
|
103
|
+
- Recreate external resources in children (DB connections, sockets, clients). Don’t share them across a fork.
|
|
104
|
+
- Memory and copy-on-write
|
|
105
|
+
- Each process has its own heap and GC. Batching reduces IPC overhead. Freeze large constants to improve CoW where possible.
|
|
106
|
+
- Backpressure
|
|
107
|
+
- ThimbleQueue is bounded; tune `queue_size` to avoid unbounded growth.
|
|
108
|
+
- Shutdown
|
|
109
|
+
- ThimbleQueue supports `close` and `close(true)` for immediate close. Avoid closing from multiple places.
|
|
110
|
+
- Error propagation
|
|
111
|
+
- Exceptions in workers are propagated back through results. For `:thread`, thread exceptions are surfaced; for `:fork`, exceptions are marshaled back and re-raised when consumed.
|
|
112
|
+
- Signal handling
|
|
113
|
+
- The main process receives signals; Thimble sends HUP to child workers when their results are consumed.
|
|
114
|
+
- Ordering
|
|
115
|
+
- Parallel stages may reorder results. If you need original order, attach sequence numbers to items and reorder at the end.
|
|
116
|
+
- Tuning
|
|
117
|
+
- Start with `max_workers` ~ number of cores for CPU-bound, higher for IO-bound. Adjust `batch_size` to minimize overhead without starving workers.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## Development
|
|
122
|
+
- Run tests: `bundle exec rspec`
|
|
123
|
+
- Linting: consider adding RuboCop (`rubocop`)
|
|
124
|
+
- Releasing: bump `Thimble::VERSION` in `lib/thimble/version.rb`, tag and push, then build and push the gem
|
|
125
|
+
|
|
126
|
+
Contributions welcome! Please open issues and PRs.
|
|
@@ -1,13 +1,19 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Thread.abort_on_exception = true
|
|
2
4
|
|
|
3
5
|
module Thimble
|
|
4
6
|
class Manager
|
|
5
7
|
attr_reader :max_workers, :batch_size, :queue_size, :worker_type
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
raise ArgumentError
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
|
|
9
|
+
def initialize(max_workers: 6, batch_size: 1000, queue_size: 1000, worker_type: :fork)
|
|
10
|
+
raise ArgumentError, 'worker type must be either :fork or :thread' unless %i[thread fork].include?(worker_type)
|
|
11
|
+
unless worker_type == :thread || Process.respond_to?(:fork)
|
|
12
|
+
raise ArgumentError, 'Your system does not respond to fork please use threads.'
|
|
13
|
+
end
|
|
14
|
+
raise ArgumentError, 'max_workers must be greater than 0' if max_workers < 1
|
|
15
|
+
raise ArgumentError, 'batch size must be greater than 0' if batch_size < 1
|
|
16
|
+
|
|
11
17
|
@worker_type = worker_type
|
|
12
18
|
@max_workers = max_workers
|
|
13
19
|
@batch_size = batch_size
|
|
@@ -16,16 +22,20 @@ module Thimble
|
|
|
16
22
|
@current_workers = {}
|
|
17
23
|
end
|
|
18
24
|
|
|
25
|
+
# @return [TrueClass, FalseClass]
|
|
19
26
|
def worker_available?
|
|
20
27
|
@current_workers.size < @max_workers
|
|
21
28
|
end
|
|
22
29
|
|
|
30
|
+
# @return [TrueClass, FalseClass]
|
|
23
31
|
def working?
|
|
24
|
-
@current_workers.size
|
|
32
|
+
@current_workers.size.positive?
|
|
25
33
|
end
|
|
26
34
|
|
|
35
|
+
# @param [Object] id
|
|
27
36
|
def sub_worker(worker, id)
|
|
28
|
-
raise
|
|
37
|
+
raise 'Worker must contain a pid!' if worker.pid.nil?
|
|
38
|
+
|
|
29
39
|
new_worker = OpenStruct.new
|
|
30
40
|
new_worker.worker = worker
|
|
31
41
|
new_worker.id = id
|
|
@@ -34,40 +44,44 @@ module Thimble
|
|
|
34
44
|
end
|
|
35
45
|
end
|
|
36
46
|
|
|
47
|
+
# @param [Object] worker
|
|
37
48
|
def rem_worker(worker)
|
|
38
49
|
@mutex.synchronize do
|
|
39
50
|
@current_workers.delete(worker.pid)
|
|
40
51
|
end
|
|
41
52
|
end
|
|
42
53
|
|
|
54
|
+
# @param [Object] id
|
|
43
55
|
def current_workers(id)
|
|
44
56
|
@mutex.synchronize do
|
|
45
|
-
@current_workers.select { |
|
|
57
|
+
@current_workers.select { |_k, v| v.id == id }
|
|
46
58
|
end
|
|
47
59
|
end
|
|
48
60
|
|
|
49
|
-
|
|
61
|
+
# @param [Object] batch
|
|
62
|
+
# @param [Proc] block
|
|
63
|
+
# @return [Object]
|
|
64
|
+
def get_worker(batch, &block)
|
|
50
65
|
@mutex.synchronize do
|
|
51
66
|
if @worker_type == :fork
|
|
52
|
-
get_fork_worker(batch, &
|
|
67
|
+
get_fork_worker(batch, &block)
|
|
53
68
|
else
|
|
54
|
-
get_thread_worker(batch, &
|
|
69
|
+
get_thread_worker(batch, &block)
|
|
55
70
|
end
|
|
56
71
|
end
|
|
57
72
|
end
|
|
58
73
|
|
|
74
|
+
# @param [Object] batch
|
|
59
75
|
def get_fork_worker(batch)
|
|
60
76
|
rd, wr = IO.pipe
|
|
61
77
|
tup = OpenStruct.new
|
|
62
78
|
pid = fork do
|
|
63
|
-
Signal.trap(
|
|
79
|
+
Signal.trap('HUP') { exit }
|
|
64
80
|
rd.close
|
|
65
81
|
t = Marshal.dump(batch.item.map do |item|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
e
|
|
70
|
-
end
|
|
82
|
+
yield item.item
|
|
83
|
+
rescue Exception => e
|
|
84
|
+
e
|
|
71
85
|
end)
|
|
72
86
|
wr.write(t)
|
|
73
87
|
wr.close
|
|
@@ -78,6 +92,8 @@ module Thimble
|
|
|
78
92
|
tup
|
|
79
93
|
end
|
|
80
94
|
|
|
95
|
+
# @param [Object] batch
|
|
96
|
+
# @param [Proc] block
|
|
81
97
|
def get_thread_worker(batch)
|
|
82
98
|
tup = OpenStruct.new
|
|
83
99
|
tup.pid = Thread.new do
|
|
@@ -89,12 +105,14 @@ module Thimble
|
|
|
89
105
|
tup
|
|
90
106
|
end
|
|
91
107
|
|
|
108
|
+
# @return [Thimble::Manager]
|
|
92
109
|
def self.deterministic
|
|
93
|
-
|
|
110
|
+
new(max_workers: 1, batch_size: 1, queue_size: 1)
|
|
94
111
|
end
|
|
95
112
|
|
|
113
|
+
# @return [Thimble::Manager]
|
|
96
114
|
def self.small
|
|
97
|
-
|
|
115
|
+
new(max_workers: 1, batch_size: 3, queue_size: 3)
|
|
98
116
|
end
|
|
99
117
|
end
|
|
100
118
|
end
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
2
4
|
module Thimble
|
|
3
5
|
class QueueItem
|
|
4
6
|
attr_reader :id, :item
|
|
5
|
-
|
|
7
|
+
|
|
8
|
+
def initialize(item, name = 'Item')
|
|
6
9
|
@id = Digest::SHA256.digest(rand(10**100).to_s + Time.now.to_i.to_s)
|
|
7
10
|
@item = item
|
|
8
11
|
@name = name
|
|
@@ -11,6 +14,5 @@ module Thimble
|
|
|
11
14
|
def to_s
|
|
12
15
|
"#{@name}: #{@item} ID: #{@id}"
|
|
13
16
|
end
|
|
14
|
-
|
|
15
17
|
end
|
|
16
|
-
end
|
|
18
|
+
end
|
|
@@ -1,41 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
|
|
2
|
-
require_relative '
|
|
3
|
-
require_relative '
|
|
4
|
-
require_relative '
|
|
3
|
+
require_relative 'manager'
|
|
4
|
+
require_relative 'thimble_queue'
|
|
5
|
+
require_relative 'queue_item'
|
|
6
|
+
require_relative 'thimble/version'
|
|
5
7
|
require 'io/wait'
|
|
6
8
|
require 'ostruct'
|
|
7
9
|
|
|
8
10
|
module Thimble
|
|
9
|
-
|
|
10
11
|
class Thimble < ThimbleQueue
|
|
11
|
-
def initialize(array, manager = Manager.new, result = nil, name =
|
|
12
|
-
raise ArgumentError
|
|
13
|
-
|
|
12
|
+
def initialize(array, manager = Manager.new, result = nil, name = 'Main')
|
|
13
|
+
raise ArgumentError, 'You need to pass a manager to Thimble!' unless manager.instance_of?(Manager)
|
|
14
|
+
|
|
15
|
+
unless array.respond_to? :each
|
|
16
|
+
raise ArgumentError,
|
|
17
|
+
'There needs to be an iterable object passed to Thimble to start.'
|
|
18
|
+
end
|
|
19
|
+
|
|
14
20
|
@result = if result.nil?
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
21
|
+
ThimbleQueue.new(array.size, 'Result')
|
|
22
|
+
else
|
|
23
|
+
result
|
|
24
|
+
end
|
|
25
|
+
unless @result.instance_of?(ThimbleQueue) && !@result.closed?
|
|
26
|
+
raise ArgumentError,
|
|
27
|
+
'result needs to be an open ThimbleQueue'
|
|
18
28
|
end
|
|
19
|
-
|
|
29
|
+
|
|
20
30
|
@manager = manager
|
|
21
31
|
@running = true
|
|
22
32
|
super(array.size, name)
|
|
23
33
|
@logger.debug("loading thimble #{name}")
|
|
24
|
-
array.each {|item| push(item)}
|
|
34
|
+
array.each { |item| push(item) }
|
|
25
35
|
@logger.debug("finished loading thimble #{name}")
|
|
26
|
-
close
|
|
36
|
+
close
|
|
27
37
|
end
|
|
28
38
|
|
|
29
39
|
# This will use the manager and transform your thimble queue.
|
|
30
|
-
# requires a block
|
|
40
|
+
# requires a block
|
|
31
41
|
# @return [ThimbleQueue]
|
|
32
|
-
def map
|
|
42
|
+
def map(&block)
|
|
33
43
|
@logger.debug("starting map in #{@name} with id #{Thread.current.object_id}")
|
|
34
44
|
@running = true
|
|
35
|
-
while @running
|
|
36
|
-
|
|
37
|
-
end
|
|
38
|
-
@result.close()
|
|
45
|
+
manage_workers(&block) while @running
|
|
46
|
+
@result.close
|
|
39
47
|
@logger.debug("finishing map in #{@name} with id #{Thread.current.object_id}")
|
|
40
48
|
@result
|
|
41
49
|
end
|
|
@@ -44,46 +52,47 @@ module Thimble
|
|
|
44
52
|
# Will return the result instantly, so you can use it for next stage processing.
|
|
45
53
|
# requires a block
|
|
46
54
|
# @return [ThimbleQueue]
|
|
47
|
-
|
|
55
|
+
# @param [Proc] block
|
|
56
|
+
def map_async(&block)
|
|
48
57
|
@logger.debug("starting async map in #{@name} with id #{Thread.current.object_id}")
|
|
49
58
|
@logger.debug("queue: #{@queue}")
|
|
50
59
|
Thimble.async do
|
|
51
|
-
map
|
|
60
|
+
map(&block)
|
|
52
61
|
end
|
|
53
62
|
@logger.debug("finished async map in #{@name} with id #{Thread.current.object_id}")
|
|
54
63
|
@result
|
|
55
64
|
end
|
|
56
65
|
|
|
57
|
-
# Will perform anything handed to this asynchronously.
|
|
66
|
+
# Will perform anything handed to this asynchronously.
|
|
58
67
|
# Requires a block
|
|
59
68
|
# @return [Thread]
|
|
60
|
-
def self.async
|
|
61
|
-
Thread.new
|
|
62
|
-
yield e
|
|
63
|
-
end
|
|
69
|
+
def self.async(&block)
|
|
70
|
+
Thread.new(&block)
|
|
64
71
|
end
|
|
65
72
|
|
|
66
73
|
private
|
|
74
|
+
|
|
67
75
|
def get_batch
|
|
68
76
|
batch = []
|
|
69
77
|
while batch.size < @manager.batch_size
|
|
70
78
|
item = self.next
|
|
71
79
|
if item.nil?
|
|
72
|
-
return nil if batch.size
|
|
73
|
-
|
|
80
|
+
return nil if batch.size.zero?
|
|
81
|
+
|
|
82
|
+
return QueueItem.new(batch, 'Batch')
|
|
74
83
|
else
|
|
75
84
|
batch << item
|
|
76
85
|
end
|
|
77
86
|
end
|
|
78
|
-
QueueItem.new(batch,
|
|
87
|
+
QueueItem.new(batch, 'Batch')
|
|
79
88
|
end
|
|
80
89
|
|
|
81
|
-
def manage_workers
|
|
82
|
-
@manager.current_workers(@id).each do |
|
|
90
|
+
def manage_workers(&block)
|
|
91
|
+
@manager.current_workers(@id).each do |_pid, pair|
|
|
83
92
|
get_result(pair.worker)
|
|
84
93
|
end
|
|
85
|
-
while
|
|
86
|
-
@manager.sub_worker(
|
|
94
|
+
while @manager.worker_available? && batch = get_batch
|
|
95
|
+
@manager.sub_worker(@manager.get_worker(batch, &block), @id)
|
|
87
96
|
end
|
|
88
97
|
@running = false if !@manager.working? && !batch
|
|
89
98
|
end
|
|
@@ -92,23 +101,26 @@ module Thimble
|
|
|
92
101
|
if @manager.worker_type == :fork
|
|
93
102
|
if tuple.reader.ready?
|
|
94
103
|
piped_result = tuple.reader.read
|
|
104
|
+
tuple.reader.close unless tuple.reader.closed?
|
|
95
105
|
loadedResult = Marshal.load(piped_result)
|
|
96
106
|
loadedResult.each { |r| raise r if r.class <= Exception }
|
|
97
107
|
push_result(loadedResult)
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
push_result(tuple.result)
|
|
108
|
+
begin
|
|
109
|
+
Process.kill('HUP', tuple.pid)
|
|
110
|
+
rescue Errno::ESRCH
|
|
111
|
+
# Process already exited; nothing to do
|
|
112
|
+
end
|
|
104
113
|
@manager.rem_worker(tuple)
|
|
105
114
|
end
|
|
115
|
+
elsif tuple.done == true
|
|
116
|
+
push_result(tuple.result)
|
|
117
|
+
@manager.rem_worker(tuple)
|
|
106
118
|
end
|
|
107
119
|
end
|
|
108
120
|
|
|
109
121
|
def push_result(result)
|
|
110
122
|
if result.respond_to? :each
|
|
111
|
-
result.each {|r| @result.push(r)}
|
|
123
|
+
result.each { |r| @result.push(r) }
|
|
112
124
|
else
|
|
113
125
|
@result.push(result)
|
|
114
126
|
end
|
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
require 'logger'
|
|
3
|
-
require_relative '
|
|
4
|
+
require_relative 'queue_item'
|
|
4
5
|
|
|
5
6
|
module Thimble
|
|
7
|
+
# noinspection RubyTooManyInstanceVariablesInspection
|
|
6
8
|
class ThimbleQueue
|
|
7
|
-
include Enumerable
|
|
8
|
-
attr_reader :size
|
|
9
9
|
def initialize(size, name)
|
|
10
|
-
|
|
10
|
+
unless size >= 1
|
|
11
|
+
raise ArgumentError, "make sure there is a size for the queue greater than 1! size received #{size}"
|
|
12
|
+
end
|
|
13
|
+
|
|
11
14
|
@id = Digest::SHA256.digest(rand(10**100).to_s + Time.now.to_i.to_s)
|
|
12
15
|
@name = name
|
|
13
16
|
@size = size
|
|
@@ -17,51 +20,56 @@ module Thimble
|
|
|
17
20
|
@close_now = false
|
|
18
21
|
@empty = ConditionVariable.new
|
|
19
22
|
@full = ConditionVariable.new
|
|
20
|
-
@logger = Logger.new(
|
|
23
|
+
@logger = Logger.new($stdout)
|
|
21
24
|
@logger.sev_threshold = Logger::UNKNOWN
|
|
22
25
|
end
|
|
23
26
|
|
|
24
|
-
|
|
27
|
+
include Enumerable
|
|
28
|
+
attr_reader :size
|
|
29
|
+
|
|
30
|
+
def set_logger(level)
|
|
25
31
|
@logger.sev_threshold = level
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
def each
|
|
29
|
-
while item = self.next
|
|
35
|
+
while (item = self.next)
|
|
30
36
|
yield item.item
|
|
31
37
|
end
|
|
32
38
|
end
|
|
33
39
|
|
|
34
40
|
# Returns the size of the ThimbleQueue
|
|
35
|
-
# @return [
|
|
41
|
+
# @return [Integer]
|
|
36
42
|
def length
|
|
37
43
|
size
|
|
38
44
|
end
|
|
39
45
|
|
|
40
46
|
# Will concatenate an enumerable to the ThimbleQueue
|
|
41
|
-
# @param [Enumerable]
|
|
42
47
|
# @return [ThimbleQueue]
|
|
48
|
+
# @param [Module<Enumerable>] other
|
|
43
49
|
def +(other)
|
|
44
|
-
raise ArgumentError
|
|
50
|
+
raise ArgumentError, '+ requires another Enumerable!' unless other.class < Enumerable
|
|
51
|
+
|
|
45
52
|
merged_thimble = ThimbleQueue.new(length + other.length, @name)
|
|
46
|
-
|
|
47
|
-
other.each {|item| merged_thimble.push(item)}
|
|
53
|
+
each { |item| merged_thimble.push(item) }
|
|
54
|
+
other.each { |item| merged_thimble.push(item) }
|
|
48
55
|
merged_thimble
|
|
49
56
|
end
|
|
50
57
|
|
|
51
58
|
# Returns the first item in the queue
|
|
52
59
|
# @return [Object]
|
|
53
60
|
def next
|
|
54
|
-
@mutex.synchronize
|
|
55
|
-
|
|
61
|
+
@mutex.synchronize do
|
|
62
|
+
until @close_now
|
|
56
63
|
a = @queue.shift
|
|
57
64
|
@logger.debug("#{@name}'s queue shifted to: #{a}")
|
|
58
65
|
if !a.nil?
|
|
59
66
|
@full.broadcast
|
|
60
67
|
@empty.broadcast
|
|
61
68
|
return a
|
|
62
|
-
else
|
|
69
|
+
else
|
|
63
70
|
@logger.debug("#{@name}'s queue is currently closed?: #{closed?}")
|
|
64
71
|
return nil if closed?
|
|
72
|
+
|
|
65
73
|
@empty.wait(@mutex)
|
|
66
74
|
end
|
|
67
75
|
end
|
|
@@ -69,46 +77,49 @@ module Thimble
|
|
|
69
77
|
end
|
|
70
78
|
|
|
71
79
|
# This will push whatever it is handed to the queue
|
|
72
|
-
# @param [Object]
|
|
73
|
-
def push(
|
|
74
|
-
raise
|
|
75
|
-
|
|
80
|
+
# @param [Object] input_item
|
|
81
|
+
def push(input_item)
|
|
82
|
+
raise 'Queue is closed!' if @closed
|
|
83
|
+
|
|
84
|
+
@logger.debug("Pushing into #{@name} values: #{input_item}")
|
|
76
85
|
@mutex.synchronize do
|
|
77
|
-
|
|
86
|
+
until offer(input_item)
|
|
78
87
|
@full.wait(@mutex)
|
|
79
88
|
@logger.debug("#{@name} is waiting on full")
|
|
80
89
|
end
|
|
81
90
|
@empty.broadcast
|
|
82
91
|
end
|
|
83
|
-
@logger.debug("Finished pushing int #{@name}: #{
|
|
92
|
+
@logger.debug("Finished pushing int #{@name}: #{input_item}")
|
|
84
93
|
end
|
|
85
94
|
|
|
86
95
|
# This will flatten any nested arrays out and feed them one at
|
|
87
96
|
# a time to the queue.
|
|
88
|
-
# @param [Object, Enumerable]
|
|
89
97
|
# @return [nil]
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
98
|
+
# @param [Object] input_item
|
|
99
|
+
def push_flat(input_item)
|
|
100
|
+
raise 'Queue is closed!' if @closed
|
|
101
|
+
|
|
102
|
+
@logger.debug("Pushing flat into #{@name} values: #{input_item}")
|
|
103
|
+
if input_item.respond_to? :each
|
|
104
|
+
input_item.each { |item| push(item) }
|
|
95
105
|
else
|
|
96
106
|
@mutex.synchronize do
|
|
97
|
-
|
|
107
|
+
until offer(input_item)
|
|
98
108
|
@logger.debug("#{@name} is waiting on full")
|
|
99
109
|
@full.wait(@mutex)
|
|
100
110
|
end
|
|
101
111
|
@empty.broadcast
|
|
102
112
|
end
|
|
103
113
|
end
|
|
104
|
-
@logger.debug("Finished pushing flat into #{@name} values: #{
|
|
114
|
+
@logger.debug("Finished pushing flat into #{@name} values: #{input_item}")
|
|
105
115
|
end
|
|
106
116
|
|
|
107
|
-
# Closes the
|
|
117
|
+
# Closes the ThimbleQueue
|
|
108
118
|
# @param [TrueClass, FalseClass]
|
|
109
119
|
# @return [nil]
|
|
110
120
|
def close(now = false)
|
|
111
|
-
raise ArgumentError
|
|
121
|
+
raise ArgumentError, 'now must be true or false' unless [true, false].include?(now)
|
|
122
|
+
|
|
112
123
|
@logger.debug("#{@name} is closing")
|
|
113
124
|
@mutex.synchronize do
|
|
114
125
|
@closed = true
|
|
@@ -123,7 +134,7 @@ module Thimble
|
|
|
123
134
|
# @return [Array[Object]]
|
|
124
135
|
def to_a
|
|
125
136
|
a = []
|
|
126
|
-
while item = self.next
|
|
137
|
+
while (item = self.next)
|
|
127
138
|
a << item.item
|
|
128
139
|
end
|
|
129
140
|
a
|
|
@@ -136,6 +147,8 @@ module Thimble
|
|
|
136
147
|
end
|
|
137
148
|
|
|
138
149
|
private
|
|
150
|
+
|
|
151
|
+
# @param [Object] x
|
|
139
152
|
def offer(x)
|
|
140
153
|
if @queue.size < @size
|
|
141
154
|
@queue << QueueItem.new(x)
|
|
@@ -146,4 +159,4 @@ module Thimble
|
|
|
146
159
|
end
|
|
147
160
|
end
|
|
148
161
|
end
|
|
149
|
-
end
|
|
162
|
+
end
|
metadata
CHANGED
|
@@ -1,15 +1,42 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: thimble
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Kovanda
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
12
|
-
dependencies:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: logger
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '0'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: ostruct
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '0'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '0'
|
|
13
40
|
description: Thimble is a ruby gem for parallelism and concurrency. It allows you
|
|
14
41
|
to decide if you want to use separate processes, or if you want to use threads in
|
|
15
42
|
ruby. It allows you to create stages with a thread safe queue, and break apart large
|
|
@@ -19,15 +46,22 @@ executables: []
|
|
|
19
46
|
extensions: []
|
|
20
47
|
extra_rdoc_files: []
|
|
21
48
|
files:
|
|
22
|
-
-
|
|
23
|
-
-
|
|
24
|
-
- lib/
|
|
25
|
-
- lib/
|
|
49
|
+
- LICENSE
|
|
50
|
+
- README.md
|
|
51
|
+
- lib/manager.rb
|
|
52
|
+
- lib/queue_item.rb
|
|
53
|
+
- lib/thimble.rb
|
|
54
|
+
- lib/thimble/version.rb
|
|
55
|
+
- lib/thimble_queue.rb
|
|
26
56
|
homepage: https://github.com/akovanda/thimble
|
|
27
57
|
licenses:
|
|
28
58
|
- MIT
|
|
29
|
-
metadata:
|
|
30
|
-
|
|
59
|
+
metadata:
|
|
60
|
+
source_code_uri: https://github.com/akovanda/thimble
|
|
61
|
+
bug_tracker_uri: https://github.com/akovanda/thimble/issues
|
|
62
|
+
changelog_uri: https://github.com/akovanda/thimble/releases
|
|
63
|
+
documentation_uri: https://github.com/akovanda/thimble#readme
|
|
64
|
+
rubygems_mfa_required: 'true'
|
|
31
65
|
rdoc_options: []
|
|
32
66
|
require_paths:
|
|
33
67
|
- lib
|
|
@@ -35,16 +69,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
35
69
|
requirements:
|
|
36
70
|
- - ">="
|
|
37
71
|
- !ruby/object:Gem::Version
|
|
38
|
-
version:
|
|
72
|
+
version: 3.0.0
|
|
39
73
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
40
74
|
requirements:
|
|
41
75
|
- - ">="
|
|
42
76
|
- !ruby/object:Gem::Version
|
|
43
77
|
version: '0'
|
|
44
78
|
requirements: []
|
|
45
|
-
|
|
46
|
-
rubygems_version: 2.6.10
|
|
47
|
-
signing_key:
|
|
79
|
+
rubygems_version: 3.6.7
|
|
48
80
|
specification_version: 4
|
|
49
81
|
summary: Concurrency and Parallelism gem that uses blocks to move data
|
|
50
82
|
test_files: []
|