shifty 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 46d6ec38da041475fa499d8d79042e685d9eb3c4ab1dbce24ab70579b466acc2
4
+ data.tar.gz: b8cc19ee79ad7321e113f8ec7311f52020566506ff8a7e84dc36c6eb840e57dd
5
+ SHA512:
6
+ metadata.gz: c11b09e67ff91d611ddedff9404d81b043f9211e0a96a452ff6d9fde0480955e65050b00fb6a71cabc867e35ef3ae28815943d371b56c1a3a1a425a80e5847f0
7
+ data.tar.gz: 46d0ef549da9cdfc57043b2563d1a4549abb44abffbe424228b18d97758e871e1ad279c9e5d8824d584a1ab66e43100a514c5ccb82fcce161280318d06762a53
data/.gitignore ADDED
@@ -0,0 +1,13 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
12
+
13
+ Gemfile.lock
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.travis.yml ADDED
@@ -0,0 +1,17 @@
1
+ env:
2
+ global:
3
+ - CC_TEST_REPORTER_ID=de10d77685b449162f3f3d568e1ab75d94c0f281afceee218502cf0995e49aa6
4
+ sudo: false
5
+ language: ruby
6
+ rvm:
7
+ - 2.5.0
8
+ - jruby-19mode
9
+ before_install: gem install bundler -v 1.16.1
10
+ before_script:
11
+ - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
12
+ - chmod +x ./cc-test-reporter
13
+ - ./cc-test-reporter before-build
14
+ script:
15
+ - bundle exec rspec
16
+ after_script:
17
+ - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ git_source(:github) {|repo_name| 'https://github.com/#{repo_name}' }
4
+
5
+ # Specify your gem's dependencies in shifty.gemspec
6
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2018 Joel Helbling
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,459 @@
1
+ [![Gem Version](https://badge.fury.io/rb/shifty.svg)](https://badge.fury.io/rb/shifty)
2
+ [![Build Status](https://travis-ci.org/joelhelbling/shifty.png)](https://travis-ci.org/joelhelbling/shifty)
3
+ [![Maintainability](https://api.codeclimate.com/v1/badges/950ded888350c1124348/maintainability)](https://codeclimate.com/github/joelhelbling/shifty/maintainability)
4
+ [![Test Coverage](https://api.codeclimate.com/v1/badges/950ded888350c1124348/test_coverage)](https://codeclimate.com/github/joelhelbling/shifty/test_coverage)
5
+
6
+ # The Shifty Framework
7
+
8
+ _"How many Ruby fibers does it take to screw in a lightbulb?"_
9
+
10
+ ## Quick Start
11
+
12
+ Add this line to your application's Gemfile:
13
+
14
+ ```ruby
15
+ gem 'shifty'
16
+ ```
17
+
18
+ And then execute:
19
+
20
+ $ bundle
21
+
22
+ Or install it yourself:
23
+
24
+ $ gem install steplader
25
+
26
+ And then use it:
27
+
28
+ ```ruby
29
+ require 'shifty'
30
+ ```
31
+
32
+ ## Shifty DSL
33
+
34
+ The DSL provides convenient shorthand for several common types of
35
+ workers. Let's look at them.
36
+
37
+ But first, be sure to include the DSL mixin:
38
+
39
+ ```ruby
40
+ include Shifty::DSL
41
+ ```
42
+
43
+ ### Source Worker
44
+
45
+ At the headwater of every Shifty pipeline, there is a source worker
46
+ which is able to generate its own work items.
47
+
48
+ Here is possibly the simplest of source workers, which provides the same
49
+ string each time it is invoked:
50
+
51
+ ```ruby
52
+ worker = source_worker { "Number 9" }
53
+
54
+ worker.shift #=> "Number 9"
55
+ worker.shift #=> "Number 9"
56
+ worker.shift #=> "Number 9"
57
+ # ...ad nauseum...
58
+ ```
59
+
60
+ Sometimes you want a worker which generates until it doesn't:
61
+
62
+ ```ruby
63
+ counter = 0
64
+ worker = source_worker do
65
+ if counter < 3
66
+ counter += 1
67
+ counter + 1000
68
+ end
69
+ end
70
+
71
+ worker.shift #=> 1001
72
+ worker.shift #=> 1002
73
+ worker.shift #=> 1003
74
+ worker.shift #=> nil
75
+ worker.shift #=> nil (and will be nil henceforth)
76
+ ```
77
+
78
+ If all you want is a source worker which generates a series of numbers
79
+ the DSL provides an easier way to do it:
80
+
81
+ ```ruby
82
+ w1 = source_worker [0,1,2]
83
+ w2 = source_worker (0..2)
84
+
85
+ w1.shift == w2.shift #=> 0
86
+ w1.shift == w2.shift #=> 1
87
+ w1.shift == w2.shift #=> 2
88
+ w1.shift == w2.shift #=> nil (and henceforth, etc.)
89
+ ```
90
+
91
+ ### Relay Worker
92
+
93
+ This is perhaps the most "normal" kind of worker in Shifty. It
94
+ accepts a value and returns some transformation of that value:
95
+
96
+ ```ruby
97
+ squarer = relay_worker do |number|
98
+ number ** 2
99
+ end
100
+ ```
101
+
102
+ Of course a relay worker needs a source from which to get values upon
103
+ which to operate:
104
+
105
+ ```ruby
106
+ source = source_worker (0..3)
107
+
108
+ squarer.supply = source
109
+ ```
110
+
111
+ Or, if you prefer, the DSL provides a vertical pipe for linking the
112
+ workers together into a pipeline:
113
+
114
+ ```ruby
115
+ pipeline = source | squarer
116
+
117
+ pipeline.shift #=> 0
118
+ pipeline.shift #=> 1
119
+ pipeline.shift #=> 4
120
+ pipeline.shift #=> 9
121
+ pipeline.shift #=> nil
122
+ ```
123
+
124
+ ### Side Worker
125
+
126
+ As we travel down the pipeline, from time to time, we will want to
127
+ stop and smell the roses, or write to a log file, or drop a beat, or
128
+ create some other kind of side-effect. That's the purpose of the
129
+ `side_worker`. A side worker will pass through the same value that
130
+ it received, but as it does so, it can perform some kind side-effect
131
+ work.
132
+
133
+ ```
134
+ source = source_worker (0..3)
135
+
136
+ evens = []
137
+ even_stasher = side_effect do |value|
138
+ if value % 2 == 0
139
+ evens << value
140
+ end
141
+ end
142
+
143
+ # re-using "squarer" from above example...
144
+ pipeline = source | even_stasher | squarer
145
+
146
+ pipeline.shift #=> 0
147
+ pipeline.shift #=> 1
148
+ pipeline.shift #=> 4
149
+ pipeline.shift #=> 9
150
+ pipeline.shift #=> nil
151
+
152
+ evens #=> [0, 2]
153
+ ```
154
+
155
+ Notice that the output is the same as the previous example, even though
156
+ we put this `even_stasher` `side_worker` in the middle of the pipeline.
157
+
158
+ However, we can also see that the `evens` array now contains the even numbers from `source` (the side effect).
159
+
160
+ _But wait,_ you want to say, _can't we still create side effects with
161
+ regular ole relay workers?_ Why, yes. Yes you can. Ruby being what
162
+ it is, there really isn't a way to prevent the implementation of any
163
+ worker from creating side effects.
164
+
165
+ _And still wait,_ you'll also be wanting to say, _isn't it possible
166
+ that a side worker could mutate the value as it's passed through?_ And
167
+ again, yes. It would be very difficult\* in the Ruby language (where
168
+ so many things are passed by reference) to perfectly prevent a side
169
+ worker from mutating the value. But please don't.
170
+
171
+ The side effect worker's purpose is to provide _intentionality_ and
172
+ clarity. When you're creating a side effect, let it be very clear.
173
+ And don't do regular, pure functional style work in the same worker.
174
+
175
+ This will make side effects easier to troubleshoot; if you pull them
176
+ out of the pipeline, the pipeline's output shouldn't change. By the
177
+ same token, if there is a problem with a side effect, troubleshooting
178
+ it will be much simpler if the side effects are already isolated and
179
+ named.
180
+
181
+ Another measure for preventing side workers from creating unwanted
182
+ side-effects is to use `:hardened` mode. That's right, `side_worker`
183
+ has a mode (which defaults to `:normal`). When set to `:hardened`,
184
+ each value is `Marshal.dump`ed and `Marshal.load`ed before being
185
+ passed to the side worker's callable. This helps to ensure that
186
+ the original value will be passed through to the next worker in the
187
+ queue in a pristine state (unmodified), and that no future or
188
+ subsequent modification can take place.
189
+
190
+ Given a source...
191
+
192
+ ```ruby
193
+ source = source_worker [[:foo], [:bar]]
194
+ ```
195
+
196
+ Consider this unsafe, unhardened side worker:
197
+
198
+ ```ruby
199
+ unsafe = side_worker { |v| v << :boo }
200
+ ```
201
+
202
+ This produces unwanted mutations, because of the `<<`.
203
+
204
+ ```ruby
205
+ pipeline = source | unsafe
206
+
207
+ pipeline.shift #=> [:foo, :boo] <-- Oh noes!
208
+ pipeline.shift #=> [:bar, :boo] <-- Disaster!
209
+ pipeline.shift #=> nil
210
+ ```
211
+
212
+ Let's harden it:
213
+
214
+ ```ruby
215
+ unsafe = side_worker(:hardened) { |v| v << :boo }
216
+ ```
217
+
218
+ The `:hardened` side worker doesn't have access to the original
219
+ value, and therefore its shenanigans are moot with respect to
220
+ the main workflow:
221
+
222
+ ```ruby
223
+ pipeline = source | unsafe
224
+
225
+ pipeline.shift #=> [:foo] <-- No changes
226
+ pipeline.shift #=> [:bar] <-- Much better
227
+ pipeline.shift #=> nil
228
+ ```
229
+
230
+ ### Filter Worker
231
+
232
+ The filter worker simply passes through the values which are given
233
+ to it, but _only_ those values which result in truthiness when your
234
+ provided callable is run against it. Values which result in
235
+ falsiness are simply discarded.
236
+
237
+ ```ruby
238
+ source = source_worker (0..5)
239
+
240
+ filter = filter_worker do |value|
241
+ value % 2 == 0
242
+ end
243
+
244
+ pipeline = source | filter
245
+
246
+ pipeline.shift #=> 0
247
+ pipeline.shift #=> 2
248
+ pipeline.shift #=> 4
249
+ pipeline.shift #=> nil
250
+ ```
251
+
252
+ ### Batch Worker
253
+
254
+ The batch worker gathers outputs into batches and the returns each
255
+ batch as its output.
256
+
257
+ ```ruby
258
+ source = source_worker (0..7)
259
+
260
+ batch = batch_worker gathering: 3
261
+
262
+ pipeline = source | batch
263
+
264
+ pipeline.shift #=> [0, 1, 2]
265
+ pipeline.shift #=> [3, 4, 5]
266
+ pipeline.shift #=> [6, 7]
267
+ pipeline.shift #=> nil
268
+ ```
269
+
270
+ Notice how the final batch doesn't have full compliment of three.
271
+
272
+ Fixed-numbered batch workers are useful for things like pagination,
273
+ perhaps, but sometimes we don't want a batch to include a specific
274
+ number of items, but to batch together all items until a certain
275
+ condition is met. So batch worker can also accept a callable:
276
+
277
+ ```ruby
278
+ source = source_worker [
279
+ "some", "rain", "must\n", "fall", "but", "ok" ]
280
+
281
+ line_reader = batch_worker do |value|
282
+ value.end_with? "\n"
283
+ end
284
+
285
+ pipeline = source | line_reader
286
+
287
+ pipeline.shift #=> ["some", "rain", "must\n"]
288
+ pipeline.shift #=> ["fall", "but", "ok"]
289
+ pipeline.shift #=> nil
290
+ ```
291
+
292
+ ### Splitter Worker
293
+
294
+ The splitter worker accepts a value from its supply, and generates an array
295
+ and then successively returns each element of the array. Once the array
296
+ has been expended, the splitter worker appeals to its supplier for another
297
+ value and the process repeats.
298
+
299
+ ```ruby
300
+ source = source_worker [
301
+ 'A bold', 'move westward' ]
302
+
303
+ splitter = splitter_worker do |value|
304
+ value.split(' ')
305
+ end
306
+
307
+ pipeline = source | splitter
308
+
309
+ pipeline.shift #=> 'A'
310
+ pipeline.shift #=> 'bold'
311
+ pipeline.shift #=> 'move'
312
+ pipeline.shift #=> 'westward'
313
+ pipeline.shift #=> nil
314
+ ```
315
+
316
+ ### Trailing Worker
317
+
318
+
319
+ The trailing worker accepts a value, and returns an array containing the
320
+ last _n_ values. This worker could be useful for things like rolling averages.
321
+
322
+ No values are returned until n values had been accumulated. New values are
323
+ _unshifted_ into the zero-th position in the array of trailing values, so
324
+ that a given value will graduate through the array until it reaches the last
325
+ position and then subsequently removed.
326
+
327
+ Maybe it's easiest to just give an example:
328
+
329
+ ```ruby
330
+ source = source_worker (0..5)
331
+ trailer = trailing_worker 4
332
+
333
+ pipeline = source | trailer
334
+
335
+ pipeline.shift #=> [3,2,1,0]
336
+ pipeline.shift #=> [4,3,2,1]
337
+ pipeline.shift #=> [5,4,3,2]
338
+ pipeline.shift #=> nil
339
+ ```
340
+
341
+ Note that as soon as a nil is received from the supplying queue, the trailing
342
+ worker simply provides a nil.
343
+
344
+
345
+ ## Origins of Shifty
346
+
347
+ Shifty was Shifty, which grew out of experimentation with Ruby fibers, after reading
348
+ [Dave Thomas' demo of Ruby fibers](http://pragdave.me/blog/2007/12/30/pipelines-using-fibers-in-ruby-19/),
349
+ wherein he created a pipeline of fiber processes, emulating the style and syntax of the
350
+ \*nix command line. I noticed that, courtesy of fibers' extremely low surface area,
351
+ fiber-to-fiber collaborators could operate with extremely low coupling. That was the
352
+ original motivation for creating the framework.
353
+
354
+ After playing around with the new framework a bit, I began to notice
355
+ other interesting characteristics of this paradigm.
356
+
357
+ ### Escalator vs Elevator
358
+
359
+ Suppose we are performing several operations on the members of a largish
360
+ collection. If we daisy-chain enumerable operators together (which is so
361
+ easy and fun with Ruby) we will notice that if something goes awry with
362
+ item number two during operation number seven, we nonetheless had to wait
363
+ through a complete run of all items through operations 1 - 6 before we
364
+ receive the bad news early in operation seven. Imagine if the first six
365
+ operations take a long time to each complete? Furthermore, what if the
366
+ operations on all incomplete items must be reversed in some way (e.g.
367
+ cleaned up or rolled back)? It would be far less messy and far more
368
+ expedient if each item could be processed though all operations before
369
+ the next one is begun.
370
+
371
+ This is the design paradigm which shifty makes easy. Although all
372
+ the workers in your assembly line can be coded in the same context (which
373
+ is one of the big selling points of the daisy-chaining of enumerable
374
+ methods,incidentally), you also get the advantage of passing each item
375
+ though the entire op-chain before starting the next.
376
+
377
+ ### Think Locally, Act Globally
378
+
379
+ Because shifty workers use fibers as their basic API, the are almost
380
+ unaware of the outside world. And they can pretty much be written as such.
381
+ At the heart of each Shifty worker is a task which you provide, which
382
+ is a callable ruby object (such as a Proc or a lambda).
383
+
384
+ The scope of the work is whatever scope existed in the task when you
385
+ initially created the worker. And if you want a worker to maintain its
386
+ own internal state, you can simply include a loop within the worker's
387
+ task, and use the `#handoff` method to pass along the worker's product at
388
+ the appropriate point in the loop.
389
+
390
+ For example:
391
+
392
+ ```ruby
393
+ realestate_maker = Shifty::Worker.new do
394
+ oceans = %w[Atlantic Pacific Indiana Arctic]
395
+ previous_ocean = "Atlantic"
396
+ while current_ocean = oceans.sample
397
+ drain current_ocean #=> let's posit that this is a long-running async process!
398
+ handoff previous_ocean
399
+ previous_ocean = current_ocean
400
+ end
401
+ ```
402
+
403
+ Anything scoped to the outside of that loop but inside the worker's task
404
+ will essentially become that worker's initial state. This means we often
405
+ don't need to bother with instance variables, accessors, and other
406
+ features of classes which deal with maintaining instances' state.
407
+
408
+ ### The Wormhole Effect
409
+
410
+ Despite the fact that the work is done in separate threads, the _scope_
411
+ of the work is the scope of the coordinating body of code. This means
412
+ that even the remaining coupling in such systems --which is primarily
413
+ just _common objects_ coupling-- this little remaining coupling is
414
+ mitigated by the possibility of having coupled code _live together_.
415
+ I call this feature the _folded collaborators_ effect.
416
+
417
+ Consider the following ~~code~~ vaporware:
418
+
419
+ ```ruby
420
+ SUBJECT = 'kitteh'
421
+
422
+ include Shifty::DSL
423
+
424
+ tweet_getter = source_worker do
425
+ twitter_api.fetch_my_tweets
426
+ end
427
+
428
+ about_me = filter_worker do |tweet|
429
+ tweet.referenced.include? SUBJECT
430
+ end
431
+
432
+ tweet_formatter = relay_worker do |tweet|
433
+ apply_format_to tweet
434
+ end
435
+
436
+ kitteh_tweets = tweet_getter | about_me | tweet_formatter
437
+
438
+ while tweet = kitteh_tweets.shift
439
+ display(tweet)
440
+ end
441
+ ```
442
+
443
+ None of these tasks have hard coupling with each other. If we were to
444
+ insert another worker between the filter and the formatter, neither of
445
+ those workers would need changes in their code, assuming the inserted
446
+ worker plays nicely with the objects they're all passing along.
447
+
448
+ Which brings me to the point: these workers to have a dependency upon the
449
+ objects they're handing off and receiving. But we have the capability to
450
+ coordinate those workers in a centralized location (such as in this code
451
+ example).
452
+
453
+ ## Shifty::Worker
454
+
455
+ The Shifty::Worker documentation (which was the old README) is now
456
+ [here](docs/shifty/worker.md).
457
+
458
+ ## Roadmap
459
+
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "shifty"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,80 @@
1
+ # Shifty::Worker
2
+
3
+ _The workhorse of the Shifty framework._
4
+
5
+ ## Workers have tasks...
6
+
7
+ Initialize with a block of code:
8
+
9
+ ```ruby
10
+ source_worker = Shifty::Worker.new { "hulk" }
11
+
12
+ source_worker.shift #=> "hulk"
13
+ ```
14
+ If you supply a worker with another worker as its supply, then you
15
+ can give it a task which accepts a value:
16
+
17
+ ```ruby
18
+ relay_worker = Shifty::Worker.new { |name| name.upcase }
19
+ relay_worker.supply = source_worker
20
+
21
+ relay_worker.shift #=> "HULK"
22
+ ```
23
+
24
+ You can also initialize a worker by passing in a callable object
25
+ as its task:
26
+
27
+ ```ruby
28
+ capitalizer = Proc.new { |name| name.capitalize }
29
+ relay_worker = Shifty::Worker.new(task: capitalizer, supply: source_worker)
30
+
31
+ relay_worker.shift #=> 'Hulk'
32
+ ```
33
+
34
+ A worker also has an accessor for its @task:
35
+
36
+ ```ruby
37
+ doofusizer = Proc.new { |name| name.gsub(/u/, 'oo') }
38
+ relay_worker.task = doofusizer
39
+
40
+ relay_worker.shift #=> 'hoolk'
41
+ ```
42
+
43
+ And finally, you can provide a task by directly overriding the
44
+ worker's #task instance method:
45
+
46
+ ```ruby
47
+ def relay_worker.task(name)
48
+ name.to_sym
49
+ end
50
+
51
+ relay_worker.shift #=> :hulk
52
+ ```
53
+
54
+ Even workers without a task have a task; all workers actually come
55
+ with a default task which simply passes on the received value unchanged:
56
+
57
+ ```ruby
58
+ useless_worker = Shifty::Worker.new(supply: source_worker)
59
+
60
+ useless_worker.shift #=> 'hulk'
61
+ ```
62
+
63
+ ## The pipeline DSL
64
+
65
+ You can stitch your workers together using the vertical pipe ("|") like so:
66
+
67
+ ```ruby
68
+ pipeline = source_worker | relay_worker | another worker
69
+ ```
70
+
71
+ ...and then just call on that pipeline (it's actually the last worker in the
72
+ chain):
73
+
74
+ ```ruby
75
+ while next_value = pipeline.shift do
76
+ do_something_with next_value
77
+ # etc.
78
+ end
79
+ ```
80
+
data/lib/shifty.rb ADDED
@@ -0,0 +1,5 @@
1
+ require_relative 'shifty/version'
2
+ require_relative 'shifty/worker'
3
+ require_relative 'shifty/gang'
4
+ require_relative 'shifty/roster'
5
+ require_relative 'shifty/dsl'
data/lib/shifty/dsl.rb ADDED
@@ -0,0 +1,180 @@
1
+ module Shifty
2
+ class WorkerInitializationError < StandardError; end
3
+
4
+ module DSL
5
+ def source_worker(argument=nil, &block)
6
+ ensure_correct_arity_for!(argument, block)
7
+
8
+ series = series_from(argument)
9
+ callable = setup_callable_for(block, series)
10
+
11
+ return Worker.new(&callable) if series.nil?
12
+
13
+ Worker.new do
14
+ series.each(&callable)
15
+
16
+ while true do
17
+ handoff nil
18
+ end
19
+ end
20
+
21
+ end
22
+
23
+ def relay_worker(&block)
24
+ ensure_regular_arity(block)
25
+
26
+ Worker.new do |value|
27
+ value && block.call(value)
28
+ end
29
+ end
30
+
31
+ def side_worker(mode=:normal, &block)
32
+ ensure_regular_arity(block)
33
+
34
+ Worker.new do |value|
35
+ value.tap do |v|
36
+ used_value = mode == :hardened ?
37
+ Marshal.load(Marshal.dump(v)) : v
38
+
39
+ v && block.call(used_value)
40
+ end
41
+ end
42
+ end
43
+
44
+ def filter_worker(argument=nil, &block)
45
+ if (block && argument.respond_to?(:call))
46
+ throw_with 'You cannot supply two callables'
47
+ end
48
+ callable = argument.respond_to?(:call) ? argument : block
49
+ ensure_callable(callable)
50
+
51
+ Worker.new do |value, supply|
52
+ while value && !callable.call(value) do
53
+ value = supply.shift
54
+ end
55
+ value
56
+ end
57
+ end
58
+
59
+ class BatchContext < OpenStruct
60
+ def batch_complete?(value, collection)
61
+ value.nil? ||
62
+ !! batch_full.call(value, collection)
63
+ end
64
+ end
65
+
66
+ def batch_worker(options = {gathering: 1}, &block)
67
+ ensure_regular_arity(block) if block
68
+ batch_full = block ||
69
+ Proc.new { |_, batch| batch.size >= options[:gathering] }
70
+
71
+ batch_context = BatchContext.new({ batch_full: batch_full })
72
+
73
+ Worker.new(context: batch_context) do |value, supply, context|
74
+ if value
75
+ context.collection = [value]
76
+ until context.batch_complete?(
77
+ context.collection.last,
78
+ context.collection
79
+ )
80
+ context.collection << supply.shift
81
+ end
82
+ context.collection.compact
83
+ end
84
+ end
85
+ end
86
+
87
+ def splitter_worker(&block)
88
+ ensure_regular_arity(block)
89
+
90
+ Worker.new do |value|
91
+ if value.nil?
92
+ value
93
+ else
94
+ parts = [block.call(value)].flatten
95
+ while parts.size > 1 do
96
+ handoff parts.shift
97
+ end
98
+ parts.shift
99
+ end
100
+ end
101
+ end
102
+
103
+ def trailing_worker(trail_length=2)
104
+ trail = []
105
+ Worker.new do |value, supply|
106
+ if value
107
+ trail.unshift value
108
+ if trail.size >= trail_length
109
+ trail.pop
110
+ end
111
+ while trail.size < trail_length
112
+ trail.unshift supply.shift
113
+ end
114
+
115
+ trail
116
+ else
117
+ value
118
+ end
119
+ end
120
+ end
121
+
122
+ def handoff(something)
123
+ Fiber.yield something
124
+ end
125
+
126
+ private
127
+
128
+ def throw_with(*msg)
129
+ raise WorkerInitializationError.new([msg].flatten.join(' '))
130
+ end
131
+
132
+ def ensure_callable(callable)
133
+ unless callable && callable.respond_to?(:call)
134
+ throw_with 'You must supply a callable'
135
+ end
136
+ end
137
+
138
+ def ensure_regular_arity(block)
139
+ if block.arity != 1
140
+ throw_with \
141
+ "Worker must accept exactly one argument (arity == 1)"
142
+ end
143
+ end
144
+
145
+ # only valid for #source_worker
146
+ def ensure_correct_arity_for!(argument, block)
147
+ return unless block
148
+ if argument
149
+ ensure_regular_arity(block)
150
+ else
151
+ if block.arity > 0
152
+ throw_with \
153
+ 'Source worker cannot accept any arguments (arity == 0)'
154
+ end
155
+ end
156
+ end
157
+
158
+ def series_from(series)
159
+ return if series.nil?
160
+ case
161
+ when series.respond_to?(:to_a)
162
+ series.to_a
163
+ when series.respond_to?(:scan)
164
+ series.scan(/./)
165
+ else
166
+ [series]
167
+ end
168
+ end
169
+
170
+ def setup_callable_for(block, series)
171
+ return block unless series
172
+ if block
173
+ return Proc.new { |value| handoff block.call(value) }
174
+ else
175
+ return Proc.new { |value| handoff value }
176
+ end
177
+ end
178
+
179
+ end
180
+ end
@@ -0,0 +1,46 @@
1
+ require 'shifty/roster'
2
+
3
+ module Shifty
4
+ class Gang
5
+ attr_accessor :workers
6
+
7
+ def initialize(workers=[])
8
+ link(workers + [])
9
+ end
10
+
11
+ def roster
12
+ workers
13
+ end
14
+
15
+ def shift
16
+ workers.last.shift
17
+ end
18
+
19
+ def ready_to_work?
20
+ workers.first.ready_to_work?
21
+ end
22
+
23
+ def supply
24
+ workers.first.supply
25
+ end
26
+
27
+ def supply=(source_queue)
28
+ workers.first.supply = source_queue
29
+ end
30
+
31
+ def supplies(subscribing_party)
32
+ subscribing_party.supply = self
33
+ end
34
+ alias_method :"|", :supplies
35
+
36
+ private
37
+
38
+ def link(workers)
39
+ @workers = [workers.shift]
40
+ while worker = workers.shift do
41
+ Roster[self] << worker
42
+ end
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,46 @@
1
+ module Shifty
2
+ module Roster
3
+ class << self
4
+ def [](gang)
5
+ RosterizedGang.new(gang)
6
+ end
7
+ end
8
+ end
9
+
10
+ class RosterizedGang
11
+ attr_reader :gang
12
+
13
+ def initialize(gang)
14
+ @gang = gang
15
+ end
16
+
17
+ def workers
18
+ gang.workers
19
+ end
20
+
21
+ def push(worker)
22
+ if worker
23
+ worker.supply = workers.last
24
+ workers << worker
25
+ end
26
+ end
27
+ alias_method :"<<", :push
28
+
29
+ def pop
30
+ gang.workers.pop.tap do |popped|
31
+ popped.supply = nil
32
+ end
33
+ end
34
+
35
+ def shift
36
+ workers.shift.tap do
37
+ workers.first.supply = nil
38
+ end
39
+ end
40
+
41
+ def unshift(worker)
42
+ workers.first.supply = worker
43
+ workers.unshift worker
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,3 @@
1
+ module Shifty
2
+ VERSION = "0.3.0"
3
+ end
@@ -0,0 +1,80 @@
1
+ require 'ostruct'
2
+
3
+ module Shifty
4
+ class Worker
5
+ attr_reader :supply
6
+
7
+ def initialize(p={}, &block)
8
+ @supply = p[:supply]
9
+ @task = block || p[:task]
10
+ @context = p[:context] || OpenStruct.new
11
+ # don't define default task here
12
+ # because we want to allow for
13
+ # an initialized worker to have
14
+ # a task injected, including
15
+ # method-based tasks.
16
+ end
17
+
18
+ def shift
19
+ ensure_ready_to_work!
20
+ workflow.resume
21
+ end
22
+
23
+ def ready_to_work?
24
+ @task && (supply || !task_accepts_a_value?)
25
+ end
26
+
27
+ def supplies(subscribing_party)
28
+ subscribing_party.supply = self
29
+ subscribing_party
30
+ end
31
+ alias_method :"|", :supplies
32
+
33
+ def supply=(supplier)
34
+ raise WorkerError.new("Worker is a source, and cannot accept a supply") unless suppliable?
35
+ @supply = supplier
36
+ end
37
+
38
+ def suppliable?
39
+ @task && @task.arity > 0
40
+ end
41
+
42
+ private
43
+
44
+ def ensure_ready_to_work!
45
+ @task ||= default_task
46
+
47
+ unless ready_to_work?
48
+ raise "This worker's task expects to receive a value from a supplier, but has no supply."
49
+ end
50
+ end
51
+
52
+ def workflow
53
+ @my_little_machine ||= Fiber.new do
54
+ loop do
55
+ value = supply && supply.shift
56
+ Fiber.yield @task.call(value, supply, @context)
57
+ end
58
+ end
59
+ end
60
+
61
+ def default_task
62
+ Proc.new { |value| value }
63
+ end
64
+
65
+ def task_accepts_a_value?
66
+ @task.arity > 0
67
+ end
68
+
69
+ def task_method_exists?
70
+ self.methods.include? :task
71
+ end
72
+
73
+ def task_method_accepts_a_value?
74
+ self.method(:task).arity > 0
75
+ end
76
+
77
+ end
78
+
79
+ class WorkerError < StandardError; end
80
+ end
data/shifty.gemspec ADDED
@@ -0,0 +1,29 @@
1
+ lib = File.expand_path('../lib', __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require 'shifty/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'shifty'
7
+ spec.version = Shifty::VERSION
8
+ spec.authors = ['Joel Helbling']
9
+ spec.email = ['joel@joelhelbling.com']
10
+
11
+ spec.summary = %q{A functional framework aimed at extremely low coupling}
12
+ spec.description = %q{Shifty provides tools for coordinating simple workers which consume a supplying queue and emit corresponding work products, valuing pure functions, carefully isolated side effects, and extremely low coupling.}
13
+ spec.homepage = 'https://github.com/joelhelbling/shifty'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
17
+ f.match(%r{^(test|spec|features)/})
18
+ end
19
+ spec.bindir = "exe"
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ["lib"]
22
+
23
+ spec.add_development_dependency 'bundler', '~> 1.16'
24
+ spec.add_development_dependency 'rake', '~> 10.0'
25
+ spec.add_development_dependency 'rspec', '~> 3.1'
26
+ spec.add_development_dependency 'rspec-given', '~> 3.8'
27
+ spec.add_development_dependency 'pry'
28
+ spec.add_development_dependency 'simplecov'
29
+ end
metadata ADDED
@@ -0,0 +1,148 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: shifty
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0
5
+ platform: ruby
6
+ authors:
7
+ - Joel Helbling
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2018-04-01 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.16'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.16'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.1'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.1'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec-given
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '3.8'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '3.8'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pry
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: simplecov
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: Shifty provides tools for coordinating simple workers which consume a
98
+ supplying queue and emit corresponding work products, valuing pure functions, carefully
99
+ isolated side effects, and extremely low coupling.
100
+ email:
101
+ - joel@joelhelbling.com
102
+ executables: []
103
+ extensions: []
104
+ extra_rdoc_files: []
105
+ files:
106
+ - ".gitignore"
107
+ - ".rspec"
108
+ - ".travis.yml"
109
+ - Gemfile
110
+ - LICENSE.txt
111
+ - README.md
112
+ - Rakefile
113
+ - bin/console
114
+ - bin/setup
115
+ - docs/shifty/worker.md
116
+ - lib/shifty.rb
117
+ - lib/shifty/dsl.rb
118
+ - lib/shifty/gang.rb
119
+ - lib/shifty/roster.rb
120
+ - lib/shifty/version.rb
121
+ - lib/shifty/worker.rb
122
+ - pkg/.gitkeep
123
+ - shifty.gemspec
124
+ homepage: https://github.com/joelhelbling/shifty
125
+ licenses:
126
+ - MIT
127
+ metadata: {}
128
+ post_install_message:
129
+ rdoc_options: []
130
+ require_paths:
131
+ - lib
132
+ required_ruby_version: !ruby/object:Gem::Requirement
133
+ requirements:
134
+ - - ">="
135
+ - !ruby/object:Gem::Version
136
+ version: '0'
137
+ required_rubygems_version: !ruby/object:Gem::Requirement
138
+ requirements:
139
+ - - ">="
140
+ - !ruby/object:Gem::Version
141
+ version: '0'
142
+ requirements: []
143
+ rubyforge_project:
144
+ rubygems_version: 2.7.3
145
+ signing_key:
146
+ specification_version: 4
147
+ summary: A functional framework aimed at extremely low coupling
148
+ test_files: []