stages 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +10 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +24 -0
- data/README +37 -0
- data/Rakefile +31 -0
- data/VERSION +1 -0
- data/lib/stage_base.rb +47 -0
- data/lib/stages.rb +3 -0
- data/lib/stages/each_element.rb +14 -0
- data/lib/stages/evens.rb +11 -0
- data/lib/stages/hash_lookup.rb +12 -0
- data/lib/stages/map.rb +7 -0
- data/lib/stages/multiples_of.rb +12 -0
- data/lib/stages/select.rb +7 -0
- data/stages.gemspec +55 -0
- data/test/helper.rb +88 -0
- data/test/test_pipeline.rb +56 -0
- data/test/test_stages.rb +51 -0
- metadata +87 -0
data/.autotest
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
Autotest.add_hook(:initialize) do |at|
|
2
|
+
%w{.git .svn .hg .swp .DS_Store ._* tmp}.each do |exception|
|
3
|
+
at.add_exception(exception)
|
4
|
+
end
|
5
|
+
|
6
|
+
at.add_mapping(%r%^dataprocessing/pipeline(/stages)?/(.*).rb$%, true) do |filename, _|
|
7
|
+
['test/test_pipeline.rb', 'test/test_stages.rb']
|
8
|
+
end
|
9
|
+
|
10
|
+
end
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://gemcutter.org/
|
3
|
+
specs:
|
4
|
+
ZenTest (4.6.2)
|
5
|
+
ansi (1.4.1)
|
6
|
+
git (1.2.5)
|
7
|
+
jeweler (1.6.4)
|
8
|
+
bundler (~> 1.0)
|
9
|
+
git (>= 1.2.5)
|
10
|
+
rake
|
11
|
+
minitest (2.10.0)
|
12
|
+
rake (0.9.2)
|
13
|
+
turn (0.8.2)
|
14
|
+
ansi (>= 1.2.2)
|
15
|
+
|
16
|
+
PLATFORMS
|
17
|
+
ruby
|
18
|
+
|
19
|
+
DEPENDENCIES
|
20
|
+
ZenTest
|
21
|
+
jeweler
|
22
|
+
minitest
|
23
|
+
rake (= 0.9.2)
|
24
|
+
turn
|
data/README
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
Stages
|
2
|
+
------
|
3
|
+
|
4
|
+
A gem for creating data pipelines out of tiny, reusable objects
|
5
|
+
|
6
|
+
Initial code stolen shamelessly from http://pragdave.blogs.pragprog.com/pragdave/2008/01/pipelines-using.html
|
7
|
+
|
8
|
+
Usage
|
9
|
+
-----
|
10
|
+
|
11
|
+
You knit your stages together with '|'. The leftmost pipeline stage will be contain a generator, which usually is an infinite loop. For ean example, look at evens.rb in our sample stages collection. If you wanted to output, for example, every even number divisible by 3, you might do:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
pipeline = Evens.new | MultiplesOf.new(3)
|
15
|
+
loop { puts pipeline.run }
|
16
|
+
```
|
17
|
+
|
18
|
+
We have included some general purpose stages, map and select, which can accomplish many pipeline operations:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
pipeline = Evens.new | Map.new{ |x| x * 3} | Select.new{ |x| x % 7 == 0}
|
22
|
+
(0..2).map{ |x| pipeline.run } #[0, 42, 84]}
|
23
|
+
```
|
24
|
+
|
25
|
+
Writing New Stages
|
26
|
+
------------------
|
27
|
+
|
28
|
+
If you are writing a stage that needs to process an element, you probably want to subclass Stage and implement handle_value.
|
29
|
+
|
30
|
+
If you are writing a generator, you probably want to subclass Stage and implement process
|
31
|
+
|
32
|
+
Stern Warnings
|
33
|
+
--------------
|
34
|
+
|
35
|
+
Returning nil from handle_value kills a pipeline. We may change this behavior in the future, but for now, it makes life easy.
|
36
|
+
|
37
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'rake'
|
11
|
+
|
12
|
+
|
13
|
+
require 'rake/testtask'
|
14
|
+
Rake::TestTask.new(:test) do |test|
|
15
|
+
test.libs << 'lib' << 'test'
|
16
|
+
test.pattern = 'test/test_*.rb'
|
17
|
+
test.verbose = true
|
18
|
+
end
|
19
|
+
|
20
|
+
require 'jeweler'
|
21
|
+
Jeweler::Tasks.new do |gem|
|
22
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
23
|
+
gem.name = "stages"
|
24
|
+
gem.homepage = "https://github.com/iGoDigital-LLC/stages"
|
25
|
+
gem.summary = "pipeline builder"
|
26
|
+
gem.description = "pipeline builder"
|
27
|
+
gem.email = "support@igodigital.com"
|
28
|
+
gem.authors = ["Nathan Acuff", "Justin Hill", "Matt Brown", "Kyle Prifogle"]
|
29
|
+
# dependencies defined in Gemfile
|
30
|
+
end
|
31
|
+
Jeweler::RubygemsDotOrgTasks.new
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1
|
data/lib/stage_base.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
module Stages
|
2
|
+
class Stage
|
3
|
+
attr_accessor :source
|
4
|
+
|
5
|
+
def initialize(&block)
|
6
|
+
@block = block
|
7
|
+
@fiber_delegate = Fiber.new do
|
8
|
+
process
|
9
|
+
die
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
@fiber_delegate.resume
|
15
|
+
end
|
16
|
+
|
17
|
+
def die
|
18
|
+
loop do
|
19
|
+
output nil
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def process
|
24
|
+
while value = input
|
25
|
+
handle_value value
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def handle_value(value)
|
30
|
+
output value
|
31
|
+
end
|
32
|
+
|
33
|
+
def input
|
34
|
+
source.run
|
35
|
+
end
|
36
|
+
|
37
|
+
def output(value)
|
38
|
+
Fiber.yield value
|
39
|
+
end
|
40
|
+
|
41
|
+
def |(other=nil)
|
42
|
+
other.source = self
|
43
|
+
other
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
data/lib/stages.rb
ADDED
data/lib/stages/evens.rb
ADDED
data/lib/stages/map.rb
ADDED
data/stages.gemspec
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "stages"
|
8
|
+
s.version = "0.0.1"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Nathan Acuff", "Justin Hill", "Matt Brown", "Kyle Prifogle"]
|
12
|
+
s.date = "2012-01-13"
|
13
|
+
s.description = "pipeline builder"
|
14
|
+
s.email = "support@igodigital.com"
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"README"
|
17
|
+
]
|
18
|
+
s.files = [
|
19
|
+
".autotest",
|
20
|
+
"Gemfile",
|
21
|
+
"Gemfile.lock",
|
22
|
+
"README",
|
23
|
+
"Rakefile",
|
24
|
+
"VERSION",
|
25
|
+
"lib/stage_base.rb",
|
26
|
+
"lib/stages.rb",
|
27
|
+
"lib/stages/each_element.rb",
|
28
|
+
"lib/stages/evens.rb",
|
29
|
+
"lib/stages/hash_lookup.rb",
|
30
|
+
"lib/stages/map.rb",
|
31
|
+
"lib/stages/multiples_of.rb",
|
32
|
+
"lib/stages/select.rb",
|
33
|
+
"stages.gemspec",
|
34
|
+
"test/helper.rb",
|
35
|
+
"test/test_pipeline.rb",
|
36
|
+
"test/test_stages.rb"
|
37
|
+
]
|
38
|
+
s.homepage = "https://github.com/iGoDigital-LLC/stages"
|
39
|
+
s.require_paths = ["lib"]
|
40
|
+
s.rubygems_version = "1.8.11"
|
41
|
+
s.summary = "pipeline builder"
|
42
|
+
|
43
|
+
if s.respond_to? :specification_version then
|
44
|
+
s.specification_version = 3
|
45
|
+
|
46
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
47
|
+
s.add_runtime_dependency(%q<rake>, ["= 0.9.2"])
|
48
|
+
else
|
49
|
+
s.add_dependency(%q<rake>, ["= 0.9.2"])
|
50
|
+
end
|
51
|
+
else
|
52
|
+
s.add_dependency(%q<rake>, ["= 0.9.2"])
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
data/test/helper.rb
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'bundler/setup'
|
3
|
+
Bundler.require(:default, :test)
|
4
|
+
require 'stages'
|
5
|
+
require 'turn'
|
6
|
+
require 'minitest/unit'
|
7
|
+
|
8
|
+
class MiniTest::Unit::TestCase
|
9
|
+
def self.test(name, &block)
|
10
|
+
define_method 'test_' + name, &block
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class MiniTest::Unit
|
15
|
+
alias :run_test_suites_old :run_test_suites
|
16
|
+
def run_test_suites(filter = /./)
|
17
|
+
@test_count, @assertion_count = 0, 0
|
18
|
+
old_sync, @@out.sync = @@out.sync, true if @@out.respond_to? :sync=
|
19
|
+
TestCase.test_suites.each do |suite|
|
20
|
+
test_cases = suite.test_methods.grep(filter)
|
21
|
+
if test_cases.size > 0
|
22
|
+
@@out.print "\n#{suite}:\n"
|
23
|
+
end
|
24
|
+
|
25
|
+
test_cases.each do |test|
|
26
|
+
inst = suite.new test
|
27
|
+
inst._assertions = 0
|
28
|
+
|
29
|
+
|
30
|
+
@broken = nil
|
31
|
+
|
32
|
+
@@out.print(case run_one inst
|
33
|
+
when :pass
|
34
|
+
@broken = false
|
35
|
+
green { "P" }
|
36
|
+
when :error
|
37
|
+
@broken = true
|
38
|
+
@@out.puts
|
39
|
+
yellow { pad_with_size "ERROR" }
|
40
|
+
when :fail
|
41
|
+
@broken = true
|
42
|
+
@@out.puts
|
43
|
+
red { pad_with_size "FAIL" }
|
44
|
+
when :skip
|
45
|
+
@broken = false
|
46
|
+
cyan { "S" }
|
47
|
+
when :timeout
|
48
|
+
@broken = true
|
49
|
+
@@out.puts
|
50
|
+
cyan { pad_with_size "TIMEOUT" }
|
51
|
+
end)
|
52
|
+
|
53
|
+
if @broken
|
54
|
+
@@out.print MiniTest::Unit.use_natural_language_case_names? ?
|
55
|
+
" #{test.gsub("test_", "").gsub(/_/, " ")}" : " #{test}"
|
56
|
+
@@out.print " (%.2fs) " % @elapsed
|
57
|
+
@@out.puts
|
58
|
+
|
59
|
+
report = @report.last
|
60
|
+
@@out.puts pad(report[:message], 10)
|
61
|
+
trace = MiniTest::filter_backtrace(report[:exception].backtrace).first
|
62
|
+
@@out.print pad(trace, 10)
|
63
|
+
|
64
|
+
@@out.puts
|
65
|
+
end
|
66
|
+
|
67
|
+
# @@out.puts
|
68
|
+
@test_count += 1
|
69
|
+
@assertion_count += inst._assertions
|
70
|
+
end
|
71
|
+
end
|
72
|
+
@@out.sync = old_sync if @@out.respond_to? :sync=
|
73
|
+
[@test_count, @assertion_count]
|
74
|
+
end
|
75
|
+
|
76
|
+
def run_one(inst)
|
77
|
+
t = Time.now
|
78
|
+
result = inst.run self
|
79
|
+
@elapsed = Time.now - t
|
80
|
+
if result == :pass && @elapsed > 5.0
|
81
|
+
result = :timeout
|
82
|
+
@report << { :message => "Test took a long time (%.2fs)" % @elapsed,
|
83
|
+
:exception => Exception.new("Long test")}
|
84
|
+
end
|
85
|
+
result
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'helper'
|
2
|
+
include Stages
|
3
|
+
|
4
|
+
class TestPipeline < MiniTest::Unit::TestCase
|
5
|
+
|
6
|
+
test 'simple basic pipeline' do
|
7
|
+
evens = Evens.new
|
8
|
+
mx3 = MultiplesOf.new(3)
|
9
|
+
mx7 = MultiplesOf.new(7)
|
10
|
+
mx3.source = evens
|
11
|
+
mx7.source = mx3
|
12
|
+
|
13
|
+
result = (0..2).map{ |x| mx7.run }
|
14
|
+
assert_equal([0, 42, 84], result)
|
15
|
+
end
|
16
|
+
|
17
|
+
test 'pipeline pipe syntax works' do
|
18
|
+
pipeline = Evens.new | MultiplesOf.new(3) | MultiplesOf.new(7)
|
19
|
+
result = (0..2).map{ |x| pipeline.run }
|
20
|
+
assert_equal([0, 42, 84], result)
|
21
|
+
end
|
22
|
+
|
23
|
+
test 'block stages work' do
|
24
|
+
pipeline = Evens.new | Map.new{ |x| x * 3} | Select.new{ |x| x % 7 == 0}
|
25
|
+
result = (0..2).map{ |x| pipeline.run }
|
26
|
+
assert_equal([0, 42, 84], result)
|
27
|
+
end
|
28
|
+
|
29
|
+
test 'exceptions do what you expect' do
|
30
|
+
begin
|
31
|
+
pipeline = Evens.new | Map.new{ |val| throw Exception.new "exception!" } | Select.new{ |v| v > 2}
|
32
|
+
pipeline.run
|
33
|
+
assert false
|
34
|
+
rescue Exception => e
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
test 'nil kills it' do
|
39
|
+
pipeline = EachElement.new([1, 2, nil, 3])
|
40
|
+
result = []
|
41
|
+
while v = pipeline.run
|
42
|
+
result << v
|
43
|
+
end
|
44
|
+
assert_equal([1, 2], result)
|
45
|
+
end
|
46
|
+
|
47
|
+
def sing
|
48
|
+
{ :do => 'doe a deer a female deer',
|
49
|
+
:re => 'ray a drop of golden sun',
|
50
|
+
:mi => 'me a name I call myself',
|
51
|
+
:fa => 'far a long long way to run',
|
52
|
+
:so => 'A needle pulling thread',
|
53
|
+
:la => 'a note to follow so',
|
54
|
+
:ti => 'a drink with jam and bread'}
|
55
|
+
end
|
56
|
+
end
|
data/test/test_stages.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'helper'
|
2
|
+
include Stages
|
3
|
+
|
4
|
+
class TestStages < MiniTest::Unit::TestCase
|
5
|
+
|
6
|
+
test 'evens' do
|
7
|
+
evens = Evens.new
|
8
|
+
result = (0..2).map{ evens.run }
|
9
|
+
assert_equal([0, 2, 4], result)
|
10
|
+
end
|
11
|
+
|
12
|
+
test 'select' do
|
13
|
+
pipeline = Evens.new | Select.new{ |val| val > 6}
|
14
|
+
result = (0..2).map{ |x| pipeline.run }
|
15
|
+
assert_equal([8, 10, 12], result)
|
16
|
+
end
|
17
|
+
|
18
|
+
test 'map' do
|
19
|
+
pipeline = Evens.new | Map.new{ |val| val * 3}
|
20
|
+
result = (0..2).map{ |x| pipeline.run }
|
21
|
+
assert_equal([0, 6, 12], result)
|
22
|
+
end
|
23
|
+
|
24
|
+
test 'multiples_of' do
|
25
|
+
pipeline = Evens.new | MultiplesOf.new(3)
|
26
|
+
result = (0..3).map{ |x| pipeline.run }
|
27
|
+
assert_equal([0, 6, 12, 18], result)
|
28
|
+
end
|
29
|
+
|
30
|
+
test 'each_element' do
|
31
|
+
pipeline = EachElement.new([1, 2, 3])
|
32
|
+
result = (0..2).map{ |x| pipeline.run }
|
33
|
+
assert_equal([1, 2, 3], result)
|
34
|
+
end
|
35
|
+
|
36
|
+
test 'hash_lookup' do
|
37
|
+
pipeline = EachElement.new([:do, :re, :mi]) | HashLookup.new(sing)
|
38
|
+
result = (0..2).map{ |x| pipeline.run }
|
39
|
+
assert_equal(['doe a deer a female deer', 'ray a drop of golden sun', 'me a name I call myself'], result)
|
40
|
+
end
|
41
|
+
|
42
|
+
def sing
|
43
|
+
{ :do => 'doe a deer a female deer',
|
44
|
+
:re => 'ray a drop of golden sun',
|
45
|
+
:mi => 'me a name I call myself',
|
46
|
+
:fa => 'far a long long way to run',
|
47
|
+
:so => 'A needle pulling thread',
|
48
|
+
:la => 'a note to follow so',
|
49
|
+
:ti => 'a drink with jam and bread'}
|
50
|
+
end
|
51
|
+
end
|
metadata
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: stages
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.1
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Nathan Acuff
|
9
|
+
- Justin Hill
|
10
|
+
- Matt Brown
|
11
|
+
- Kyle Prifogle
|
12
|
+
autorequire:
|
13
|
+
bindir: bin
|
14
|
+
cert_chain: []
|
15
|
+
|
16
|
+
date: 2012-01-13 00:00:00 Z
|
17
|
+
dependencies:
|
18
|
+
- !ruby/object:Gem::Dependency
|
19
|
+
name: rake
|
20
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
21
|
+
none: false
|
22
|
+
requirements:
|
23
|
+
- - "="
|
24
|
+
- !ruby/object:Gem::Version
|
25
|
+
version: 0.9.2
|
26
|
+
type: :runtime
|
27
|
+
prerelease: false
|
28
|
+
version_requirements: *id001
|
29
|
+
description: pipeline builder
|
30
|
+
email: support@igodigital.com
|
31
|
+
executables: []
|
32
|
+
|
33
|
+
extensions: []
|
34
|
+
|
35
|
+
extra_rdoc_files:
|
36
|
+
- README
|
37
|
+
files:
|
38
|
+
- .autotest
|
39
|
+
- Gemfile
|
40
|
+
- Gemfile.lock
|
41
|
+
- README
|
42
|
+
- Rakefile
|
43
|
+
- VERSION
|
44
|
+
- lib/stage_base.rb
|
45
|
+
- lib/stages.rb
|
46
|
+
- lib/stages/each_element.rb
|
47
|
+
- lib/stages/evens.rb
|
48
|
+
- lib/stages/hash_lookup.rb
|
49
|
+
- lib/stages/map.rb
|
50
|
+
- lib/stages/multiples_of.rb
|
51
|
+
- lib/stages/select.rb
|
52
|
+
- stages.gemspec
|
53
|
+
- test/helper.rb
|
54
|
+
- test/test_pipeline.rb
|
55
|
+
- test/test_stages.rb
|
56
|
+
homepage: https://github.com/iGoDigital-LLC/stages
|
57
|
+
licenses: []
|
58
|
+
|
59
|
+
post_install_message:
|
60
|
+
rdoc_options: []
|
61
|
+
|
62
|
+
require_paths:
|
63
|
+
- lib
|
64
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
hash: 4101338441187827743
|
70
|
+
segments:
|
71
|
+
- 0
|
72
|
+
version: "0"
|
73
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: "0"
|
79
|
+
requirements: []
|
80
|
+
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 1.8.11
|
83
|
+
signing_key:
|
84
|
+
specification_version: 3
|
85
|
+
summary: pipeline builder
|
86
|
+
test_files: []
|
87
|
+
|