ford 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/README +7 -0
- data/Rakefile +2 -0
- data/examples/01.rb +37 -0
- data/ford.gemspec +23 -0
- data/lib/ford/config.rb +42 -0
- data/lib/ford/stage.rb +138 -0
- data/lib/ford/version.rb +3 -0
- data/lib/ford.rb +6 -0
- metadata +82 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
Ford is inspired in Henry Ford's techniques of production (Fordism), which enabled the mass production of vehicles in the early 20th century through assembly lines.
|
2
|
+
|
3
|
+
In the context of software development, pipeline is a powerful framework to speed up and parallelize repetitive tasks.
|
4
|
+
|
5
|
+
Ford will help you to parallelize any ruby script through the concept of specialized and distributed Stages.
|
6
|
+
|
7
|
+
Warning: Ford is not released. It's under development.
|
data/Rakefile
ADDED
data/examples/01.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'ford'
|
3
|
+
|
4
|
+
module MyPipeline
|
5
|
+
|
6
|
+
class Stage1 < Ford::Stage
|
7
|
+
|
8
|
+
# Override method run
|
9
|
+
def run
|
10
|
+
|
11
|
+
10.times do |i|
|
12
|
+
enqueue_to Stage2, 'obj'
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
class Stage2 < Ford::Stage
|
20
|
+
|
21
|
+
def consume_input
|
22
|
+
sleep 1 # fake some processing
|
23
|
+
puts @input
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.run
|
29
|
+
MyPipeline::Stage1.init_stage(:threads => 1, :debug => true)
|
30
|
+
MyPipeline::Stage2.init_stage(:threads => 3, :debug => true)
|
31
|
+
|
32
|
+
Ford.join
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
MyPipeline.run
|
data/ford.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "ford/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.add_dependency 'logger'
|
7
|
+
|
8
|
+
s.name = "ford"
|
9
|
+
s.version = Ford::VERSION
|
10
|
+
s.platform = Gem::Platform::RUBY
|
11
|
+
s.authors = ["Bruno Bueno","Débora Setton", "Rafael Barbolo", "Rafael Ivan"]
|
12
|
+
s.email = ["bruno.bueno@infosimples.com.br", "debora.setton@infosimples.com.br", "rafael.barbolo@infosimples.com.br", "rafael.ivan@infosimples.com.br"]
|
13
|
+
s.homepage = "http://www.infosimples.com.br/en/"
|
14
|
+
s.summary = %q{Ruby scalable pipeline framework}
|
15
|
+
s.description = %q{Ford allows you to split a ruby script into stages of a scalable and performant pipeline}
|
16
|
+
|
17
|
+
s.rubyforge_project = "ford"
|
18
|
+
|
19
|
+
s.files = `git ls-files`.split("\n")
|
20
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
21
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
22
|
+
s.require_paths = ["lib"]
|
23
|
+
end
|
data/lib/ford/config.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
module Ford
|
2
|
+
|
3
|
+
#
|
4
|
+
# Config class.
|
5
|
+
# Based on http://mjijackson.com/2010/02/flexible-ruby-config-objects
|
6
|
+
#
|
7
|
+
class Config
|
8
|
+
|
9
|
+
def initialize(data={})
|
10
|
+
@data = {}
|
11
|
+
update!(data)
|
12
|
+
end
|
13
|
+
|
14
|
+
def update!(data)
|
15
|
+
data.each do |key, value|
|
16
|
+
self[key] = value
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def [](key)
|
21
|
+
@data[key.to_sym]
|
22
|
+
end
|
23
|
+
|
24
|
+
def []=(key, value)
|
25
|
+
if value.class == Hash
|
26
|
+
@data[key.to_sym] = Config.new(value)
|
27
|
+
else
|
28
|
+
@data[key.to_sym] = value
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def method_missing(sym, *args)
|
33
|
+
if sym.to_s =~ /(.+)=$/
|
34
|
+
self[$1] = args.first
|
35
|
+
else
|
36
|
+
self[sym]
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/lib/ford/stage.rb
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Ford
|
4
|
+
|
5
|
+
#
|
6
|
+
# Ford threads
|
7
|
+
#
|
8
|
+
@@threads = []
|
9
|
+
def self.threads
|
10
|
+
@@threads
|
11
|
+
end
|
12
|
+
|
13
|
+
#
|
14
|
+
# Join all threads and wait them to finish
|
15
|
+
#
|
16
|
+
def self.join
|
17
|
+
@@threads.each {|t| t.join}
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# The class Ford::Stage can be extended by each stage in a pipeline
|
22
|
+
#
|
23
|
+
# It has built-in structures and functions that helps building a pipeline
|
24
|
+
#
|
25
|
+
class Stage
|
26
|
+
|
27
|
+
attr_accessor :config, :logger, :input
|
28
|
+
|
29
|
+
#
|
30
|
+
# Create a queue for each Stage subclass
|
31
|
+
#
|
32
|
+
@queue = Queue.new
|
33
|
+
def self.queue
|
34
|
+
@queue
|
35
|
+
end
|
36
|
+
def self.queue=queue
|
37
|
+
@queue = queue
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.inherited(subclass)
|
41
|
+
subclass.queue = Queue.new
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
#
|
46
|
+
# Create a stage in thread mode
|
47
|
+
#
|
48
|
+
def self.init_stage(options={})
|
49
|
+
options = {
|
50
|
+
:threads => 1
|
51
|
+
}.merge(options)
|
52
|
+
|
53
|
+
options[:threads].times do |tid|
|
54
|
+
options = options.clone
|
55
|
+
options[:thread_id] = tid
|
56
|
+
|
57
|
+
# Create a new thread
|
58
|
+
t = Thread.new {
|
59
|
+
obj = nil
|
60
|
+
begin
|
61
|
+
obj = self.new(options)
|
62
|
+
obj.run
|
63
|
+
rescue Exception => exc
|
64
|
+
obj.logger.fatal("\nFailed to execute the #{self.class}'s thread (#{tid})")
|
65
|
+
obj.logger.fatal("was consuming: #{obj.input}")
|
66
|
+
obj.logger.fatal("#{exc}\n#{exc.backtrace.join("\n")}")
|
67
|
+
end
|
68
|
+
}
|
69
|
+
|
70
|
+
Ford.threads.push t
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
#
|
78
|
+
# Initialize the stage
|
79
|
+
#
|
80
|
+
def initialize(options={})
|
81
|
+
data = {
|
82
|
+
:debug => false, # If true, logs messages during execution
|
83
|
+
:log_to => STDOUT, # Logging path or IO instance
|
84
|
+
:input_stage => self.class # Reference to the input Stage (normally, itself). Will load objs from its queue.
|
85
|
+
}.merge(options)
|
86
|
+
|
87
|
+
@config = Ford::Config.new(data) # instance configuration
|
88
|
+
@logger = Logger.new(@config.log_to) # instance logger
|
89
|
+
@logger.level = @config.debug ? Logger::DEBUG : Logger::INFO
|
90
|
+
end
|
91
|
+
|
92
|
+
#
|
93
|
+
# Run the stage
|
94
|
+
#
|
95
|
+
def run
|
96
|
+
while (@input = pop_input)
|
97
|
+
start_consume_at = Time.now
|
98
|
+
|
99
|
+
logger.debug("Consuming...(#{config.thread_id})")
|
100
|
+
consume_input
|
101
|
+
|
102
|
+
logger.debug("Consumed in #{Time.now - start_consume_at} seconds (#{config.thread_id})")
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
#
|
107
|
+
# When using the default run, consume_input should be implemented.
|
108
|
+
#
|
109
|
+
def consume_input
|
110
|
+
raise 'Must implement!'
|
111
|
+
end
|
112
|
+
|
113
|
+
#
|
114
|
+
# Pop an object from the input queue
|
115
|
+
#
|
116
|
+
def pop_input
|
117
|
+
@config.input_stage.queue.pop
|
118
|
+
end
|
119
|
+
|
120
|
+
#
|
121
|
+
# Enqueue an object in the stage's queue
|
122
|
+
#
|
123
|
+
def enqueue_to(stage_class, obj)
|
124
|
+
stage_class.queue.push obj
|
125
|
+
logger.debug("Enqueued into #{stage_class}'s queue (#{config.thread_id})")
|
126
|
+
end
|
127
|
+
|
128
|
+
#
|
129
|
+
# Enqueue an object in the current stage's queue
|
130
|
+
#
|
131
|
+
def enqueue_back(obj)
|
132
|
+
self.class.queue.push obj
|
133
|
+
logger.debug("Enqueued back (#{config.thread_id})")
|
134
|
+
end
|
135
|
+
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
data/lib/ford/version.rb
ADDED
data/lib/ford.rb
ADDED
metadata
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ford
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.2
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Bruno Bueno
|
9
|
+
- "D\xC3\xA9bora Setton"
|
10
|
+
- Rafael Barbolo
|
11
|
+
- Rafael Ivan
|
12
|
+
autorequire:
|
13
|
+
bindir: bin
|
14
|
+
cert_chain: []
|
15
|
+
|
16
|
+
date: 2011-07-11 00:00:00 -03:00
|
17
|
+
default_executable:
|
18
|
+
dependencies:
|
19
|
+
- !ruby/object:Gem::Dependency
|
20
|
+
name: logger
|
21
|
+
prerelease: false
|
22
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
23
|
+
none: false
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: "0"
|
28
|
+
type: :runtime
|
29
|
+
version_requirements: *id001
|
30
|
+
description: Ford allows you to split a ruby script into stages of a scalable and performant pipeline
|
31
|
+
email:
|
32
|
+
- bruno.bueno@infosimples.com.br
|
33
|
+
- debora.setton@infosimples.com.br
|
34
|
+
- rafael.barbolo@infosimples.com.br
|
35
|
+
- rafael.ivan@infosimples.com.br
|
36
|
+
executables: []
|
37
|
+
|
38
|
+
extensions: []
|
39
|
+
|
40
|
+
extra_rdoc_files: []
|
41
|
+
|
42
|
+
files:
|
43
|
+
- .gitignore
|
44
|
+
- Gemfile
|
45
|
+
- README
|
46
|
+
- Rakefile
|
47
|
+
- examples/01.rb
|
48
|
+
- ford.gemspec
|
49
|
+
- lib/ford.rb
|
50
|
+
- lib/ford/config.rb
|
51
|
+
- lib/ford/stage.rb
|
52
|
+
- lib/ford/version.rb
|
53
|
+
has_rdoc: true
|
54
|
+
homepage: http://www.infosimples.com.br/en/
|
55
|
+
licenses: []
|
56
|
+
|
57
|
+
post_install_message:
|
58
|
+
rdoc_options: []
|
59
|
+
|
60
|
+
require_paths:
|
61
|
+
- lib
|
62
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
63
|
+
none: false
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: "0"
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: "0"
|
74
|
+
requirements: []
|
75
|
+
|
76
|
+
rubyforge_project: ford
|
77
|
+
rubygems_version: 1.5.2
|
78
|
+
signing_key:
|
79
|
+
specification_version: 3
|
80
|
+
summary: Ruby scalable pipeline framework
|
81
|
+
test_files: []
|
82
|
+
|