ford 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in ford.gemspec
4
+ gemspec
data/README ADDED
@@ -0,0 +1,7 @@
1
+ Ford is inspired in Henry Ford's techniques of production (Fordism), which enabled the mass production of vehicles in the early 20th century through assembly lines.
2
+
3
+ In the context of software development, pipeline is a powerful framework to speed up and parallelize repetitive tasks.
4
+
5
+ Ford will help you to parallelize any ruby script through the concept of specialized and distributed Stages.
6
+
7
+ Warning: Ford is not released. It's under development.
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
data/examples/01.rb ADDED
@@ -0,0 +1,37 @@
1
+ require 'rubygems'
2
+ require 'ford'
3
+
4
+ module MyPipeline
5
+
6
+ class Stage1 < Ford::Stage
7
+
8
+ # Override method run
9
+ def run
10
+
11
+ 10.times do |i|
12
+ enqueue_to Stage2, 'obj'
13
+ end
14
+
15
+ end
16
+
17
+ end
18
+
19
+ class Stage2 < Ford::Stage
20
+
21
+ def consume_input
22
+ sleep 1 # fake some processing
23
+ puts @input
24
+ end
25
+
26
+ end
27
+
28
+ def self.run
29
+ MyPipeline::Stage1.init_stage(:threads => 1, :debug => true)
30
+ MyPipeline::Stage2.init_stage(:threads => 3, :debug => true)
31
+
32
+ Ford.join
33
+ end
34
+
35
+ end
36
+
37
+ MyPipeline.run
data/ford.gemspec ADDED
@@ -0,0 +1,23 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "ford/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.add_dependency 'logger'
7
+
8
+ s.name = "ford"
9
+ s.version = Ford::VERSION
10
+ s.platform = Gem::Platform::RUBY
11
+ s.authors = ["Bruno Bueno","Débora Setton", "Rafael Barbolo", "Rafael Ivan"]
12
+ s.email = ["bruno.bueno@infosimples.com.br", "debora.setton@infosimples.com.br", "rafael.barbolo@infosimples.com.br", "rafael.ivan@infosimples.com.br"]
13
+ s.homepage = "http://www.infosimples.com.br/en/"
14
+ s.summary = %q{Ruby scalable pipeline framework}
15
+ s.description = %q{Ford allows you to split a ruby script into stages of a scalable and performant pipeline}
16
+
17
+ s.rubyforge_project = "ford"
18
+
19
+ s.files = `git ls-files`.split("\n")
20
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
21
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
22
+ s.require_paths = ["lib"]
23
+ end
@@ -0,0 +1,42 @@
1
+ module Ford
2
+
3
+ #
4
+ # Config class.
5
+ # Based on http://mjijackson.com/2010/02/flexible-ruby-config-objects
6
+ #
7
+ class Config
8
+
9
+ def initialize(data={})
10
+ @data = {}
11
+ update!(data)
12
+ end
13
+
14
+ def update!(data)
15
+ data.each do |key, value|
16
+ self[key] = value
17
+ end
18
+ end
19
+
20
+ def [](key)
21
+ @data[key.to_sym]
22
+ end
23
+
24
+ def []=(key, value)
25
+ if value.class == Hash
26
+ @data[key.to_sym] = Config.new(value)
27
+ else
28
+ @data[key.to_sym] = value
29
+ end
30
+ end
31
+
32
+ def method_missing(sym, *args)
33
+ if sym.to_s =~ /(.+)=$/
34
+ self[$1] = args.first
35
+ else
36
+ self[sym]
37
+ end
38
+ end
39
+
40
+ end
41
+
42
+ end
data/lib/ford/stage.rb ADDED
@@ -0,0 +1,138 @@
1
+ require 'logger'
2
+
3
+ module Ford
4
+
5
+ #
6
+ # Ford threads
7
+ #
8
+ @@threads = []
9
+ def self.threads
10
+ @@threads
11
+ end
12
+
13
+ #
14
+ # Join all threads and wait them to finish
15
+ #
16
+ def self.join
17
+ @@threads.each {|t| t.join}
18
+ end
19
+
20
+ #
21
+ # The class Ford::Stage can be extended by each stage in a pipeline
22
+ #
23
+ # It has built-in structures and functions that helps building a pipeline
24
+ #
25
+ class Stage
26
+
27
+ attr_accessor :config, :logger, :input
28
+
29
+ #
30
+ # Create a queue for each Stage subclass
31
+ #
32
+ @queue = Queue.new
33
+ def self.queue
34
+ @queue
35
+ end
36
+ def self.queue=queue
37
+ @queue = queue
38
+ end
39
+
40
+ def self.inherited(subclass)
41
+ subclass.queue = Queue.new
42
+ end
43
+
44
+
45
+ #
46
+ # Create a stage in thread mode
47
+ #
48
+ def self.init_stage(options={})
49
+ options = {
50
+ :threads => 1
51
+ }.merge(options)
52
+
53
+ options[:threads].times do |tid|
54
+ options = options.clone
55
+ options[:thread_id] = tid
56
+
57
+ # Create a new thread
58
+ t = Thread.new {
59
+ obj = nil
60
+ begin
61
+ obj = self.new(options)
62
+ obj.run
63
+ rescue Exception => exc
64
+ obj.logger.fatal("\nFailed to execute the #{self.class}'s thread (#{tid})")
65
+ obj.logger.fatal("was consuming: #{obj.input}")
66
+ obj.logger.fatal("#{exc}\n#{exc.backtrace.join("\n")}")
67
+ end
68
+ }
69
+
70
+ Ford.threads.push t
71
+
72
+ end
73
+
74
+ end
75
+
76
+
77
+ #
78
+ # Initialize the stage
79
+ #
80
+ def initialize(options={})
81
+ data = {
82
+ :debug => false, # If true, logs messages during execution
83
+ :log_to => STDOUT, # Logging path or IO instance
84
+ :input_stage => self.class # Reference to the input Stage (normally, itself). Will load objs from its queue.
85
+ }.merge(options)
86
+
87
+ @config = Ford::Config.new(data) # instance configuration
88
+ @logger = Logger.new(@config.log_to) # instance logger
89
+ @logger.level = @config.debug ? Logger::DEBUG : Logger::INFO
90
+ end
91
+
92
+ #
93
+ # Run the stage
94
+ #
95
+ def run
96
+ while (@input = pop_input)
97
+ start_consume_at = Time.now
98
+
99
+ logger.debug("Consuming...(#{config.thread_id})")
100
+ consume_input
101
+
102
+ logger.debug("Consumed in #{Time.now - start_consume_at} seconds (#{config.thread_id})")
103
+ end
104
+ end
105
+
106
+ #
107
+ # When using the default run, consume_input should be implemented.
108
+ #
109
+ def consume_input
110
+ raise 'Must implement!'
111
+ end
112
+
113
+ #
114
+ # Pop an object from the input queue
115
+ #
116
+ def pop_input
117
+ @config.input_stage.queue.pop
118
+ end
119
+
120
+ #
121
+ # Enqueue an object in the stage's queue
122
+ #
123
+ def enqueue_to(stage_class, obj)
124
+ stage_class.queue.push obj
125
+ logger.debug("Enqueued into #{stage_class}'s queue (#{config.thread_id})")
126
+ end
127
+
128
+ #
129
+ # Enqueue an object in the current stage's queue
130
+ #
131
+ def enqueue_back(obj)
132
+ self.class.queue.push obj
133
+ logger.debug("Enqueued back (#{config.thread_id})")
134
+ end
135
+
136
+ end
137
+
138
+ end
@@ -0,0 +1,3 @@
1
+ module Ford
2
+ VERSION = "0.0.2"
3
+ end
data/lib/ford.rb ADDED
@@ -0,0 +1,6 @@
1
+ module Ford
2
+
3
+ end
4
+
5
+ require 'ford/config'
6
+ require 'ford/stage'
metadata ADDED
@@ -0,0 +1,82 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ford
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 0.0.2
6
+ platform: ruby
7
+ authors:
8
+ - Bruno Bueno
9
+ - "D\xC3\xA9bora Setton"
10
+ - Rafael Barbolo
11
+ - Rafael Ivan
12
+ autorequire:
13
+ bindir: bin
14
+ cert_chain: []
15
+
16
+ date: 2011-07-11 00:00:00 -03:00
17
+ default_executable:
18
+ dependencies:
19
+ - !ruby/object:Gem::Dependency
20
+ name: logger
21
+ prerelease: false
22
+ requirement: &id001 !ruby/object:Gem::Requirement
23
+ none: false
24
+ requirements:
25
+ - - ">="
26
+ - !ruby/object:Gem::Version
27
+ version: "0"
28
+ type: :runtime
29
+ version_requirements: *id001
30
+ description: Ford allows you to split a ruby script into stages of a scalable and performant pipeline
31
+ email:
32
+ - bruno.bueno@infosimples.com.br
33
+ - debora.setton@infosimples.com.br
34
+ - rafael.barbolo@infosimples.com.br
35
+ - rafael.ivan@infosimples.com.br
36
+ executables: []
37
+
38
+ extensions: []
39
+
40
+ extra_rdoc_files: []
41
+
42
+ files:
43
+ - .gitignore
44
+ - Gemfile
45
+ - README
46
+ - Rakefile
47
+ - examples/01.rb
48
+ - ford.gemspec
49
+ - lib/ford.rb
50
+ - lib/ford/config.rb
51
+ - lib/ford/stage.rb
52
+ - lib/ford/version.rb
53
+ has_rdoc: true
54
+ homepage: http://www.infosimples.com.br/en/
55
+ licenses: []
56
+
57
+ post_install_message:
58
+ rdoc_options: []
59
+
60
+ require_paths:
61
+ - lib
62
+ required_ruby_version: !ruby/object:Gem::Requirement
63
+ none: false
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: "0"
68
+ required_rubygems_version: !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ version: "0"
74
+ requirements: []
75
+
76
+ rubyforge_project: ford
77
+ rubygems_version: 1.5.2
78
+ signing_key:
79
+ specification_version: 3
80
+ summary: Ruby scalable pipeline framework
81
+ test_files: []
82
+