ford 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ford.gemspec
+gemspec

data/README

@@ -0,0 +1,7 @@
+Ford is inspired in Henry Ford's techniques of production (Fordism), which enabled the mass production of vehicles in the early 20th century through assembly lines.
+
+In the context of software development, pipeline is a powerful framework to speed up and parallelize repetitive tasks.
+
+Ford will help you to parallelize any ruby script through the concept of specialized and distributed Stages.
+
+Warning: Ford is not released. It's under development.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/examples/01.rb

@@ -0,0 +1,37 @@
+require 'rubygems'
+require 'ford'
+
+module MyPipeline
+  
+  class Stage1 < Ford::Stage
+    
+    # Override method run
+    def run
+      
+      10.times do |i|
+        enqueue_to Stage2, 'obj'
+      end
+      
+    end
+    
+  end
+  
+  class Stage2 < Ford::Stage
+    
+    def consume_input
+      sleep 1 # fake some processing
+      puts @input
+    end
+    
+  end
+  
+  def self.run
+    MyPipeline::Stage1.init_stage(:threads => 1, :debug => true)
+    MyPipeline::Stage2.init_stage(:threads => 3, :debug => true)
+    
+    Ford.join
+  end
+  
+end
+
+MyPipeline.run

data/ford.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ford/version"
+
+Gem::Specification.new do |s|
+  s.add_dependency 'logger'
+  
+  s.name        = "ford"
+  s.version     = Ford::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Bruno Bueno","Débora Setton", "Rafael Barbolo", "Rafael Ivan"]
+  s.email       = ["bruno.bueno@infosimples.com.br", "debora.setton@infosimples.com.br", "rafael.barbolo@infosimples.com.br", "rafael.ivan@infosimples.com.br"]
+  s.homepage    = "http://www.infosimples.com.br/en/"
+  s.summary     = %q{Ruby scalable pipeline framework}
+  s.description = %q{Ford allows you to split a ruby script into stages of a scalable and performant pipeline}
+
+  s.rubyforge_project = "ford"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/ford/config.rb

@@ -0,0 +1,42 @@
+module Ford
+
+  #
+  # Config class.
+  # Based on http://mjijackson.com/2010/02/flexible-ruby-config-objects
+  #
+  class Config
+
+    def initialize(data={})
+      @data = {}
+      update!(data)
+    end
+
+    def update!(data)
+      data.each do |key, value|
+        self[key] = value
+      end
+    end
+
+    def [](key)
+      @data[key.to_sym]
+    end
+
+    def []=(key, value)
+      if value.class == Hash
+        @data[key.to_sym] = Config.new(value)
+      else
+        @data[key.to_sym] = value
+      end
+    end
+
+    def method_missing(sym, *args)
+      if sym.to_s =~ /(.+)=$/
+        self[$1] = args.first
+      else
+        self[sym]
+      end
+    end
+
+  end
+
+end

data/lib/ford/stage.rb

@@ -0,0 +1,138 @@
+require 'logger'
+
+module Ford
+  
+  #
+  # Ford threads
+  #
+  @@threads = []
+  def self.threads
+    @@threads
+  end
+  
+  #
+  # Join all threads and wait them to finish
+  #
+  def self.join
+    @@threads.each {|t| t.join}
+  end
+
+  #
+  # The class Ford::Stage can be extended by each stage in a pipeline
+  #
+  # It has built-in structures and functions that helps building a pipeline
+  #
+  class Stage
+    
+    attr_accessor :config, :logger, :input
+    
+    #
+    # Create a queue for each Stage subclass
+    #
+    @queue = Queue.new
+    def self.queue
+      @queue
+    end
+    def self.queue=queue
+      @queue = queue
+    end
+    
+    def self.inherited(subclass)
+      subclass.queue = Queue.new
+    end
+    
+    
+    #
+    # Create a stage in thread mode
+    #
+    def self.init_stage(options={})
+      options = {
+        :threads => 1
+      }.merge(options)
+      
+      options[:threads].times do |tid|
+        options = options.clone
+        options[:thread_id] = tid
+        
+        # Create a new thread
+        t = Thread.new {
+          obj = nil
+          begin
+            obj = self.new(options)
+            obj.run
+          rescue Exception => exc
+            obj.logger.fatal("\nFailed to execute the #{self.class}'s thread (#{tid})")
+            obj.logger.fatal("was consuming: #{obj.input}")
+            obj.logger.fatal("#{exc}\n#{exc.backtrace.join("\n")}")
+          end
+        }
+        
+        Ford.threads.push t
+        
+      end
+      
+    end
+    
+    
+    #
+    # Initialize the stage 
+    #
+    def initialize(options={})
+      data = {
+        :debug => false, # If true, logs messages during execution
+        :log_to => STDOUT, # Logging path or IO instance
+        :input_stage => self.class # Reference to the input Stage (normally, itself). Will load objs from its queue.
+      }.merge(options)
+      
+      @config = Ford::Config.new(data) # instance configuration
+      @logger = Logger.new(@config.log_to) # instance logger
+      @logger.level = @config.debug ? Logger::DEBUG : Logger::INFO      
+    end
+    
+    #
+    # Run the stage
+    #
+    def run      
+      while (@input = pop_input)
+        start_consume_at = Time.now
+        
+        logger.debug("Consuming...(#{config.thread_id})")
+        consume_input
+        
+        logger.debug("Consumed in #{Time.now - start_consume_at} seconds (#{config.thread_id})")
+      end      
+    end
+    
+    #
+    # When using the default run, consume_input should be implemented.
+    #
+    def consume_input
+      raise 'Must implement!'
+    end
+    
+    #
+    # Pop an object from the input queue
+    #
+    def pop_input
+      @config.input_stage.queue.pop
+    end
+    
+    #
+    # Enqueue an object in the stage's queue
+    #
+    def enqueue_to(stage_class, obj)
+      stage_class.queue.push obj
+      logger.debug("Enqueued into #{stage_class}'s queue (#{config.thread_id})")
+    end
+    
+    #
+    # Enqueue an object in the current stage's queue
+    #
+    def enqueue_back(obj)
+      self.class.queue.push obj
+      logger.debug("Enqueued back (#{config.thread_id})")
+    end
+    
+  end
+  
+end

data/lib/ford/version.rb

@@ -0,0 +1,3 @@
+module Ford
+  VERSION = "0.0.2"
+end

data/lib/ford.rb

@@ -0,0 +1,6 @@
+module Ford
+  
+end
+
+require 'ford/config'
+require 'ford/stage'

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: ford
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Bruno Bueno
+- "D\xC3\xA9bora Setton"
+- Rafael Barbolo
+- Rafael Ivan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-11 00:00:00 -03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: logger
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Ford allows you to split a ruby script into stages of a scalable and performant pipeline
+email: 
+- bruno.bueno@infosimples.com.br
+- debora.setton@infosimples.com.br
+- rafael.barbolo@infosimples.com.br
+- rafael.ivan@infosimples.com.br
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- examples/01.rb
+- ford.gemspec
+- lib/ford.rb
+- lib/ford/config.rb
+- lib/ford/stage.rb
+- lib/ford/version.rb
+has_rdoc: true
+homepage: http://www.infosimples.com.br/en/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: ford
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: Ruby scalable pipeline framework
+test_files: []
+