pipelines 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 616bf7c8beb7ad8ba7f11cce119849f592fb3f74
+  data.tar.gz: 52bec1e786aeb7f9a7594c7e295001ab9e7ffec1
+SHA512:
+  metadata.gz: 8196a97dcb6f7116527af650626054f9bb81b1c31af55dba17b317b025e76ea7beac0d297c77aa9857edf1faf6a23ef4b498323cbfcd5301700155943cb629e2
+  data.tar.gz: a426d7a00fd930213491055ffffd293dba7fbce8c04f454773cced30cc2977088d54899b1b53a7d53c21b0304a9a4739fec9631cc92ac947c8f3952f87dd2e5a

data/lib/pipelines.rb

@@ -0,0 +1,314 @@
+require 'yaml'
+require 'monitor'
+
+class Pipeline
+  attr :dir
+  attr :ended_at
+  attr :exception
+  attr :invocations
+  attr :name
+  attr :input
+  attr :output
+  attr :started_at
+  attr :stats
+  attr :thread_lock
+  attr :threads
+  attr :topdir
+
+  def initialize(*args)
+    options = args.last.is_a?(Hash) ? args.pop : {}
+    dir = args.first
+
+    @topdir = @parent ? @parent.topdir : dir
+    @dir = dir
+    @type = options.delete(:type) || :serial
+    @parent = options.delete(:parent) # This is nil only for the top level pipeline.
+
+    @serial_count = 0
+    @parallel_count = 0
+
+    @thread_lock = @parent ? @parent.thread_lock : Monitor.new
+    @stats = @parent ? @parent.stats : {}
+
+    @name = underscore self.class.name.split('::')[-1]
+    @order = options.delete(:order)
+
+    @input = options.delete(:input)
+    @output = serial? ? nil : []
+
+    @threads = []
+    Thread.abort_on_exception = true
+
+    @options = options
+    @step = nil
+    @ended_at = nil
+    @started_at = options[:started_at] || Time.now
+    @invocations = 0
+
+    unless @dir.nil?
+      Dir.mkdir(@dir) unless Dir.exists?(@dir)
+    end
+
+    if @parent.nil? # This is the top level pipeline.
+      class << self
+        alias_method :unlocked_run, :run
+
+        def run(*args, &block)
+          lock
+          output = unlocked_run(*args, &block)
+          unlock
+          notify
+
+	  output
+        end
+      end
+    end
+  end
+
+
+  def serial(args=nil, &block)
+    pipeline(:serial, args, &block)
+  end
+
+
+  def parallel(args=nil, &block)
+    pipeline(:parallel, args, &block)
+  end
+
+
+  def invoke(klass, *args)
+    @invocations += 1
+   
+    dir = File.join(@dir, "#{description(klass)}") unless @dir.nil?
+    segment = klass.new dir, :order => @order, :parent => self
+
+    output_file = segment_cache segment
+    if output_file && File.exists?(output_file)
+      segment.puts "\033[0;35mSkipping\033[0m"
+      output = YAML.load(File.read(output_file))[:data]
+
+      if serial?
+        @output = output
+        @input = output
+      elsif parallel?
+        @thread_lock.synchronize do
+          @output << output
+        end
+      end
+    else
+      segment.puts "\033[0;32mRunning\033[0m"
+
+      if serial?
+        dispatch(segment, output_file, *args)
+        @input = @output
+      elsif parallel?
+        thread = Thread.new do
+          dispatch(segment, output_file, *args)
+        end
+        @threads << thread
+      end
+    end
+  end
+
+
+  def description(klass)
+    if @order.nil?
+      "#{@invocations}-#{underscore(klass.name.split('::')[-1])}"
+    else
+      "#{@order}-#{@invocations}-#{underscore(klass.name.split('::')[-1])}"
+    end
+  end
+
+
+  def puts(string='')
+    @thread_lock.synchronize do
+      unlocked_puts(string)
+    end
+
+    nil # Behave like Kernel.puts
+  end
+
+
+  def unlocked_puts(string='')
+    if self.class == Pipeline || @parent.nil? # If @parent is missing, it is in the top level block.
+      if @order
+        Kernel.puts "\033[32m[#{@order}][#{@invocations}]\033[0m #{string}"
+      else
+        Kernel.puts "\033[33m[#{@invocations}]\033[0m #{string}"
+      end
+    else
+      if @order
+        Kernel.puts "\033[32m[#{@order}][#{@name}][#{@parent.invocations}]\033[0m #{string}"
+      else
+        Kernel.puts "\033[32m[#{@name}][#{@parent.invocations}]\033[0m #{string}"
+      end
+    end
+
+    STDOUT.flush
+  end
+
+
+  private
+
+  def serial?
+    @type == :serial
+  end
+
+
+  def parallel?
+    @type == :parallel
+  end
+
+
+  def pipeline(mode, args=nil, &block)
+    begin
+      if parallel? # When inside parallel.
+        thread = Thread.new do
+          pipeline = child(mode, args)
+          pipeline.instance_eval &block
+          pipeline.threads.each { |thread| thread.join } # Could be a parallel block inside a parallel block.
+          @thread_lock.synchronize do
+            @output << (mode == :parallel ? pipeline.output.flatten(1) : pipeline.output)
+          end
+        end
+        @threads << thread
+      elsif serial?
+        pipeline = child(mode, args)
+        pipeline.instance_eval &block
+        pipeline.threads.each { |thread| thread.join }
+        @output = (mode == :parallel ? pipeline.output.flatten(1) : pipeline.output)
+        @input = @output
+      end
+    rescue Exception => e
+      @exception = e
+      notify
+      raise
+    end
+  end
+
+
+  def notify
+    # This should be implemented in the subclasses.
+  end
+
+
+  def run
+    # This should be implemented in the subclasses.
+  end
+
+
+  def run_with_args(segment, args, options)
+    if args.empty?
+      if @invocations > 1
+        if options.empty?
+          segment.send :run, @input
+        else
+          segment.send :run, @input, options
+        end
+      else
+        if options.empty?
+          segment.send :run
+        else
+          segment.send :run, options
+        end
+      end
+    else
+      if options.empty?
+        segment.send :run, *args
+      else
+      	segment.send :run, *args, options
+      end
+    end
+  end
+
+
+  def dispatch(segment, output_file, *args)
+    options = args.last.is_a?(Hash) ? args.pop : {}
+
+    output = if segment.method(:run).arity > 0 # Optional arguments result in negative arity.
+               run_with_args(segment, args, options)
+             elsif segment.method(:run).arity < 0
+               run_with_args(segment, args, options)
+             else
+               segment.send :run
+             end
+
+    if output_file
+      File.open(output_file, "w") do |f|
+        f.write({:data => output}.to_yaml)
+      end
+    end
+
+    if serial?
+      @output = output
+    elsif parallel?
+      @thread_lock.synchronize do
+        @output << output
+      end
+    end
+  end
+
+
+  def segment_cache(segment)
+    File.join(@dir, "#{description(segment.class)}.yaml") if segment.dir
+  end
+
+
+  def child(type, args=nil)
+    order = if type == :serial
+              @serial_count += 1
+              "#{@order}S#{@serial_count}"
+            elsif type == :parallel
+              @parallel_count += 1
+              "#{@order}P#{@parallel_count}"
+            end
+
+    Pipeline.new(@dir, :type => type, :input => args || @input, :parent => self, :order => order, :started_at => started_at)
+  end
+
+
+  def underscore(string)
+    string.gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
+        gsub(/([a-z\d])([A-Z])/, '\1_\2').
+        tr("-", "_").downcase
+  end
+
+
+  def lock
+    return if @dir.nil?
+
+    lock = File.join @dir, 'lock'
+    if !@options[:force] && File.exists?(lock)
+      raise "Another instance of the pipelines seems to be running.\nPlease remove #{lock} if that is not the case."
+    end
+
+    File.open(lock, 'w') do |f|
+      f.write $$
+    end
+  end
+
+
+  def unlock
+    unless @dir.nil?
+      lock = File.join @dir, 'lock'
+      File.delete lock
+    end
+
+    @ended_at = Time.now
+  end
+
+
+  def set_stats(values)
+    @thread_lock.synchronize do 
+      @stats[@name] = values
+    end
+  end
+
+
+  def get_stats
+    @thread_lock.synchronize do
+      @stats[@name]
+    end
+  end
+end

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification
+name: pipelines
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Sujoy Gupta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-25 00:00:00.000000000 Z
+dependencies: []
+description: A simple way to build a pipeline of tasks. These tasks can be configured
+  to run in serial, parallel or any combination thereof.
+email: sujoyg@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/pipelines.rb
+homepage: https://github.com/sujoyg/pipelines
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.4
+signing_key: 
+specification_version: 4
+summary: Pipelines
+test_files: []