andromeda 0.1

data/.gitignore

@@ -0,0 +1,11 @@
+.idea
+.bundle
+.yardoc
+doc
+db
+pkg
+html
+package
+coverage
+bin
+*.gem

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm --create gemset use andromeda
+export JRUBY_OPTS="$JRUBY_OPTS -Xcompat.version=1.9"

data/AUTHORS

@@ -0,0 +1 @@
+Stefan Plantikow <stefanp@moviepilot.com>

data/Gemfile

@@ -0,0 +1,22 @@
+source "http://rubygems.org"
+
+gem 'json', '>=1.6.5'
+gem 'threadpool'
+gem 'facter'
+gem 'atomic'
+
+group :development do
+  gem 'rake'
+  gem 'redcarpet', :require => false
+  gem 'yard', :require => false
+  gem 'irbtools', :require => false
+end
+
+group :jruby do
+  gem 'maruku'
+end
+
+group :test do
+  gem 'rspec', '2.6.0'
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -0,0 +1,85 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    atomic (1.0.0)
+    awesome_print (1.0.2)
+    boson (1.1.1)
+    clipboard (1.0.1)
+    coderay (1.0.6)
+    diff-lcs (1.1.3)
+    every_day_irb (1.2.2)
+    facter (1.6.8)
+    fancy_irb (0.7.2)
+      paint (>= 0.8.1)
+      unicode-display_width (>= 0.1.1)
+    g (1.6.0)
+      ruby_gntp
+    hirb (0.6.2)
+    interactive_editor (0.0.10)
+      spoon (>= 0.0.1)
+    irbtools (1.2.2)
+      awesome_print (~> 1.0.2)
+      boson (~> 1.1.1)
+      clipboard (~> 1.0.1)
+      coderay (~> 1.0.5)
+      every_day_irb (>= 1.2.2)
+      fancy_irb (>= 0.7.2)
+      g (>= 1.5.0)
+      hirb (~> 0.6.1)
+      interactive_editor (>= 0.0.10)
+      method_locator (>= 0.0.4)
+      method_source (>= 0.7.0)
+      methodfinder (>= 1.2.5)
+      ori (~> 0.1.0)
+      paint (>= 0.8.4)
+      sketches (>= 0.1.1)
+      wirb (>= 0.4.2)
+      zucker (>= 12.1)
+    json (1.6.6)
+    maruku (0.6.0)
+      syntax (>= 1.0.0)
+    method_locator (0.0.4)
+    method_source (0.7.1)
+    methodfinder (1.2.5)
+    multi_json (1.3.2)
+    ori (0.1.0)
+    paint (0.8.4)
+    rake (0.9.2.2)
+    redcarpet (2.1.1)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby_gntp (0.3.4)
+    simplecov (0.6.2)
+      multi_json (~> 1.3)
+      simplecov-html (~> 0.5.3)
+    simplecov-html (0.5.3)
+    sketches (0.1.1)
+    spoon (0.0.1)
+    syntax (1.0.0)
+    threadpool (0.1.0.1)
+    unicode-display_width (0.1.1)
+    wirb (0.4.2)
+    yard (0.7.5)
+    zucker (12.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  atomic
+  facter
+  irbtools
+  json (>= 1.6.5)
+  maruku
+  rake
+  redcarpet
+  rspec (= 2.6.0)
+  simplecov
+  threadpool
+  yard

data/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2012 Stefan Plantikow
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,28 @@
+# andromeda
+
+Andromeda is a ultra light weight multicore stream processing framework based on a small dataflow DSL
+
+It is currently untested and undocumented.  
+
+Below is an example that writes events to a file and reads them back in, to give an idea of what it does:
+
+    require 'andromeda'
+    w = Andromeda::CommandoWriter.new path: '/tmp/some_file'
+    w << (Commando.new :test)
+    w << (Commando.new :test, weight: 40)    
+    w << (Commando.new :test, height: 20)        
+    w << :close
+
+    r = Andromeda::CommandParser.new path: '/tmp/some_file'
+    # make r process events using a global thread pool of num_cpus threads
+    r.pool = :global
+    t = Andromeda::Tee.new
+    # make r output to t
+    r >> t
+    # start reading
+    r << :start
+    # t will log to a Logger.new(STDERR) by default
+
+There is much more, dig the source, luke!
+
+

data/Rakefile

@@ -0,0 +1,39 @@
+require 'bundler/gem_tasks'
+
+require 'rspec'
+require 'rspec/core/rake_task'
+
+require 'yard'
+require 'yard/rake/yardoc_task'
+
+desc 'Run all rspecs'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.fail_on_error = true
+  spec.verbose       = false
+#  spec.rspec_opts    = ['--backtrace']
+end
+
+desc 'Run yardoc over project sources'
+YARD::Rake::YardocTask.new(:ydoc) do |t|
+  t.options = ['--verbose']	
+  t.files   = ['lib/**/*.rb', '-', 'README.md', 'AUTHORS', 'LICENSE.txt']
+end
+
+#RDoc::Task.new(:rdoc) do |rdoc|
+#  # rdoc.main = "README.rdoc"
+#  rdoc.rdoc_files.include("lib/**/*.rb")
+#end
+
+desc 'Run irb in project environment'
+task :console do
+  require 'irb'
+  ARGV.clear
+  IRB.conf[:USE_READLINE] = false if ENV['JRUBY_OPTS'] =~ /--ng/
+  IRB.start
+end
+
+task :doc => :ydoc
+task :docs => :ydoc
+task :test => :spec
+task :tests => :spec
+task :irb => :console

data/andromeda.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'andromeda/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'andromeda'
+  s.version     = Andromeda::VERSION
+  s.summary     = 'Ultra light weight multicore stream processing framework based on a dataflow DSL'
+  s.description = s.summary
+  s.author      = 'Stefan Plantikow'
+  s.email       = 'stefanp@moviepilot.com'
+  s.homepage    = 'https://github.com/moviepilot/andromeda'
+  s.rubyforge_project = 'andromeda'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.bindir      = 'script'
+  s.executables = `git ls-files -- script/*`.split("\n").map{ |f| File.basename(f) }
+  s.licenses = ['PUBLIC DOMAIN WITHOUT ANY WARRANTY']
+end

data/lib/andromeda.rb

@@ -0,0 +1,14 @@
+require 'json'
+require 'logger'
+require 'threadpool'
+require 'facter'
+require 'thread'
+Facter.loadfacts
+
+require 'andromeda/id'
+require 'andromeda/pools'
+require 'andromeda/scope'
+require 'andromeda/andromeda'
+require 'andromeda/helpers'
+require 'andromeda/join'
+require 'andromeda/commando'

data/lib/andromeda/andromeda.rb

@@ -0,0 +1,225 @@
+# TODO
+#  - Turn into separate gem
+#  - Write Tests
+#  - Write fusor for synchronization (extra class that blocks until ready to submit)
+#  - Write docs, add yard support for documenting bases, attrs, etc.
+#  - Make nice slideshow and become very famous and rich. yay!
+module Andromeda
+
+  module Internal
+    class Transplanting
+      attr_reader :orig
+
+      def initialize(init_opts = nil)
+        @opts = init_opts
+        @orig = self
+      end
+
+      protected
+
+      def transplant(new_opts = nil)
+        return self if @opts == new_opts
+        obj = self.clone
+        obj.instance_variable_set '@opts', new_opts
+        obj
+      end
+    end
+  end
+
+  class Dest < Internal::Transplanting
+    attr_reader :base
+    attr_reader :meth
+
+    def initialize(base, meth, init_opts = nil)
+      super init_opts
+      raise ArgumentError, "'#{meth}' is not a symbol" unless meth.kind_of?(Symbol)
+      raise ArgumentError, "'#{base}' is not a base" unless base.kind_of?(Base)
+      raise NoMethodError, "'#{base}' does not respond to '#{meth}'" unless base.respond_to?(meth)
+      @base  = base
+      @meth  = meth
+    end
+
+    def <<(chunk, opts = {}) ; submit chunk, opts ; self end
+    def submit(chunk, opts = {}) ; submit_to nil, chunk, opts end
+    def submit_now(chunk, opts = {}) ; submit_to :local, chunk, opts end
+
+    def submit_to(target_pool, chunk, opts = {})
+      if @opts
+        new_opts = @opts.clone
+        opts.each { |k, v| new_opts[k] = v }
+      else
+        new_opts = opts
+      end
+      new_opts[:scope] ||= Scope.new
+      new_opts[:mark]  ||= Id.zero
+      base.transplant(new_opts).process target_pool, new_opts[:scope], self.meth, chunk
+      new_opts
+    end
+
+    def entry ; self end
+  end
+
+  class Base < Internal::Transplanting
+    attr_reader :id
+    attr_reader :opts
+
+    attr_accessor :log
+    attr_accessor :mark
+    attr_accessor :emit
+    attr_accessor :pool
+
+    attr_reader :trace_enter
+    attr_reader :trace_exit
+
+    def initialize(config = {})
+      super config[:init_opts]
+      @id = Id.gen
+      set_from_config init_from_config, config      
+      @trace_enter ||= init_trace_hash :enter
+      @trace_exit  ||= init_trace_hash :emit
+      @pool        ||= init_pool_config
+    end 
+
+    def trace=(new_trace)
+      raise ArgumentError, "'#{new_trace}' is not a Hash" unless new_trace.kind_of?(Hash)
+      @trace = new_trace
+    end
+
+    def init_trace_hash(kind) ; {} end
+    def init_pool_config ; nil end
+    def init_from_config ; [:readers, :writers] end
+
+    def log ; @log = Logger.new(STDERR) unless @log ; @log end
+    def mark ; @mark = Id.zero unless @mark ; @mark end
+
+    def on_enter(c)
+      emit << c rescue nil
+    end   
+
+    # @param [nil, :local, :spawn, :single, :fifo, :default, :global, #process, #process_base] pool_descr 
+    #     If nil, uses self.pool as pool_descr and continues. If that is nil, too, defaults to :local.
+    #     If #process_base, uses target_pool.process_base(meth) as pool_descr and continues.
+    #     If pool_descr is :spawn, uses SpawnPool.default_pool
+    #     If pool_descr is :single, uses PoolSupport.new_single_pool
+    #     If pool_descr is :fifo, uses PoolSupport.new_fifo_pool
+    #     If pool_descr is :global, uses the globally shared PoolSupport.global_pool
+    #     If pool_descr is :default uses PoolSupport.new_default_pool
+    #     Finally, if #process, runs by calling #process.  If :local, runs in current thread. 
+    #     Otherwise, the behaviour is undefined.
+    def process(pool_descr, scope, meth, chunk)
+      this = self
+      run target_pool(pool_descr, meth), scope, meth, chunk  do
+        begin
+          enter_level = trace_enter[meth]
+          exit_level  = trace_exit[meth]
+          trace :enter, enter_level, meth, chunk if enter_level
+          send meth, chunk
+          trace :exit, exit_level, meth, chunk if exit_level
+        rescue Exception => e
+          handle_exception meth, chunk, e
+        ensure
+          scope.leave if scope
+        end
+      end
+    end
+
+    def check_mark
+      mark = @opts[:mark]
+      raise RuntimeError, 'invalid mark' if mark && !mark.zero?
+    end
+
+    def intern(dest) ; dest.transplant(opts) end
+
+    def dest(name)
+      result = if self.respond_to?(name)
+        then intern self.send(name)
+        else Dest.new self, "on_#{name}".to_sym, opts  end
+      raise ArgumentError, "unknown or invalid dest: '#{name}'" unless result.kind_of?(Dest)
+      result
+    end
+
+    def trace(kind, level, method, chunk) 
+      log_ = log     
+      log_.send level, "TRACE #{id.to_s} :#{kind} :#{method} chunk: '#{chunk}'" if log_
+    end
+
+    protected
+
+    def set_from_config(what, config = {})
+      init_readers = what.include? :readers
+      init_writers = what.include? :writers
+      config.each_pair do |k, v|
+        k = k.to_sym rescue nil
+        if init_writers
+          writer = "#{k}=".to_sym rescue nil
+          if writer &&  self.respond_to?(writer)
+            then self.send writer, v 
+            else instance_variable_set "@#{k}", v if init_readers && self.respond_to?(k) end
+        else
+          instance_variable_set "@#{k}", v if init_readers && self.respond_to?(k)  
+        end
+      end
+    end
+
+    def mark_opts
+      if @mark && !@mark.zero?
+        if @opts[:mark]
+          then @opts[:mark] = @opts[:mark].xor(mark)
+          else @opts[:mark] = mark end
+      end
+    end
+
+    def run(pool, scope, meth, chunk, &thunk)
+      scope.enter
+      begin
+        if pool && pool.respond_to?(:process)
+          then pool.process(&thunk)
+          else thunk.call end
+      rescue Exception => e
+        handle_exception meth, chunk, e
+        scope.leave if scope
+      end
+      self
+    end
+
+    def target_pool(pool_descr, meth)
+      pool_descr = pool unless pool_descr
+      case pool_descr
+          when nil then :local
+          when :local then :local
+          when :spawn then SpawnPool.default_pool
+          when :global then  PoolSupport.global_pool
+          when :default then PoolSupport.new_default_pool
+          when :single then PoolSupport.new_single_pool
+          when :fifo then PoolSupport.new_fifo_pool 
+          else 
+             if pool_descr.respond_to(:process_base) 
+              then pool_descr.process_base(meth) 
+              else pool_descr end
+      end
+    end
+
+    def handle_exception(meth, chunk, e)
+      if log
+        trace = ''
+        e.backtrace.each { |s| trace << "\n    #{s}" }
+        log.error "Caught '#{e}' when processing chunk: '#{chunk}' via meth: '#{meth}' with backtrace: '#{trace}'"
+      end
+    end
+
+    public
+
+    def >>(dest) ; self.emit = dest.entry end
+
+    def drop ; self.emit = nil end
+
+    def entry ; dest(:enter) end
+    alias_method :exit, :emit
+
+    def <<(chunk, opts = {}) ; entry.<< chunk, opts end
+    def submit(chunk, opts = {}) ; entry.submit chunk, opts end
+    def submit_now(chunk, opts = {}) ; entry.submit_now chunk, opts end
+    def submit_to(target_pool, chunk, opts = {}) ; entry.submit_to target_pool, chunk, opts end
+  end
+end
+

data/lib/andromeda/commando.rb

@@ -0,0 +1,106 @@
+module Andromeda
+
+	class CommandoBase < Base
+		attr_reader :file
+		attr_reader :path
+	end
+
+	class Commando
+		attr_reader :cmd
+		attr_reader :data
+		attr_reader :time
+
+		def initialize(cmd, data = {})
+			raise ArgumentError unless cmd.kind_of?(Symbol)
+			@cmd  = cmd
+			@data = data
+			@time = Time.now.to_i
+		end
+
+		def to_hash
+			{ :cmd => cmd, :data => data, :time => time }
+		end
+	end
+
+	class CommandoWriter < CommandoBase
+
+		def initialize(config = {})
+			super config
+			@mode ||= 'a+'
+			@file   = File.open path, @mode
+		end
+
+		def on_enter(c)
+			if c == :close
+				file.sync
+				file.fsync rescue nil
+				file.close
+			else
+				c = c.to_hash if c.kind_of?(Commando)				
+				cmd  = c[:cmd]
+				raise ArgumentError, "invalid commando" unless cmd.kind_of?(Symbol)
+				data = c[:data]
+				str  = if data then data.to_json else '' end
+				len  = str.length
+				len += 1 unless str.end_with?('\n')
+				tim  = c[:time] if c[:time]
+				tim  = Time.now unless tim
+				tim  = tim.to_i unless tim.kind_of?(Fixnum)
+				file.write ">>> ANDROMEDA_COMMANDO :#{cmd} TIME #{tim} LEN #{len.to_i} START\n"
+				file.write(str) if data
+				if str.end_with?('\n')
+					file.write "<<< ANDROMEDA_COMMANDO :#{cmd} END\n"
+				else
+					file.write "\n<<< ANDROMEDA_COMMANDO :#{cmd} END\n"
+				end
+			end
+		end
+	end
+
+	class CommandoParser < CommandoBase
+
+		def initialize(config = {})
+			super config
+			@file = File.open path, 'r'
+		end
+
+		def on_enter(c)
+			parser        = dest(:parse)
+			start_matcher = />>> ANDROMEDA_COMMANDO :(\w+) TIME (\d+) LEN (\d+) START/
+			end_matcher   = /<<< ANDROMEDA_COMMANDO :(\w+) END/
+			while (line = file.gets)
+				line  = line.chomp
+				match = start_matcher.match line
+				if match
+					cmd = match[1].to_sym
+					tim = match[2].to_i
+					len = match[3].to_i
+					buf = if len == 0 then '' else file.gets end
+					while buf.length < len
+						log.debug line
+						buf << line
+					end
+					line  = file.gets.chomp
+					match = end_matcher.match line
+					if match
+						end_cmd = match[1].to_sym
+						raise ArgumentError, "command name mismatch between START ('#{cmd}') and END ('#{end_cmd}')" unless cmd == end_cmd
+						raise ArgumentError, "length mismatch" unless len == buf.length
+						h = { :cmd => end_cmd, :data => buf, :time => tim }
+						parser << h
+					else
+						raise ArgumentError, "garbage commando end: '#{line}'"
+					end
+				else
+					raise ArgumentError, "garbage commando start: '#{line}'"
+				end
+			end
+		end
+
+		def on_parse(c)
+			data = c[:data]
+			c[:data] = if data.chomp == '' then nil else JSON::parse(data) end
+			emit << c rescue nil
+		end
+	end
+end

data/lib/andromeda/helpers.rb

@@ -0,0 +1,134 @@
+module Andromeda
+
+
+	class Join < Base
+
+		def initialize(config = {})
+			super config
+			@mutex = Mutex.new
+			@cv    = ConditionVariable.new
+			#  box value to keep ref after clone
+			@state = [ state_init ]
+		end
+
+		protected
+
+		def state_init ; {} end
+		def state_key(chunk) ;  chunk end
+		def state_complete?(state) ; state end
+		def state_updatable?(state, chunk) ;  state[state_key(chunk)] == nil end
+		def state_update(state, chunk) ; state[state_key(chunk)] = chunk end
+
+	    def run(pool, scope, meth, chunk, &thunk)
+	    	@mutex.synchronize {
+	    		state = @state[0]
+	    		while true
+	    			if state_updatable?(state, chunk)
+	    				@state[0] = (state = state_update(state, chunk))
+	    				if state_complete?(state)
+	    					super pool, scope, meth, state, &thunk
+	    					@state[0] = state_init
+	    				end
+						cv.signal
+						return
+	    			else
+	    				cv.wait
+	    			end
+	    		end
+	    	}
+	    end
+
+	end
+
+	class Transf < Base
+		attr_accessor :filter
+		attr_accessor :mapper
+		attr_accessor :reducer
+
+		def output(c)
+			filter_  = filter
+			mapper_  = mapper
+			reducer_ = reducer
+			if !filter_ || filter_.call(c)
+				c = if mapper_ then mapper_.call c else [c] end
+				c = reducer_.call c if reducer_
+				yield c
+			end
+		end
+
+		def on_enter(c)
+			output(c) { |o| super o } 
+		end
+	end
+
+	class Tee < Transf
+		attr_accessor :level
+
+		def init_pool_config ; :local end
+
+		def initialize(config = {})
+			super config
+			@level ||= :info
+		end
+
+		def on_enter(c)
+			log_   = log
+			level_ = level
+			log_.send level, "#{c}" if log_ && level_
+			super c
+		end
+	end
+
+	class Targeting < Transf
+		attr_accessor :targets
+
+		def initialize(config = {})
+			super config
+			@targets ||= {}
+		end
+
+		def target_values
+			t = targets
+			if t.kind_of?(Hash)	then t.values else t end
+		end
+
+		def switch_target(c)
+			switch_ = switch
+			switch_ = switch_.call(c) if switch_
+			targets[switch_]
+		end
+	end
+
+	class Broadc < Targeting
+		def on_enter(c)
+			output(c) do |o|
+				target_values { |t| intern(t) << o rescue nil }
+			end
+		end		
+	end
+
+	class Switch < Targeting
+		attr_accessor :switch
+
+		def on_enter(c)
+			target_ = intern(switch_target c) rescue emit
+			output(c) { |o| target_ << o }
+		end
+	end
+
+	class Router < Targeting
+		def on_enter(c)
+			target_ = intern(switch_target c[0]) rescue emit
+			output(c[1]) { |o| target_ << o }
+		end
+	end
+
+	class FifoBase < Base
+		def init_pool_config ; :fifo end
+	end
+
+	class LocalBase < Base
+		def init_pool_config ; :local end
+	end
+
+end

data/lib/andromeda/id.rb

@@ -0,0 +1,92 @@
+module Andromeda
+
+	# Generator for random xorable ids (used in marks)
+	class Id
+	  # Default length if generated ids
+	  NUM_BYTES = 12
+
+	  protected
+
+	  def initialize(len = NUM_BYTES, random = true, init_data = nil)
+	    raise ArgumentError unless len.kind_of?(Fixnum)
+	    raise ArgumentError unless len >= 0
+
+    	@data = if init_data
+	    	init_data
+	    else
+		    if random
+		      then len.times.map { Id.rnd_byte }
+		      else len.times.map { 0 } end
+		 end
+	  end
+
+	  public
+
+	  def length ; @data.length end
+
+	  def zero?
+	    each { |b| return false unless b == 0 }
+	    true
+	  end
+
+	  def each ; this = self ; 0.upto(length-1).each { |i| yield this[i] } end
+
+	  def each_with_index ; this = self ; 0.upto(length-1).each { |i| yield i, this[i] } end
+
+	  def zip_bytes(b)
+	    return ArgumentError unless same_id_kind?(b)
+	    a = self
+	    0.upto(length-1).each { |i| yield a[i], b[i] }
+	  end
+	  def [](key) ; @data[key] end
+
+	  def same_id_kind?(obj) ; obj.kind_of?(Id) && obj.length == self.length end
+
+	  # Compare self to b
+	  # @param [Id] b
+	  def eq?(b)
+	    zip_bytes(b) { |i,j| return false if i != j }
+	    true
+	  end
+
+	  alias_method :==, :eq?
+
+	  # xor self and b's ids component-wise
+	  # @param [Array<Fixnum>] b
+	  # @return [Id]
+	  def xor(b)
+	    r = []
+	    zip_bytes(b) { |i,j| r << (i ^ j) }
+	    Id.new r.length, false, r
+	  end
+
+	  def to_s
+	    r = "#<#{self.class}:"
+	    each { |b| r << Id.twochars(b.to_s(16)) }
+	    r << '>'
+	    r
+	  end
+
+	  # @param [Fixnum] length
+	  # @return [Id] random id
+	  def self.gen(length = NUM_BYTES) ;  Id.new length, true end
+
+	  # @param [Fixnum] length
+	  # @return [Id] empty (zero) id
+	  def self.zero(length = NUM_BYTES) ; Id.new length, false end
+
+	  private
+
+	  def self.rnd_byte ; Random.rand(256) end
+
+	  def self.twochars(s)
+	  	case s.length
+	  		when 0 then '00'
+	  		when 1 then "0#{s}"
+	  		else s
+	  	end
+	  end
+
+	end
+
+end

data/lib/andromeda/join.rb

@@ -0,0 +1,48 @@
+module Andromeda
+
+	# untested as in not at all but should would perfectly fine according to theory
+
+	class Join < Base
+
+		def initialize(config = {})
+			super config
+			@mutex = Mutex.new
+			@cv    = ConditionVariable.new
+			#  box value to keep ref after clone
+			@state = [ state_init ]
+		end
+
+		protected
+
+		def state_init ; {} end
+
+		# typical usages need to override these two
+		def state_key(chunk) ; chunk end
+		def state_complete?(state)  state end
+
+		def state_updatable?(state, chunk) ;  state[state_key(chunk)] == nil end
+		def state_update(state, chunk) ; state[state_key(chunk)] = chunk; state end
+
+	    def run(pool, scope, meth, chunk, &thunk)
+	    	@mutex.synchronize do
+	    		state = @state[0]
+	    		while true
+	    			if state_updatable?(state, chunk)
+	    				@state[0] = (state = state_update(state, chunk))
+	    				if state_complete?(state)
+	    					@state[0] = state_init
+							cv.signal
+	    					return super pool, scope, meth, state, &thunk
+	    				else
+	    					cv.signal
+	    					return self
+	    				end
+	    			else
+	    				cv.wait @mutex
+	    			end
+	    		end
+	    	end
+	    end
+
+	end
+end	

data/lib/andromeda/pools.rb

@@ -0,0 +1,69 @@
+module Andromeda
+
+  # Helper class for easily obtaining a thread pool with num_processors threads
+  class PoolSupport
+    # @return [Fixnum] number of processors as determined by Facter
+    def self.num_processors ; Facter.sp_number_processors.strip.to_i end
+
+    # @return [ThreadPool] a new thread pool with num_processors threads
+    def self.new_default_pool ; ThreadPool.new self.num_processors end
+
+    # @return [ThreadPool] a globally shared thread pool with num_processors threads
+    def self.global_pool(reset = false)
+      @pool = self.new_default_pool unless @pool || reset
+      @pool
+    end
+
+    # @return [ThreadPool] of size 1
+    def self.new_single_pool ; ThreadPool.new(1) end
+
+    # @return [ThreadPool] that guarantees fifo processing of requests
+    def self.new_fifo_pool ; new_single_pool end
+  end
+
+  # Fake thread pool that spawns an unlimited number of threads
+  class SpawnPool
+    def process(&block) ; Thread.new &block end
+
+    # @return [SpawnPool] a globally shared SpawnPool instance
+    def self.default_pool
+      @pool ||= SpawnPool.new
+      @pool
+    end
+
+    # Does nothing
+    def shutdown ; end
+  end
+
+  # Caching factory for thread pools
+  class PoolFactory < Hash
+
+    attr_reader :pool_maker
+
+    # @yield [Proc] factory/maker for building thread pools for a given key
+    def initialize(&pool_maker)
+      @pool_maker = pool_maker
+    end
+
+    def [](key)
+      current = super key
+      if ! current
+        current   = pool_maker.call key
+        self[key] = current
+      end
+      current
+    end
+
+    def []=(key, value)
+      raise ArgumentError, "Not a ThreadPool" unless value.respond_to?(:process)
+      super key, value
+    end
+
+    def shutdown
+      values.each { |pool| pool.shutdown }
+    end
+
+    alias_method :process_stage, :[]
+  end
+
+end

data/lib/andromeda/scope.rb

@@ -0,0 +1,38 @@
+require 'atomic'
+
+module Andromeda
+
+	class Scope
+
+		def initialize(init_value = 0)
+			raise ArgumentError unless init_value.kind_of?(Fixnum)
+			@count = Atomic.new init_value
+		end
+
+		def value ; @count.value end
+
+		def enter(amount = 1)
+			raise ArgumentError unless amount.kind_of?(Fixnum)
+			raise ArgumentError unless amount >= 0
+			@count.update { |v| v + amount }
+		end
+
+		def leave(amount = 1)
+			raise ArgumentError unless amount >= 0
+			raise ArgumentError unless amount.kind_of?(Fixnum)
+			@count.update { |v| v - amount }
+		end
+
+		def wait_while(&test)
+			while test.call(value)
+				Thread::pass
+			end
+		end
+
+		def wait_for(val = 0)
+			raise ArgumentError unless val.kind_of?(Fixnum)
+			wait_while { |v| v != val }
+		end
+	end
+
+end

data/lib/andromeda/version.rb

@@ -0,0 +1,3 @@
+module Andromeda
+  VERSION = '0.1'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'bundler'
+Bundler.require
+
+require 'simplecov'
+SimpleCov.start

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: andromeda
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Stefan Plantikow
+autorequire: 
+bindir: script
+cert_chain: []
+date: 2012-04-24 00:00:00.000000000 Z
+dependencies: []
+description: Ultra light weight multicore stream processing framework based on a dataflow
+  DSL
+email: stefanp@moviepilot.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- AUTHORS
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- andromeda.gemspec
+- lib/andromeda.rb
+- lib/andromeda/andromeda.rb
+- lib/andromeda/commando.rb
+- lib/andromeda/helpers.rb
+- lib/andromeda/id.rb
+- lib/andromeda/join.rb
+- lib/andromeda/pools.rb
+- lib/andromeda/scope.rb
+- lib/andromeda/version.rb
+- spec/spec_helper.rb
+homepage: https://github.com/moviepilot/andromeda
+licenses:
+- PUBLIC DOMAIN WITHOUT ANY WARRANTY
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: andromeda
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Ultra light weight multicore stream processing framework based on a dataflow
+  DSL
+test_files:
+- spec/spec_helper.rb
+has_rdoc: