andromeda 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +11 -0
- data/.rvmrc +2 -0
- data/AUTHORS +1 -0
- data/Gemfile +22 -0
- data/Gemfile.lock +85 -0
- data/LICENSE.txt +21 -0
- data/README.md +28 -0
- data/Rakefile +39 -0
- data/andromeda.gemspec +23 -0
- data/lib/andromeda.rb +14 -0
- data/lib/andromeda/andromeda.rb +225 -0
- data/lib/andromeda/commando.rb +106 -0
- data/lib/andromeda/helpers.rb +134 -0
- data/lib/andromeda/id.rb +92 -0
- data/lib/andromeda/join.rb +48 -0
- data/lib/andromeda/pools.rb +69 -0
- data/lib/andromeda/scope.rb +38 -0
- data/lib/andromeda/version.rb +3 -0
- data/spec/spec_helper.rb +5 -0
- metadata +68 -0
data/.gitignore
ADDED
data/.rvmrc
ADDED
data/AUTHORS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Stefan Plantikow <stefanp@moviepilot.com>
|
data/Gemfile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
gem 'json', '>=1.6.5'
|
4
|
+
gem 'threadpool'
|
5
|
+
gem 'facter'
|
6
|
+
gem 'atomic'
|
7
|
+
|
8
|
+
group :development do
|
9
|
+
gem 'rake'
|
10
|
+
gem 'redcarpet', :require => false
|
11
|
+
gem 'yard', :require => false
|
12
|
+
gem 'irbtools', :require => false
|
13
|
+
end
|
14
|
+
|
15
|
+
group :jruby do
|
16
|
+
gem 'maruku'
|
17
|
+
end
|
18
|
+
|
19
|
+
group :test do
|
20
|
+
gem 'rspec', '2.6.0'
|
21
|
+
gem 'simplecov'
|
22
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
atomic (1.0.0)
|
5
|
+
awesome_print (1.0.2)
|
6
|
+
boson (1.1.1)
|
7
|
+
clipboard (1.0.1)
|
8
|
+
coderay (1.0.6)
|
9
|
+
diff-lcs (1.1.3)
|
10
|
+
every_day_irb (1.2.2)
|
11
|
+
facter (1.6.8)
|
12
|
+
fancy_irb (0.7.2)
|
13
|
+
paint (>= 0.8.1)
|
14
|
+
unicode-display_width (>= 0.1.1)
|
15
|
+
g (1.6.0)
|
16
|
+
ruby_gntp
|
17
|
+
hirb (0.6.2)
|
18
|
+
interactive_editor (0.0.10)
|
19
|
+
spoon (>= 0.0.1)
|
20
|
+
irbtools (1.2.2)
|
21
|
+
awesome_print (~> 1.0.2)
|
22
|
+
boson (~> 1.1.1)
|
23
|
+
clipboard (~> 1.0.1)
|
24
|
+
coderay (~> 1.0.5)
|
25
|
+
every_day_irb (>= 1.2.2)
|
26
|
+
fancy_irb (>= 0.7.2)
|
27
|
+
g (>= 1.5.0)
|
28
|
+
hirb (~> 0.6.1)
|
29
|
+
interactive_editor (>= 0.0.10)
|
30
|
+
method_locator (>= 0.0.4)
|
31
|
+
method_source (>= 0.7.0)
|
32
|
+
methodfinder (>= 1.2.5)
|
33
|
+
ori (~> 0.1.0)
|
34
|
+
paint (>= 0.8.4)
|
35
|
+
sketches (>= 0.1.1)
|
36
|
+
wirb (>= 0.4.2)
|
37
|
+
zucker (>= 12.1)
|
38
|
+
json (1.6.6)
|
39
|
+
maruku (0.6.0)
|
40
|
+
syntax (>= 1.0.0)
|
41
|
+
method_locator (0.0.4)
|
42
|
+
method_source (0.7.1)
|
43
|
+
methodfinder (1.2.5)
|
44
|
+
multi_json (1.3.2)
|
45
|
+
ori (0.1.0)
|
46
|
+
paint (0.8.4)
|
47
|
+
rake (0.9.2.2)
|
48
|
+
redcarpet (2.1.1)
|
49
|
+
rspec (2.6.0)
|
50
|
+
rspec-core (~> 2.6.0)
|
51
|
+
rspec-expectations (~> 2.6.0)
|
52
|
+
rspec-mocks (~> 2.6.0)
|
53
|
+
rspec-core (2.6.4)
|
54
|
+
rspec-expectations (2.6.0)
|
55
|
+
diff-lcs (~> 1.1.2)
|
56
|
+
rspec-mocks (2.6.0)
|
57
|
+
ruby_gntp (0.3.4)
|
58
|
+
simplecov (0.6.2)
|
59
|
+
multi_json (~> 1.3)
|
60
|
+
simplecov-html (~> 0.5.3)
|
61
|
+
simplecov-html (0.5.3)
|
62
|
+
sketches (0.1.1)
|
63
|
+
spoon (0.0.1)
|
64
|
+
syntax (1.0.0)
|
65
|
+
threadpool (0.1.0.1)
|
66
|
+
unicode-display_width (0.1.1)
|
67
|
+
wirb (0.4.2)
|
68
|
+
yard (0.7.5)
|
69
|
+
zucker (12.1)
|
70
|
+
|
71
|
+
PLATFORMS
|
72
|
+
ruby
|
73
|
+
|
74
|
+
DEPENDENCIES
|
75
|
+
atomic
|
76
|
+
facter
|
77
|
+
irbtools
|
78
|
+
json (>= 1.6.5)
|
79
|
+
maruku
|
80
|
+
rake
|
81
|
+
redcarpet
|
82
|
+
rspec (= 2.6.0)
|
83
|
+
simplecov
|
84
|
+
threadpool
|
85
|
+
yard
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2012 Stefan Plantikow
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
|
+
|
data/README.md
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# andromeda
|
2
|
+
|
3
|
+
Andromeda is a ultra light weight multicore stream processing framework based on a small dataflow DSL
|
4
|
+
|
5
|
+
It is currently untested and undocumented.
|
6
|
+
|
7
|
+
Below is an example that writes events to a file and reads them back in, to give an idea of what it does:
|
8
|
+
|
9
|
+
require 'andromeda'
|
10
|
+
w = Andromeda::CommandoWriter.new path: '/tmp/some_file'
|
11
|
+
w << (Commando.new :test)
|
12
|
+
w << (Commando.new :test, weight: 40)
|
13
|
+
w << (Commando.new :test, height: 20)
|
14
|
+
w << :close
|
15
|
+
|
16
|
+
r = Andromeda::CommandParser.new path: '/tmp/some_file'
|
17
|
+
# make r process events using a global thread pool of num_cpus threads
|
18
|
+
r.pool = :global
|
19
|
+
t = Andromeda::Tee.new
|
20
|
+
# make r output to t
|
21
|
+
r >> t
|
22
|
+
# start reading
|
23
|
+
r << :start
|
24
|
+
# t will log to a Logger.new(STDERR) by default
|
25
|
+
|
26
|
+
There is much more, dig the source, luke!
|
27
|
+
|
28
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
|
3
|
+
require 'rspec'
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
|
6
|
+
require 'yard'
|
7
|
+
require 'yard/rake/yardoc_task'
|
8
|
+
|
9
|
+
desc 'Run all rspecs'
|
10
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
11
|
+
spec.fail_on_error = true
|
12
|
+
spec.verbose = false
|
13
|
+
# spec.rspec_opts = ['--backtrace']
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Run yardoc over project sources'
|
17
|
+
YARD::Rake::YardocTask.new(:ydoc) do |t|
|
18
|
+
t.options = ['--verbose']
|
19
|
+
t.files = ['lib/**/*.rb', '-', 'README.md', 'AUTHORS', 'LICENSE.txt']
|
20
|
+
end
|
21
|
+
|
22
|
+
#RDoc::Task.new(:rdoc) do |rdoc|
|
23
|
+
# # rdoc.main = "README.rdoc"
|
24
|
+
# rdoc.rdoc_files.include("lib/**/*.rb")
|
25
|
+
#end
|
26
|
+
|
27
|
+
desc 'Run irb in project environment'
|
28
|
+
task :console do
|
29
|
+
require 'irb'
|
30
|
+
ARGV.clear
|
31
|
+
IRB.conf[:USE_READLINE] = false if ENV['JRUBY_OPTS'] =~ /--ng/
|
32
|
+
IRB.start
|
33
|
+
end
|
34
|
+
|
35
|
+
task :doc => :ydoc
|
36
|
+
task :docs => :ydoc
|
37
|
+
task :test => :spec
|
38
|
+
task :tests => :spec
|
39
|
+
task :irb => :console
|
data/andromeda.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require 'andromeda/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'andromeda'
|
7
|
+
s.version = Andromeda::VERSION
|
8
|
+
s.summary = 'Ultra light weight multicore stream processing framework based on a dataflow DSL'
|
9
|
+
s.description = s.summary
|
10
|
+
s.author = 'Stefan Plantikow'
|
11
|
+
s.email = 'stefanp@moviepilot.com'
|
12
|
+
s.homepage = 'https://github.com/moviepilot/andromeda'
|
13
|
+
s.rubyforge_project = 'andromeda'
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.bindir = 'script'
|
21
|
+
s.executables = `git ls-files -- script/*`.split("\n").map{ |f| File.basename(f) }
|
22
|
+
s.licenses = ['PUBLIC DOMAIN WITHOUT ANY WARRANTY']
|
23
|
+
end
|
data/lib/andromeda.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'logger'
|
3
|
+
require 'threadpool'
|
4
|
+
require 'facter'
|
5
|
+
require 'thread'
|
6
|
+
Facter.loadfacts
|
7
|
+
|
8
|
+
require 'andromeda/id'
|
9
|
+
require 'andromeda/pools'
|
10
|
+
require 'andromeda/scope'
|
11
|
+
require 'andromeda/andromeda'
|
12
|
+
require 'andromeda/helpers'
|
13
|
+
require 'andromeda/join'
|
14
|
+
require 'andromeda/commando'
|
@@ -0,0 +1,225 @@
|
|
1
|
+
# TODO
|
2
|
+
# - Turn into separate gem
|
3
|
+
# - Write Tests
|
4
|
+
# - Write fusor for synchronization (extra class that blocks until ready to submit)
|
5
|
+
# - Write docs, add yard support for documenting bases, attrs, etc.
|
6
|
+
# - Make nice slideshow and become very famous and rich. yay!
|
7
|
+
module Andromeda
|
8
|
+
|
9
|
+
module Internal
|
10
|
+
class Transplanting
|
11
|
+
attr_reader :orig
|
12
|
+
|
13
|
+
def initialize(init_opts = nil)
|
14
|
+
@opts = init_opts
|
15
|
+
@orig = self
|
16
|
+
end
|
17
|
+
|
18
|
+
protected
|
19
|
+
|
20
|
+
def transplant(new_opts = nil)
|
21
|
+
return self if @opts == new_opts
|
22
|
+
obj = self.clone
|
23
|
+
obj.instance_variable_set '@opts', new_opts
|
24
|
+
obj
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
class Dest < Internal::Transplanting
|
30
|
+
attr_reader :base
|
31
|
+
attr_reader :meth
|
32
|
+
|
33
|
+
def initialize(base, meth, init_opts = nil)
|
34
|
+
super init_opts
|
35
|
+
raise ArgumentError, "'#{meth}' is not a symbol" unless meth.kind_of?(Symbol)
|
36
|
+
raise ArgumentError, "'#{base}' is not a base" unless base.kind_of?(Base)
|
37
|
+
raise NoMethodError, "'#{base}' does not respond to '#{meth}'" unless base.respond_to?(meth)
|
38
|
+
@base = base
|
39
|
+
@meth = meth
|
40
|
+
end
|
41
|
+
|
42
|
+
def <<(chunk, opts = {}) ; submit chunk, opts ; self end
|
43
|
+
def submit(chunk, opts = {}) ; submit_to nil, chunk, opts end
|
44
|
+
def submit_now(chunk, opts = {}) ; submit_to :local, chunk, opts end
|
45
|
+
|
46
|
+
def submit_to(target_pool, chunk, opts = {})
|
47
|
+
if @opts
|
48
|
+
new_opts = @opts.clone
|
49
|
+
opts.each { |k, v| new_opts[k] = v }
|
50
|
+
else
|
51
|
+
new_opts = opts
|
52
|
+
end
|
53
|
+
new_opts[:scope] ||= Scope.new
|
54
|
+
new_opts[:mark] ||= Id.zero
|
55
|
+
base.transplant(new_opts).process target_pool, new_opts[:scope], self.meth, chunk
|
56
|
+
new_opts
|
57
|
+
end
|
58
|
+
|
59
|
+
def entry ; self end
|
60
|
+
end
|
61
|
+
|
62
|
+
class Base < Internal::Transplanting
|
63
|
+
attr_reader :id
|
64
|
+
attr_reader :opts
|
65
|
+
|
66
|
+
attr_accessor :log
|
67
|
+
attr_accessor :mark
|
68
|
+
attr_accessor :emit
|
69
|
+
attr_accessor :pool
|
70
|
+
|
71
|
+
attr_reader :trace_enter
|
72
|
+
attr_reader :trace_exit
|
73
|
+
|
74
|
+
def initialize(config = {})
|
75
|
+
super config[:init_opts]
|
76
|
+
@id = Id.gen
|
77
|
+
set_from_config init_from_config, config
|
78
|
+
@trace_enter ||= init_trace_hash :enter
|
79
|
+
@trace_exit ||= init_trace_hash :emit
|
80
|
+
@pool ||= init_pool_config
|
81
|
+
end
|
82
|
+
|
83
|
+
def trace=(new_trace)
|
84
|
+
raise ArgumentError, "'#{new_trace}' is not a Hash" unless new_trace.kind_of?(Hash)
|
85
|
+
@trace = new_trace
|
86
|
+
end
|
87
|
+
|
88
|
+
def init_trace_hash(kind) ; {} end
|
89
|
+
def init_pool_config ; nil end
|
90
|
+
def init_from_config ; [:readers, :writers] end
|
91
|
+
|
92
|
+
def log ; @log = Logger.new(STDERR) unless @log ; @log end
|
93
|
+
def mark ; @mark = Id.zero unless @mark ; @mark end
|
94
|
+
|
95
|
+
def on_enter(c)
|
96
|
+
emit << c rescue nil
|
97
|
+
end
|
98
|
+
|
99
|
+
# @param [nil, :local, :spawn, :single, :fifo, :default, :global, #process, #process_base] pool_descr
|
100
|
+
# If nil, uses self.pool as pool_descr and continues. If that is nil, too, defaults to :local.
|
101
|
+
# If #process_base, uses target_pool.process_base(meth) as pool_descr and continues.
|
102
|
+
# If pool_descr is :spawn, uses SpawnPool.default_pool
|
103
|
+
# If pool_descr is :single, uses PoolSupport.new_single_pool
|
104
|
+
# If pool_descr is :fifo, uses PoolSupport.new_fifo_pool
|
105
|
+
# If pool_descr is :global, uses the globally shared PoolSupport.global_pool
|
106
|
+
# If pool_descr is :default uses PoolSupport.new_default_pool
|
107
|
+
# Finally, if #process, runs by calling #process. If :local, runs in current thread.
|
108
|
+
# Otherwise, the behaviour is undefined.
|
109
|
+
def process(pool_descr, scope, meth, chunk)
|
110
|
+
this = self
|
111
|
+
run target_pool(pool_descr, meth), scope, meth, chunk do
|
112
|
+
begin
|
113
|
+
enter_level = trace_enter[meth]
|
114
|
+
exit_level = trace_exit[meth]
|
115
|
+
trace :enter, enter_level, meth, chunk if enter_level
|
116
|
+
send meth, chunk
|
117
|
+
trace :exit, exit_level, meth, chunk if exit_level
|
118
|
+
rescue Exception => e
|
119
|
+
handle_exception meth, chunk, e
|
120
|
+
ensure
|
121
|
+
scope.leave if scope
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def check_mark
|
127
|
+
mark = @opts[:mark]
|
128
|
+
raise RuntimeError, 'invalid mark' if mark && !mark.zero?
|
129
|
+
end
|
130
|
+
|
131
|
+
def intern(dest) ; dest.transplant(opts) end
|
132
|
+
|
133
|
+
def dest(name)
|
134
|
+
result = if self.respond_to?(name)
|
135
|
+
then intern self.send(name)
|
136
|
+
else Dest.new self, "on_#{name}".to_sym, opts end
|
137
|
+
raise ArgumentError, "unknown or invalid dest: '#{name}'" unless result.kind_of?(Dest)
|
138
|
+
result
|
139
|
+
end
|
140
|
+
|
141
|
+
def trace(kind, level, method, chunk)
|
142
|
+
log_ = log
|
143
|
+
log_.send level, "TRACE #{id.to_s} :#{kind} :#{method} chunk: '#{chunk}'" if log_
|
144
|
+
end
|
145
|
+
|
146
|
+
protected
|
147
|
+
|
148
|
+
def set_from_config(what, config = {})
|
149
|
+
init_readers = what.include? :readers
|
150
|
+
init_writers = what.include? :writers
|
151
|
+
config.each_pair do |k, v|
|
152
|
+
k = k.to_sym rescue nil
|
153
|
+
if init_writers
|
154
|
+
writer = "#{k}=".to_sym rescue nil
|
155
|
+
if writer && self.respond_to?(writer)
|
156
|
+
then self.send writer, v
|
157
|
+
else instance_variable_set "@#{k}", v if init_readers && self.respond_to?(k) end
|
158
|
+
else
|
159
|
+
instance_variable_set "@#{k}", v if init_readers && self.respond_to?(k)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def mark_opts
|
165
|
+
if @mark && !@mark.zero?
|
166
|
+
if @opts[:mark]
|
167
|
+
then @opts[:mark] = @opts[:mark].xor(mark)
|
168
|
+
else @opts[:mark] = mark end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def run(pool, scope, meth, chunk, &thunk)
|
173
|
+
scope.enter
|
174
|
+
begin
|
175
|
+
if pool && pool.respond_to?(:process)
|
176
|
+
then pool.process(&thunk)
|
177
|
+
else thunk.call end
|
178
|
+
rescue Exception => e
|
179
|
+
handle_exception meth, chunk, e
|
180
|
+
scope.leave if scope
|
181
|
+
end
|
182
|
+
self
|
183
|
+
end
|
184
|
+
|
185
|
+
def target_pool(pool_descr, meth)
|
186
|
+
pool_descr = pool unless pool_descr
|
187
|
+
case pool_descr
|
188
|
+
when nil then :local
|
189
|
+
when :local then :local
|
190
|
+
when :spawn then SpawnPool.default_pool
|
191
|
+
when :global then PoolSupport.global_pool
|
192
|
+
when :default then PoolSupport.new_default_pool
|
193
|
+
when :single then PoolSupport.new_single_pool
|
194
|
+
when :fifo then PoolSupport.new_fifo_pool
|
195
|
+
else
|
196
|
+
if pool_descr.respond_to(:process_base)
|
197
|
+
then pool_descr.process_base(meth)
|
198
|
+
else pool_descr end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def handle_exception(meth, chunk, e)
|
203
|
+
if log
|
204
|
+
trace = ''
|
205
|
+
e.backtrace.each { |s| trace << "\n #{s}" }
|
206
|
+
log.error "Caught '#{e}' when processing chunk: '#{chunk}' via meth: '#{meth}' with backtrace: '#{trace}'"
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
public
|
211
|
+
|
212
|
+
def >>(dest) ; self.emit = dest.entry end
|
213
|
+
|
214
|
+
def drop ; self.emit = nil end
|
215
|
+
|
216
|
+
def entry ; dest(:enter) end
|
217
|
+
alias_method :exit, :emit
|
218
|
+
|
219
|
+
def <<(chunk, opts = {}) ; entry.<< chunk, opts end
|
220
|
+
def submit(chunk, opts = {}) ; entry.submit chunk, opts end
|
221
|
+
def submit_now(chunk, opts = {}) ; entry.submit_now chunk, opts end
|
222
|
+
def submit_to(target_pool, chunk, opts = {}) ; entry.submit_to target_pool, chunk, opts end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
@@ -0,0 +1,106 @@
|
|
1
|
+
module Andromeda
|
2
|
+
|
3
|
+
class CommandoBase < Base
|
4
|
+
attr_reader :file
|
5
|
+
attr_reader :path
|
6
|
+
end
|
7
|
+
|
8
|
+
class Commando
|
9
|
+
attr_reader :cmd
|
10
|
+
attr_reader :data
|
11
|
+
attr_reader :time
|
12
|
+
|
13
|
+
def initialize(cmd, data = {})
|
14
|
+
raise ArgumentError unless cmd.kind_of?(Symbol)
|
15
|
+
@cmd = cmd
|
16
|
+
@data = data
|
17
|
+
@time = Time.now.to_i
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_hash
|
21
|
+
{ :cmd => cmd, :data => data, :time => time }
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class CommandoWriter < CommandoBase
|
26
|
+
|
27
|
+
def initialize(config = {})
|
28
|
+
super config
|
29
|
+
@mode ||= 'a+'
|
30
|
+
@file = File.open path, @mode
|
31
|
+
end
|
32
|
+
|
33
|
+
def on_enter(c)
|
34
|
+
if c == :close
|
35
|
+
file.sync
|
36
|
+
file.fsync rescue nil
|
37
|
+
file.close
|
38
|
+
else
|
39
|
+
c = c.to_hash if c.kind_of?(Commando)
|
40
|
+
cmd = c[:cmd]
|
41
|
+
raise ArgumentError, "invalid commando" unless cmd.kind_of?(Symbol)
|
42
|
+
data = c[:data]
|
43
|
+
str = if data then data.to_json else '' end
|
44
|
+
len = str.length
|
45
|
+
len += 1 unless str.end_with?('\n')
|
46
|
+
tim = c[:time] if c[:time]
|
47
|
+
tim = Time.now unless tim
|
48
|
+
tim = tim.to_i unless tim.kind_of?(Fixnum)
|
49
|
+
file.write ">>> ANDROMEDA_COMMANDO :#{cmd} TIME #{tim} LEN #{len.to_i} START\n"
|
50
|
+
file.write(str) if data
|
51
|
+
if str.end_with?('\n')
|
52
|
+
file.write "<<< ANDROMEDA_COMMANDO :#{cmd} END\n"
|
53
|
+
else
|
54
|
+
file.write "\n<<< ANDROMEDA_COMMANDO :#{cmd} END\n"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
class CommandoParser < CommandoBase
|
61
|
+
|
62
|
+
def initialize(config = {})
|
63
|
+
super config
|
64
|
+
@file = File.open path, 'r'
|
65
|
+
end
|
66
|
+
|
67
|
+
def on_enter(c)
|
68
|
+
parser = dest(:parse)
|
69
|
+
start_matcher = />>> ANDROMEDA_COMMANDO :(\w+) TIME (\d+) LEN (\d+) START/
|
70
|
+
end_matcher = /<<< ANDROMEDA_COMMANDO :(\w+) END/
|
71
|
+
while (line = file.gets)
|
72
|
+
line = line.chomp
|
73
|
+
match = start_matcher.match line
|
74
|
+
if match
|
75
|
+
cmd = match[1].to_sym
|
76
|
+
tim = match[2].to_i
|
77
|
+
len = match[3].to_i
|
78
|
+
buf = if len == 0 then '' else file.gets end
|
79
|
+
while buf.length < len
|
80
|
+
log.debug line
|
81
|
+
buf << line
|
82
|
+
end
|
83
|
+
line = file.gets.chomp
|
84
|
+
match = end_matcher.match line
|
85
|
+
if match
|
86
|
+
end_cmd = match[1].to_sym
|
87
|
+
raise ArgumentError, "command name mismatch between START ('#{cmd}') and END ('#{end_cmd}')" unless cmd == end_cmd
|
88
|
+
raise ArgumentError, "length mismatch" unless len == buf.length
|
89
|
+
h = { :cmd => end_cmd, :data => buf, :time => tim }
|
90
|
+
parser << h
|
91
|
+
else
|
92
|
+
raise ArgumentError, "garbage commando end: '#{line}'"
|
93
|
+
end
|
94
|
+
else
|
95
|
+
raise ArgumentError, "garbage commando start: '#{line}'"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def on_parse(c)
|
101
|
+
data = c[:data]
|
102
|
+
c[:data] = if data.chomp == '' then nil else JSON::parse(data) end
|
103
|
+
emit << c rescue nil
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
module Andromeda
|
2
|
+
|
3
|
+
|
4
|
+
class Join < Base
|
5
|
+
|
6
|
+
def initialize(config = {})
|
7
|
+
super config
|
8
|
+
@mutex = Mutex.new
|
9
|
+
@cv = ConditionVariable.new
|
10
|
+
# box value to keep ref after clone
|
11
|
+
@state = [ state_init ]
|
12
|
+
end
|
13
|
+
|
14
|
+
protected
|
15
|
+
|
16
|
+
def state_init ; {} end
|
17
|
+
def state_key(chunk) ; chunk end
|
18
|
+
def state_complete?(state) ; state end
|
19
|
+
def state_updatable?(state, chunk) ; state[state_key(chunk)] == nil end
|
20
|
+
def state_update(state, chunk) ; state[state_key(chunk)] = chunk end
|
21
|
+
|
22
|
+
def run(pool, scope, meth, chunk, &thunk)
|
23
|
+
@mutex.synchronize {
|
24
|
+
state = @state[0]
|
25
|
+
while true
|
26
|
+
if state_updatable?(state, chunk)
|
27
|
+
@state[0] = (state = state_update(state, chunk))
|
28
|
+
if state_complete?(state)
|
29
|
+
super pool, scope, meth, state, &thunk
|
30
|
+
@state[0] = state_init
|
31
|
+
end
|
32
|
+
cv.signal
|
33
|
+
return
|
34
|
+
else
|
35
|
+
cv.wait
|
36
|
+
end
|
37
|
+
end
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
class Transf < Base
|
44
|
+
attr_accessor :filter
|
45
|
+
attr_accessor :mapper
|
46
|
+
attr_accessor :reducer
|
47
|
+
|
48
|
+
def output(c)
|
49
|
+
filter_ = filter
|
50
|
+
mapper_ = mapper
|
51
|
+
reducer_ = reducer
|
52
|
+
if !filter_ || filter_.call(c)
|
53
|
+
c = if mapper_ then mapper_.call c else [c] end
|
54
|
+
c = reducer_.call c if reducer_
|
55
|
+
yield c
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def on_enter(c)
|
60
|
+
output(c) { |o| super o }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class Tee < Transf
|
65
|
+
attr_accessor :level
|
66
|
+
|
67
|
+
def init_pool_config ; :local end
|
68
|
+
|
69
|
+
def initialize(config = {})
|
70
|
+
super config
|
71
|
+
@level ||= :info
|
72
|
+
end
|
73
|
+
|
74
|
+
def on_enter(c)
|
75
|
+
log_ = log
|
76
|
+
level_ = level
|
77
|
+
log_.send level, "#{c}" if log_ && level_
|
78
|
+
super c
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
class Targeting < Transf
|
83
|
+
attr_accessor :targets
|
84
|
+
|
85
|
+
def initialize(config = {})
|
86
|
+
super config
|
87
|
+
@targets ||= {}
|
88
|
+
end
|
89
|
+
|
90
|
+
def target_values
|
91
|
+
t = targets
|
92
|
+
if t.kind_of?(Hash) then t.values else t end
|
93
|
+
end
|
94
|
+
|
95
|
+
def switch_target(c)
|
96
|
+
switch_ = switch
|
97
|
+
switch_ = switch_.call(c) if switch_
|
98
|
+
targets[switch_]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
class Broadc < Targeting
|
103
|
+
def on_enter(c)
|
104
|
+
output(c) do |o|
|
105
|
+
target_values { |t| intern(t) << o rescue nil }
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
class Switch < Targeting
|
111
|
+
attr_accessor :switch
|
112
|
+
|
113
|
+
def on_enter(c)
|
114
|
+
target_ = intern(switch_target c) rescue emit
|
115
|
+
output(c) { |o| target_ << o }
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
class Router < Targeting
|
120
|
+
def on_enter(c)
|
121
|
+
target_ = intern(switch_target c[0]) rescue emit
|
122
|
+
output(c[1]) { |o| target_ << o }
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
class FifoBase < Base
|
127
|
+
def init_pool_config ; :fifo end
|
128
|
+
end
|
129
|
+
|
130
|
+
class LocalBase < Base
|
131
|
+
def init_pool_config ; :local end
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
data/lib/andromeda/id.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
module Andromeda
|
2
|
+
|
3
|
+
# Generator for random xorable ids (used in marks)
|
4
|
+
class Id
|
5
|
+
# Default length if generated ids
|
6
|
+
NUM_BYTES = 12
|
7
|
+
|
8
|
+
protected
|
9
|
+
|
10
|
+
def initialize(len = NUM_BYTES, random = true, init_data = nil)
|
11
|
+
raise ArgumentError unless len.kind_of?(Fixnum)
|
12
|
+
raise ArgumentError unless len >= 0
|
13
|
+
|
14
|
+
@data = if init_data
|
15
|
+
init_data
|
16
|
+
else
|
17
|
+
if random
|
18
|
+
then len.times.map { Id.rnd_byte }
|
19
|
+
else len.times.map { 0 } end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
public
|
24
|
+
|
25
|
+
def length ; @data.length end
|
26
|
+
|
27
|
+
def zero?
|
28
|
+
each { |b| return false unless b == 0 }
|
29
|
+
true
|
30
|
+
end
|
31
|
+
|
32
|
+
def each ; this = self ; 0.upto(length-1).each { |i| yield this[i] } end
|
33
|
+
|
34
|
+
def each_with_index ; this = self ; 0.upto(length-1).each { |i| yield i, this[i] } end
|
35
|
+
|
36
|
+
def zip_bytes(b)
|
37
|
+
return ArgumentError unless same_id_kind?(b)
|
38
|
+
a = self
|
39
|
+
0.upto(length-1).each { |i| yield a[i], b[i] }
|
40
|
+
end
|
41
|
+
def [](key) ; @data[key] end
|
42
|
+
|
43
|
+
def same_id_kind?(obj) ; obj.kind_of?(Id) && obj.length == self.length end
|
44
|
+
|
45
|
+
# Compare self to b
|
46
|
+
# @param [Id] b
|
47
|
+
def eq?(b)
|
48
|
+
zip_bytes(b) { |i,j| return false if i != j }
|
49
|
+
true
|
50
|
+
end
|
51
|
+
|
52
|
+
alias_method :==, :eq?
|
53
|
+
|
54
|
+
# xor self and b's ids component-wise
|
55
|
+
# @param [Array<Fixnum>] b
|
56
|
+
# @return [Id]
|
57
|
+
def xor(b)
|
58
|
+
r = []
|
59
|
+
zip_bytes(b) { |i,j| r << (i ^ j) }
|
60
|
+
Id.new r.length, false, r
|
61
|
+
end
|
62
|
+
|
63
|
+
def to_s
|
64
|
+
r = "#<#{self.class}:"
|
65
|
+
each { |b| r << Id.twochars(b.to_s(16)) }
|
66
|
+
r << '>'
|
67
|
+
r
|
68
|
+
end
|
69
|
+
|
70
|
+
# @param [Fixnum] length
|
71
|
+
# @return [Id] random id
|
72
|
+
def self.gen(length = NUM_BYTES) ; Id.new length, true end
|
73
|
+
|
74
|
+
# @param [Fixnum] length
|
75
|
+
# @return [Id] empty (zero) id
|
76
|
+
def self.zero(length = NUM_BYTES) ; Id.new length, false end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def self.rnd_byte ; Random.rand(256) end
|
81
|
+
|
82
|
+
def self.twochars(s)
|
83
|
+
case s.length
|
84
|
+
when 0 then '00'
|
85
|
+
when 1 then "0#{s}"
|
86
|
+
else s
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module Andromeda
|
2
|
+
|
3
|
+
# untested as in not at all but should would perfectly fine according to theory
|
4
|
+
|
5
|
+
class Join < Base
|
6
|
+
|
7
|
+
def initialize(config = {})
|
8
|
+
super config
|
9
|
+
@mutex = Mutex.new
|
10
|
+
@cv = ConditionVariable.new
|
11
|
+
# box value to keep ref after clone
|
12
|
+
@state = [ state_init ]
|
13
|
+
end
|
14
|
+
|
15
|
+
protected
|
16
|
+
|
17
|
+
def state_init ; {} end
|
18
|
+
|
19
|
+
# typical usages need to override these two
|
20
|
+
def state_key(chunk) ; chunk end
|
21
|
+
def state_complete?(state) state end
|
22
|
+
|
23
|
+
def state_updatable?(state, chunk) ; state[state_key(chunk)] == nil end
|
24
|
+
def state_update(state, chunk) ; state[state_key(chunk)] = chunk; state end
|
25
|
+
|
26
|
+
def run(pool, scope, meth, chunk, &thunk)
|
27
|
+
@mutex.synchronize do
|
28
|
+
state = @state[0]
|
29
|
+
while true
|
30
|
+
if state_updatable?(state, chunk)
|
31
|
+
@state[0] = (state = state_update(state, chunk))
|
32
|
+
if state_complete?(state)
|
33
|
+
@state[0] = state_init
|
34
|
+
cv.signal
|
35
|
+
return super pool, scope, meth, state, &thunk
|
36
|
+
else
|
37
|
+
cv.signal
|
38
|
+
return self
|
39
|
+
end
|
40
|
+
else
|
41
|
+
cv.wait @mutex
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Andromeda
|
2
|
+
|
3
|
+
# Helper class for easily obtaining a thread pool with num_processors threads
|
4
|
+
class PoolSupport
|
5
|
+
# @return [Fixnum] number of processors as determined by Facter
|
6
|
+
def self.num_processors ; Facter.sp_number_processors.strip.to_i end
|
7
|
+
|
8
|
+
# @return [ThreadPool] a new thread pool with num_processors threads
|
9
|
+
def self.new_default_pool ; ThreadPool.new self.num_processors end
|
10
|
+
|
11
|
+
# @return [ThreadPool] a globally shared thread pool with num_processors threads
|
12
|
+
def self.global_pool(reset = false)
|
13
|
+
@pool = self.new_default_pool unless @pool || reset
|
14
|
+
@pool
|
15
|
+
end
|
16
|
+
|
17
|
+
# @return [ThreadPool] of size 1
|
18
|
+
def self.new_single_pool ; ThreadPool.new(1) end
|
19
|
+
|
20
|
+
# @return [ThreadPool] that guarantees fifo processing of requests
|
21
|
+
def self.new_fifo_pool ; new_single_pool end
|
22
|
+
end
|
23
|
+
|
24
|
+
# Fake thread pool that spawns an unlimited number of threads
|
25
|
+
class SpawnPool
|
26
|
+
def process(&block) ; Thread.new &block end
|
27
|
+
|
28
|
+
# @return [SpawnPool] a globally shared SpawnPool instance
|
29
|
+
def self.default_pool
|
30
|
+
@pool ||= SpawnPool.new
|
31
|
+
@pool
|
32
|
+
end
|
33
|
+
|
34
|
+
# Does nothing
|
35
|
+
def shutdown ; end
|
36
|
+
end
|
37
|
+
|
38
|
+
# Caching factory for thread pools
|
39
|
+
class PoolFactory < Hash
|
40
|
+
|
41
|
+
attr_reader :pool_maker
|
42
|
+
|
43
|
+
# @yield [Proc] factory/maker for building thread pools for a given key
|
44
|
+
def initialize(&pool_maker)
|
45
|
+
@pool_maker = pool_maker
|
46
|
+
end
|
47
|
+
|
48
|
+
def [](key)
|
49
|
+
current = super key
|
50
|
+
if ! current
|
51
|
+
current = pool_maker.call key
|
52
|
+
self[key] = current
|
53
|
+
end
|
54
|
+
current
|
55
|
+
end
|
56
|
+
|
57
|
+
def []=(key, value)
|
58
|
+
raise ArgumentError, "Not a ThreadPool" unless value.respond_to?(:process)
|
59
|
+
super key, value
|
60
|
+
end
|
61
|
+
|
62
|
+
def shutdown
|
63
|
+
values.each { |pool| pool.shutdown }
|
64
|
+
end
|
65
|
+
|
66
|
+
alias_method :process_stage, :[]
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'atomic'
|
2
|
+
|
3
|
+
module Andromeda
|
4
|
+
|
5
|
+
class Scope
|
6
|
+
|
7
|
+
def initialize(init_value = 0)
|
8
|
+
raise ArgumentError unless init_value.kind_of?(Fixnum)
|
9
|
+
@count = Atomic.new init_value
|
10
|
+
end
|
11
|
+
|
12
|
+
def value ; @count.value end
|
13
|
+
|
14
|
+
def enter(amount = 1)
|
15
|
+
raise ArgumentError unless amount.kind_of?(Fixnum)
|
16
|
+
raise ArgumentError unless amount >= 0
|
17
|
+
@count.update { |v| v + amount }
|
18
|
+
end
|
19
|
+
|
20
|
+
def leave(amount = 1)
|
21
|
+
raise ArgumentError unless amount >= 0
|
22
|
+
raise ArgumentError unless amount.kind_of?(Fixnum)
|
23
|
+
@count.update { |v| v - amount }
|
24
|
+
end
|
25
|
+
|
26
|
+
def wait_while(&test)
|
27
|
+
while test.call(value)
|
28
|
+
Thread::pass
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def wait_for(val = 0)
|
33
|
+
raise ArgumentError unless val.kind_of?(Fixnum)
|
34
|
+
wait_while { |v| v != val }
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: andromeda
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: '0.1'
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Stefan Plantikow
|
9
|
+
autorequire:
|
10
|
+
bindir: script
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-04-24 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: Ultra light weight multicore stream processing framework based on a dataflow
|
15
|
+
DSL
|
16
|
+
email: stefanp@moviepilot.com
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- .gitignore
|
22
|
+
- .rvmrc
|
23
|
+
- AUTHORS
|
24
|
+
- Gemfile
|
25
|
+
- Gemfile.lock
|
26
|
+
- LICENSE.txt
|
27
|
+
- README.md
|
28
|
+
- Rakefile
|
29
|
+
- andromeda.gemspec
|
30
|
+
- lib/andromeda.rb
|
31
|
+
- lib/andromeda/andromeda.rb
|
32
|
+
- lib/andromeda/commando.rb
|
33
|
+
- lib/andromeda/helpers.rb
|
34
|
+
- lib/andromeda/id.rb
|
35
|
+
- lib/andromeda/join.rb
|
36
|
+
- lib/andromeda/pools.rb
|
37
|
+
- lib/andromeda/scope.rb
|
38
|
+
- lib/andromeda/version.rb
|
39
|
+
- spec/spec_helper.rb
|
40
|
+
homepage: https://github.com/moviepilot/andromeda
|
41
|
+
licenses:
|
42
|
+
- PUBLIC DOMAIN WITHOUT ANY WARRANTY
|
43
|
+
post_install_message:
|
44
|
+
rdoc_options: []
|
45
|
+
require_paths:
|
46
|
+
- lib
|
47
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
48
|
+
none: false
|
49
|
+
requirements:
|
50
|
+
- - ! '>='
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '0'
|
53
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ! '>='
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: '0'
|
59
|
+
requirements: []
|
60
|
+
rubyforge_project: andromeda
|
61
|
+
rubygems_version: 1.8.17
|
62
|
+
signing_key:
|
63
|
+
specification_version: 3
|
64
|
+
summary: Ultra light weight multicore stream processing framework based on a dataflow
|
65
|
+
DSL
|
66
|
+
test_files:
|
67
|
+
- spec/spec_helper.rb
|
68
|
+
has_rdoc:
|