andromeda 0.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/.rvmrc +1 -0
- data/CHANGELOG.md +49 -0
- data/Gemfile +17 -9
- data/Gemfile.lock +5 -0
- data/LICENSE.txt +1 -2
- data/README.md +150 -15
- data/ROADMAP.md +73 -0
- data/Rakefile +4 -6
- data/andromeda.gemspec +2 -2
- data/lib/andromeda.rb +52 -9
- data/lib/andromeda/atom.rb +105 -0
- data/lib/andromeda/cmd.rb +242 -0
- data/lib/andromeda/common.rb +0 -0
- data/lib/andromeda/copy_clone.rb +44 -0
- data/lib/andromeda/error.rb +42 -0
- data/lib/andromeda/guide.rb +50 -0
- data/lib/andromeda/guide_track.rb +98 -0
- data/lib/andromeda/id.rb +10 -83
- data/lib/andromeda/impl/atom.rb +47 -0
- data/lib/andromeda/impl/class_attr.rb +31 -0
- data/lib/andromeda/impl/proto_plan.rb +219 -0
- data/lib/andromeda/impl/to_s.rb +48 -0
- data/lib/andromeda/impl/xor_id.rb +89 -0
- data/lib/andromeda/kit.rb +172 -0
- data/lib/andromeda/map_reduce.rb +3 -0
- data/lib/andromeda/plan.rb +130 -0
- data/lib/andromeda/pool_guide.rb +70 -0
- data/lib/andromeda/spot.rb +132 -0
- data/lib/andromeda/sugar.rb +41 -0
- data/lib/andromeda/sync.rb +68 -0
- data/lib/andromeda/version.rb +1 -1
- data/yard_extensions/andromeda.rb +28 -0
- metadata +30 -13
- data/lib/andromeda/andromeda.rb +0 -225
- data/lib/andromeda/commando.rb +0 -106
- data/lib/andromeda/helpers.rb +0 -134
- data/lib/andromeda/join.rb +0 -48
- data/lib/andromeda/pools.rb +0 -69
- data/lib/andromeda/scope.rb +0 -38
data/lib/andromeda/commando.rb
DELETED
@@ -1,106 +0,0 @@
|
|
1
|
-
module Andromeda
|
2
|
-
|
3
|
-
class CommandoBase < Base
|
4
|
-
attr_reader :file
|
5
|
-
attr_reader :path
|
6
|
-
end
|
7
|
-
|
8
|
-
class Commando
|
9
|
-
attr_reader :cmd
|
10
|
-
attr_reader :data
|
11
|
-
attr_reader :time
|
12
|
-
|
13
|
-
def initialize(cmd, data = {})
|
14
|
-
raise ArgumentError unless cmd.kind_of?(Symbol)
|
15
|
-
@cmd = cmd
|
16
|
-
@data = data
|
17
|
-
@time = Time.now.to_i
|
18
|
-
end
|
19
|
-
|
20
|
-
def to_hash
|
21
|
-
{ :cmd => cmd, :data => data, :time => time }
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
class CommandoWriter < CommandoBase
|
26
|
-
|
27
|
-
def initialize(config = {})
|
28
|
-
super config
|
29
|
-
@mode ||= 'a+'
|
30
|
-
@file = File.open path, @mode
|
31
|
-
end
|
32
|
-
|
33
|
-
def on_enter(c)
|
34
|
-
if c == :close
|
35
|
-
file.sync
|
36
|
-
file.fsync rescue nil
|
37
|
-
file.close
|
38
|
-
else
|
39
|
-
c = c.to_hash if c.kind_of?(Commando)
|
40
|
-
cmd = c[:cmd]
|
41
|
-
raise ArgumentError, "invalid commando" unless cmd.kind_of?(Symbol)
|
42
|
-
data = c[:data]
|
43
|
-
str = if data then data.to_json else '' end
|
44
|
-
len = str.length
|
45
|
-
len += 1 unless str.end_with?('\n')
|
46
|
-
tim = c[:time] if c[:time]
|
47
|
-
tim = Time.now unless tim
|
48
|
-
tim = tim.to_i unless tim.kind_of?(Fixnum)
|
49
|
-
file.write ">>> ANDROMEDA_COMMANDO :#{cmd} TIME #{tim} LEN #{len.to_i} START\n"
|
50
|
-
file.write(str) if data
|
51
|
-
if str.end_with?('\n')
|
52
|
-
file.write "<<< ANDROMEDA_COMMANDO :#{cmd} END\n"
|
53
|
-
else
|
54
|
-
file.write "\n<<< ANDROMEDA_COMMANDO :#{cmd} END\n"
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
class CommandoParser < CommandoBase
|
61
|
-
|
62
|
-
def initialize(config = {})
|
63
|
-
super config
|
64
|
-
@file = File.open path, 'r'
|
65
|
-
end
|
66
|
-
|
67
|
-
def on_enter(c)
|
68
|
-
parser = dest(:parse)
|
69
|
-
start_matcher = />>> ANDROMEDA_COMMANDO :(\w+) TIME (\d+) LEN (\d+) START/
|
70
|
-
end_matcher = /<<< ANDROMEDA_COMMANDO :(\w+) END/
|
71
|
-
while (line = file.gets)
|
72
|
-
line = line.chomp
|
73
|
-
match = start_matcher.match line
|
74
|
-
if match
|
75
|
-
cmd = match[1].to_sym
|
76
|
-
tim = match[2].to_i
|
77
|
-
len = match[3].to_i
|
78
|
-
buf = if len == 0 then '' else file.gets end
|
79
|
-
while buf.length < len
|
80
|
-
log.debug line
|
81
|
-
buf << line
|
82
|
-
end
|
83
|
-
line = file.gets.chomp
|
84
|
-
match = end_matcher.match line
|
85
|
-
if match
|
86
|
-
end_cmd = match[1].to_sym
|
87
|
-
raise ArgumentError, "command name mismatch between START ('#{cmd}') and END ('#{end_cmd}')" unless cmd == end_cmd
|
88
|
-
raise ArgumentError, "length mismatch" unless len == buf.length
|
89
|
-
h = { :cmd => end_cmd, :data => buf, :time => tim }
|
90
|
-
parser << h
|
91
|
-
else
|
92
|
-
raise ArgumentError, "garbage commando end: '#{line}'"
|
93
|
-
end
|
94
|
-
else
|
95
|
-
raise ArgumentError, "garbage commando start: '#{line}'"
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
def on_parse(c)
|
101
|
-
data = c[:data]
|
102
|
-
c[:data] = if data.chomp == '' then nil else JSON::parse(data) end
|
103
|
-
emit << c rescue nil
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
data/lib/andromeda/helpers.rb
DELETED
@@ -1,134 +0,0 @@
|
|
1
|
-
module Andromeda
|
2
|
-
|
3
|
-
|
4
|
-
class Join < Base
|
5
|
-
|
6
|
-
def initialize(config = {})
|
7
|
-
super config
|
8
|
-
@mutex = Mutex.new
|
9
|
-
@cv = ConditionVariable.new
|
10
|
-
# box value to keep ref after clone
|
11
|
-
@state = [ state_init ]
|
12
|
-
end
|
13
|
-
|
14
|
-
protected
|
15
|
-
|
16
|
-
def state_init ; {} end
|
17
|
-
def state_key(chunk) ; chunk end
|
18
|
-
def state_complete?(state) ; state end
|
19
|
-
def state_updatable?(state, chunk) ; state[state_key(chunk)] == nil end
|
20
|
-
def state_update(state, chunk) ; state[state_key(chunk)] = chunk end
|
21
|
-
|
22
|
-
def run(pool, scope, meth, chunk, &thunk)
|
23
|
-
@mutex.synchronize {
|
24
|
-
state = @state[0]
|
25
|
-
while true
|
26
|
-
if state_updatable?(state, chunk)
|
27
|
-
@state[0] = (state = state_update(state, chunk))
|
28
|
-
if state_complete?(state)
|
29
|
-
super pool, scope, meth, state, &thunk
|
30
|
-
@state[0] = state_init
|
31
|
-
end
|
32
|
-
cv.signal
|
33
|
-
return
|
34
|
-
else
|
35
|
-
cv.wait
|
36
|
-
end
|
37
|
-
end
|
38
|
-
}
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
|
43
|
-
class Transf < Base
|
44
|
-
attr_accessor :filter
|
45
|
-
attr_accessor :mapper
|
46
|
-
attr_accessor :reducer
|
47
|
-
|
48
|
-
def output(c)
|
49
|
-
filter_ = filter
|
50
|
-
mapper_ = mapper
|
51
|
-
reducer_ = reducer
|
52
|
-
if !filter_ || filter_.call(c)
|
53
|
-
c = if mapper_ then mapper_.call c else [c] end
|
54
|
-
c = reducer_.call c if reducer_
|
55
|
-
yield c
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def on_enter(c)
|
60
|
-
output(c) { |o| super o }
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
class Tee < Transf
|
65
|
-
attr_accessor :level
|
66
|
-
|
67
|
-
def init_pool_config ; :local end
|
68
|
-
|
69
|
-
def initialize(config = {})
|
70
|
-
super config
|
71
|
-
@level ||= :info
|
72
|
-
end
|
73
|
-
|
74
|
-
def on_enter(c)
|
75
|
-
log_ = log
|
76
|
-
level_ = level
|
77
|
-
log_.send level, "#{c}" if log_ && level_
|
78
|
-
super c
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
class Targeting < Transf
|
83
|
-
attr_accessor :targets
|
84
|
-
|
85
|
-
def initialize(config = {})
|
86
|
-
super config
|
87
|
-
@targets ||= {}
|
88
|
-
end
|
89
|
-
|
90
|
-
def target_values
|
91
|
-
t = targets
|
92
|
-
if t.kind_of?(Hash) then t.values else t end
|
93
|
-
end
|
94
|
-
|
95
|
-
def switch_target(c)
|
96
|
-
switch_ = switch
|
97
|
-
switch_ = switch_.call(c) if switch_
|
98
|
-
targets[switch_]
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
class Broadc < Targeting
|
103
|
-
def on_enter(c)
|
104
|
-
output(c) do |o|
|
105
|
-
target_values { |t| intern(t) << o rescue nil }
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
class Switch < Targeting
|
111
|
-
attr_accessor :switch
|
112
|
-
|
113
|
-
def on_enter(c)
|
114
|
-
target_ = intern(switch_target c) rescue emit
|
115
|
-
output(c) { |o| target_ << o }
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
class Router < Targeting
|
120
|
-
def on_enter(c)
|
121
|
-
target_ = intern(switch_target c[0]) rescue emit
|
122
|
-
output(c[1]) { |o| target_ << o }
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
class FifoBase < Base
|
127
|
-
def init_pool_config ; :fifo end
|
128
|
-
end
|
129
|
-
|
130
|
-
class LocalBase < Base
|
131
|
-
def init_pool_config ; :local end
|
132
|
-
end
|
133
|
-
|
134
|
-
end
|
data/lib/andromeda/join.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
module Andromeda
|
2
|
-
|
3
|
-
# untested as in not at all but should would perfectly fine according to theory
|
4
|
-
|
5
|
-
class Join < Base
|
6
|
-
|
7
|
-
def initialize(config = {})
|
8
|
-
super config
|
9
|
-
@mutex = Mutex.new
|
10
|
-
@cv = ConditionVariable.new
|
11
|
-
# box value to keep ref after clone
|
12
|
-
@state = [ state_init ]
|
13
|
-
end
|
14
|
-
|
15
|
-
protected
|
16
|
-
|
17
|
-
def state_init ; {} end
|
18
|
-
|
19
|
-
# typical usages need to override these two
|
20
|
-
def state_key(chunk) ; chunk end
|
21
|
-
def state_complete?(state) state end
|
22
|
-
|
23
|
-
def state_updatable?(state, chunk) ; state[state_key(chunk)] == nil end
|
24
|
-
def state_update(state, chunk) ; state[state_key(chunk)] = chunk; state end
|
25
|
-
|
26
|
-
def run(pool, scope, meth, chunk, &thunk)
|
27
|
-
@mutex.synchronize do
|
28
|
-
state = @state[0]
|
29
|
-
while true
|
30
|
-
if state_updatable?(state, chunk)
|
31
|
-
@state[0] = (state = state_update(state, chunk))
|
32
|
-
if state_complete?(state)
|
33
|
-
@state[0] = state_init
|
34
|
-
cv.signal
|
35
|
-
return super pool, scope, meth, state, &thunk
|
36
|
-
else
|
37
|
-
cv.signal
|
38
|
-
return self
|
39
|
-
end
|
40
|
-
else
|
41
|
-
cv.wait @mutex
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
end
|
48
|
-
end
|
data/lib/andromeda/pools.rb
DELETED
@@ -1,69 +0,0 @@
|
|
1
|
-
module Andromeda
|
2
|
-
|
3
|
-
# Helper class for easily obtaining a thread pool with num_processors threads
|
4
|
-
class PoolSupport
|
5
|
-
# @return [Fixnum] number of processors as determined by Facter
|
6
|
-
def self.num_processors ; Facter.sp_number_processors.strip.to_i end
|
7
|
-
|
8
|
-
# @return [ThreadPool] a new thread pool with num_processors threads
|
9
|
-
def self.new_default_pool ; ThreadPool.new self.num_processors end
|
10
|
-
|
11
|
-
# @return [ThreadPool] a globally shared thread pool with num_processors threads
|
12
|
-
def self.global_pool(reset = false)
|
13
|
-
@pool = self.new_default_pool unless @pool || reset
|
14
|
-
@pool
|
15
|
-
end
|
16
|
-
|
17
|
-
# @return [ThreadPool] of size 1
|
18
|
-
def self.new_single_pool ; ThreadPool.new(1) end
|
19
|
-
|
20
|
-
# @return [ThreadPool] that guarantees fifo processing of requests
|
21
|
-
def self.new_fifo_pool ; new_single_pool end
|
22
|
-
end
|
23
|
-
|
24
|
-
# Fake thread pool that spawns an unlimited number of threads
|
25
|
-
class SpawnPool
|
26
|
-
def process(&block) ; Thread.new &block end
|
27
|
-
|
28
|
-
# @return [SpawnPool] a globally shared SpawnPool instance
|
29
|
-
def self.default_pool
|
30
|
-
@pool ||= SpawnPool.new
|
31
|
-
@pool
|
32
|
-
end
|
33
|
-
|
34
|
-
# Does nothing
|
35
|
-
def shutdown ; end
|
36
|
-
end
|
37
|
-
|
38
|
-
# Caching factory for thread pools
|
39
|
-
class PoolFactory < Hash
|
40
|
-
|
41
|
-
attr_reader :pool_maker
|
42
|
-
|
43
|
-
# @yield [Proc] factory/maker for building thread pools for a given key
|
44
|
-
def initialize(&pool_maker)
|
45
|
-
@pool_maker = pool_maker
|
46
|
-
end
|
47
|
-
|
48
|
-
def [](key)
|
49
|
-
current = super key
|
50
|
-
if ! current
|
51
|
-
current = pool_maker.call key
|
52
|
-
self[key] = current
|
53
|
-
end
|
54
|
-
current
|
55
|
-
end
|
56
|
-
|
57
|
-
def []=(key, value)
|
58
|
-
raise ArgumentError, "Not a ThreadPool" unless value.respond_to?(:process)
|
59
|
-
super key, value
|
60
|
-
end
|
61
|
-
|
62
|
-
def shutdown
|
63
|
-
values.each { |pool| pool.shutdown }
|
64
|
-
end
|
65
|
-
|
66
|
-
alias_method :process_stage, :[]
|
67
|
-
end
|
68
|
-
|
69
|
-
end
|
data/lib/andromeda/scope.rb
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
require 'atomic'
|
2
|
-
|
3
|
-
module Andromeda
|
4
|
-
|
5
|
-
class Scope
|
6
|
-
|
7
|
-
def initialize(init_value = 0)
|
8
|
-
raise ArgumentError unless init_value.kind_of?(Fixnum)
|
9
|
-
@count = Atomic.new init_value
|
10
|
-
end
|
11
|
-
|
12
|
-
def value ; @count.value end
|
13
|
-
|
14
|
-
def enter(amount = 1)
|
15
|
-
raise ArgumentError unless amount.kind_of?(Fixnum)
|
16
|
-
raise ArgumentError unless amount >= 0
|
17
|
-
@count.update { |v| v + amount }
|
18
|
-
end
|
19
|
-
|
20
|
-
def leave(amount = 1)
|
21
|
-
raise ArgumentError unless amount >= 0
|
22
|
-
raise ArgumentError unless amount.kind_of?(Fixnum)
|
23
|
-
@count.update { |v| v - amount }
|
24
|
-
end
|
25
|
-
|
26
|
-
def wait_while(&test)
|
27
|
-
while test.call(value)
|
28
|
-
Thread::pass
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def wait_for(val = 0)
|
33
|
-
raise ArgumentError unless val.kind_of?(Fixnum)
|
34
|
-
wait_while { |v| v != val }
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
end
|