br-nanite 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +201 -0
- data/README.rdoc +356 -0
- data/Rakefile +74 -0
- data/TODO +24 -0
- data/bin/nanite-admin +59 -0
- data/bin/nanite-agent +30 -0
- data/bin/nanite-mapper +22 -0
- data/lib/nanite.rb +58 -0
- data/lib/nanite/actor.rb +64 -0
- data/lib/nanite/actor_registry.rb +25 -0
- data/lib/nanite/admin.rb +147 -0
- data/lib/nanite/agent.rb +163 -0
- data/lib/nanite/amqp.rb +47 -0
- data/lib/nanite/cluster.rb +110 -0
- data/lib/nanite/config.rb +80 -0
- data/lib/nanite/console.rb +39 -0
- data/lib/nanite/daemonize.rb +12 -0
- data/lib/nanite/dispatcher.rb +59 -0
- data/lib/nanite/identity.rb +16 -0
- data/lib/nanite/job.rb +50 -0
- data/lib/nanite/log.rb +23 -0
- data/lib/nanite/mapper.rb +214 -0
- data/lib/nanite/packets.rb +192 -0
- data/lib/nanite/reaper.rb +30 -0
- data/lib/nanite/serializer.rb +40 -0
- data/lib/nanite/streaming.rb +125 -0
- data/lib/nanite/util.rb +51 -0
- metadata +104 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
module Nanite
|
2
|
+
class Reaper
|
3
|
+
|
4
|
+
def initialize(frequency=2)
|
5
|
+
@timeouts = {}
|
6
|
+
EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
|
7
|
+
end
|
8
|
+
|
9
|
+
def timeout(token, seconds, &blk)
|
10
|
+
@timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
|
11
|
+
end
|
12
|
+
|
13
|
+
def reset(token)
|
14
|
+
@timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def reap
|
20
|
+
@timeouts.reject! do |token, data|
|
21
|
+
if Time.now > data[:timestamp]
|
22
|
+
data[:callback].call
|
23
|
+
true
|
24
|
+
else
|
25
|
+
false
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Nanite
|
2
|
+
class Serializer
|
3
|
+
|
4
|
+
class SerializationError < StandardError
|
5
|
+
attr_accessor :action, :packet
|
6
|
+
def initialize(action, packet)
|
7
|
+
@action, @packet = action, packet
|
8
|
+
super("Could not #{action} #{packet.inspect} using #{SERIALIZERS.keys.join(', ')}")
|
9
|
+
end
|
10
|
+
end # SerializationError
|
11
|
+
|
12
|
+
def initialize(preferred_format="marshal")
|
13
|
+
preferred_format ||= "marshal"
|
14
|
+
preferred_serializer = SERIALIZERS[preferred_format.to_sym]
|
15
|
+
@serializers = SERIALIZERS.values.clone
|
16
|
+
@serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer
|
17
|
+
end
|
18
|
+
|
19
|
+
def dump(packet)
|
20
|
+
cascade_serializers(:dump, packet)
|
21
|
+
end
|
22
|
+
|
23
|
+
def load(packet)
|
24
|
+
cascade_serializers(:load, packet)
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze
|
30
|
+
|
31
|
+
def cascade_serializers(action, packet)
|
32
|
+
@serializers.map do |serializer|
|
33
|
+
o = serializer.send(action, packet) rescue nil
|
34
|
+
return o if o
|
35
|
+
end
|
36
|
+
raise SerializationError.new(action, packet)
|
37
|
+
end
|
38
|
+
|
39
|
+
end # Serializer
|
40
|
+
end # Nanite
|
@@ -0,0 +1,125 @@
|
|
1
|
+
module Nanite
|
2
|
+
# Nanite actors can transfer files to each other.
|
3
|
+
#
|
4
|
+
# ==== Options
|
5
|
+
#
|
6
|
+
# filename : you guessed it, name of the file!
|
7
|
+
# domain : part of the routing key used to locate receiver(s)
|
8
|
+
# destination : is a name of the file as it gonna be stored at the destination
|
9
|
+
# meta :
|
10
|
+
#
|
11
|
+
# File streaming is done in chunks. When file streaming starts,
|
12
|
+
# Nanite::FileStart packet is sent, followed by one or more (usually more ;))
|
13
|
+
# Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
|
14
|
+
# Nanite::FileEnd packet is sent.
|
15
|
+
#
|
16
|
+
# 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
|
17
|
+
# more than that in one operation via socket.
|
18
|
+
#
|
19
|
+
# ==== Domains
|
20
|
+
#
|
21
|
+
# Streaming happens using a topic exchange called 'file broadcast', with keys
|
22
|
+
# formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
|
23
|
+
# receivers find each other in the cluster. Default domain is 'global'.
|
24
|
+
#
|
25
|
+
# Domains also serve as a way to register a callback Nanite agent executes once file
|
26
|
+
# streaming is completed. If a callback with name of domain is registered, it is called.
|
27
|
+
#
|
28
|
+
# Callbacks are registered by passing a block to subscribe_to_files method.
|
29
|
+
module FileStreaming
|
30
|
+
def broadcast_file(filename, options = {})
|
31
|
+
if File.exist?(filename)
|
32
|
+
File.open(filename, 'rb') do |file|
|
33
|
+
broadcast_data(filename, file, options)
|
34
|
+
end
|
35
|
+
else
|
36
|
+
return "file not found"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def broadcast_data(filename, io, options = {})
|
41
|
+
domain = options[:domain] || 'global'
|
42
|
+
filename = File.basename(filename)
|
43
|
+
dest = options[:destination] || filename
|
44
|
+
sent = 0
|
45
|
+
|
46
|
+
begin
|
47
|
+
file_push = Nanite::FileStart.new(filename, dest)
|
48
|
+
amq.topic('file broadcast').publish(dump_packet(file_push), :key => "nanite.filepeer.#{domain}")
|
49
|
+
res = Nanite::FileChunk.new(file_push.token)
|
50
|
+
while chunk = io.read(16384)
|
51
|
+
res.chunk = chunk
|
52
|
+
amq.topic('file broadcast').publish(dump_packet(res), :key => "nanite.filepeer.#{domain}")
|
53
|
+
sent += chunk.length
|
54
|
+
end
|
55
|
+
fend = Nanite::FileEnd.new(file_push.token, options[:meta])
|
56
|
+
amq.topic('file broadcast').publish(dump_packet(fend), :key => "nanite.filepeer.#{domain}")
|
57
|
+
""
|
58
|
+
ensure
|
59
|
+
io.close
|
60
|
+
end
|
61
|
+
|
62
|
+
sent
|
63
|
+
end
|
64
|
+
|
65
|
+
# FileState represents a file download in progress.
|
66
|
+
# It incapsulates the following information:
|
67
|
+
#
|
68
|
+
# * unique operation token
|
69
|
+
# * domain (namespace for file streaming operations)
|
70
|
+
# * file IO chunks are written to on receiver's side
|
71
|
+
class FileState
|
72
|
+
|
73
|
+
def initialize(token, dest, domain, write, blk)
|
74
|
+
@token = token
|
75
|
+
@cb = blk
|
76
|
+
@domain = domain
|
77
|
+
@write = write
|
78
|
+
|
79
|
+
if write
|
80
|
+
@filename = File.join(Nanite.agent.file_root, dest)
|
81
|
+
@dest = File.open(@filename, 'wb')
|
82
|
+
else
|
83
|
+
@dest = dest
|
84
|
+
end
|
85
|
+
|
86
|
+
@data = ""
|
87
|
+
end
|
88
|
+
|
89
|
+
def handle_packet(packet)
|
90
|
+
case packet
|
91
|
+
when Nanite::FileChunk
|
92
|
+
log.debug "written chunk to #{@dest.inspect}"
|
93
|
+
@data << packet.chunk
|
94
|
+
|
95
|
+
if @write
|
96
|
+
@dest.write(packet.chunk)
|
97
|
+
end
|
98
|
+
when Nanite::FileEnd
|
99
|
+
log.debug "#{@dest.inspect} receiving is completed"
|
100
|
+
if @write
|
101
|
+
@dest.close
|
102
|
+
end
|
103
|
+
|
104
|
+
@cb.call(@data, @dest, packet.meta)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
|
110
|
+
def subscribe_to_files(domain='global', write=false, &blk)
|
111
|
+
log.info "subscribing to file broadcasts for #{domain}"
|
112
|
+
@files ||= {}
|
113
|
+
amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
|
114
|
+
case msg = load_packet(packet)
|
115
|
+
when FileStart
|
116
|
+
@files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
|
117
|
+
when FileChunk, FileEnd
|
118
|
+
if file = @files[msg.token]
|
119
|
+
file.handle_packet(msg)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
data/lib/nanite/util.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
class String
|
2
|
+
##
|
3
|
+
# Convert to snake case.
|
4
|
+
#
|
5
|
+
# "FooBar".snake_case #=> "foo_bar"
|
6
|
+
# "HeadlineCNNNews".snake_case #=> "headline_cnn_news"
|
7
|
+
# "CNN".snake_case #=> "cnn"
|
8
|
+
#
|
9
|
+
# @return [String] Receiver converted to snake case.
|
10
|
+
#
|
11
|
+
# @api public
|
12
|
+
def snake_case
|
13
|
+
return self.downcase if self =~ /^[A-Z]+$/
|
14
|
+
self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/
|
15
|
+
return $+.downcase
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# Convert a constant name to a path, assuming a conventional structure.
|
20
|
+
#
|
21
|
+
# "FooBar::Baz".to_const_path # => "foo_bar/baz"
|
22
|
+
#
|
23
|
+
# @return [String] Path to the file containing the constant named by receiver
|
24
|
+
# (constantized string), assuming a conventional structure.
|
25
|
+
#
|
26
|
+
# @api public
|
27
|
+
def to_const_path
|
28
|
+
snake_case.gsub(/::/, "/")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class Object
|
33
|
+
module InstanceExecHelper; end
|
34
|
+
include InstanceExecHelper
|
35
|
+
def instance_exec(*args, &block)
|
36
|
+
begin
|
37
|
+
old_critical, Thread.critical = Thread.critical, true
|
38
|
+
n = 0
|
39
|
+
n += 1 while respond_to?(mname="__instance_exec#{n}")
|
40
|
+
InstanceExecHelper.module_eval{ define_method(mname, &block) }
|
41
|
+
ensure
|
42
|
+
Thread.critical = old_critical
|
43
|
+
end
|
44
|
+
begin
|
45
|
+
ret = send(mname, *args)
|
46
|
+
ensure
|
47
|
+
InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
|
48
|
+
end
|
49
|
+
ret
|
50
|
+
end
|
51
|
+
end
|
metadata
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: br-nanite
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ezra Zygmuntowicz
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-12-17 00:00:00 -08:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: extlib
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: amqp
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.6.0
|
34
|
+
version:
|
35
|
+
description: self assembling fabric of ruby daemons
|
36
|
+
email: ezra@engineyard.com
|
37
|
+
executables:
|
38
|
+
- nanite-agent
|
39
|
+
- nanite-mapper
|
40
|
+
- nanite-admin
|
41
|
+
extensions: []
|
42
|
+
|
43
|
+
extra_rdoc_files:
|
44
|
+
- README.rdoc
|
45
|
+
- LICENSE
|
46
|
+
- TODO
|
47
|
+
files:
|
48
|
+
- LICENSE
|
49
|
+
- README.rdoc
|
50
|
+
- Rakefile
|
51
|
+
- TODO
|
52
|
+
- lib/nanite/actor.rb
|
53
|
+
- lib/nanite/actor_registry.rb
|
54
|
+
- lib/nanite/admin.rb
|
55
|
+
- lib/nanite/agent.rb
|
56
|
+
- lib/nanite/amqp.rb
|
57
|
+
- lib/nanite/cluster.rb
|
58
|
+
- lib/nanite/config.rb
|
59
|
+
- lib/nanite/console.rb
|
60
|
+
- lib/nanite/daemonize.rb
|
61
|
+
- lib/nanite/dispatcher.rb
|
62
|
+
- lib/nanite/identity.rb
|
63
|
+
- lib/nanite/job.rb
|
64
|
+
- lib/nanite/log.rb
|
65
|
+
- lib/nanite/mapper.rb
|
66
|
+
- lib/nanite/packets.rb
|
67
|
+
- lib/nanite/reaper.rb
|
68
|
+
- lib/nanite/serializer.rb
|
69
|
+
- lib/nanite/streaming.rb
|
70
|
+
- lib/nanite/util.rb
|
71
|
+
- lib/nanite.rb
|
72
|
+
- bin/nanite-admin
|
73
|
+
- bin/nanite-agent
|
74
|
+
- bin/nanite-mapper
|
75
|
+
has_rdoc: true
|
76
|
+
homepage: http://github.com/ezmobius/nanite
|
77
|
+
licenses: []
|
78
|
+
|
79
|
+
post_install_message:
|
80
|
+
rdoc_options: []
|
81
|
+
|
82
|
+
require_paths:
|
83
|
+
- lib
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
89
|
+
version:
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: "0"
|
95
|
+
version:
|
96
|
+
requirements: []
|
97
|
+
|
98
|
+
rubyforge_project:
|
99
|
+
rubygems_version: 1.3.5
|
100
|
+
signing_key:
|
101
|
+
specification_version: 3
|
102
|
+
summary: self assembling fabric of ruby daemons
|
103
|
+
test_files: []
|
104
|
+
|