br-nanite 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,30 @@
1
+ module Nanite
2
+ class Reaper
3
+
4
+ def initialize(frequency=2)
5
+ @timeouts = {}
6
+ EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
7
+ end
8
+
9
+ def timeout(token, seconds, &blk)
10
+ @timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
11
+ end
12
+
13
+ def reset(token)
14
+ @timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
15
+ end
16
+
17
+ private
18
+
19
+ def reap
20
+ @timeouts.reject! do |token, data|
21
+ if Time.now > data[:timestamp]
22
+ data[:callback].call
23
+ true
24
+ else
25
+ false
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,40 @@
1
+ module Nanite
2
+ class Serializer
3
+
4
+ class SerializationError < StandardError
5
+ attr_accessor :action, :packet
6
+ def initialize(action, packet)
7
+ @action, @packet = action, packet
8
+ super("Could not #{action} #{packet.inspect} using #{SERIALIZERS.keys.join(', ')}")
9
+ end
10
+ end # SerializationError
11
+
12
+ def initialize(preferred_format="marshal")
13
+ preferred_format ||= "marshal"
14
+ preferred_serializer = SERIALIZERS[preferred_format.to_sym]
15
+ @serializers = SERIALIZERS.values.clone
16
+ @serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer
17
+ end
18
+
19
+ def dump(packet)
20
+ cascade_serializers(:dump, packet)
21
+ end
22
+
23
+ def load(packet)
24
+ cascade_serializers(:load, packet)
25
+ end
26
+
27
+ private
28
+
29
+ SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze
30
+
31
+ def cascade_serializers(action, packet)
32
+ @serializers.map do |serializer|
33
+ o = serializer.send(action, packet) rescue nil
34
+ return o if o
35
+ end
36
+ raise SerializationError.new(action, packet)
37
+ end
38
+
39
+ end # Serializer
40
+ end # Nanite
@@ -0,0 +1,125 @@
1
+ module Nanite
2
+ # Nanite actors can transfer files to each other.
3
+ #
4
+ # ==== Options
5
+ #
6
+ # filename : you guessed it, name of the file!
7
+ # domain : part of the routing key used to locate receiver(s)
8
+ # destination : is a name of the file as it gonna be stored at the destination
9
+ # meta :
10
+ #
11
+ # File streaming is done in chunks. When file streaming starts,
12
+ # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
13
+ # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
14
+ # Nanite::FileEnd packet is sent.
15
+ #
16
+ # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
17
+ # more than that in one operation via socket.
18
+ #
19
+ # ==== Domains
20
+ #
21
+ # Streaming happens using a topic exchange called 'file broadcast', with keys
22
+ # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
23
+ # receivers find each other in the cluster. Default domain is 'global'.
24
+ #
25
+ # Domains also serve as a way to register a callback Nanite agent executes once file
26
+ # streaming is completed. If a callback with name of domain is registered, it is called.
27
+ #
28
+ # Callbacks are registered by passing a block to subscribe_to_files method.
29
+ module FileStreaming
30
+ def broadcast_file(filename, options = {})
31
+ if File.exist?(filename)
32
+ File.open(filename, 'rb') do |file|
33
+ broadcast_data(filename, file, options)
34
+ end
35
+ else
36
+ return "file not found"
37
+ end
38
+ end
39
+
40
+ def broadcast_data(filename, io, options = {})
41
+ domain = options[:domain] || 'global'
42
+ filename = File.basename(filename)
43
+ dest = options[:destination] || filename
44
+ sent = 0
45
+
46
+ begin
47
+ file_push = Nanite::FileStart.new(filename, dest)
48
+ amq.topic('file broadcast').publish(dump_packet(file_push), :key => "nanite.filepeer.#{domain}")
49
+ res = Nanite::FileChunk.new(file_push.token)
50
+ while chunk = io.read(16384)
51
+ res.chunk = chunk
52
+ amq.topic('file broadcast').publish(dump_packet(res), :key => "nanite.filepeer.#{domain}")
53
+ sent += chunk.length
54
+ end
55
+ fend = Nanite::FileEnd.new(file_push.token, options[:meta])
56
+ amq.topic('file broadcast').publish(dump_packet(fend), :key => "nanite.filepeer.#{domain}")
57
+ ""
58
+ ensure
59
+ io.close
60
+ end
61
+
62
+ sent
63
+ end
64
+
65
+ # FileState represents a file download in progress.
66
+ # It incapsulates the following information:
67
+ #
68
+ # * unique operation token
69
+ # * domain (namespace for file streaming operations)
70
+ # * file IO chunks are written to on receiver's side
71
+ class FileState
72
+
73
+ def initialize(token, dest, domain, write, blk)
74
+ @token = token
75
+ @cb = blk
76
+ @domain = domain
77
+ @write = write
78
+
79
+ if write
80
+ @filename = File.join(Nanite.agent.file_root, dest)
81
+ @dest = File.open(@filename, 'wb')
82
+ else
83
+ @dest = dest
84
+ end
85
+
86
+ @data = ""
87
+ end
88
+
89
+ def handle_packet(packet)
90
+ case packet
91
+ when Nanite::FileChunk
92
+ log.debug "written chunk to #{@dest.inspect}"
93
+ @data << packet.chunk
94
+
95
+ if @write
96
+ @dest.write(packet.chunk)
97
+ end
98
+ when Nanite::FileEnd
99
+ log.debug "#{@dest.inspect} receiving is completed"
100
+ if @write
101
+ @dest.close
102
+ end
103
+
104
+ @cb.call(@data, @dest, packet.meta)
105
+ end
106
+ end
107
+
108
+ end
109
+
110
+ def subscribe_to_files(domain='global', write=false, &blk)
111
+ log.info "subscribing to file broadcasts for #{domain}"
112
+ @files ||= {}
113
+ amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
114
+ case msg = load_packet(packet)
115
+ when FileStart
116
+ @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
117
+ when FileChunk, FileEnd
118
+ if file = @files[msg.token]
119
+ file.handle_packet(msg)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,51 @@
1
+ class String
2
+ ##
3
+ # Convert to snake case.
4
+ #
5
+ # "FooBar".snake_case #=> "foo_bar"
6
+ # "HeadlineCNNNews".snake_case #=> "headline_cnn_news"
7
+ # "CNN".snake_case #=> "cnn"
8
+ #
9
+ # @return [String] Receiver converted to snake case.
10
+ #
11
+ # @api public
12
+ def snake_case
13
+ return self.downcase if self =~ /^[A-Z]+$/
14
+ self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/
15
+ return $+.downcase
16
+ end
17
+
18
+ ##
19
+ # Convert a constant name to a path, assuming a conventional structure.
20
+ #
21
+ # "FooBar::Baz".to_const_path # => "foo_bar/baz"
22
+ #
23
+ # @return [String] Path to the file containing the constant named by receiver
24
+ # (constantized string), assuming a conventional structure.
25
+ #
26
+ # @api public
27
+ def to_const_path
28
+ snake_case.gsub(/::/, "/")
29
+ end
30
+ end
31
+
32
+ class Object
33
+ module InstanceExecHelper; end
34
+ include InstanceExecHelper
35
+ def instance_exec(*args, &block)
36
+ begin
37
+ old_critical, Thread.critical = Thread.critical, true
38
+ n = 0
39
+ n += 1 while respond_to?(mname="__instance_exec#{n}")
40
+ InstanceExecHelper.module_eval{ define_method(mname, &block) }
41
+ ensure
42
+ Thread.critical = old_critical
43
+ end
44
+ begin
45
+ ret = send(mname, *args)
46
+ ensure
47
+ InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
48
+ end
49
+ ret
50
+ end
51
+ end
metadata ADDED
@@ -0,0 +1,104 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: br-nanite
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0
5
+ platform: ruby
6
+ authors:
7
+ - Ezra Zygmuntowicz
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-12-17 00:00:00 -08:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: extlib
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: "0"
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: amqp
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.6.0
34
+ version:
35
+ description: self assembling fabric of ruby daemons
36
+ email: ezra@engineyard.com
37
+ executables:
38
+ - nanite-agent
39
+ - nanite-mapper
40
+ - nanite-admin
41
+ extensions: []
42
+
43
+ extra_rdoc_files:
44
+ - README.rdoc
45
+ - LICENSE
46
+ - TODO
47
+ files:
48
+ - LICENSE
49
+ - README.rdoc
50
+ - Rakefile
51
+ - TODO
52
+ - lib/nanite/actor.rb
53
+ - lib/nanite/actor_registry.rb
54
+ - lib/nanite/admin.rb
55
+ - lib/nanite/agent.rb
56
+ - lib/nanite/amqp.rb
57
+ - lib/nanite/cluster.rb
58
+ - lib/nanite/config.rb
59
+ - lib/nanite/console.rb
60
+ - lib/nanite/daemonize.rb
61
+ - lib/nanite/dispatcher.rb
62
+ - lib/nanite/identity.rb
63
+ - lib/nanite/job.rb
64
+ - lib/nanite/log.rb
65
+ - lib/nanite/mapper.rb
66
+ - lib/nanite/packets.rb
67
+ - lib/nanite/reaper.rb
68
+ - lib/nanite/serializer.rb
69
+ - lib/nanite/streaming.rb
70
+ - lib/nanite/util.rb
71
+ - lib/nanite.rb
72
+ - bin/nanite-admin
73
+ - bin/nanite-agent
74
+ - bin/nanite-mapper
75
+ has_rdoc: true
76
+ homepage: http://github.com/ezmobius/nanite
77
+ licenses: []
78
+
79
+ post_install_message:
80
+ rdoc_options: []
81
+
82
+ require_paths:
83
+ - lib
84
+ required_ruby_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: "0"
89
+ version:
90
+ required_rubygems_version: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: "0"
95
+ version:
96
+ requirements: []
97
+
98
+ rubyforge_project:
99
+ rubygems_version: 1.3.5
100
+ signing_key:
101
+ specification_version: 3
102
+ summary: self assembling fabric of ruby daemons
103
+ test_files: []
104
+