bitbroker 0.0.0 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 52b3d5bc540ac0446b4b6fe9eff2a4f64a806f11
4
- data.tar.gz: 78a2121112db9467da940e40afd59d0dc57b262c
3
+ metadata.gz: ca31c10d84f52d9aaebce0fb53b9ed19afcfbb43
4
+ data.tar.gz: 2195c0fd65b83741ece9d8163e2c2b56a0ee590a
5
5
  SHA512:
6
- metadata.gz: e7b33204425476104f7218dcfa634284085a65cb8605c9e93b70e3d51357560ba4a57d8a2b52bf0448193099a71841f0c710906afd5e2834af4a46a74d51ecde
7
- data.tar.gz: 29d50c8abf6d4c7722bbc099ef4f8add9efc655acc7ec947f6b65d26efb840d0fd9bce5c58d67451f21da1931628b2ce8913abcb81359ca8003d15d3570fbd62
6
+ metadata.gz: 5e2a4671faf3d8907d9811febb0a958eaf932afc9251baf8be8dc533bc7fefad5c4cf7bdb162d7332ac7b9bad4dd6020018b748a1b634f75a8aec924f8d4ade2
7
+ data.tar.gz: bec351376c5e7ee3ef35895efb8baf6e70faf23da2608ab3b6c3d8af2faaf3caccbdba84c8973803ec6c9f633ced8a7f1f881efe694e73850f1151380a949299
data/.gitignore CHANGED
@@ -6,5 +6,7 @@
6
6
  /doc/
7
7
  /pkg/
8
8
  /spec/reports/
9
+ /spec/.test
9
10
  /tmp/
10
11
  /vendor/
12
+ *.swp
data/.rspec CHANGED
@@ -1,2 +1,3 @@
1
1
  --format documentation
2
2
  --color
3
+ -r turnip/rspec
data/Gemfile CHANGED
@@ -1,4 +1,10 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in bitbroker.gemspec
4
+ gem 'rspec'
4
5
  gem 'gemspec'
6
+ gem 'msgpack'
7
+ gem 'fileutils'
8
+ gem 'listen', '~> 3.0'
9
+ gem 'macaddr', '1.7.1'
10
+ gem 'turnip'
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bitbroker'
4
+
5
+ PIDFILE = BitBroker::Config.path_pid
6
+ BitBroker::Log.level = Logger::DEBUG
7
+
8
+ if FileTest.exists? PIDFILE
9
+ puts "(NOTICE) bitbroker is now running"
10
+ exit
11
+ end
12
+
13
+ BitBroker::Manager.start
14
+
15
+ puts "bitbroker is started."
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bitbroker'
4
+
5
+ PIDFILE = BitBroker::Config.path_pid
6
+
7
+ def is_stopped? pid
8
+ begin
9
+ Process.getpgid(pid)
10
+ false
11
+ rescue Errno::ESRCH => _
12
+ true
13
+ end
14
+ end
15
+
16
+ unless FileTest.exists? PIDFILE
17
+ puts "(NOTICE) bitbroker is not running"
18
+ exit
19
+ end
20
+
21
+ pids = File.read(PIDFILE).split(/\n/).map{|x| x.to_i}
22
+ pids.each do |pid|
23
+ Process.kill("TERM", pid)
24
+ end
25
+
26
+ while pids.size > 0
27
+ pid = pids.shift
28
+ unless is_stopped? pid
29
+ pids.push(pid)
30
+ end
31
+ end
32
+ File.unlink PIDFILE
33
+
34
+ puts "bitbroker is stopped."
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
23
23
  end
24
24
 
25
25
  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
26
- spec.bindir = "exe"
26
+ spec.bindir = "bin"
27
27
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
28
  spec.require_paths = ["lib"]
29
29
 
@@ -1,5 +1,15 @@
1
1
  require "bitbroker/version"
2
+ require 'bitbroker/solvant'
3
+ require 'bitbroker/observer'
4
+ require 'bitbroker/broker'
5
+ require 'bitbroker/metadata'
6
+ require 'bitbroker/manager'
7
+ require 'bitbroker/manager_impl'
8
+ require 'bitbroker/config'
9
+ require 'bitbroker/log'
2
10
 
3
11
  module Bitbroker
4
- # Your code goes here...
12
+ def self.version
13
+ VERSION
14
+ end
5
15
  end
@@ -0,0 +1,108 @@
1
+ require 'bunny'
2
+ require 'msgpack'
3
+ require 'openssl'
4
+
5
+ module BitBroker
6
+ # This method communicate with AMQP for transmitting and receiving data
7
+ class Broker
8
+ RKEY_DATA = 'data'
9
+ RKEY_METADATA = 'metadata'
10
+ ENCRYPT_ALGORITHM = "AES-256-CBC"
11
+
12
+ def initialize(config)
13
+ @connection = Bunny.new(:host => config[:mqconfig]['host'],
14
+ :vhost => config[:mqconfig]['vhost'],
15
+ :user => config[:mqconfig]['user'],
16
+ :password => config[:mqconfig]['passwd'])
17
+ @connection.start
18
+ @channel = @connection.create_channel
19
+ @exchange = @channel.direct(config[:label])
20
+ @passwd = config[:passwd].to_s + config[:label]
21
+ end
22
+
23
+ def finish
24
+ @channel.close
25
+ @connection.close
26
+ end
27
+ end
28
+
29
+ class Publisher < Broker
30
+ def initialize(config)
31
+ super(config)
32
+ end
33
+
34
+ def send_data(data)
35
+ send(RKEY_DATA, data)
36
+ end
37
+ def send_metadata(data)
38
+ send(RKEY_METADATA, data)
39
+ end
40
+ def send_p_data(dest, data)
41
+ send(RKEY_DATA + dest, data)
42
+ end
43
+ def send_p_metadata(dest, data)
44
+ send(RKEY_METADATA + dest, data)
45
+ end
46
+
47
+ private
48
+ def send(rkey, data)
49
+ @exchange.publish(encode(MessagePack.pack({
50
+ 'data' => data,
51
+ 'from' => Mac.addr,
52
+ })), :routing_key => rkey)
53
+ end
54
+
55
+ def encode(data)
56
+ cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
57
+ cipher.encrypt
58
+ cipher.pkcs5_keyivgen(@passwd)
59
+ cipher.update(data) + cipher.final
60
+ end
61
+ end
62
+
63
+ class Subscriber < Broker
64
+ SIGNAL_TERM = 'TERM'
65
+
66
+ def initialize(config)
67
+ super(config)
68
+ end
69
+
70
+ def recv_data(&block)
71
+ recv(RKEY_DATA, &block)
72
+ end
73
+ def recv_metadata(&block)
74
+ recv(RKEY_METADATA, &block)
75
+ end
76
+ def recv_p_data(&block)
77
+ recv(RKEY_DATA + Mac.addr, &block)
78
+ end
79
+ def recv_p_metadata(&block)
80
+ recv(RKEY_METADATA + Mac.addr, &block)
81
+ end
82
+
83
+ private
84
+ def recv(rkey, &block)
85
+ queue = @channel.queue('', :exclusive => true)
86
+ queue.bind(@exchange, :routing_key => rkey)
87
+ begin
88
+ queue.subscribe(:block => true) do |info, prop, binary|
89
+ msg = MessagePack.unpack(decode(binary))
90
+
91
+ if msg['from'] != Mac.addr
92
+ block.call(msg['data'], msg['from'])
93
+ end
94
+ end
95
+ rescue OpenSSL::Cipher::CipherError => e
96
+ Log.warn("[Subscriber] #{e.to_s}")
97
+ rescue Exception => _
98
+ finish
99
+ end
100
+ end
101
+ def decode(encrypted_data)
102
+ cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
103
+ cipher.decrypt
104
+ cipher.pkcs5_keyivgen(@passwd)
105
+ cipher.update(encrypted_data) + cipher.final
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,26 @@
1
+ require 'yaml'
2
+
3
+ module BitBroker
4
+ class Config
5
+ DEFAULT_PATH_CONFIG = "#{ENV['HOME']}/.bitbroker/config.yml"
6
+ DEFAULT_PATH_PID = "#{ENV['HOME']}/.bitbroker/pid"
7
+
8
+ def self.[](param)
9
+ YAML.load_file(path_config)[param]
10
+ end
11
+
12
+ def self.path_config
13
+ @path_config ||= DEFAULT_PATH_CONFIG
14
+ end
15
+ def self.path_pid
16
+ @path_pid ||= DEFAULT_PATH_PID
17
+ end
18
+
19
+ def self.set_config(path)
20
+ unless FileTest.exist? path
21
+ raise InvalidFile path
22
+ end
23
+ @path_config = path
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,5 @@
1
+ module BitBroker
2
+ class InvalidArguemnt < StandardError; end
3
+ class DiscomfortDirectoryStructure < StandardError; end
4
+ class InvalidFile < StandardError; end
5
+ end
@@ -0,0 +1,24 @@
1
+ require 'logger'
2
+
3
+ module BitBroker
4
+ class Log
5
+ DEFAULT_LOGFILE = '/tmp/bitbroker.log'
6
+
7
+ def self.method_missing(m, *args, &block)
8
+ logger.method(m).call(*args)
9
+ end
10
+
11
+ private
12
+ def self.logger
13
+ @logger ||= init_logger
14
+ end
15
+
16
+ def self.init_logger
17
+ Logger.new(logfile)
18
+ end
19
+
20
+ def self.logfile
21
+ BitBroker::Config['logfile'] != nil ? BitBroker::Config['logfile'] : DEFAULT_LOGFILE
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,84 @@
1
+ require 'yaml'
2
+ require 'macaddr'
3
+ require 'msgpack'
4
+ require 'time'
5
+ require 'thread'
6
+
7
+ require 'bitbroker/manager_impl'
8
+
9
+ module BitBroker
10
+ ### This object is created for each directory
11
+ class Manager < ManagerImpl
12
+
13
+ def self.start
14
+ BitBroker::Config['directories'].each do |entry|
15
+ fork do
16
+ Process.daemon
17
+ File.open(PIDFILE, 'a') do |f|
18
+ f.write("#{$$}\n")
19
+ end
20
+
21
+ begin
22
+ manager = BitBroker::Manager.new({
23
+ :mqconfig => BitBroker::Config['mqconfig'],
24
+ :path => entry['path'],
25
+ :name => entry['name'],
26
+ })
27
+
28
+ manager.start
29
+ manager.advertise
30
+
31
+ loop {}
32
+ rescue Exception => _
33
+ manager.stop
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ def initialize(opts)
40
+ super(opts)
41
+ end
42
+
43
+ def advertise
44
+ @metadata.advertise(@publisher)
45
+ end
46
+
47
+ def start
48
+ # start observer that watches changing of local file-system
49
+ @observer = do_start_observer
50
+
51
+ # start receivers that consume message of remote nodes
52
+ @metadata_receiver = do_start_metadata_receiver
53
+ @p_metadata_receiver = do_start_p_metadata_receiver
54
+
55
+ @data_receiver = do_start_data_receiver
56
+ @p_data_receiver = do_start_p_data_receiver
57
+
58
+ # start collector that maintains the shared directory will be same with remote ones.
59
+ @collector = do_start_collector
60
+ end
61
+
62
+ def stop
63
+ # for observer
64
+ @observer.raise 'stop'
65
+ @observer.join
66
+
67
+ # for receiver
68
+ @metadata_receiver.raise "stop"
69
+ @metadata_receiver.join
70
+
71
+ @p_metadata_receiver.raise "stop"
72
+ @p_metadata_receiver.join
73
+
74
+ @data_receiver.raise "stop"
75
+ @data_receiver.join
76
+
77
+ @p_data_receiver.raise "stop"
78
+ @p_data_receiver.join
79
+
80
+ # for collector
81
+ @collector.kill
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,278 @@
1
+ module BitBroker
2
+ class ManagerImpl
3
+ def initialize(opts)
4
+ # validate user created arguments
5
+ validate(opts)
6
+
7
+ ### prepare brokers
8
+ @config = {
9
+ :mqconfig => opts[:mqconfig],
10
+ :label => opts[:name],
11
+ :dirpath => form_dirpath(opts[:path]),
12
+ }
13
+
14
+ @metadata = Metadata.new(@config[:dirpath])
15
+
16
+ @publisher = Publisher.new(@config)
17
+
18
+ @deficients = @suggestions = []
19
+ @semaphore = Mutex.new
20
+
21
+ # internal variable in this class to know who modified/crated file
22
+ @file_activities = []
23
+ end
24
+
25
+ def form_dirpath path
26
+ path[-1] == '/' ? form_dirpath(path.chop) : path
27
+ end
28
+ def validate(opts)
29
+ raise InvalidArgument("Specified path is not directory") unless File.directory?(opts[:path])
30
+ end
31
+
32
+ def do_start_observer
33
+ def handle_add(path)
34
+ Log.debug("[ManagerImpl] (handle_add) path:#{path}")
35
+
36
+ rpath = @metadata.get_rpath(path)
37
+ if obj = @file_activities.find {|x| x.path == rpath}
38
+ @file_activities.delete(obj)
39
+ else
40
+ # create metadata info
41
+ @metadata.create(rpath)
42
+
43
+ # upload target file
44
+ Solvant.new(@metadata.dir, rpath).upload(@publisher)
45
+ end
46
+ end
47
+
48
+ def handle_mod(path)
49
+ Log.debug("[ManagerImpl] (handle_mod) path:#{path}")
50
+
51
+ rpath = @metadata.get_rpath(path)
52
+ if obj = @file_activities.find {|x| x.path == rpath}
53
+ @file_activities.delete(obj)
54
+ else
55
+ # upload target file
56
+ Solvant.new(@metadata.dir, rpath).upload(@publisher)
57
+
58
+ # update fileinfo
59
+ @metadata.get_with_path(rpath).update
60
+
61
+ @metadata.advertise(@publisher)
62
+ end
63
+ end
64
+
65
+ def handle_rem(path)
66
+ rpath = @metadata.get_rpath(path)
67
+
68
+ #@metadata.remove_with_path(rpath)
69
+ file = @metadata.get_with_path(rpath)
70
+ if file != nil
71
+ Log.debug("[ManagerImpl] (handle_rem) path:#{path}")
72
+
73
+ file.remove
74
+ @metadata.advertise(@publisher)
75
+ end
76
+ end
77
+
78
+ Thread.new do
79
+ Observer.new(@config[:dirpath]) do |mod, add, rem|
80
+ mod.each {|x| handle_mod(x)}
81
+ add.each {|x| handle_add(x)}
82
+ rem.each {|x| handle_rem(x)}
83
+ end
84
+ end
85
+ end
86
+
87
+ def do_start_collector
88
+ Thread.new do
89
+ loop do
90
+ deficient = @deficients.first
91
+ if deficient != nil
92
+ candidates = @suggestions.select { |x| x['path'] == deficient['path'] }
93
+ if candidates.size > 0
94
+ candidate = candidates[rand(candidates.size)]
95
+
96
+ @metadata.request(@publisher, [candidate], candidate['from'])
97
+
98
+ @semaphore.synchronize do
99
+ @suggestions = @suggestions.reject {|x| x['path'] == deficient['path']}
100
+ @deficients.delete(deficient)
101
+ end
102
+ end
103
+ end
104
+ Thread.pass
105
+ end
106
+ end
107
+ end
108
+
109
+ def do_start_metadata_receiver
110
+ Thread.new do
111
+ receiver = Subscriber.new(@config)
112
+ receiver.recv_metadata do |msg, from|
113
+ case msg['type']
114
+ when Metadata::TYPE_ADVERTISE then
115
+ receive_advertise(msg['data'], from)
116
+ when Metadata::TYPE_REQUEST_ALL then
117
+ receive_request_all(msg['data'], from)
118
+ end
119
+ end
120
+ end
121
+ end
122
+
123
+ def do_start_p_metadata_receiver
124
+ Thread.new do
125
+ receiver = Subscriber.new(@config)
126
+ receiver.recv_p_metadata do |msg, from|
127
+ case msg['type']
128
+ when Metadata::TYPE_SUGGESTION then
129
+ receive_suggestion(msg['data'], from)
130
+ when Metadata::TYPE_REQUEST then
131
+ receive_request(msg['data'], from)
132
+ end
133
+ end
134
+ end
135
+ end
136
+
137
+ def do_start_data_receiver
138
+ Thread.new do
139
+ receiver = Subscriber.new(@config)
140
+ receiver.recv_data do |binary, from|
141
+ path = MessagePack.unpack(binary)['path']
142
+
143
+ Log.debug("[ManagerImpl] (data_receiver) path: #{path}")
144
+
145
+ @file_activities.push(FileActivity.create(path))
146
+
147
+ Solvant.load_binary(@config[:dirpath], binary)
148
+ end
149
+ end
150
+ end
151
+ def do_start_p_data_receiver
152
+ Thread.new do
153
+ receiver = Subscriber.new(@config)
154
+ receiver.recv_p_data do |binary, from|
155
+ path = MessagePack.unpack(binary)['path']
156
+
157
+ Log.debug("[ManagerImpl] (p_data_receiver) path: #{path}")
158
+
159
+ @file_activities.push(FileActivity.create(path))
160
+
161
+ Solvant.load_binary(@config[:dirpath], binary)
162
+ end
163
+ end
164
+ end
165
+
166
+ def receive_advertise(data, from)
167
+ def updated?(remote)
168
+ case f = @metadata.get_with_path(remote['path'])
169
+ when nil # this means target file doesn't exist in local.
170
+ true
171
+ else
172
+ f.size != remote['size'] and
173
+ not f.removed?
174
+ end
175
+ end
176
+
177
+ def removed?(remote)
178
+ case f = @metadata.get_with_path(remote['path'])
179
+ when nil
180
+ false
181
+ else
182
+ remote['status'].to_i & Metadata::FileInfo::STATUS_REMOVED > 0
183
+ end
184
+ end
185
+ Log.debug("[ManagerImpl] (receive_advertise) <#{from}> data:#{data}")
186
+
187
+ deficients = []
188
+ data.each do |remote|
189
+ if removed? remote
190
+ Log.debug("[ManagerImpl] (receive_advertise) remove: #{remote}")
191
+
192
+ # set file_activities
193
+ @file_activities.push(FileActivity.remove(remote['path']))
194
+
195
+ # remove FileInfo object which metadata has
196
+ @metadata.remove_with_path(remote['path'])
197
+
198
+ # remove actual file in local FS
199
+ Solvant.new(@config[:dirpath], remote['path']).remove
200
+ else updated? remote
201
+ deficients.push(remote)
202
+
203
+ fpath = "#{@config[:dirpath]}/#{remote['path']}"
204
+ if FileTest.exist? fpath
205
+ Log.debug("[ManagerImpl] trancated(#{fpath}, #{remote['size']})")
206
+
207
+ # truncate files when target file is cut down
208
+ File.truncate(fpath, remote['size'])
209
+ end
210
+ end
211
+ end
212
+
213
+ # request all deficients files
214
+ @metadata.request_all(@publisher, deficients)
215
+
216
+ # record deficient files to get it from remote node
217
+ @semaphore.synchronize do
218
+ @deficients += deficients
219
+ end
220
+ end
221
+
222
+ def receive_request_all(data, from)
223
+ def has_file?(remote)
224
+ @metadata.get_with_path(remote['path']) != nil
225
+ end
226
+ Log.debug("[ManagerImpl] (receive_request_all) <#{from}> data:#{data}")
227
+
228
+ files = data.map {|f| @metadata.get_with_path(f['path'])}.select{|x| x != nil}
229
+ if files != []
230
+ Log.debug("[ManagerImpl] (receive_request_all) files:#{files}")
231
+ @metadata.suggestion(@publisher, files.map{|x| x.to_h}, from)
232
+ end
233
+ end
234
+
235
+ def receive_suggestion(data, from)
236
+ Log.debug("[ManagerImpl] (receive_suggestion) <#{from}> data:#{data}")
237
+
238
+ data.each {|x| x['from'] = from}
239
+ @semaphore.synchronize do
240
+ @suggestions += data
241
+ end
242
+ end
243
+
244
+ def receive_request(data, from)
245
+ Log.debug("[ManagerImpl] (receive_request) <#{from}> data:#{data}")
246
+
247
+ data.each do |remote|
248
+ f = @metadata.get_with_path(remote['path'])
249
+
250
+ Solvant.new(@config[:dirpath], f.path).upload_to(@publisher, from)
251
+ end
252
+ end
253
+
254
+ class FileActivity
255
+ STATUS_REMOVED = 1 << 0
256
+
257
+ attr_reader :path, :mtime
258
+
259
+ def initialize(path, mtime, status = 0)
260
+ @path = path
261
+ @mtime = mtime
262
+ @status = status
263
+ end
264
+
265
+ def removed?
266
+ @status & STATUS_REMOVED > 0
267
+ end
268
+
269
+ def self.create(path)
270
+ self.new(path, FileTest.exist?(path) ? File.mtime(path) : Time.now)
271
+ end
272
+
273
+ def self.remove(path)
274
+ self.new(path, nil, STATUS_REMOVED)
275
+ end
276
+ end
277
+ end
278
+ end
@@ -0,0 +1,122 @@
1
+ require 'msgpack'
2
+
3
+ module BitBroker
4
+ class Metadata
5
+ # describes message types
6
+ TYPE_ADVERTISE = 1<<0
7
+ TYPE_REQUEST_ALL = 1<<1
8
+ TYPE_SUGGESTION = 1<<2
9
+ TYPE_REQUEST = 1<<3
10
+
11
+ attr_reader :dir
12
+
13
+ def initialize(dir)
14
+ @dir = dir
15
+ @files = scanning_files(@dir).map do |path|
16
+ FileInfo.new(@dir, get_rpath(path))
17
+ end
18
+ end
19
+
20
+ def get_with_path(path)
21
+ @files.select{|f| f.path == path}.first
22
+ end
23
+ def remove_with_path(path)
24
+ @files.reject!{|f| f.path == path}
25
+ end
26
+ def create(path)
27
+ if get_with_path(path) == nil
28
+ @files.push(FileInfo.new(@dir, path))
29
+ else
30
+ puts "Warning: #{path} is already created"
31
+ end
32
+ end
33
+
34
+ ### sending message for broker
35
+ def advertise(broker)
36
+ broker.send_metadata({
37
+ :type => TYPE_ADVERTISE,
38
+ :data => @files.map{|x| x.to_h },
39
+ })
40
+ end
41
+ def request_all(broker, files)
42
+ broker.send_metadata({
43
+ :type => TYPE_REQUEST_ALL,
44
+ :data => files,
45
+ })
46
+ end
47
+ def suggestion(broker, files, dest)
48
+ broker.send_p_metadata(dest, {
49
+ :type => TYPE_SUGGESTION,
50
+ :data => files,
51
+ })
52
+ end
53
+ def request(broker, files, dest)
54
+ broker.send_p_metadata(dest, {
55
+ :type => TYPE_REQUEST,
56
+ :data => files,
57
+ })
58
+ end
59
+
60
+ ### utility methods
61
+ def get_rpath(path)
62
+ raise DiscomfortDirectoryStructure unless !!path.match(/^#{@dir}/)
63
+ path.split(@dir).last
64
+ end
65
+
66
+ private
67
+ def scanning_files(current_dir, &block)
68
+ arr = []
69
+ Dir.foreach(current_dir) do |f|
70
+ if /^\.+$/ !~ f
71
+ path = "#{current_dir}/#{f}"
72
+
73
+ if File.directory? f
74
+ arr += scanning(path)
75
+ else
76
+ arr.push(path)
77
+ end
78
+ end
79
+ end
80
+ arr
81
+ end
82
+ class FileInfo
83
+ attr_reader :path, :size, :mtime
84
+
85
+ # describes file status
86
+ STATUS_REMOVED = 1 << 0
87
+
88
+ def initialize(dirpath, filepath)
89
+ @fpath = "#{dirpath}/#{filepath}"
90
+ @path = filepath
91
+ @status = 0
92
+
93
+ self.update
94
+ end
95
+ def update
96
+ if FileTest.exist? @fpath
97
+ file = File.new(@fpath)
98
+
99
+ @size = file.size
100
+ @mtime = file.mtime
101
+ else
102
+ @size = 0
103
+ @mtime = Time.new(0)
104
+ end
105
+ end
106
+ def removed?
107
+ @status & STATUS_REMOVED > 0
108
+ end
109
+ def remove
110
+ @status |= STATUS_REMOVED
111
+ end
112
+ def to_h
113
+ {
114
+ 'path' => @path,
115
+ 'status' => @status,
116
+ 'size' => @size,
117
+ 'mtime' => @mtime.to_s,
118
+ }
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,19 @@
1
+ require 'listen'
2
+
3
+ module BitBroker
4
+ class Observer
5
+ def initialize(dir, &block)
6
+ @target_dir = dir
7
+
8
+ @listener = Listen.to(dir) do |mod, add, rem|
9
+ block.call(mod, add, rem)
10
+ end
11
+
12
+ @listener.start
13
+ end
14
+
15
+ def stop
16
+ @listener.stop
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,84 @@
1
+ require 'msgpack'
2
+ require 'fileutils'
3
+
4
+ module BitBroker
5
+ class Solvant
6
+ DEFAULT_CHUNK_SIZE = 1<<20
7
+
8
+ attr_reader :chunks
9
+
10
+ def initialize(dirpath, r_path, chunk_size = DEFAULT_CHUNK_SIZE)
11
+ @f_path = "#{dirpath}/#{r_path}"
12
+
13
+ # Validate target file at first
14
+ if not FileTest.exist? @f_path
15
+ FileUtils.touch(@f_path)
16
+ end
17
+
18
+ # separate per chunk
19
+ @chunks = []
20
+ chunk_splitter(File::Stat.new(@f_path).size, chunk_size) do |offset, size|
21
+ @chunks.push(Chunk.new({
22
+ :r_path => r_path,
23
+ :f_path => @f_path,
24
+ :size => size,
25
+ :offset => offset,
26
+ :chunk_size => chunk_size,
27
+ }))
28
+ end
29
+ end
30
+
31
+ # This defines operations to manipulate actual Flie object on FileSystem
32
+ def remove
33
+ File.unlink(@f_path)
34
+ end
35
+
36
+ def upload broker
37
+ @chunks.each do |chunk|
38
+ broker.send_data(chunk.serialize)
39
+ end
40
+ end
41
+
42
+ def upload_to(broker, dest)
43
+ @chunks.each do |chunk|
44
+ broker.send_p_data(dest, chunk.serialize)
45
+ end
46
+ end
47
+
48
+ def self.load_binary(dirpath, binary)
49
+ data = MessagePack.unpack(binary)
50
+ offset = data['offset'] * data['chunk_size']
51
+
52
+ File.binwrite(dirpath + data['path'], data['data'], offset)
53
+ end
54
+
55
+ private
56
+ def chunk_splitter(total_size, chunk_size, &block)
57
+ last = total_size / chunk_size
58
+ (0..last).each do |i|
59
+ size = (i == last) ? total_size - chunk_size * i : chunk_size
60
+
61
+ block.call(i, size)
62
+ end
63
+ end
64
+
65
+ class Chunk
66
+ def initialize(opts)
67
+ @r_path = opts[:r_path]
68
+ @f_path = opts[:f_path]
69
+ @size = opts[:size]
70
+ @offset = opts[:offset]
71
+ @chunk_size = opts[:chunk_size]
72
+ end
73
+
74
+ def serialize
75
+ MessagePack.pack({
76
+ 'path' => @r_path,
77
+ 'data' => File.binread(@f_path, @size, @offset * @chunk_size),
78
+ 'offset' => @offset,
79
+ 'chunk_size' => @chunk_size,
80
+ })
81
+ end
82
+ end
83
+ end
84
+ end
@@ -1,3 +1,3 @@
1
1
  module Bitbroker
2
- VERSION = "0.0.0"
2
+ VERSION = "0.1.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bitbroker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiroyasu OHYAMA
8
8
  autorequire:
9
- bindir: exe
9
+ bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-02 00:00:00.000000000 Z
11
+ date: 2015-12-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -66,10 +66,21 @@ files:
66
66
  - LICENSE.txt
67
67
  - README.md
68
68
  - Rakefile
69
+ - bin/bitbroker-start
70
+ - bin/bitbroker-stop
69
71
  - bin/console
70
72
  - bin/setup
71
73
  - bitbroker.gemspec
72
74
  - lib/bitbroker.rb
75
+ - lib/bitbroker/broker.rb
76
+ - lib/bitbroker/config.rb
77
+ - lib/bitbroker/exception.rb
78
+ - lib/bitbroker/log.rb
79
+ - lib/bitbroker/manager.rb
80
+ - lib/bitbroker/manager_impl.rb
81
+ - lib/bitbroker/metadata.rb
82
+ - lib/bitbroker/observer.rb
83
+ - lib/bitbroker/solvant.rb
73
84
  - lib/bitbroker/version.rb
74
85
  homepage: https://github.com/userlocalhost2000/bitbroker
75
86
  licenses: