bitbroker 0.0.0 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 52b3d5bc540ac0446b4b6fe9eff2a4f64a806f11
4
- data.tar.gz: 78a2121112db9467da940e40afd59d0dc57b262c
3
+ metadata.gz: ca31c10d84f52d9aaebce0fb53b9ed19afcfbb43
4
+ data.tar.gz: 2195c0fd65b83741ece9d8163e2c2b56a0ee590a
5
5
  SHA512:
6
- metadata.gz: e7b33204425476104f7218dcfa634284085a65cb8605c9e93b70e3d51357560ba4a57d8a2b52bf0448193099a71841f0c710906afd5e2834af4a46a74d51ecde
7
- data.tar.gz: 29d50c8abf6d4c7722bbc099ef4f8add9efc655acc7ec947f6b65d26efb840d0fd9bce5c58d67451f21da1931628b2ce8913abcb81359ca8003d15d3570fbd62
6
+ metadata.gz: 5e2a4671faf3d8907d9811febb0a958eaf932afc9251baf8be8dc533bc7fefad5c4cf7bdb162d7332ac7b9bad4dd6020018b748a1b634f75a8aec924f8d4ade2
7
+ data.tar.gz: bec351376c5e7ee3ef35895efb8baf6e70faf23da2608ab3b6c3d8af2faaf3caccbdba84c8973803ec6c9f633ced8a7f1f881efe694e73850f1151380a949299
data/.gitignore CHANGED
@@ -6,5 +6,7 @@
6
6
  /doc/
7
7
  /pkg/
8
8
  /spec/reports/
9
+ /spec/.test
9
10
  /tmp/
10
11
  /vendor/
12
+ *.swp
data/.rspec CHANGED
@@ -1,2 +1,3 @@
1
1
  --format documentation
2
2
  --color
3
+ -r turnip/rspec
data/Gemfile CHANGED
@@ -1,4 +1,10 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in bitbroker.gemspec
4
+ gem 'rspec'
4
5
  gem 'gemspec'
6
+ gem 'msgpack'
7
+ gem 'fileutils'
8
+ gem 'listen', '~> 3.0'
9
+ gem 'macaddr', '1.7.1'
10
+ gem 'turnip'
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bitbroker'
4
+
5
+ PIDFILE = BitBroker::Config.path_pid
6
+ BitBroker::Log.level = Logger::DEBUG
7
+
8
+ if FileTest.exists? PIDFILE
9
+ puts "(NOTICE) bitbroker is now running"
10
+ exit
11
+ end
12
+
13
+ BitBroker::Manager.start
14
+
15
+ puts "bitbroker is started."
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bitbroker'
4
+
5
+ PIDFILE = BitBroker::Config.path_pid
6
+
7
+ def is_stopped? pid
8
+ begin
9
+ Process.getpgid(pid)
10
+ false
11
+ rescue Errno::ESRCH => _
12
+ true
13
+ end
14
+ end
15
+
16
+ unless FileTest.exists? PIDFILE
17
+ puts "(NOTICE) bitbroker is not running"
18
+ exit
19
+ end
20
+
21
+ pids = File.read(PIDFILE).split(/\n/).map{|x| x.to_i}
22
+ pids.each do |pid|
23
+ Process.kill("TERM", pid)
24
+ end
25
+
26
+ while pids.size > 0
27
+ pid = pids.shift
28
+ unless is_stopped? pid
29
+ pids.push(pid)
30
+ end
31
+ end
32
+ File.unlink PIDFILE
33
+
34
+ puts "bitbroker is stopped."
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
23
23
  end
24
24
 
25
25
  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
26
- spec.bindir = "exe"
26
+ spec.bindir = "bin"
27
27
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
28
  spec.require_paths = ["lib"]
29
29
 
@@ -1,5 +1,15 @@
1
1
  require "bitbroker/version"
2
+ require 'bitbroker/solvant'
3
+ require 'bitbroker/observer'
4
+ require 'bitbroker/broker'
5
+ require 'bitbroker/metadata'
6
+ require 'bitbroker/manager'
7
+ require 'bitbroker/manager_impl'
8
+ require 'bitbroker/config'
9
+ require 'bitbroker/log'
2
10
 
3
11
  module Bitbroker
4
- # Your code goes here...
12
+ def self.version
13
+ VERSION
14
+ end
5
15
  end
@@ -0,0 +1,108 @@
1
+ require 'bunny'
2
+ require 'msgpack'
3
+ require 'openssl'
4
+
5
+ module BitBroker
6
+ # This method communicate with AMQP for transmitting and receiving data
7
+ class Broker
8
+ RKEY_DATA = 'data'
9
+ RKEY_METADATA = 'metadata'
10
+ ENCRYPT_ALGORITHM = "AES-256-CBC"
11
+
12
+ def initialize(config)
13
+ @connection = Bunny.new(:host => config[:mqconfig]['host'],
14
+ :vhost => config[:mqconfig]['vhost'],
15
+ :user => config[:mqconfig]['user'],
16
+ :password => config[:mqconfig]['passwd'])
17
+ @connection.start
18
+ @channel = @connection.create_channel
19
+ @exchange = @channel.direct(config[:label])
20
+ @passwd = config[:passwd].to_s + config[:label]
21
+ end
22
+
23
+ def finish
24
+ @channel.close
25
+ @connection.close
26
+ end
27
+ end
28
+
29
+ class Publisher < Broker
30
+ def initialize(config)
31
+ super(config)
32
+ end
33
+
34
+ def send_data(data)
35
+ send(RKEY_DATA, data)
36
+ end
37
+ def send_metadata(data)
38
+ send(RKEY_METADATA, data)
39
+ end
40
+ def send_p_data(dest, data)
41
+ send(RKEY_DATA + dest, data)
42
+ end
43
+ def send_p_metadata(dest, data)
44
+ send(RKEY_METADATA + dest, data)
45
+ end
46
+
47
+ private
48
+ def send(rkey, data)
49
+ @exchange.publish(encode(MessagePack.pack({
50
+ 'data' => data,
51
+ 'from' => Mac.addr,
52
+ })), :routing_key => rkey)
53
+ end
54
+
55
+ def encode(data)
56
+ cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
57
+ cipher.encrypt
58
+ cipher.pkcs5_keyivgen(@passwd)
59
+ cipher.update(data) + cipher.final
60
+ end
61
+ end
62
+
63
+ class Subscriber < Broker
64
+ SIGNAL_TERM = 'TERM'
65
+
66
+ def initialize(config)
67
+ super(config)
68
+ end
69
+
70
+ def recv_data(&block)
71
+ recv(RKEY_DATA, &block)
72
+ end
73
+ def recv_metadata(&block)
74
+ recv(RKEY_METADATA, &block)
75
+ end
76
+ def recv_p_data(&block)
77
+ recv(RKEY_DATA + Mac.addr, &block)
78
+ end
79
+ def recv_p_metadata(&block)
80
+ recv(RKEY_METADATA + Mac.addr, &block)
81
+ end
82
+
83
+ private
84
+ def recv(rkey, &block)
85
+ queue = @channel.queue('', :exclusive => true)
86
+ queue.bind(@exchange, :routing_key => rkey)
87
+ begin
88
+ queue.subscribe(:block => true) do |info, prop, binary|
89
+ msg = MessagePack.unpack(decode(binary))
90
+
91
+ if msg['from'] != Mac.addr
92
+ block.call(msg['data'], msg['from'])
93
+ end
94
+ end
95
+ rescue OpenSSL::Cipher::CipherError => e
96
+ Log.warn("[Subscriber] #{e.to_s}")
97
+ rescue Exception => _
98
+ finish
99
+ end
100
+ end
101
+ def decode(encrypted_data)
102
+ cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
103
+ cipher.decrypt
104
+ cipher.pkcs5_keyivgen(@passwd)
105
+ cipher.update(encrypted_data) + cipher.final
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,26 @@
1
+ require 'yaml'
2
+
3
+ module BitBroker
4
+ class Config
5
+ DEFAULT_PATH_CONFIG = "#{ENV['HOME']}/.bitbroker/config.yml"
6
+ DEFAULT_PATH_PID = "#{ENV['HOME']}/.bitbroker/pid"
7
+
8
+ def self.[](param)
9
+ YAML.load_file(path_config)[param]
10
+ end
11
+
12
+ def self.path_config
13
+ @path_config ||= DEFAULT_PATH_CONFIG
14
+ end
15
+ def self.path_pid
16
+ @path_pid ||= DEFAULT_PATH_PID
17
+ end
18
+
19
+ def self.set_config(path)
20
+ unless FileTest.exist? path
21
+ raise InvalidFile path
22
+ end
23
+ @path_config = path
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,5 @@
1
+ module BitBroker
2
+ class InvalidArguemnt < StandardError; end
3
+ class DiscomfortDirectoryStructure < StandardError; end
4
+ class InvalidFile < StandardError; end
5
+ end
@@ -0,0 +1,24 @@
1
+ require 'logger'
2
+
3
+ module BitBroker
4
+ class Log
5
+ DEFAULT_LOGFILE = '/tmp/bitbroker.log'
6
+
7
+ def self.method_missing(m, *args, &block)
8
+ logger.method(m).call(*args)
9
+ end
10
+
11
+ private
12
+ def self.logger
13
+ @logger ||= init_logger
14
+ end
15
+
16
+ def self.init_logger
17
+ Logger.new(logfile)
18
+ end
19
+
20
+ def self.logfile
21
+ BitBroker::Config['logfile'] != nil ? BitBroker::Config['logfile'] : DEFAULT_LOGFILE
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,84 @@
1
+ require 'yaml'
2
+ require 'macaddr'
3
+ require 'msgpack'
4
+ require 'time'
5
+ require 'thread'
6
+
7
+ require 'bitbroker/manager_impl'
8
+
9
+ module BitBroker
10
+ ### This object is created for each directory
11
+ class Manager < ManagerImpl
12
+
13
+ def self.start
14
+ BitBroker::Config['directories'].each do |entry|
15
+ fork do
16
+ Process.daemon
17
+ File.open(PIDFILE, 'a') do |f|
18
+ f.write("#{$$}\n")
19
+ end
20
+
21
+ begin
22
+ manager = BitBroker::Manager.new({
23
+ :mqconfig => BitBroker::Config['mqconfig'],
24
+ :path => entry['path'],
25
+ :name => entry['name'],
26
+ })
27
+
28
+ manager.start
29
+ manager.advertise
30
+
31
+ loop {}
32
+ rescue Exception => _
33
+ manager.stop
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ def initialize(opts)
40
+ super(opts)
41
+ end
42
+
43
+ def advertise
44
+ @metadata.advertise(@publisher)
45
+ end
46
+
47
+ def start
48
+ # start observer that watches changing of local file-system
49
+ @observer = do_start_observer
50
+
51
+ # start receivers that consume message of remote nodes
52
+ @metadata_receiver = do_start_metadata_receiver
53
+ @p_metadata_receiver = do_start_p_metadata_receiver
54
+
55
+ @data_receiver = do_start_data_receiver
56
+ @p_data_receiver = do_start_p_data_receiver
57
+
58
+ # start collector that maintains the shared directory will be same with remote ones.
59
+ @collector = do_start_collector
60
+ end
61
+
62
+ def stop
63
+ # for observer
64
+ @observer.raise 'stop'
65
+ @observer.join
66
+
67
+ # for receiver
68
+ @metadata_receiver.raise "stop"
69
+ @metadata_receiver.join
70
+
71
+ @p_metadata_receiver.raise "stop"
72
+ @p_metadata_receiver.join
73
+
74
+ @data_receiver.raise "stop"
75
+ @data_receiver.join
76
+
77
+ @p_data_receiver.raise "stop"
78
+ @p_data_receiver.join
79
+
80
+ # for collector
81
+ @collector.kill
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,278 @@
1
+ module BitBroker
2
+ class ManagerImpl
3
+ def initialize(opts)
4
+ # validate user created arguments
5
+ validate(opts)
6
+
7
+ ### prepare brokers
8
+ @config = {
9
+ :mqconfig => opts[:mqconfig],
10
+ :label => opts[:name],
11
+ :dirpath => form_dirpath(opts[:path]),
12
+ }
13
+
14
+ @metadata = Metadata.new(@config[:dirpath])
15
+
16
+ @publisher = Publisher.new(@config)
17
+
18
+ @deficients = @suggestions = []
19
+ @semaphore = Mutex.new
20
+
21
+ # internal variable in this class to know who modified/crated file
22
+ @file_activities = []
23
+ end
24
+
25
+ def form_dirpath path
26
+ path[-1] == '/' ? form_dirpath(path.chop) : path
27
+ end
28
+ def validate(opts)
29
+ raise InvalidArgument("Specified path is not directory") unless File.directory?(opts[:path])
30
+ end
31
+
32
+ def do_start_observer
33
+ def handle_add(path)
34
+ Log.debug("[ManagerImpl] (handle_add) path:#{path}")
35
+
36
+ rpath = @metadata.get_rpath(path)
37
+ if obj = @file_activities.find {|x| x.path == rpath}
38
+ @file_activities.delete(obj)
39
+ else
40
+ # create metadata info
41
+ @metadata.create(rpath)
42
+
43
+ # upload target file
44
+ Solvant.new(@metadata.dir, rpath).upload(@publisher)
45
+ end
46
+ end
47
+
48
+ def handle_mod(path)
49
+ Log.debug("[ManagerImpl] (handle_mod) path:#{path}")
50
+
51
+ rpath = @metadata.get_rpath(path)
52
+ if obj = @file_activities.find {|x| x.path == rpath}
53
+ @file_activities.delete(obj)
54
+ else
55
+ # upload target file
56
+ Solvant.new(@metadata.dir, rpath).upload(@publisher)
57
+
58
+ # update fileinfo
59
+ @metadata.get_with_path(rpath).update
60
+
61
+ @metadata.advertise(@publisher)
62
+ end
63
+ end
64
+
65
+ def handle_rem(path)
66
+ rpath = @metadata.get_rpath(path)
67
+
68
+ #@metadata.remove_with_path(rpath)
69
+ file = @metadata.get_with_path(rpath)
70
+ if file != nil
71
+ Log.debug("[ManagerImpl] (handle_rem) path:#{path}")
72
+
73
+ file.remove
74
+ @metadata.advertise(@publisher)
75
+ end
76
+ end
77
+
78
+ Thread.new do
79
+ Observer.new(@config[:dirpath]) do |mod, add, rem|
80
+ mod.each {|x| handle_mod(x)}
81
+ add.each {|x| handle_add(x)}
82
+ rem.each {|x| handle_rem(x)}
83
+ end
84
+ end
85
+ end
86
+
87
+ def do_start_collector
88
+ Thread.new do
89
+ loop do
90
+ deficient = @deficients.first
91
+ if deficient != nil
92
+ candidates = @suggestions.select { |x| x['path'] == deficient['path'] }
93
+ if candidates.size > 0
94
+ candidate = candidates[rand(candidates.size)]
95
+
96
+ @metadata.request(@publisher, [candidate], candidate['from'])
97
+
98
+ @semaphore.synchronize do
99
+ @suggestions = @suggestions.reject {|x| x['path'] == deficient['path']}
100
+ @deficients.delete(deficient)
101
+ end
102
+ end
103
+ end
104
+ Thread.pass
105
+ end
106
+ end
107
+ end
108
+
109
+ def do_start_metadata_receiver
110
+ Thread.new do
111
+ receiver = Subscriber.new(@config)
112
+ receiver.recv_metadata do |msg, from|
113
+ case msg['type']
114
+ when Metadata::TYPE_ADVERTISE then
115
+ receive_advertise(msg['data'], from)
116
+ when Metadata::TYPE_REQUEST_ALL then
117
+ receive_request_all(msg['data'], from)
118
+ end
119
+ end
120
+ end
121
+ end
122
+
123
+ def do_start_p_metadata_receiver
124
+ Thread.new do
125
+ receiver = Subscriber.new(@config)
126
+ receiver.recv_p_metadata do |msg, from|
127
+ case msg['type']
128
+ when Metadata::TYPE_SUGGESTION then
129
+ receive_suggestion(msg['data'], from)
130
+ when Metadata::TYPE_REQUEST then
131
+ receive_request(msg['data'], from)
132
+ end
133
+ end
134
+ end
135
+ end
136
+
137
+ def do_start_data_receiver
138
+ Thread.new do
139
+ receiver = Subscriber.new(@config)
140
+ receiver.recv_data do |binary, from|
141
+ path = MessagePack.unpack(binary)['path']
142
+
143
+ Log.debug("[ManagerImpl] (data_receiver) path: #{path}")
144
+
145
+ @file_activities.push(FileActivity.create(path))
146
+
147
+ Solvant.load_binary(@config[:dirpath], binary)
148
+ end
149
+ end
150
+ end
151
+ def do_start_p_data_receiver
152
+ Thread.new do
153
+ receiver = Subscriber.new(@config)
154
+ receiver.recv_p_data do |binary, from|
155
+ path = MessagePack.unpack(binary)['path']
156
+
157
+ Log.debug("[ManagerImpl] (p_data_receiver) path: #{path}")
158
+
159
+ @file_activities.push(FileActivity.create(path))
160
+
161
+ Solvant.load_binary(@config[:dirpath], binary)
162
+ end
163
+ end
164
+ end
165
+
166
+ def receive_advertise(data, from)
167
+ def updated?(remote)
168
+ case f = @metadata.get_with_path(remote['path'])
169
+ when nil # this means target file doesn't exist in local.
170
+ true
171
+ else
172
+ f.size != remote['size'] and
173
+ not f.removed?
174
+ end
175
+ end
176
+
177
+ def removed?(remote)
178
+ case f = @metadata.get_with_path(remote['path'])
179
+ when nil
180
+ false
181
+ else
182
+ remote['status'].to_i & Metadata::FileInfo::STATUS_REMOVED > 0
183
+ end
184
+ end
185
+ Log.debug("[ManagerImpl] (receive_advertise) <#{from}> data:#{data}")
186
+
187
+ deficients = []
188
+ data.each do |remote|
189
+ if removed? remote
190
+ Log.debug("[ManagerImpl] (receive_advertise) remove: #{remote}")
191
+
192
+ # set file_activities
193
+ @file_activities.push(FileActivity.remove(remote['path']))
194
+
195
+ # remove FileInfo object which metadata has
196
+ @metadata.remove_with_path(remote['path'])
197
+
198
+ # remove actual file in local FS
199
+ Solvant.new(@config[:dirpath], remote['path']).remove
200
+ else updated? remote
201
+ deficients.push(remote)
202
+
203
+ fpath = "#{@config[:dirpath]}/#{remote['path']}"
204
+ if FileTest.exist? fpath
205
+ Log.debug("[ManagerImpl] trancated(#{fpath}, #{remote['size']})")
206
+
207
+ # truncate files when target file is cut down
208
+ File.truncate(fpath, remote['size'])
209
+ end
210
+ end
211
+ end
212
+
213
+ # request all deficients files
214
+ @metadata.request_all(@publisher, deficients)
215
+
216
+ # record deficient files to get it from remote node
217
+ @semaphore.synchronize do
218
+ @deficients += deficients
219
+ end
220
+ end
221
+
222
+ def receive_request_all(data, from)
223
+ def has_file?(remote)
224
+ @metadata.get_with_path(remote['path']) != nil
225
+ end
226
+ Log.debug("[ManagerImpl] (receive_request_all) <#{from}> data:#{data}")
227
+
228
+ files = data.map {|f| @metadata.get_with_path(f['path'])}.select{|x| x != nil}
229
+ if files != []
230
+ Log.debug("[ManagerImpl] (receive_request_all) files:#{files}")
231
+ @metadata.suggestion(@publisher, files.map{|x| x.to_h}, from)
232
+ end
233
+ end
234
+
235
+ def receive_suggestion(data, from)
236
+ Log.debug("[ManagerImpl] (receive_suggestion) <#{from}> data:#{data}")
237
+
238
+ data.each {|x| x['from'] = from}
239
+ @semaphore.synchronize do
240
+ @suggestions += data
241
+ end
242
+ end
243
+
244
+ def receive_request(data, from)
245
+ Log.debug("[ManagerImpl] (receive_request) <#{from}> data:#{data}")
246
+
247
+ data.each do |remote|
248
+ f = @metadata.get_with_path(remote['path'])
249
+
250
+ Solvant.new(@config[:dirpath], f.path).upload_to(@publisher, from)
251
+ end
252
+ end
253
+
254
+ class FileActivity
255
+ STATUS_REMOVED = 1 << 0
256
+
257
+ attr_reader :path, :mtime
258
+
259
+ def initialize(path, mtime, status = 0)
260
+ @path = path
261
+ @mtime = mtime
262
+ @status = status
263
+ end
264
+
265
+ def removed?
266
+ @status & STATUS_REMOVED > 0
267
+ end
268
+
269
+ def self.create(path)
270
+ self.new(path, FileTest.exist?(path) ? File.mtime(path) : Time.now)
271
+ end
272
+
273
+ def self.remove(path)
274
+ self.new(path, nil, STATUS_REMOVED)
275
+ end
276
+ end
277
+ end
278
+ end
@@ -0,0 +1,122 @@
1
+ require 'msgpack'
2
+
3
+ module BitBroker
4
+ class Metadata
5
+ # describes message types
6
+ TYPE_ADVERTISE = 1<<0
7
+ TYPE_REQUEST_ALL = 1<<1
8
+ TYPE_SUGGESTION = 1<<2
9
+ TYPE_REQUEST = 1<<3
10
+
11
+ attr_reader :dir
12
+
13
+ def initialize(dir)
14
+ @dir = dir
15
+ @files = scanning_files(@dir).map do |path|
16
+ FileInfo.new(@dir, get_rpath(path))
17
+ end
18
+ end
19
+
20
+ def get_with_path(path)
21
+ @files.select{|f| f.path == path}.first
22
+ end
23
+ def remove_with_path(path)
24
+ @files.reject!{|f| f.path == path}
25
+ end
26
+ def create(path)
27
+ if get_with_path(path) == nil
28
+ @files.push(FileInfo.new(@dir, path))
29
+ else
30
+ puts "Warning: #{path} is already created"
31
+ end
32
+ end
33
+
34
+ ### sending message for broker
35
+ def advertise(broker)
36
+ broker.send_metadata({
37
+ :type => TYPE_ADVERTISE,
38
+ :data => @files.map{|x| x.to_h },
39
+ })
40
+ end
41
+ def request_all(broker, files)
42
+ broker.send_metadata({
43
+ :type => TYPE_REQUEST_ALL,
44
+ :data => files,
45
+ })
46
+ end
47
+ def suggestion(broker, files, dest)
48
+ broker.send_p_metadata(dest, {
49
+ :type => TYPE_SUGGESTION,
50
+ :data => files,
51
+ })
52
+ end
53
+ def request(broker, files, dest)
54
+ broker.send_p_metadata(dest, {
55
+ :type => TYPE_REQUEST,
56
+ :data => files,
57
+ })
58
+ end
59
+
60
+ ### utility methods
61
+ def get_rpath(path)
62
+ raise DiscomfortDirectoryStructure unless !!path.match(/^#{@dir}/)
63
+ path.split(@dir).last
64
+ end
65
+
66
+ private
67
+ def scanning_files(current_dir, &block)
68
+ arr = []
69
+ Dir.foreach(current_dir) do |f|
70
+ if /^\.+$/ !~ f
71
+ path = "#{current_dir}/#{f}"
72
+
73
+ if File.directory? f
74
+ arr += scanning(path)
75
+ else
76
+ arr.push(path)
77
+ end
78
+ end
79
+ end
80
+ arr
81
+ end
82
+ class FileInfo
83
+ attr_reader :path, :size, :mtime
84
+
85
+ # describes file status
86
+ STATUS_REMOVED = 1 << 0
87
+
88
+ def initialize(dirpath, filepath)
89
+ @fpath = "#{dirpath}/#{filepath}"
90
+ @path = filepath
91
+ @status = 0
92
+
93
+ self.update
94
+ end
95
+ def update
96
+ if FileTest.exist? @fpath
97
+ file = File.new(@fpath)
98
+
99
+ @size = file.size
100
+ @mtime = file.mtime
101
+ else
102
+ @size = 0
103
+ @mtime = Time.new(0)
104
+ end
105
+ end
106
+ def removed?
107
+ @status & STATUS_REMOVED > 0
108
+ end
109
+ def remove
110
+ @status |= STATUS_REMOVED
111
+ end
112
+ def to_h
113
+ {
114
+ 'path' => @path,
115
+ 'status' => @status,
116
+ 'size' => @size,
117
+ 'mtime' => @mtime.to_s,
118
+ }
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,19 @@
1
+ require 'listen'
2
+
3
+ module BitBroker
4
+ class Observer
5
+ def initialize(dir, &block)
6
+ @target_dir = dir
7
+
8
+ @listener = Listen.to(dir) do |mod, add, rem|
9
+ block.call(mod, add, rem)
10
+ end
11
+
12
+ @listener.start
13
+ end
14
+
15
+ def stop
16
+ @listener.stop
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,84 @@
1
+ require 'msgpack'
2
+ require 'fileutils'
3
+
4
+ module BitBroker
5
+ class Solvant
6
+ DEFAULT_CHUNK_SIZE = 1<<20
7
+
8
+ attr_reader :chunks
9
+
10
+ def initialize(dirpath, r_path, chunk_size = DEFAULT_CHUNK_SIZE)
11
+ @f_path = "#{dirpath}/#{r_path}"
12
+
13
+ # Validate target file at first
14
+ if not FileTest.exist? @f_path
15
+ FileUtils.touch(@f_path)
16
+ end
17
+
18
+ # separate per chunk
19
+ @chunks = []
20
+ chunk_splitter(File::Stat.new(@f_path).size, chunk_size) do |offset, size|
21
+ @chunks.push(Chunk.new({
22
+ :r_path => r_path,
23
+ :f_path => @f_path,
24
+ :size => size,
25
+ :offset => offset,
26
+ :chunk_size => chunk_size,
27
+ }))
28
+ end
29
+ end
30
+
31
+ # This defines operations to manipulate actual Flie object on FileSystem
32
+ def remove
33
+ File.unlink(@f_path)
34
+ end
35
+
36
+ def upload broker
37
+ @chunks.each do |chunk|
38
+ broker.send_data(chunk.serialize)
39
+ end
40
+ end
41
+
42
+ def upload_to(broker, dest)
43
+ @chunks.each do |chunk|
44
+ broker.send_p_data(dest, chunk.serialize)
45
+ end
46
+ end
47
+
48
+ def self.load_binary(dirpath, binary)
49
+ data = MessagePack.unpack(binary)
50
+ offset = data['offset'] * data['chunk_size']
51
+
52
+ File.binwrite(dirpath + data['path'], data['data'], offset)
53
+ end
54
+
55
+ private
56
+ def chunk_splitter(total_size, chunk_size, &block)
57
+ last = total_size / chunk_size
58
+ (0..last).each do |i|
59
+ size = (i == last) ? total_size - chunk_size * i : chunk_size
60
+
61
+ block.call(i, size)
62
+ end
63
+ end
64
+
65
+ class Chunk
66
+ def initialize(opts)
67
+ @r_path = opts[:r_path]
68
+ @f_path = opts[:f_path]
69
+ @size = opts[:size]
70
+ @offset = opts[:offset]
71
+ @chunk_size = opts[:chunk_size]
72
+ end
73
+
74
+ def serialize
75
+ MessagePack.pack({
76
+ 'path' => @r_path,
77
+ 'data' => File.binread(@f_path, @size, @offset * @chunk_size),
78
+ 'offset' => @offset,
79
+ 'chunk_size' => @chunk_size,
80
+ })
81
+ end
82
+ end
83
+ end
84
+ end
@@ -1,3 +1,3 @@
1
1
  module Bitbroker
2
- VERSION = "0.0.0"
2
+ VERSION = "0.1.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bitbroker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiroyasu OHYAMA
8
8
  autorequire:
9
- bindir: exe
9
+ bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-02 00:00:00.000000000 Z
11
+ date: 2015-12-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -66,10 +66,21 @@ files:
66
66
  - LICENSE.txt
67
67
  - README.md
68
68
  - Rakefile
69
+ - bin/bitbroker-start
70
+ - bin/bitbroker-stop
69
71
  - bin/console
70
72
  - bin/setup
71
73
  - bitbroker.gemspec
72
74
  - lib/bitbroker.rb
75
+ - lib/bitbroker/broker.rb
76
+ - lib/bitbroker/config.rb
77
+ - lib/bitbroker/exception.rb
78
+ - lib/bitbroker/log.rb
79
+ - lib/bitbroker/manager.rb
80
+ - lib/bitbroker/manager_impl.rb
81
+ - lib/bitbroker/metadata.rb
82
+ - lib/bitbroker/observer.rb
83
+ - lib/bitbroker/solvant.rb
73
84
  - lib/bitbroker/version.rb
74
85
  homepage: https://github.com/userlocalhost2000/bitbroker
75
86
  licenses: