bitbroker 0.0.0 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/Gemfile +6 -0
- data/bin/bitbroker-start +15 -0
- data/bin/bitbroker-stop +34 -0
- data/bitbroker.gemspec +1 -1
- data/lib/bitbroker.rb +11 -1
- data/lib/bitbroker/broker.rb +108 -0
- data/lib/bitbroker/config.rb +26 -0
- data/lib/bitbroker/exception.rb +5 -0
- data/lib/bitbroker/log.rb +24 -0
- data/lib/bitbroker/manager.rb +84 -0
- data/lib/bitbroker/manager_impl.rb +278 -0
- data/lib/bitbroker/metadata.rb +122 -0
- data/lib/bitbroker/observer.rb +19 -0
- data/lib/bitbroker/solvant.rb +84 -0
- data/lib/bitbroker/version.rb +1 -1
- metadata +14 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca31c10d84f52d9aaebce0fb53b9ed19afcfbb43
|
4
|
+
data.tar.gz: 2195c0fd65b83741ece9d8163e2c2b56a0ee590a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e2a4671faf3d8907d9811febb0a958eaf932afc9251baf8be8dc533bc7fefad5c4cf7bdb162d7332ac7b9bad4dd6020018b748a1b634f75a8aec924f8d4ade2
|
7
|
+
data.tar.gz: bec351376c5e7ee3ef35895efb8baf6e70faf23da2608ab3b6c3d8af2faaf3caccbdba84c8973803ec6c9f633ced8a7f1f881efe694e73850f1151380a949299
|
data/.gitignore
CHANGED
data/.rspec
CHANGED
data/Gemfile
CHANGED
data/bin/bitbroker-start
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bitbroker'
|
4
|
+
|
5
|
+
PIDFILE = BitBroker::Config.path_pid
|
6
|
+
BitBroker::Log.level = Logger::DEBUG
|
7
|
+
|
8
|
+
if FileTest.exists? PIDFILE
|
9
|
+
puts "(NOTICE) bitbroker is now running"
|
10
|
+
exit
|
11
|
+
end
|
12
|
+
|
13
|
+
BitBroker::Manager.start
|
14
|
+
|
15
|
+
puts "bitbroker is started."
|
data/bin/bitbroker-stop
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bitbroker'
|
4
|
+
|
5
|
+
PIDFILE = BitBroker::Config.path_pid
|
6
|
+
|
7
|
+
def is_stopped? pid
|
8
|
+
begin
|
9
|
+
Process.getpgid(pid)
|
10
|
+
false
|
11
|
+
rescue Errno::ESRCH => _
|
12
|
+
true
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
unless FileTest.exists? PIDFILE
|
17
|
+
puts "(NOTICE) bitbroker is not running"
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
|
21
|
+
pids = File.read(PIDFILE).split(/\n/).map{|x| x.to_i}
|
22
|
+
pids.each do |pid|
|
23
|
+
Process.kill("TERM", pid)
|
24
|
+
end
|
25
|
+
|
26
|
+
while pids.size > 0
|
27
|
+
pid = pids.shift
|
28
|
+
unless is_stopped? pid
|
29
|
+
pids.push(pid)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
File.unlink PIDFILE
|
33
|
+
|
34
|
+
puts "bitbroker is stopped."
|
data/bitbroker.gemspec
CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
end
|
24
24
|
|
25
25
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
26
|
-
spec.bindir = "
|
26
|
+
spec.bindir = "bin"
|
27
27
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
28
28
|
spec.require_paths = ["lib"]
|
29
29
|
|
data/lib/bitbroker.rb
CHANGED
@@ -1,5 +1,15 @@
|
|
1
1
|
require "bitbroker/version"
|
2
|
+
require 'bitbroker/solvant'
|
3
|
+
require 'bitbroker/observer'
|
4
|
+
require 'bitbroker/broker'
|
5
|
+
require 'bitbroker/metadata'
|
6
|
+
require 'bitbroker/manager'
|
7
|
+
require 'bitbroker/manager_impl'
|
8
|
+
require 'bitbroker/config'
|
9
|
+
require 'bitbroker/log'
|
2
10
|
|
3
11
|
module Bitbroker
|
4
|
-
|
12
|
+
def self.version
|
13
|
+
VERSION
|
14
|
+
end
|
5
15
|
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require 'bunny'
|
2
|
+
require 'msgpack'
|
3
|
+
require 'openssl'
|
4
|
+
|
5
|
+
module BitBroker
|
6
|
+
# This method communicate with AMQP for transmitting and receiving data
|
7
|
+
class Broker
|
8
|
+
RKEY_DATA = 'data'
|
9
|
+
RKEY_METADATA = 'metadata'
|
10
|
+
ENCRYPT_ALGORITHM = "AES-256-CBC"
|
11
|
+
|
12
|
+
def initialize(config)
|
13
|
+
@connection = Bunny.new(:host => config[:mqconfig]['host'],
|
14
|
+
:vhost => config[:mqconfig]['vhost'],
|
15
|
+
:user => config[:mqconfig]['user'],
|
16
|
+
:password => config[:mqconfig]['passwd'])
|
17
|
+
@connection.start
|
18
|
+
@channel = @connection.create_channel
|
19
|
+
@exchange = @channel.direct(config[:label])
|
20
|
+
@passwd = config[:passwd].to_s + config[:label]
|
21
|
+
end
|
22
|
+
|
23
|
+
def finish
|
24
|
+
@channel.close
|
25
|
+
@connection.close
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
class Publisher < Broker
|
30
|
+
def initialize(config)
|
31
|
+
super(config)
|
32
|
+
end
|
33
|
+
|
34
|
+
def send_data(data)
|
35
|
+
send(RKEY_DATA, data)
|
36
|
+
end
|
37
|
+
def send_metadata(data)
|
38
|
+
send(RKEY_METADATA, data)
|
39
|
+
end
|
40
|
+
def send_p_data(dest, data)
|
41
|
+
send(RKEY_DATA + dest, data)
|
42
|
+
end
|
43
|
+
def send_p_metadata(dest, data)
|
44
|
+
send(RKEY_METADATA + dest, data)
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
def send(rkey, data)
|
49
|
+
@exchange.publish(encode(MessagePack.pack({
|
50
|
+
'data' => data,
|
51
|
+
'from' => Mac.addr,
|
52
|
+
})), :routing_key => rkey)
|
53
|
+
end
|
54
|
+
|
55
|
+
def encode(data)
|
56
|
+
cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
|
57
|
+
cipher.encrypt
|
58
|
+
cipher.pkcs5_keyivgen(@passwd)
|
59
|
+
cipher.update(data) + cipher.final
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
class Subscriber < Broker
|
64
|
+
SIGNAL_TERM = 'TERM'
|
65
|
+
|
66
|
+
def initialize(config)
|
67
|
+
super(config)
|
68
|
+
end
|
69
|
+
|
70
|
+
def recv_data(&block)
|
71
|
+
recv(RKEY_DATA, &block)
|
72
|
+
end
|
73
|
+
def recv_metadata(&block)
|
74
|
+
recv(RKEY_METADATA, &block)
|
75
|
+
end
|
76
|
+
def recv_p_data(&block)
|
77
|
+
recv(RKEY_DATA + Mac.addr, &block)
|
78
|
+
end
|
79
|
+
def recv_p_metadata(&block)
|
80
|
+
recv(RKEY_METADATA + Mac.addr, &block)
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
def recv(rkey, &block)
|
85
|
+
queue = @channel.queue('', :exclusive => true)
|
86
|
+
queue.bind(@exchange, :routing_key => rkey)
|
87
|
+
begin
|
88
|
+
queue.subscribe(:block => true) do |info, prop, binary|
|
89
|
+
msg = MessagePack.unpack(decode(binary))
|
90
|
+
|
91
|
+
if msg['from'] != Mac.addr
|
92
|
+
block.call(msg['data'], msg['from'])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
rescue OpenSSL::Cipher::CipherError => e
|
96
|
+
Log.warn("[Subscriber] #{e.to_s}")
|
97
|
+
rescue Exception => _
|
98
|
+
finish
|
99
|
+
end
|
100
|
+
end
|
101
|
+
def decode(encrypted_data)
|
102
|
+
cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
|
103
|
+
cipher.decrypt
|
104
|
+
cipher.pkcs5_keyivgen(@passwd)
|
105
|
+
cipher.update(encrypted_data) + cipher.final
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Config
|
5
|
+
DEFAULT_PATH_CONFIG = "#{ENV['HOME']}/.bitbroker/config.yml"
|
6
|
+
DEFAULT_PATH_PID = "#{ENV['HOME']}/.bitbroker/pid"
|
7
|
+
|
8
|
+
def self.[](param)
|
9
|
+
YAML.load_file(path_config)[param]
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.path_config
|
13
|
+
@path_config ||= DEFAULT_PATH_CONFIG
|
14
|
+
end
|
15
|
+
def self.path_pid
|
16
|
+
@path_pid ||= DEFAULT_PATH_PID
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.set_config(path)
|
20
|
+
unless FileTest.exist? path
|
21
|
+
raise InvalidFile path
|
22
|
+
end
|
23
|
+
@path_config = path
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Log
|
5
|
+
DEFAULT_LOGFILE = '/tmp/bitbroker.log'
|
6
|
+
|
7
|
+
def self.method_missing(m, *args, &block)
|
8
|
+
logger.method(m).call(*args)
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
def self.logger
|
13
|
+
@logger ||= init_logger
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.init_logger
|
17
|
+
Logger.new(logfile)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.logfile
|
21
|
+
BitBroker::Config['logfile'] != nil ? BitBroker::Config['logfile'] : DEFAULT_LOGFILE
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'macaddr'
|
3
|
+
require 'msgpack'
|
4
|
+
require 'time'
|
5
|
+
require 'thread'
|
6
|
+
|
7
|
+
require 'bitbroker/manager_impl'
|
8
|
+
|
9
|
+
module BitBroker
|
10
|
+
### This object is created for each directory
|
11
|
+
class Manager < ManagerImpl
|
12
|
+
|
13
|
+
def self.start
|
14
|
+
BitBroker::Config['directories'].each do |entry|
|
15
|
+
fork do
|
16
|
+
Process.daemon
|
17
|
+
File.open(PIDFILE, 'a') do |f|
|
18
|
+
f.write("#{$$}\n")
|
19
|
+
end
|
20
|
+
|
21
|
+
begin
|
22
|
+
manager = BitBroker::Manager.new({
|
23
|
+
:mqconfig => BitBroker::Config['mqconfig'],
|
24
|
+
:path => entry['path'],
|
25
|
+
:name => entry['name'],
|
26
|
+
})
|
27
|
+
|
28
|
+
manager.start
|
29
|
+
manager.advertise
|
30
|
+
|
31
|
+
loop {}
|
32
|
+
rescue Exception => _
|
33
|
+
manager.stop
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize(opts)
|
40
|
+
super(opts)
|
41
|
+
end
|
42
|
+
|
43
|
+
def advertise
|
44
|
+
@metadata.advertise(@publisher)
|
45
|
+
end
|
46
|
+
|
47
|
+
def start
|
48
|
+
# start observer that watches changing of local file-system
|
49
|
+
@observer = do_start_observer
|
50
|
+
|
51
|
+
# start receivers that consume message of remote nodes
|
52
|
+
@metadata_receiver = do_start_metadata_receiver
|
53
|
+
@p_metadata_receiver = do_start_p_metadata_receiver
|
54
|
+
|
55
|
+
@data_receiver = do_start_data_receiver
|
56
|
+
@p_data_receiver = do_start_p_data_receiver
|
57
|
+
|
58
|
+
# start collector that maintains the shared directory will be same with remote ones.
|
59
|
+
@collector = do_start_collector
|
60
|
+
end
|
61
|
+
|
62
|
+
def stop
|
63
|
+
# for observer
|
64
|
+
@observer.raise 'stop'
|
65
|
+
@observer.join
|
66
|
+
|
67
|
+
# for receiver
|
68
|
+
@metadata_receiver.raise "stop"
|
69
|
+
@metadata_receiver.join
|
70
|
+
|
71
|
+
@p_metadata_receiver.raise "stop"
|
72
|
+
@p_metadata_receiver.join
|
73
|
+
|
74
|
+
@data_receiver.raise "stop"
|
75
|
+
@data_receiver.join
|
76
|
+
|
77
|
+
@p_data_receiver.raise "stop"
|
78
|
+
@p_data_receiver.join
|
79
|
+
|
80
|
+
# for collector
|
81
|
+
@collector.kill
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,278 @@
|
|
1
|
+
module BitBroker
|
2
|
+
class ManagerImpl
|
3
|
+
def initialize(opts)
|
4
|
+
# validate user created arguments
|
5
|
+
validate(opts)
|
6
|
+
|
7
|
+
### prepare brokers
|
8
|
+
@config = {
|
9
|
+
:mqconfig => opts[:mqconfig],
|
10
|
+
:label => opts[:name],
|
11
|
+
:dirpath => form_dirpath(opts[:path]),
|
12
|
+
}
|
13
|
+
|
14
|
+
@metadata = Metadata.new(@config[:dirpath])
|
15
|
+
|
16
|
+
@publisher = Publisher.new(@config)
|
17
|
+
|
18
|
+
@deficients = @suggestions = []
|
19
|
+
@semaphore = Mutex.new
|
20
|
+
|
21
|
+
# internal variable in this class to know who modified/crated file
|
22
|
+
@file_activities = []
|
23
|
+
end
|
24
|
+
|
25
|
+
def form_dirpath path
|
26
|
+
path[-1] == '/' ? form_dirpath(path.chop) : path
|
27
|
+
end
|
28
|
+
def validate(opts)
|
29
|
+
raise InvalidArgument("Specified path is not directory") unless File.directory?(opts[:path])
|
30
|
+
end
|
31
|
+
|
32
|
+
def do_start_observer
|
33
|
+
def handle_add(path)
|
34
|
+
Log.debug("[ManagerImpl] (handle_add) path:#{path}")
|
35
|
+
|
36
|
+
rpath = @metadata.get_rpath(path)
|
37
|
+
if obj = @file_activities.find {|x| x.path == rpath}
|
38
|
+
@file_activities.delete(obj)
|
39
|
+
else
|
40
|
+
# create metadata info
|
41
|
+
@metadata.create(rpath)
|
42
|
+
|
43
|
+
# upload target file
|
44
|
+
Solvant.new(@metadata.dir, rpath).upload(@publisher)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def handle_mod(path)
|
49
|
+
Log.debug("[ManagerImpl] (handle_mod) path:#{path}")
|
50
|
+
|
51
|
+
rpath = @metadata.get_rpath(path)
|
52
|
+
if obj = @file_activities.find {|x| x.path == rpath}
|
53
|
+
@file_activities.delete(obj)
|
54
|
+
else
|
55
|
+
# upload target file
|
56
|
+
Solvant.new(@metadata.dir, rpath).upload(@publisher)
|
57
|
+
|
58
|
+
# update fileinfo
|
59
|
+
@metadata.get_with_path(rpath).update
|
60
|
+
|
61
|
+
@metadata.advertise(@publisher)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def handle_rem(path)
|
66
|
+
rpath = @metadata.get_rpath(path)
|
67
|
+
|
68
|
+
#@metadata.remove_with_path(rpath)
|
69
|
+
file = @metadata.get_with_path(rpath)
|
70
|
+
if file != nil
|
71
|
+
Log.debug("[ManagerImpl] (handle_rem) path:#{path}")
|
72
|
+
|
73
|
+
file.remove
|
74
|
+
@metadata.advertise(@publisher)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
Thread.new do
|
79
|
+
Observer.new(@config[:dirpath]) do |mod, add, rem|
|
80
|
+
mod.each {|x| handle_mod(x)}
|
81
|
+
add.each {|x| handle_add(x)}
|
82
|
+
rem.each {|x| handle_rem(x)}
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def do_start_collector
|
88
|
+
Thread.new do
|
89
|
+
loop do
|
90
|
+
deficient = @deficients.first
|
91
|
+
if deficient != nil
|
92
|
+
candidates = @suggestions.select { |x| x['path'] == deficient['path'] }
|
93
|
+
if candidates.size > 0
|
94
|
+
candidate = candidates[rand(candidates.size)]
|
95
|
+
|
96
|
+
@metadata.request(@publisher, [candidate], candidate['from'])
|
97
|
+
|
98
|
+
@semaphore.synchronize do
|
99
|
+
@suggestions = @suggestions.reject {|x| x['path'] == deficient['path']}
|
100
|
+
@deficients.delete(deficient)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
Thread.pass
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def do_start_metadata_receiver
|
110
|
+
Thread.new do
|
111
|
+
receiver = Subscriber.new(@config)
|
112
|
+
receiver.recv_metadata do |msg, from|
|
113
|
+
case msg['type']
|
114
|
+
when Metadata::TYPE_ADVERTISE then
|
115
|
+
receive_advertise(msg['data'], from)
|
116
|
+
when Metadata::TYPE_REQUEST_ALL then
|
117
|
+
receive_request_all(msg['data'], from)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def do_start_p_metadata_receiver
|
124
|
+
Thread.new do
|
125
|
+
receiver = Subscriber.new(@config)
|
126
|
+
receiver.recv_p_metadata do |msg, from|
|
127
|
+
case msg['type']
|
128
|
+
when Metadata::TYPE_SUGGESTION then
|
129
|
+
receive_suggestion(msg['data'], from)
|
130
|
+
when Metadata::TYPE_REQUEST then
|
131
|
+
receive_request(msg['data'], from)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def do_start_data_receiver
|
138
|
+
Thread.new do
|
139
|
+
receiver = Subscriber.new(@config)
|
140
|
+
receiver.recv_data do |binary, from|
|
141
|
+
path = MessagePack.unpack(binary)['path']
|
142
|
+
|
143
|
+
Log.debug("[ManagerImpl] (data_receiver) path: #{path}")
|
144
|
+
|
145
|
+
@file_activities.push(FileActivity.create(path))
|
146
|
+
|
147
|
+
Solvant.load_binary(@config[:dirpath], binary)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
def do_start_p_data_receiver
|
152
|
+
Thread.new do
|
153
|
+
receiver = Subscriber.new(@config)
|
154
|
+
receiver.recv_p_data do |binary, from|
|
155
|
+
path = MessagePack.unpack(binary)['path']
|
156
|
+
|
157
|
+
Log.debug("[ManagerImpl] (p_data_receiver) path: #{path}")
|
158
|
+
|
159
|
+
@file_activities.push(FileActivity.create(path))
|
160
|
+
|
161
|
+
Solvant.load_binary(@config[:dirpath], binary)
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
def receive_advertise(data, from)
|
167
|
+
def updated?(remote)
|
168
|
+
case f = @metadata.get_with_path(remote['path'])
|
169
|
+
when nil # this means target file doesn't exist in local.
|
170
|
+
true
|
171
|
+
else
|
172
|
+
f.size != remote['size'] and
|
173
|
+
not f.removed?
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def removed?(remote)
|
178
|
+
case f = @metadata.get_with_path(remote['path'])
|
179
|
+
when nil
|
180
|
+
false
|
181
|
+
else
|
182
|
+
remote['status'].to_i & Metadata::FileInfo::STATUS_REMOVED > 0
|
183
|
+
end
|
184
|
+
end
|
185
|
+
Log.debug("[ManagerImpl] (receive_advertise) <#{from}> data:#{data}")
|
186
|
+
|
187
|
+
deficients = []
|
188
|
+
data.each do |remote|
|
189
|
+
if removed? remote
|
190
|
+
Log.debug("[ManagerImpl] (receive_advertise) remove: #{remote}")
|
191
|
+
|
192
|
+
# set file_activities
|
193
|
+
@file_activities.push(FileActivity.remove(remote['path']))
|
194
|
+
|
195
|
+
# remove FileInfo object which metadata has
|
196
|
+
@metadata.remove_with_path(remote['path'])
|
197
|
+
|
198
|
+
# remove actual file in local FS
|
199
|
+
Solvant.new(@config[:dirpath], remote['path']).remove
|
200
|
+
else updated? remote
|
201
|
+
deficients.push(remote)
|
202
|
+
|
203
|
+
fpath = "#{@config[:dirpath]}/#{remote['path']}"
|
204
|
+
if FileTest.exist? fpath
|
205
|
+
Log.debug("[ManagerImpl] trancated(#{fpath}, #{remote['size']})")
|
206
|
+
|
207
|
+
# truncate files when target file is cut down
|
208
|
+
File.truncate(fpath, remote['size'])
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# request all deficients files
|
214
|
+
@metadata.request_all(@publisher, deficients)
|
215
|
+
|
216
|
+
# record deficient files to get it from remote node
|
217
|
+
@semaphore.synchronize do
|
218
|
+
@deficients += deficients
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def receive_request_all(data, from)
|
223
|
+
def has_file?(remote)
|
224
|
+
@metadata.get_with_path(remote['path']) != nil
|
225
|
+
end
|
226
|
+
Log.debug("[ManagerImpl] (receive_request_all) <#{from}> data:#{data}")
|
227
|
+
|
228
|
+
files = data.map {|f| @metadata.get_with_path(f['path'])}.select{|x| x != nil}
|
229
|
+
if files != []
|
230
|
+
Log.debug("[ManagerImpl] (receive_request_all) files:#{files}")
|
231
|
+
@metadata.suggestion(@publisher, files.map{|x| x.to_h}, from)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def receive_suggestion(data, from)
|
236
|
+
Log.debug("[ManagerImpl] (receive_suggestion) <#{from}> data:#{data}")
|
237
|
+
|
238
|
+
data.each {|x| x['from'] = from}
|
239
|
+
@semaphore.synchronize do
|
240
|
+
@suggestions += data
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def receive_request(data, from)
|
245
|
+
Log.debug("[ManagerImpl] (receive_request) <#{from}> data:#{data}")
|
246
|
+
|
247
|
+
data.each do |remote|
|
248
|
+
f = @metadata.get_with_path(remote['path'])
|
249
|
+
|
250
|
+
Solvant.new(@config[:dirpath], f.path).upload_to(@publisher, from)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
class FileActivity
|
255
|
+
STATUS_REMOVED = 1 << 0
|
256
|
+
|
257
|
+
attr_reader :path, :mtime
|
258
|
+
|
259
|
+
def initialize(path, mtime, status = 0)
|
260
|
+
@path = path
|
261
|
+
@mtime = mtime
|
262
|
+
@status = status
|
263
|
+
end
|
264
|
+
|
265
|
+
def removed?
|
266
|
+
@status & STATUS_REMOVED > 0
|
267
|
+
end
|
268
|
+
|
269
|
+
def self.create(path)
|
270
|
+
self.new(path, FileTest.exist?(path) ? File.mtime(path) : Time.now)
|
271
|
+
end
|
272
|
+
|
273
|
+
def self.remove(path)
|
274
|
+
self.new(path, nil, STATUS_REMOVED)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require 'msgpack'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Metadata
|
5
|
+
# describes message types
|
6
|
+
TYPE_ADVERTISE = 1<<0
|
7
|
+
TYPE_REQUEST_ALL = 1<<1
|
8
|
+
TYPE_SUGGESTION = 1<<2
|
9
|
+
TYPE_REQUEST = 1<<3
|
10
|
+
|
11
|
+
attr_reader :dir
|
12
|
+
|
13
|
+
def initialize(dir)
|
14
|
+
@dir = dir
|
15
|
+
@files = scanning_files(@dir).map do |path|
|
16
|
+
FileInfo.new(@dir, get_rpath(path))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_with_path(path)
|
21
|
+
@files.select{|f| f.path == path}.first
|
22
|
+
end
|
23
|
+
def remove_with_path(path)
|
24
|
+
@files.reject!{|f| f.path == path}
|
25
|
+
end
|
26
|
+
def create(path)
|
27
|
+
if get_with_path(path) == nil
|
28
|
+
@files.push(FileInfo.new(@dir, path))
|
29
|
+
else
|
30
|
+
puts "Warning: #{path} is already created"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
### sending message for broker
|
35
|
+
def advertise(broker)
|
36
|
+
broker.send_metadata({
|
37
|
+
:type => TYPE_ADVERTISE,
|
38
|
+
:data => @files.map{|x| x.to_h },
|
39
|
+
})
|
40
|
+
end
|
41
|
+
def request_all(broker, files)
|
42
|
+
broker.send_metadata({
|
43
|
+
:type => TYPE_REQUEST_ALL,
|
44
|
+
:data => files,
|
45
|
+
})
|
46
|
+
end
|
47
|
+
def suggestion(broker, files, dest)
|
48
|
+
broker.send_p_metadata(dest, {
|
49
|
+
:type => TYPE_SUGGESTION,
|
50
|
+
:data => files,
|
51
|
+
})
|
52
|
+
end
|
53
|
+
def request(broker, files, dest)
|
54
|
+
broker.send_p_metadata(dest, {
|
55
|
+
:type => TYPE_REQUEST,
|
56
|
+
:data => files,
|
57
|
+
})
|
58
|
+
end
|
59
|
+
|
60
|
+
### utility methods
|
61
|
+
def get_rpath(path)
|
62
|
+
raise DiscomfortDirectoryStructure unless !!path.match(/^#{@dir}/)
|
63
|
+
path.split(@dir).last
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
def scanning_files(current_dir, &block)
|
68
|
+
arr = []
|
69
|
+
Dir.foreach(current_dir) do |f|
|
70
|
+
if /^\.+$/ !~ f
|
71
|
+
path = "#{current_dir}/#{f}"
|
72
|
+
|
73
|
+
if File.directory? f
|
74
|
+
arr += scanning(path)
|
75
|
+
else
|
76
|
+
arr.push(path)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
arr
|
81
|
+
end
|
82
|
+
class FileInfo
|
83
|
+
attr_reader :path, :size, :mtime
|
84
|
+
|
85
|
+
# describes file status
|
86
|
+
STATUS_REMOVED = 1 << 0
|
87
|
+
|
88
|
+
def initialize(dirpath, filepath)
|
89
|
+
@fpath = "#{dirpath}/#{filepath}"
|
90
|
+
@path = filepath
|
91
|
+
@status = 0
|
92
|
+
|
93
|
+
self.update
|
94
|
+
end
|
95
|
+
def update
|
96
|
+
if FileTest.exist? @fpath
|
97
|
+
file = File.new(@fpath)
|
98
|
+
|
99
|
+
@size = file.size
|
100
|
+
@mtime = file.mtime
|
101
|
+
else
|
102
|
+
@size = 0
|
103
|
+
@mtime = Time.new(0)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
def removed?
|
107
|
+
@status & STATUS_REMOVED > 0
|
108
|
+
end
|
109
|
+
def remove
|
110
|
+
@status |= STATUS_REMOVED
|
111
|
+
end
|
112
|
+
def to_h
|
113
|
+
{
|
114
|
+
'path' => @path,
|
115
|
+
'status' => @status,
|
116
|
+
'size' => @size,
|
117
|
+
'mtime' => @mtime.to_s,
|
118
|
+
}
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'listen'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Observer
|
5
|
+
def initialize(dir, &block)
|
6
|
+
@target_dir = dir
|
7
|
+
|
8
|
+
@listener = Listen.to(dir) do |mod, add, rem|
|
9
|
+
block.call(mod, add, rem)
|
10
|
+
end
|
11
|
+
|
12
|
+
@listener.start
|
13
|
+
end
|
14
|
+
|
15
|
+
def stop
|
16
|
+
@listener.stop
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'msgpack'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module BitBroker
|
5
|
+
class Solvant
|
6
|
+
DEFAULT_CHUNK_SIZE = 1<<20
|
7
|
+
|
8
|
+
attr_reader :chunks
|
9
|
+
|
10
|
+
def initialize(dirpath, r_path, chunk_size = DEFAULT_CHUNK_SIZE)
|
11
|
+
@f_path = "#{dirpath}/#{r_path}"
|
12
|
+
|
13
|
+
# Validate target file at first
|
14
|
+
if not FileTest.exist? @f_path
|
15
|
+
FileUtils.touch(@f_path)
|
16
|
+
end
|
17
|
+
|
18
|
+
# separate per chunk
|
19
|
+
@chunks = []
|
20
|
+
chunk_splitter(File::Stat.new(@f_path).size, chunk_size) do |offset, size|
|
21
|
+
@chunks.push(Chunk.new({
|
22
|
+
:r_path => r_path,
|
23
|
+
:f_path => @f_path,
|
24
|
+
:size => size,
|
25
|
+
:offset => offset,
|
26
|
+
:chunk_size => chunk_size,
|
27
|
+
}))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# This defines operations to manipulate actual Flie object on FileSystem
|
32
|
+
def remove
|
33
|
+
File.unlink(@f_path)
|
34
|
+
end
|
35
|
+
|
36
|
+
def upload broker
|
37
|
+
@chunks.each do |chunk|
|
38
|
+
broker.send_data(chunk.serialize)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def upload_to(broker, dest)
|
43
|
+
@chunks.each do |chunk|
|
44
|
+
broker.send_p_data(dest, chunk.serialize)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.load_binary(dirpath, binary)
|
49
|
+
data = MessagePack.unpack(binary)
|
50
|
+
offset = data['offset'] * data['chunk_size']
|
51
|
+
|
52
|
+
File.binwrite(dirpath + data['path'], data['data'], offset)
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
def chunk_splitter(total_size, chunk_size, &block)
|
57
|
+
last = total_size / chunk_size
|
58
|
+
(0..last).each do |i|
|
59
|
+
size = (i == last) ? total_size - chunk_size * i : chunk_size
|
60
|
+
|
61
|
+
block.call(i, size)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class Chunk
|
66
|
+
def initialize(opts)
|
67
|
+
@r_path = opts[:r_path]
|
68
|
+
@f_path = opts[:f_path]
|
69
|
+
@size = opts[:size]
|
70
|
+
@offset = opts[:offset]
|
71
|
+
@chunk_size = opts[:chunk_size]
|
72
|
+
end
|
73
|
+
|
74
|
+
def serialize
|
75
|
+
MessagePack.pack({
|
76
|
+
'path' => @r_path,
|
77
|
+
'data' => File.binread(@f_path, @size, @offset * @chunk_size),
|
78
|
+
'offset' => @offset,
|
79
|
+
'chunk_size' => @chunk_size,
|
80
|
+
})
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/bitbroker/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bitbroker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroyasu OHYAMA
|
8
8
|
autorequire:
|
9
|
-
bindir:
|
9
|
+
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-12-
|
11
|
+
date: 2015-12-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,10 +66,21 @@ files:
|
|
66
66
|
- LICENSE.txt
|
67
67
|
- README.md
|
68
68
|
- Rakefile
|
69
|
+
- bin/bitbroker-start
|
70
|
+
- bin/bitbroker-stop
|
69
71
|
- bin/console
|
70
72
|
- bin/setup
|
71
73
|
- bitbroker.gemspec
|
72
74
|
- lib/bitbroker.rb
|
75
|
+
- lib/bitbroker/broker.rb
|
76
|
+
- lib/bitbroker/config.rb
|
77
|
+
- lib/bitbroker/exception.rb
|
78
|
+
- lib/bitbroker/log.rb
|
79
|
+
- lib/bitbroker/manager.rb
|
80
|
+
- lib/bitbroker/manager_impl.rb
|
81
|
+
- lib/bitbroker/metadata.rb
|
82
|
+
- lib/bitbroker/observer.rb
|
83
|
+
- lib/bitbroker/solvant.rb
|
73
84
|
- lib/bitbroker/version.rb
|
74
85
|
homepage: https://github.com/userlocalhost2000/bitbroker
|
75
86
|
licenses:
|