bitbroker 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/Gemfile +6 -0
- data/bin/bitbroker-start +15 -0
- data/bin/bitbroker-stop +34 -0
- data/bitbroker.gemspec +1 -1
- data/lib/bitbroker.rb +11 -1
- data/lib/bitbroker/broker.rb +108 -0
- data/lib/bitbroker/config.rb +26 -0
- data/lib/bitbroker/exception.rb +5 -0
- data/lib/bitbroker/log.rb +24 -0
- data/lib/bitbroker/manager.rb +84 -0
- data/lib/bitbroker/manager_impl.rb +278 -0
- data/lib/bitbroker/metadata.rb +122 -0
- data/lib/bitbroker/observer.rb +19 -0
- data/lib/bitbroker/solvant.rb +84 -0
- data/lib/bitbroker/version.rb +1 -1
- metadata +14 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca31c10d84f52d9aaebce0fb53b9ed19afcfbb43
|
4
|
+
data.tar.gz: 2195c0fd65b83741ece9d8163e2c2b56a0ee590a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e2a4671faf3d8907d9811febb0a958eaf932afc9251baf8be8dc533bc7fefad5c4cf7bdb162d7332ac7b9bad4dd6020018b748a1b634f75a8aec924f8d4ade2
|
7
|
+
data.tar.gz: bec351376c5e7ee3ef35895efb8baf6e70faf23da2608ab3b6c3d8af2faaf3caccbdba84c8973803ec6c9f633ced8a7f1f881efe694e73850f1151380a949299
|
data/.gitignore
CHANGED
data/.rspec
CHANGED
data/Gemfile
CHANGED
data/bin/bitbroker-start
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bitbroker'
|
4
|
+
|
5
|
+
PIDFILE = BitBroker::Config.path_pid
|
6
|
+
BitBroker::Log.level = Logger::DEBUG
|
7
|
+
|
8
|
+
if FileTest.exists? PIDFILE
|
9
|
+
puts "(NOTICE) bitbroker is now running"
|
10
|
+
exit
|
11
|
+
end
|
12
|
+
|
13
|
+
BitBroker::Manager.start
|
14
|
+
|
15
|
+
puts "bitbroker is started."
|
data/bin/bitbroker-stop
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bitbroker'
|
4
|
+
|
5
|
+
PIDFILE = BitBroker::Config.path_pid
|
6
|
+
|
7
|
+
def is_stopped? pid
|
8
|
+
begin
|
9
|
+
Process.getpgid(pid)
|
10
|
+
false
|
11
|
+
rescue Errno::ESRCH => _
|
12
|
+
true
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
unless FileTest.exists? PIDFILE
|
17
|
+
puts "(NOTICE) bitbroker is not running"
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
|
21
|
+
pids = File.read(PIDFILE).split(/\n/).map{|x| x.to_i}
|
22
|
+
pids.each do |pid|
|
23
|
+
Process.kill("TERM", pid)
|
24
|
+
end
|
25
|
+
|
26
|
+
while pids.size > 0
|
27
|
+
pid = pids.shift
|
28
|
+
unless is_stopped? pid
|
29
|
+
pids.push(pid)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
File.unlink PIDFILE
|
33
|
+
|
34
|
+
puts "bitbroker is stopped."
|
data/bitbroker.gemspec
CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
end
|
24
24
|
|
25
25
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
26
|
-
spec.bindir = "
|
26
|
+
spec.bindir = "bin"
|
27
27
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
28
28
|
spec.require_paths = ["lib"]
|
29
29
|
|
data/lib/bitbroker.rb
CHANGED
@@ -1,5 +1,15 @@
|
|
1
1
|
require "bitbroker/version"
|
2
|
+
require 'bitbroker/solvant'
|
3
|
+
require 'bitbroker/observer'
|
4
|
+
require 'bitbroker/broker'
|
5
|
+
require 'bitbroker/metadata'
|
6
|
+
require 'bitbroker/manager'
|
7
|
+
require 'bitbroker/manager_impl'
|
8
|
+
require 'bitbroker/config'
|
9
|
+
require 'bitbroker/log'
|
2
10
|
|
3
11
|
module Bitbroker
|
4
|
-
|
12
|
+
def self.version
|
13
|
+
VERSION
|
14
|
+
end
|
5
15
|
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require 'bunny'
|
2
|
+
require 'msgpack'
|
3
|
+
require 'openssl'
|
4
|
+
|
5
|
+
module BitBroker
|
6
|
+
# This method communicate with AMQP for transmitting and receiving data
|
7
|
+
class Broker
|
8
|
+
RKEY_DATA = 'data'
|
9
|
+
RKEY_METADATA = 'metadata'
|
10
|
+
ENCRYPT_ALGORITHM = "AES-256-CBC"
|
11
|
+
|
12
|
+
def initialize(config)
|
13
|
+
@connection = Bunny.new(:host => config[:mqconfig]['host'],
|
14
|
+
:vhost => config[:mqconfig]['vhost'],
|
15
|
+
:user => config[:mqconfig]['user'],
|
16
|
+
:password => config[:mqconfig]['passwd'])
|
17
|
+
@connection.start
|
18
|
+
@channel = @connection.create_channel
|
19
|
+
@exchange = @channel.direct(config[:label])
|
20
|
+
@passwd = config[:passwd].to_s + config[:label]
|
21
|
+
end
|
22
|
+
|
23
|
+
def finish
|
24
|
+
@channel.close
|
25
|
+
@connection.close
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
class Publisher < Broker
|
30
|
+
def initialize(config)
|
31
|
+
super(config)
|
32
|
+
end
|
33
|
+
|
34
|
+
def send_data(data)
|
35
|
+
send(RKEY_DATA, data)
|
36
|
+
end
|
37
|
+
def send_metadata(data)
|
38
|
+
send(RKEY_METADATA, data)
|
39
|
+
end
|
40
|
+
def send_p_data(dest, data)
|
41
|
+
send(RKEY_DATA + dest, data)
|
42
|
+
end
|
43
|
+
def send_p_metadata(dest, data)
|
44
|
+
send(RKEY_METADATA + dest, data)
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
def send(rkey, data)
|
49
|
+
@exchange.publish(encode(MessagePack.pack({
|
50
|
+
'data' => data,
|
51
|
+
'from' => Mac.addr,
|
52
|
+
})), :routing_key => rkey)
|
53
|
+
end
|
54
|
+
|
55
|
+
def encode(data)
|
56
|
+
cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
|
57
|
+
cipher.encrypt
|
58
|
+
cipher.pkcs5_keyivgen(@passwd)
|
59
|
+
cipher.update(data) + cipher.final
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
class Subscriber < Broker
|
64
|
+
SIGNAL_TERM = 'TERM'
|
65
|
+
|
66
|
+
def initialize(config)
|
67
|
+
super(config)
|
68
|
+
end
|
69
|
+
|
70
|
+
def recv_data(&block)
|
71
|
+
recv(RKEY_DATA, &block)
|
72
|
+
end
|
73
|
+
def recv_metadata(&block)
|
74
|
+
recv(RKEY_METADATA, &block)
|
75
|
+
end
|
76
|
+
def recv_p_data(&block)
|
77
|
+
recv(RKEY_DATA + Mac.addr, &block)
|
78
|
+
end
|
79
|
+
def recv_p_metadata(&block)
|
80
|
+
recv(RKEY_METADATA + Mac.addr, &block)
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
def recv(rkey, &block)
|
85
|
+
queue = @channel.queue('', :exclusive => true)
|
86
|
+
queue.bind(@exchange, :routing_key => rkey)
|
87
|
+
begin
|
88
|
+
queue.subscribe(:block => true) do |info, prop, binary|
|
89
|
+
msg = MessagePack.unpack(decode(binary))
|
90
|
+
|
91
|
+
if msg['from'] != Mac.addr
|
92
|
+
block.call(msg['data'], msg['from'])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
rescue OpenSSL::Cipher::CipherError => e
|
96
|
+
Log.warn("[Subscriber] #{e.to_s}")
|
97
|
+
rescue Exception => _
|
98
|
+
finish
|
99
|
+
end
|
100
|
+
end
|
101
|
+
def decode(encrypted_data)
|
102
|
+
cipher = OpenSSL::Cipher::Cipher.new(ENCRYPT_ALGORITHM)
|
103
|
+
cipher.decrypt
|
104
|
+
cipher.pkcs5_keyivgen(@passwd)
|
105
|
+
cipher.update(encrypted_data) + cipher.final
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Config
|
5
|
+
DEFAULT_PATH_CONFIG = "#{ENV['HOME']}/.bitbroker/config.yml"
|
6
|
+
DEFAULT_PATH_PID = "#{ENV['HOME']}/.bitbroker/pid"
|
7
|
+
|
8
|
+
def self.[](param)
|
9
|
+
YAML.load_file(path_config)[param]
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.path_config
|
13
|
+
@path_config ||= DEFAULT_PATH_CONFIG
|
14
|
+
end
|
15
|
+
def self.path_pid
|
16
|
+
@path_pid ||= DEFAULT_PATH_PID
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.set_config(path)
|
20
|
+
unless FileTest.exist? path
|
21
|
+
raise InvalidFile path
|
22
|
+
end
|
23
|
+
@path_config = path
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Log
|
5
|
+
DEFAULT_LOGFILE = '/tmp/bitbroker.log'
|
6
|
+
|
7
|
+
def self.method_missing(m, *args, &block)
|
8
|
+
logger.method(m).call(*args)
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
def self.logger
|
13
|
+
@logger ||= init_logger
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.init_logger
|
17
|
+
Logger.new(logfile)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.logfile
|
21
|
+
BitBroker::Config['logfile'] != nil ? BitBroker::Config['logfile'] : DEFAULT_LOGFILE
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'macaddr'
|
3
|
+
require 'msgpack'
|
4
|
+
require 'time'
|
5
|
+
require 'thread'
|
6
|
+
|
7
|
+
require 'bitbroker/manager_impl'
|
8
|
+
|
9
|
+
module BitBroker
|
10
|
+
### This object is created for each directory
|
11
|
+
class Manager < ManagerImpl
|
12
|
+
|
13
|
+
def self.start
|
14
|
+
BitBroker::Config['directories'].each do |entry|
|
15
|
+
fork do
|
16
|
+
Process.daemon
|
17
|
+
File.open(PIDFILE, 'a') do |f|
|
18
|
+
f.write("#{$$}\n")
|
19
|
+
end
|
20
|
+
|
21
|
+
begin
|
22
|
+
manager = BitBroker::Manager.new({
|
23
|
+
:mqconfig => BitBroker::Config['mqconfig'],
|
24
|
+
:path => entry['path'],
|
25
|
+
:name => entry['name'],
|
26
|
+
})
|
27
|
+
|
28
|
+
manager.start
|
29
|
+
manager.advertise
|
30
|
+
|
31
|
+
loop {}
|
32
|
+
rescue Exception => _
|
33
|
+
manager.stop
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize(opts)
|
40
|
+
super(opts)
|
41
|
+
end
|
42
|
+
|
43
|
+
def advertise
|
44
|
+
@metadata.advertise(@publisher)
|
45
|
+
end
|
46
|
+
|
47
|
+
def start
|
48
|
+
# start observer that watches changing of local file-system
|
49
|
+
@observer = do_start_observer
|
50
|
+
|
51
|
+
# start receivers that consume message of remote nodes
|
52
|
+
@metadata_receiver = do_start_metadata_receiver
|
53
|
+
@p_metadata_receiver = do_start_p_metadata_receiver
|
54
|
+
|
55
|
+
@data_receiver = do_start_data_receiver
|
56
|
+
@p_data_receiver = do_start_p_data_receiver
|
57
|
+
|
58
|
+
# start collector that maintains the shared directory will be same with remote ones.
|
59
|
+
@collector = do_start_collector
|
60
|
+
end
|
61
|
+
|
62
|
+
def stop
|
63
|
+
# for observer
|
64
|
+
@observer.raise 'stop'
|
65
|
+
@observer.join
|
66
|
+
|
67
|
+
# for receiver
|
68
|
+
@metadata_receiver.raise "stop"
|
69
|
+
@metadata_receiver.join
|
70
|
+
|
71
|
+
@p_metadata_receiver.raise "stop"
|
72
|
+
@p_metadata_receiver.join
|
73
|
+
|
74
|
+
@data_receiver.raise "stop"
|
75
|
+
@data_receiver.join
|
76
|
+
|
77
|
+
@p_data_receiver.raise "stop"
|
78
|
+
@p_data_receiver.join
|
79
|
+
|
80
|
+
# for collector
|
81
|
+
@collector.kill
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,278 @@
|
|
1
|
+
module BitBroker
|
2
|
+
class ManagerImpl
|
3
|
+
def initialize(opts)
|
4
|
+
# validate user created arguments
|
5
|
+
validate(opts)
|
6
|
+
|
7
|
+
### prepare brokers
|
8
|
+
@config = {
|
9
|
+
:mqconfig => opts[:mqconfig],
|
10
|
+
:label => opts[:name],
|
11
|
+
:dirpath => form_dirpath(opts[:path]),
|
12
|
+
}
|
13
|
+
|
14
|
+
@metadata = Metadata.new(@config[:dirpath])
|
15
|
+
|
16
|
+
@publisher = Publisher.new(@config)
|
17
|
+
|
18
|
+
@deficients = @suggestions = []
|
19
|
+
@semaphore = Mutex.new
|
20
|
+
|
21
|
+
# internal variable in this class to know who modified/crated file
|
22
|
+
@file_activities = []
|
23
|
+
end
|
24
|
+
|
25
|
+
def form_dirpath path
|
26
|
+
path[-1] == '/' ? form_dirpath(path.chop) : path
|
27
|
+
end
|
28
|
+
def validate(opts)
|
29
|
+
raise InvalidArgument("Specified path is not directory") unless File.directory?(opts[:path])
|
30
|
+
end
|
31
|
+
|
32
|
+
def do_start_observer
|
33
|
+
def handle_add(path)
|
34
|
+
Log.debug("[ManagerImpl] (handle_add) path:#{path}")
|
35
|
+
|
36
|
+
rpath = @metadata.get_rpath(path)
|
37
|
+
if obj = @file_activities.find {|x| x.path == rpath}
|
38
|
+
@file_activities.delete(obj)
|
39
|
+
else
|
40
|
+
# create metadata info
|
41
|
+
@metadata.create(rpath)
|
42
|
+
|
43
|
+
# upload target file
|
44
|
+
Solvant.new(@metadata.dir, rpath).upload(@publisher)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def handle_mod(path)
|
49
|
+
Log.debug("[ManagerImpl] (handle_mod) path:#{path}")
|
50
|
+
|
51
|
+
rpath = @metadata.get_rpath(path)
|
52
|
+
if obj = @file_activities.find {|x| x.path == rpath}
|
53
|
+
@file_activities.delete(obj)
|
54
|
+
else
|
55
|
+
# upload target file
|
56
|
+
Solvant.new(@metadata.dir, rpath).upload(@publisher)
|
57
|
+
|
58
|
+
# update fileinfo
|
59
|
+
@metadata.get_with_path(rpath).update
|
60
|
+
|
61
|
+
@metadata.advertise(@publisher)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def handle_rem(path)
|
66
|
+
rpath = @metadata.get_rpath(path)
|
67
|
+
|
68
|
+
#@metadata.remove_with_path(rpath)
|
69
|
+
file = @metadata.get_with_path(rpath)
|
70
|
+
if file != nil
|
71
|
+
Log.debug("[ManagerImpl] (handle_rem) path:#{path}")
|
72
|
+
|
73
|
+
file.remove
|
74
|
+
@metadata.advertise(@publisher)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
Thread.new do
|
79
|
+
Observer.new(@config[:dirpath]) do |mod, add, rem|
|
80
|
+
mod.each {|x| handle_mod(x)}
|
81
|
+
add.each {|x| handle_add(x)}
|
82
|
+
rem.each {|x| handle_rem(x)}
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def do_start_collector
|
88
|
+
Thread.new do
|
89
|
+
loop do
|
90
|
+
deficient = @deficients.first
|
91
|
+
if deficient != nil
|
92
|
+
candidates = @suggestions.select { |x| x['path'] == deficient['path'] }
|
93
|
+
if candidates.size > 0
|
94
|
+
candidate = candidates[rand(candidates.size)]
|
95
|
+
|
96
|
+
@metadata.request(@publisher, [candidate], candidate['from'])
|
97
|
+
|
98
|
+
@semaphore.synchronize do
|
99
|
+
@suggestions = @suggestions.reject {|x| x['path'] == deficient['path']}
|
100
|
+
@deficients.delete(deficient)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
Thread.pass
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def do_start_metadata_receiver
|
110
|
+
Thread.new do
|
111
|
+
receiver = Subscriber.new(@config)
|
112
|
+
receiver.recv_metadata do |msg, from|
|
113
|
+
case msg['type']
|
114
|
+
when Metadata::TYPE_ADVERTISE then
|
115
|
+
receive_advertise(msg['data'], from)
|
116
|
+
when Metadata::TYPE_REQUEST_ALL then
|
117
|
+
receive_request_all(msg['data'], from)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def do_start_p_metadata_receiver
|
124
|
+
Thread.new do
|
125
|
+
receiver = Subscriber.new(@config)
|
126
|
+
receiver.recv_p_metadata do |msg, from|
|
127
|
+
case msg['type']
|
128
|
+
when Metadata::TYPE_SUGGESTION then
|
129
|
+
receive_suggestion(msg['data'], from)
|
130
|
+
when Metadata::TYPE_REQUEST then
|
131
|
+
receive_request(msg['data'], from)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def do_start_data_receiver
|
138
|
+
Thread.new do
|
139
|
+
receiver = Subscriber.new(@config)
|
140
|
+
receiver.recv_data do |binary, from|
|
141
|
+
path = MessagePack.unpack(binary)['path']
|
142
|
+
|
143
|
+
Log.debug("[ManagerImpl] (data_receiver) path: #{path}")
|
144
|
+
|
145
|
+
@file_activities.push(FileActivity.create(path))
|
146
|
+
|
147
|
+
Solvant.load_binary(@config[:dirpath], binary)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
def do_start_p_data_receiver
|
152
|
+
Thread.new do
|
153
|
+
receiver = Subscriber.new(@config)
|
154
|
+
receiver.recv_p_data do |binary, from|
|
155
|
+
path = MessagePack.unpack(binary)['path']
|
156
|
+
|
157
|
+
Log.debug("[ManagerImpl] (p_data_receiver) path: #{path}")
|
158
|
+
|
159
|
+
@file_activities.push(FileActivity.create(path))
|
160
|
+
|
161
|
+
Solvant.load_binary(@config[:dirpath], binary)
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
def receive_advertise(data, from)
|
167
|
+
def updated?(remote)
|
168
|
+
case f = @metadata.get_with_path(remote['path'])
|
169
|
+
when nil # this means target file doesn't exist in local.
|
170
|
+
true
|
171
|
+
else
|
172
|
+
f.size != remote['size'] and
|
173
|
+
not f.removed?
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def removed?(remote)
|
178
|
+
case f = @metadata.get_with_path(remote['path'])
|
179
|
+
when nil
|
180
|
+
false
|
181
|
+
else
|
182
|
+
remote['status'].to_i & Metadata::FileInfo::STATUS_REMOVED > 0
|
183
|
+
end
|
184
|
+
end
|
185
|
+
Log.debug("[ManagerImpl] (receive_advertise) <#{from}> data:#{data}")
|
186
|
+
|
187
|
+
deficients = []
|
188
|
+
data.each do |remote|
|
189
|
+
if removed? remote
|
190
|
+
Log.debug("[ManagerImpl] (receive_advertise) remove: #{remote}")
|
191
|
+
|
192
|
+
# set file_activities
|
193
|
+
@file_activities.push(FileActivity.remove(remote['path']))
|
194
|
+
|
195
|
+
# remove FileInfo object which metadata has
|
196
|
+
@metadata.remove_with_path(remote['path'])
|
197
|
+
|
198
|
+
# remove actual file in local FS
|
199
|
+
Solvant.new(@config[:dirpath], remote['path']).remove
|
200
|
+
else updated? remote
|
201
|
+
deficients.push(remote)
|
202
|
+
|
203
|
+
fpath = "#{@config[:dirpath]}/#{remote['path']}"
|
204
|
+
if FileTest.exist? fpath
|
205
|
+
Log.debug("[ManagerImpl] trancated(#{fpath}, #{remote['size']})")
|
206
|
+
|
207
|
+
# truncate files when target file is cut down
|
208
|
+
File.truncate(fpath, remote['size'])
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# request all deficients files
|
214
|
+
@metadata.request_all(@publisher, deficients)
|
215
|
+
|
216
|
+
# record deficient files to get it from remote node
|
217
|
+
@semaphore.synchronize do
|
218
|
+
@deficients += deficients
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def receive_request_all(data, from)
|
223
|
+
def has_file?(remote)
|
224
|
+
@metadata.get_with_path(remote['path']) != nil
|
225
|
+
end
|
226
|
+
Log.debug("[ManagerImpl] (receive_request_all) <#{from}> data:#{data}")
|
227
|
+
|
228
|
+
files = data.map {|f| @metadata.get_with_path(f['path'])}.select{|x| x != nil}
|
229
|
+
if files != []
|
230
|
+
Log.debug("[ManagerImpl] (receive_request_all) files:#{files}")
|
231
|
+
@metadata.suggestion(@publisher, files.map{|x| x.to_h}, from)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def receive_suggestion(data, from)
|
236
|
+
Log.debug("[ManagerImpl] (receive_suggestion) <#{from}> data:#{data}")
|
237
|
+
|
238
|
+
data.each {|x| x['from'] = from}
|
239
|
+
@semaphore.synchronize do
|
240
|
+
@suggestions += data
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def receive_request(data, from)
|
245
|
+
Log.debug("[ManagerImpl] (receive_request) <#{from}> data:#{data}")
|
246
|
+
|
247
|
+
data.each do |remote|
|
248
|
+
f = @metadata.get_with_path(remote['path'])
|
249
|
+
|
250
|
+
Solvant.new(@config[:dirpath], f.path).upload_to(@publisher, from)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
class FileActivity
|
255
|
+
STATUS_REMOVED = 1 << 0
|
256
|
+
|
257
|
+
attr_reader :path, :mtime
|
258
|
+
|
259
|
+
def initialize(path, mtime, status = 0)
|
260
|
+
@path = path
|
261
|
+
@mtime = mtime
|
262
|
+
@status = status
|
263
|
+
end
|
264
|
+
|
265
|
+
def removed?
|
266
|
+
@status & STATUS_REMOVED > 0
|
267
|
+
end
|
268
|
+
|
269
|
+
def self.create(path)
|
270
|
+
self.new(path, FileTest.exist?(path) ? File.mtime(path) : Time.now)
|
271
|
+
end
|
272
|
+
|
273
|
+
def self.remove(path)
|
274
|
+
self.new(path, nil, STATUS_REMOVED)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require 'msgpack'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Metadata
|
5
|
+
# describes message types
|
6
|
+
TYPE_ADVERTISE = 1<<0
|
7
|
+
TYPE_REQUEST_ALL = 1<<1
|
8
|
+
TYPE_SUGGESTION = 1<<2
|
9
|
+
TYPE_REQUEST = 1<<3
|
10
|
+
|
11
|
+
attr_reader :dir
|
12
|
+
|
13
|
+
def initialize(dir)
|
14
|
+
@dir = dir
|
15
|
+
@files = scanning_files(@dir).map do |path|
|
16
|
+
FileInfo.new(@dir, get_rpath(path))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_with_path(path)
|
21
|
+
@files.select{|f| f.path == path}.first
|
22
|
+
end
|
23
|
+
def remove_with_path(path)
|
24
|
+
@files.reject!{|f| f.path == path}
|
25
|
+
end
|
26
|
+
def create(path)
|
27
|
+
if get_with_path(path) == nil
|
28
|
+
@files.push(FileInfo.new(@dir, path))
|
29
|
+
else
|
30
|
+
puts "Warning: #{path} is already created"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
### sending message for broker
|
35
|
+
def advertise(broker)
|
36
|
+
broker.send_metadata({
|
37
|
+
:type => TYPE_ADVERTISE,
|
38
|
+
:data => @files.map{|x| x.to_h },
|
39
|
+
})
|
40
|
+
end
|
41
|
+
def request_all(broker, files)
|
42
|
+
broker.send_metadata({
|
43
|
+
:type => TYPE_REQUEST_ALL,
|
44
|
+
:data => files,
|
45
|
+
})
|
46
|
+
end
|
47
|
+
def suggestion(broker, files, dest)
|
48
|
+
broker.send_p_metadata(dest, {
|
49
|
+
:type => TYPE_SUGGESTION,
|
50
|
+
:data => files,
|
51
|
+
})
|
52
|
+
end
|
53
|
+
def request(broker, files, dest)
|
54
|
+
broker.send_p_metadata(dest, {
|
55
|
+
:type => TYPE_REQUEST,
|
56
|
+
:data => files,
|
57
|
+
})
|
58
|
+
end
|
59
|
+
|
60
|
+
### utility methods
|
61
|
+
def get_rpath(path)
|
62
|
+
raise DiscomfortDirectoryStructure unless !!path.match(/^#{@dir}/)
|
63
|
+
path.split(@dir).last
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
def scanning_files(current_dir, &block)
|
68
|
+
arr = []
|
69
|
+
Dir.foreach(current_dir) do |f|
|
70
|
+
if /^\.+$/ !~ f
|
71
|
+
path = "#{current_dir}/#{f}"
|
72
|
+
|
73
|
+
if File.directory? f
|
74
|
+
arr += scanning(path)
|
75
|
+
else
|
76
|
+
arr.push(path)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
arr
|
81
|
+
end
|
82
|
+
class FileInfo
|
83
|
+
attr_reader :path, :size, :mtime
|
84
|
+
|
85
|
+
# describes file status
|
86
|
+
STATUS_REMOVED = 1 << 0
|
87
|
+
|
88
|
+
def initialize(dirpath, filepath)
|
89
|
+
@fpath = "#{dirpath}/#{filepath}"
|
90
|
+
@path = filepath
|
91
|
+
@status = 0
|
92
|
+
|
93
|
+
self.update
|
94
|
+
end
|
95
|
+
def update
|
96
|
+
if FileTest.exist? @fpath
|
97
|
+
file = File.new(@fpath)
|
98
|
+
|
99
|
+
@size = file.size
|
100
|
+
@mtime = file.mtime
|
101
|
+
else
|
102
|
+
@size = 0
|
103
|
+
@mtime = Time.new(0)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
def removed?
|
107
|
+
@status & STATUS_REMOVED > 0
|
108
|
+
end
|
109
|
+
def remove
|
110
|
+
@status |= STATUS_REMOVED
|
111
|
+
end
|
112
|
+
def to_h
|
113
|
+
{
|
114
|
+
'path' => @path,
|
115
|
+
'status' => @status,
|
116
|
+
'size' => @size,
|
117
|
+
'mtime' => @mtime.to_s,
|
118
|
+
}
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'listen'
|
2
|
+
|
3
|
+
module BitBroker
|
4
|
+
class Observer
|
5
|
+
def initialize(dir, &block)
|
6
|
+
@target_dir = dir
|
7
|
+
|
8
|
+
@listener = Listen.to(dir) do |mod, add, rem|
|
9
|
+
block.call(mod, add, rem)
|
10
|
+
end
|
11
|
+
|
12
|
+
@listener.start
|
13
|
+
end
|
14
|
+
|
15
|
+
def stop
|
16
|
+
@listener.stop
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'msgpack'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module BitBroker
|
5
|
+
class Solvant
|
6
|
+
DEFAULT_CHUNK_SIZE = 1<<20
|
7
|
+
|
8
|
+
attr_reader :chunks
|
9
|
+
|
10
|
+
def initialize(dirpath, r_path, chunk_size = DEFAULT_CHUNK_SIZE)
|
11
|
+
@f_path = "#{dirpath}/#{r_path}"
|
12
|
+
|
13
|
+
# Validate target file at first
|
14
|
+
if not FileTest.exist? @f_path
|
15
|
+
FileUtils.touch(@f_path)
|
16
|
+
end
|
17
|
+
|
18
|
+
# separate per chunk
|
19
|
+
@chunks = []
|
20
|
+
chunk_splitter(File::Stat.new(@f_path).size, chunk_size) do |offset, size|
|
21
|
+
@chunks.push(Chunk.new({
|
22
|
+
:r_path => r_path,
|
23
|
+
:f_path => @f_path,
|
24
|
+
:size => size,
|
25
|
+
:offset => offset,
|
26
|
+
:chunk_size => chunk_size,
|
27
|
+
}))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# This defines operations to manipulate actual Flie object on FileSystem
|
32
|
+
def remove
|
33
|
+
File.unlink(@f_path)
|
34
|
+
end
|
35
|
+
|
36
|
+
def upload broker
|
37
|
+
@chunks.each do |chunk|
|
38
|
+
broker.send_data(chunk.serialize)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def upload_to(broker, dest)
|
43
|
+
@chunks.each do |chunk|
|
44
|
+
broker.send_p_data(dest, chunk.serialize)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.load_binary(dirpath, binary)
|
49
|
+
data = MessagePack.unpack(binary)
|
50
|
+
offset = data['offset'] * data['chunk_size']
|
51
|
+
|
52
|
+
File.binwrite(dirpath + data['path'], data['data'], offset)
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
def chunk_splitter(total_size, chunk_size, &block)
|
57
|
+
last = total_size / chunk_size
|
58
|
+
(0..last).each do |i|
|
59
|
+
size = (i == last) ? total_size - chunk_size * i : chunk_size
|
60
|
+
|
61
|
+
block.call(i, size)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class Chunk
|
66
|
+
def initialize(opts)
|
67
|
+
@r_path = opts[:r_path]
|
68
|
+
@f_path = opts[:f_path]
|
69
|
+
@size = opts[:size]
|
70
|
+
@offset = opts[:offset]
|
71
|
+
@chunk_size = opts[:chunk_size]
|
72
|
+
end
|
73
|
+
|
74
|
+
def serialize
|
75
|
+
MessagePack.pack({
|
76
|
+
'path' => @r_path,
|
77
|
+
'data' => File.binread(@f_path, @size, @offset * @chunk_size),
|
78
|
+
'offset' => @offset,
|
79
|
+
'chunk_size' => @chunk_size,
|
80
|
+
})
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/bitbroker/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bitbroker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroyasu OHYAMA
|
8
8
|
autorequire:
|
9
|
-
bindir:
|
9
|
+
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-12-
|
11
|
+
date: 2015-12-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,10 +66,21 @@ files:
|
|
66
66
|
- LICENSE.txt
|
67
67
|
- README.md
|
68
68
|
- Rakefile
|
69
|
+
- bin/bitbroker-start
|
70
|
+
- bin/bitbroker-stop
|
69
71
|
- bin/console
|
70
72
|
- bin/setup
|
71
73
|
- bitbroker.gemspec
|
72
74
|
- lib/bitbroker.rb
|
75
|
+
- lib/bitbroker/broker.rb
|
76
|
+
- lib/bitbroker/config.rb
|
77
|
+
- lib/bitbroker/exception.rb
|
78
|
+
- lib/bitbroker/log.rb
|
79
|
+
- lib/bitbroker/manager.rb
|
80
|
+
- lib/bitbroker/manager_impl.rb
|
81
|
+
- lib/bitbroker/metadata.rb
|
82
|
+
- lib/bitbroker/observer.rb
|
83
|
+
- lib/bitbroker/solvant.rb
|
73
84
|
- lib/bitbroker/version.rb
|
74
85
|
homepage: https://github.com/userlocalhost2000/bitbroker
|
75
86
|
licenses:
|