mdbm_dist_support 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +24 -20
- data/lib/mdbm_dist_support/distributer.rb +5 -5
- data/lib/mdbm_dist_support/validator.rb +1 -1
- data/lib/mdbm_dist_support/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f0fdf4ee1e6251dbe55f3c8b8a4a19e3b3cc6707
|
4
|
+
data.tar.gz: 6ebe15b54bb62a91cb57403df2905131444308b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 70c5653b264ee60a02677c1a49eddd972cce8a458a7f6ccd755620944544b2e4ee027dd196e52ee74861ca58b2f91ea1a216a1c8cdb69011828c138b1c2a684b
|
7
|
+
data.tar.gz: '09f17e87473de8ac15774babfb85a7f6f9541940c88715f9904cf1db563cb91e9d37466ff302b023f5812b39f6911b0e0862cd3b205efb42357e308ac276a276'
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -24,31 +24,35 @@ Use like this.
|
|
24
24
|
require 'mdbm_dist_support'
|
25
25
|
|
26
26
|
MdbmDistSupport::Distributer.new do |m|
|
27
|
-
m.lock_path
|
28
|
-
m.meta_path
|
29
|
-
m.local_path
|
30
|
-
m.dist_path
|
31
|
-
m.cmd_print
|
32
|
-
m.cmd_gen
|
33
|
-
m.cmd_rep
|
34
|
-
m.full_mode
|
35
|
-
m.
|
36
|
-
|
27
|
+
m.lock_path = '/tmp/hoge.lck'
|
28
|
+
m.meta_path = '/tmp/hoge_meta.mdbm'
|
29
|
+
m.local_path = '/tmp/hoge_local.mdbm'
|
30
|
+
m.dist_path = '/tmp/hoge_dist.mdbm'
|
31
|
+
m.cmd_print = '/tmp/print.rb'
|
32
|
+
m.cmd_gen = :mdbm_store_func
|
33
|
+
m.cmd_rep = '/usr/local/bin/mdbm_replace'
|
34
|
+
m.full_mode = true
|
35
|
+
m.dist_servers = [
|
36
|
+
:host => 'remote1', :port => 22, :user => 'foo', :key => '/home/foo/.ssh/id_rsa',
|
37
|
+
:host => 'remote2', :port => 22, :user => 'hee', :key => '/home/hee/.ssh/id_rsa',
|
38
|
+
:host => 'remote3', :port => 22, :user => 'hyo', :key => '/home/hyo/.ssh/id_rsa',
|
39
|
+
]
|
40
|
+
m.meta_incr_key = 'when_processed_fetched_max_date'
|
37
41
|
end.run_dist
|
38
42
|
```
|
39
43
|
#### settings @run_dist
|
40
44
|
|name|value|memo|
|
41
45
|
|:-----------|:------------|:------------|
|
42
|
-
|lock_path|
|
43
|
-
|meta_path|
|
44
|
-
|local_path|
|
45
|
-
|dist_path|mdbm path in remote
|
46
|
+
|lock_path|lock file path||
|
47
|
+
|meta_path|meta mdbm path||
|
48
|
+
|local_path|mdbm path||
|
49
|
+
|dist_path|mdbm path in remote dist_server_hosts||
|
46
50
|
|cmd_print|print command path|need to make|
|
47
|
-
|cmd_gen|generate mdbm command path or
|
51
|
+
|cmd_gen|generate mdbm command path or `:mdbm_store_func`|need to make if you want to generate mdbm othert than _String:String_ set like _int64:String_, _int32:int64_.if you set `:mdbm_store_func` only, set use mdbm.stroe() function.|
|
48
52
|
|cmd_rep|replace mdbm command path|[mdbm_replace](https://github.com/yahoo/mdbm/blob/master/gendoc/mdbm_replace.rst)|
|
49
|
-
|full_mode
|
50
|
-
|
|
51
|
-
|meta_incr_key|meta mdbm key
|
53
|
+
|full_mode|`true`: full `false`: incremental||
|
54
|
+
|dist_servers|distribute server info array||
|
55
|
+
|meta_incr_key|meta mdbm key using incremental mode||
|
52
56
|
|
53
57
|
#### print.rb
|
54
58
|
```ruby
|
@@ -65,8 +69,8 @@ end.run_print_after(fetched_max_date)
|
|
65
69
|
#### settings @run_print_after
|
66
70
|
|name|value|
|
67
71
|
|:-----------|:------------|
|
68
|
-
|meta_path|
|
69
|
-
|meta_incr_key|meta mdbm key
|
72
|
+
|meta_path|meta mdbm path|
|
73
|
+
|meta_incr_key|meta mdbm key using incremental mode|
|
70
74
|
|
71
75
|
## Requires
|
72
76
|
* [mdbm](https://github.com/yahoo/mdbm)
|
@@ -10,7 +10,7 @@ module MdbmDistSupport
|
|
10
10
|
class Distributer
|
11
11
|
attr_accessor :lock_path, :lock, :meta_path, :meta, :local_path,\
|
12
12
|
:dist_path, :cmd_print, :cmd_gen, :cmd_rep,\
|
13
|
-
:
|
13
|
+
:dist_servers, :meta_incr_key, :full_mode
|
14
14
|
|
15
15
|
include MdbmDistSupport::CustomLogger
|
16
16
|
|
@@ -55,7 +55,7 @@ module MdbmDistSupport
|
|
55
55
|
def do_mdbm_store(f)
|
56
56
|
f.each_line do |s|
|
57
57
|
kv = s.split("\t")
|
58
|
-
m = Mdbm.new(@local_path, Mdbm::MDBM_O_RDWR | Mdbm::MDBM_O_CREAT,
|
58
|
+
m = Mdbm.new(@local_path, Mdbm::MDBM_O_RDWR | Mdbm::MDBM_O_CREAT, 0644, 0, 0)
|
59
59
|
m.store(kv[0], kv[1], Mdbm::MDBM_REPLACE)
|
60
60
|
end
|
61
61
|
end
|
@@ -65,9 +65,9 @@ module MdbmDistSupport
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def dist
|
68
|
-
@
|
69
|
-
cmd_exec %(scp -
|
70
|
-
cmd_exec %(ssh -
|
68
|
+
@dist_servers.each do |s|
|
69
|
+
cmd_exec %(scp -P #{s[:port]} -i #{s[:key]} #{@local_path} #{s[:user]}@#{s[:host]}:#{@dist_path}.tmp)
|
70
|
+
cmd_exec %(ssh -p #{s[:port]} -i #{s[:key]} #{s[:user]}@#{s[:host]} \" #{@cmd_rep} #{@dist_path} #{@dist_path}.tmp && chmod 777 #{@dist_path}\")
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
@@ -5,7 +5,7 @@ module MdbmDistSupport
|
|
5
5
|
class Validator
|
6
6
|
RUN_DIST_REQUIRE_INSTANCE_VARS =
|
7
7
|
%i[@lock_path @meta_path @local_path @dist_path @cmd_print
|
8
|
-
@cmd_gen @cmd_rep @full_mode @
|
8
|
+
@cmd_gen @cmd_rep @full_mode @dist_servers
|
9
9
|
@meta_incr_key].freeze
|
10
10
|
|
11
11
|
RUN_PRINT_AFTER_REQUIRE_INSTANCE_VARS =
|