ccp 0.3.4 → 0.3.5
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/lib/ccp/kvs/core.rb +10 -1
- data/lib/ccp/kvs/tch.rb +2 -1
- data/lib/ccp/kvs/tokyo/cabinet.rb +12 -3
- data/lib/ccp/kvs/tokyo/state_machine.rb +24 -21
- data/lib/ccp/receivers/skippable.rb +9 -1
- data/lib/ccp/storage.rb +17 -4
- data/lib/ccp/version.rb +1 -1
- data/spec/kvs/core_spec.rb +2 -0
- data/spec/kvs/kvs_spec.rb +12 -2
- data/spec/kvs/tokyo/cabinet_spec.rb +13 -3
- data/spec/storage/loadable_spec.rb +11 -3
- metadata +4 -4
data/.gitignore
CHANGED
data/lib/ccp/kvs/core.rb
CHANGED
@@ -5,7 +5,6 @@ module Ccp
|
|
5
5
|
def set(k,v) ; raise NotImplementedError, "subclass resposibility"; end
|
6
6
|
def del(k) ; raise NotImplementedError, "subclass resposibility"; end
|
7
7
|
def keys ; raise NotImplementedError, "subclass resposibility"; end
|
8
|
-
def read! ; keys.inject({}){|h,k| h[k] = get(k); h } ; end
|
9
8
|
def exist?(k); !! get(k) ; end
|
10
9
|
def key?(k) ; exist?(k) ; end
|
11
10
|
|
@@ -32,6 +31,16 @@ module Ccp
|
|
32
31
|
def self.open(*args); new.tap{|kvs| kvs.open(*args)}; end
|
33
32
|
end
|
34
33
|
end
|
34
|
+
|
35
|
+
# bulk operation
|
36
|
+
def read ; keys.inject({}){|h,k| h[k] = get(k); h } ; end
|
37
|
+
def write(h) ; h.each_pair{|k,v| set(k,v)} ; end
|
38
|
+
|
39
|
+
# backward compat (until 0.3.6)
|
40
|
+
def read!
|
41
|
+
STDERR.puts "DEPRECATION WARNING: #{self.class}#read! will be removed in 0.3.6, use read instead"
|
42
|
+
read
|
43
|
+
end
|
35
44
|
end
|
36
45
|
end
|
37
46
|
end
|
data/lib/ccp/kvs/tch.rb
CHANGED
@@ -7,10 +7,11 @@ module Ccp
|
|
7
7
|
def set(k,v) ; W{ super }; end
|
8
8
|
def del(k) ; W{ super }; end
|
9
9
|
def count ; R{ super }; end
|
10
|
-
def read! ; R{ super }; end
|
11
10
|
def keys ; R{ super }; end
|
12
11
|
def first_key; R{ super }; end
|
13
12
|
def first ; R{ super }; end
|
13
|
+
def read ; R{ super }; end
|
14
|
+
def write(h) ; W{ super }; end
|
14
15
|
end
|
15
16
|
end
|
16
17
|
end
|
@@ -60,10 +60,10 @@ module Ccp
|
|
60
60
|
######################################################################
|
61
61
|
### bulk operations (not DRY but fast)
|
62
62
|
|
63
|
-
def read
|
64
|
-
tryR("read
|
63
|
+
def read
|
64
|
+
tryR("read")
|
65
65
|
hash = {}
|
66
|
-
@db.iterinit or tokyo_error!("read
|
66
|
+
@db.iterinit or tokyo_error!("read: ")
|
67
67
|
while k = @db.iternext
|
68
68
|
v = @db.get(k) or tokyo_error!("get(%s): " % k)
|
69
69
|
hash[k] = decode(v)
|
@@ -71,6 +71,15 @@ module Ccp
|
|
71
71
|
return hash
|
72
72
|
end
|
73
73
|
|
74
|
+
def write(h)
|
75
|
+
tryW("write")
|
76
|
+
h.each_pair do |k,v|
|
77
|
+
val = encode(v)
|
78
|
+
@db[k.to_s] = val or tokyo_error!("write(%s): " % k)
|
79
|
+
end
|
80
|
+
return h
|
81
|
+
end
|
82
|
+
|
74
83
|
######################################################################
|
75
84
|
### iterator
|
76
85
|
|
@@ -11,20 +11,20 @@ module Ccp
|
|
11
11
|
READABLE = 2
|
12
12
|
WRITABLE = 3
|
13
13
|
|
14
|
+
LOCKED_BY = proc{|c| Array(c).select{|i| i !~ %r{/ruby/[^/]+/gems/}}[0,5].join("\n") || c rescue c}
|
15
|
+
|
14
16
|
def state
|
15
17
|
@state || CLOSED
|
16
18
|
end
|
17
19
|
|
18
20
|
def locker_info
|
19
|
-
pretty = proc{|c| Array(c).find{|i| i !~ %r{/ruby/[^/]+/gems/}} || c}
|
20
|
-
|
21
21
|
if CONNECTIONS[@source]
|
22
|
-
return
|
22
|
+
return LOCKED_BY[CONNECTIONS[@source]]
|
23
23
|
end
|
24
24
|
|
25
25
|
target = File.basename(@source)
|
26
26
|
CONNECTIONS.each_pair do |file, reason|
|
27
|
-
return
|
27
|
+
return LOCKED_BY[reason] if File.basename(file) == target
|
28
28
|
end
|
29
29
|
|
30
30
|
if CONNECTIONS.any?
|
@@ -34,67 +34,70 @@ module Ccp
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
-
def open(mode)
|
37
|
+
def open(mode, locker = nil)
|
38
38
|
Pathname(@source.to_s).parent.mkpath
|
39
39
|
|
40
40
|
# open and mark filename for threading error
|
41
41
|
if @db.open(@source.to_s, mode)
|
42
|
-
|
42
|
+
locker ||= (caller rescue "???")
|
43
|
+
STDERR.puts "LOCK: #{@source} by [#{LOCKED_BY[locker]}]" if @debug
|
44
|
+
CONNECTIONS[@db.path.to_s] = locker
|
43
45
|
elsif threading_error?
|
44
|
-
raise Tokyo::Locked, "%s is locked by %s" % [@source, locker_info]
|
46
|
+
raise Tokyo::Locked, "%s is locked by [%s]" % [@source, locker_info]
|
45
47
|
else
|
46
48
|
tokyo_error!("%s#open(%s,%s): " % [self.class, @source, mode])
|
47
49
|
end
|
48
50
|
end
|
49
51
|
|
50
|
-
def __close__
|
52
|
+
def __close__(locker = nil)
|
51
53
|
@db.close
|
52
54
|
CONNECTIONS[@db.path] = nil
|
55
|
+
STDERR.puts "UNLOCK: #{@source} by [#{LOCKED_BY[locker || caller]}]" if @debug
|
53
56
|
end
|
54
57
|
|
55
|
-
def close
|
56
|
-
C!
|
58
|
+
def close(locker = nil)
|
59
|
+
C!(locker)
|
57
60
|
end
|
58
61
|
|
59
|
-
def C!
|
62
|
+
def C!(locker = nil)
|
60
63
|
case state
|
61
64
|
when CLOSED ; # NOP
|
62
65
|
when READABLE,
|
63
|
-
WRITABLE ; __close__; @state = CLOSED
|
66
|
+
WRITABLE ; __close__(locker); @state = CLOSED
|
64
67
|
else ; raise "unknown state: #{state}"
|
65
68
|
end
|
66
69
|
end
|
67
70
|
|
68
|
-
def R!
|
71
|
+
def R!(locker = nil)
|
69
72
|
case state
|
70
|
-
when CLOSED ; open(HDB::OREADER); @state = READABLE
|
73
|
+
when CLOSED ; open(HDB::OREADER, locker); @state = READABLE
|
71
74
|
when READABLE ; # NOP
|
72
75
|
when WRITABLE ; # NOP
|
73
76
|
else ; raise "unknown state: #{state}"
|
74
77
|
end
|
75
78
|
end
|
76
79
|
|
77
|
-
def W!
|
80
|
+
def W!(locker = nil)
|
78
81
|
case state
|
79
|
-
when CLOSED ; open(HDB::OCREAT | HDB::OWRITER); @state = WRITABLE
|
80
|
-
when READABLE ; C
|
82
|
+
when CLOSED ; open(HDB::OCREAT | HDB::OWRITER, locker); @state = WRITABLE
|
83
|
+
when READABLE ; C!(locker); W!(locker)
|
81
84
|
when WRITABLE ; # NOP
|
82
85
|
else ; raise "unknown state: #{state}"
|
83
86
|
end
|
84
87
|
end
|
85
88
|
|
86
|
-
def R(&block)
|
89
|
+
def R(locker = nil, &block)
|
87
90
|
case state
|
88
|
-
when CLOSED ; begin; R!(); yield; ensure; close; end
|
91
|
+
when CLOSED ; begin; R!(locker); yield; ensure; close(locker); end
|
89
92
|
when READABLE ; yield
|
90
93
|
when WRITABLE ; yield
|
91
94
|
else ; raise "unknown state: #{state}"
|
92
95
|
end
|
93
96
|
end
|
94
97
|
|
95
|
-
def W(&block)
|
98
|
+
def W(locker = nil, &block)
|
96
99
|
case state
|
97
|
-
when CLOSED ; begin; W!(); yield; ensure; close; end
|
100
|
+
when CLOSED ; begin; W!(locker); yield; ensure; close(locker); end
|
98
101
|
when READABLE ; raise "reopen from read to write is not permitted"
|
99
102
|
# TODO: close -> W -> close -> R ???
|
100
103
|
when WRITABLE ; yield
|
@@ -2,7 +2,10 @@ module Ccp
|
|
2
2
|
module Receivers
|
3
3
|
module Skippable
|
4
4
|
def execute(cmd)
|
5
|
-
|
5
|
+
if skip?(cmd)
|
6
|
+
notify_skip(cmd)
|
7
|
+
return false
|
8
|
+
end
|
6
9
|
super
|
7
10
|
end
|
8
11
|
|
@@ -11,6 +14,11 @@ module Ccp
|
|
11
14
|
key = "skip_%s" % cmd.class.name.underscore.gsub("/","_")
|
12
15
|
data.set?(key)
|
13
16
|
end
|
17
|
+
|
18
|
+
def notify_skip(cmd)
|
19
|
+
@logger ||= data.set?(:logger) ? data[:logger] : Logger.new(STDOUT)
|
20
|
+
@logger.debug Utils::Colorize.pink("[SKIP] #{cmd.class}")
|
21
|
+
end
|
14
22
|
end
|
15
23
|
end
|
16
24
|
end
|
data/lib/ccp/storage.rb
CHANGED
@@ -10,7 +10,7 @@ module Ccp
|
|
10
10
|
end
|
11
11
|
|
12
12
|
attr_reader :source, :kvs, :codec, :path
|
13
|
-
delegate :get, :set, :del, :keys, :read
|
13
|
+
delegate :get, :set, :del, :keys, :read, :to=>"@kvs"
|
14
14
|
|
15
15
|
def initialize(source, kvs, codec)
|
16
16
|
@source = source
|
@@ -43,20 +43,33 @@ module Ccp
|
|
43
43
|
)
|
44
44
|
end
|
45
45
|
|
46
|
+
def close
|
47
|
+
@tables.each_pair do |_,kvs|
|
48
|
+
kvs.close
|
49
|
+
end
|
50
|
+
@tables = {}
|
51
|
+
end
|
52
|
+
|
46
53
|
######################################################################
|
47
54
|
### kvs
|
48
55
|
|
49
|
-
def read
|
56
|
+
def read
|
50
57
|
if @path.directory?
|
51
58
|
tables
|
52
59
|
hash = {}
|
53
60
|
@tables.each_pair do |k, kvs|
|
54
|
-
hash[k] = kvs.read
|
61
|
+
hash[k] = kvs.read
|
55
62
|
end
|
56
63
|
return hash
|
57
64
|
else
|
58
|
-
return @kvs.read
|
65
|
+
return @kvs.read
|
59
66
|
end
|
60
67
|
end
|
68
|
+
|
69
|
+
# backward compat (until 0.3.6)
|
70
|
+
def read!
|
71
|
+
STDERR.puts "DEPRECATION WARNING: #{self.class}#read! will be removed in 0.3.6, use read instead"
|
72
|
+
read
|
73
|
+
end
|
61
74
|
end
|
62
75
|
end
|
data/lib/ccp/version.rb
CHANGED
data/spec/kvs/core_spec.rb
CHANGED
data/spec/kvs/kvs_spec.rb
CHANGED
@@ -72,13 +72,23 @@ Ccp::Kvs.each do |klass|
|
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
75
|
-
describe "#read
|
75
|
+
describe "#read" do
|
76
76
|
specify do
|
77
77
|
kvs.touch
|
78
78
|
kvs.codec! :json
|
79
79
|
kvs.set(:a, 1)
|
80
80
|
kvs.set(:b, ["x", 0])
|
81
|
-
kvs.read
|
81
|
+
kvs.read.should == {"a"=>1, "b"=>["x", 0]}
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
describe "#write" do
|
86
|
+
specify do
|
87
|
+
kvs.touch
|
88
|
+
kvs.codec! :json
|
89
|
+
kvs.write("a"=>1, "b"=>["x", 0])
|
90
|
+
kvs["a"].should == 1
|
91
|
+
kvs["b"].should == ["x", 0]
|
82
92
|
end
|
83
93
|
end
|
84
94
|
|
@@ -206,15 +206,25 @@ describe Ccp::Kvs::Tokyo::Cabinet do
|
|
206
206
|
end
|
207
207
|
|
208
208
|
######################################################################
|
209
|
-
### read
|
209
|
+
### read / write
|
210
210
|
|
211
|
-
describe "#read
|
211
|
+
describe "#read" do
|
212
212
|
specify do
|
213
213
|
put(:foo, 1)
|
214
214
|
put(:bar, 2)
|
215
215
|
|
216
216
|
kvs.R!
|
217
|
-
kvs.read
|
217
|
+
kvs.read.should == {"foo" => "1", "bar" => "2"}
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
describe "#write" do
|
222
|
+
specify do
|
223
|
+
kvs.W!
|
224
|
+
kvs.write("foo" => "1", "bar" => "2")
|
225
|
+
|
226
|
+
kvs.get("foo").should == "1"
|
227
|
+
kvs.get("bar").should == "2"
|
218
228
|
end
|
219
229
|
end
|
220
230
|
|
@@ -37,7 +37,7 @@ describe Ccp::Storage do
|
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
-
describe "#read
|
40
|
+
describe "#read" do
|
41
41
|
before { FileUtils.rm_rf(tmp_path) if tmp_path.directory? }
|
42
42
|
|
43
43
|
context "(file)" do
|
@@ -50,7 +50,7 @@ describe Ccp::Storage do
|
|
50
50
|
system("tchmgr put #{tch} b 0.1")
|
51
51
|
}
|
52
52
|
specify do
|
53
|
-
subject.read
|
53
|
+
subject.read.should == {"a" => [1, 2], "b" => 0.1}
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
@@ -66,13 +66,21 @@ describe Ccp::Storage do
|
|
66
66
|
system("tchmgr put #{tch}/b.json.tch y 0.1")
|
67
67
|
}
|
68
68
|
specify do
|
69
|
-
subject.read
|
69
|
+
subject.read.should == {
|
70
70
|
"a" => {"x" => [1, 2], "y" => []},
|
71
71
|
"b" => {"y" => 0.1},
|
72
72
|
}
|
73
73
|
end
|
74
74
|
end
|
75
|
+
end
|
76
|
+
|
77
|
+
describe "#close" do
|
78
|
+
before { FileUtils.rm_rf(tmp_path) if tmp_path.directory? }
|
75
79
|
|
80
|
+
let(:tch) { tmp_path + "foo.json.tch" }
|
81
|
+
subject { Ccp::Storage.new(tch, Ccp::Kvs::Tch, Ccp::Serializers::Json) }
|
82
|
+
|
83
|
+
it { should respond_to(:close) }
|
76
84
|
end
|
77
85
|
|
78
86
|
# it { should respond_to(:tables) }
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ccp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 25
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 3
|
9
|
-
-
|
10
|
-
version: 0.3.
|
9
|
+
- 5
|
10
|
+
version: 0.3.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- maiha
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2013-
|
18
|
+
date: 2013-10-29 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: activesupport
|