ccp 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/lib/ccp/kvs/core.rb +10 -1
- data/lib/ccp/kvs/tch.rb +2 -1
- data/lib/ccp/kvs/tokyo/cabinet.rb +12 -3
- data/lib/ccp/kvs/tokyo/state_machine.rb +24 -21
- data/lib/ccp/receivers/skippable.rb +9 -1
- data/lib/ccp/storage.rb +17 -4
- data/lib/ccp/version.rb +1 -1
- data/spec/kvs/core_spec.rb +2 -0
- data/spec/kvs/kvs_spec.rb +12 -2
- data/spec/kvs/tokyo/cabinet_spec.rb +13 -3
- data/spec/storage/loadable_spec.rb +11 -3
- metadata +4 -4
data/.gitignore
CHANGED
data/lib/ccp/kvs/core.rb
CHANGED
@@ -5,7 +5,6 @@ module Ccp
|
|
5
5
|
def set(k,v) ; raise NotImplementedError, "subclass resposibility"; end
|
6
6
|
def del(k) ; raise NotImplementedError, "subclass resposibility"; end
|
7
7
|
def keys ; raise NotImplementedError, "subclass resposibility"; end
|
8
|
-
def read! ; keys.inject({}){|h,k| h[k] = get(k); h } ; end
|
9
8
|
def exist?(k); !! get(k) ; end
|
10
9
|
def key?(k) ; exist?(k) ; end
|
11
10
|
|
@@ -32,6 +31,16 @@ module Ccp
|
|
32
31
|
def self.open(*args); new.tap{|kvs| kvs.open(*args)}; end
|
33
32
|
end
|
34
33
|
end
|
34
|
+
|
35
|
+
# bulk operation
|
36
|
+
def read ; keys.inject({}){|h,k| h[k] = get(k); h } ; end
|
37
|
+
def write(h) ; h.each_pair{|k,v| set(k,v)} ; end
|
38
|
+
|
39
|
+
# backward compat (until 0.3.6)
|
40
|
+
def read!
|
41
|
+
STDERR.puts "DEPRECATION WARNING: #{self.class}#read! will be removed in 0.3.6, use read instead"
|
42
|
+
read
|
43
|
+
end
|
35
44
|
end
|
36
45
|
end
|
37
46
|
end
|
data/lib/ccp/kvs/tch.rb
CHANGED
@@ -7,10 +7,11 @@ module Ccp
|
|
7
7
|
def set(k,v) ; W{ super }; end
|
8
8
|
def del(k) ; W{ super }; end
|
9
9
|
def count ; R{ super }; end
|
10
|
-
def read! ; R{ super }; end
|
11
10
|
def keys ; R{ super }; end
|
12
11
|
def first_key; R{ super }; end
|
13
12
|
def first ; R{ super }; end
|
13
|
+
def read ; R{ super }; end
|
14
|
+
def write(h) ; W{ super }; end
|
14
15
|
end
|
15
16
|
end
|
16
17
|
end
|
@@ -60,10 +60,10 @@ module Ccp
|
|
60
60
|
######################################################################
|
61
61
|
### bulk operations (not DRY but fast)
|
62
62
|
|
63
|
-
def read
|
64
|
-
tryR("read
|
63
|
+
def read
|
64
|
+
tryR("read")
|
65
65
|
hash = {}
|
66
|
-
@db.iterinit or tokyo_error!("read
|
66
|
+
@db.iterinit or tokyo_error!("read: ")
|
67
67
|
while k = @db.iternext
|
68
68
|
v = @db.get(k) or tokyo_error!("get(%s): " % k)
|
69
69
|
hash[k] = decode(v)
|
@@ -71,6 +71,15 @@ module Ccp
|
|
71
71
|
return hash
|
72
72
|
end
|
73
73
|
|
74
|
+
def write(h)
|
75
|
+
tryW("write")
|
76
|
+
h.each_pair do |k,v|
|
77
|
+
val = encode(v)
|
78
|
+
@db[k.to_s] = val or tokyo_error!("write(%s): " % k)
|
79
|
+
end
|
80
|
+
return h
|
81
|
+
end
|
82
|
+
|
74
83
|
######################################################################
|
75
84
|
### iterator
|
76
85
|
|
@@ -11,20 +11,20 @@ module Ccp
|
|
11
11
|
READABLE = 2
|
12
12
|
WRITABLE = 3
|
13
13
|
|
14
|
+
LOCKED_BY = proc{|c| Array(c).select{|i| i !~ %r{/ruby/[^/]+/gems/}}[0,5].join("\n") || c rescue c}
|
15
|
+
|
14
16
|
def state
|
15
17
|
@state || CLOSED
|
16
18
|
end
|
17
19
|
|
18
20
|
def locker_info
|
19
|
-
pretty = proc{|c| Array(c).find{|i| i !~ %r{/ruby/[^/]+/gems/}} || c}
|
20
|
-
|
21
21
|
if CONNECTIONS[@source]
|
22
|
-
return
|
22
|
+
return LOCKED_BY[CONNECTIONS[@source]]
|
23
23
|
end
|
24
24
|
|
25
25
|
target = File.basename(@source)
|
26
26
|
CONNECTIONS.each_pair do |file, reason|
|
27
|
-
return
|
27
|
+
return LOCKED_BY[reason] if File.basename(file) == target
|
28
28
|
end
|
29
29
|
|
30
30
|
if CONNECTIONS.any?
|
@@ -34,67 +34,70 @@ module Ccp
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
-
def open(mode)
|
37
|
+
def open(mode, locker = nil)
|
38
38
|
Pathname(@source.to_s).parent.mkpath
|
39
39
|
|
40
40
|
# open and mark filename for threading error
|
41
41
|
if @db.open(@source.to_s, mode)
|
42
|
-
|
42
|
+
locker ||= (caller rescue "???")
|
43
|
+
STDERR.puts "LOCK: #{@source} by [#{LOCKED_BY[locker]}]" if @debug
|
44
|
+
CONNECTIONS[@db.path.to_s] = locker
|
43
45
|
elsif threading_error?
|
44
|
-
raise Tokyo::Locked, "%s is locked by %s" % [@source, locker_info]
|
46
|
+
raise Tokyo::Locked, "%s is locked by [%s]" % [@source, locker_info]
|
45
47
|
else
|
46
48
|
tokyo_error!("%s#open(%s,%s): " % [self.class, @source, mode])
|
47
49
|
end
|
48
50
|
end
|
49
51
|
|
50
|
-
def __close__
|
52
|
+
def __close__(locker = nil)
|
51
53
|
@db.close
|
52
54
|
CONNECTIONS[@db.path] = nil
|
55
|
+
STDERR.puts "UNLOCK: #{@source} by [#{LOCKED_BY[locker || caller]}]" if @debug
|
53
56
|
end
|
54
57
|
|
55
|
-
def close
|
56
|
-
C!
|
58
|
+
def close(locker = nil)
|
59
|
+
C!(locker)
|
57
60
|
end
|
58
61
|
|
59
|
-
def C!
|
62
|
+
def C!(locker = nil)
|
60
63
|
case state
|
61
64
|
when CLOSED ; # NOP
|
62
65
|
when READABLE,
|
63
|
-
WRITABLE ; __close__; @state = CLOSED
|
66
|
+
WRITABLE ; __close__(locker); @state = CLOSED
|
64
67
|
else ; raise "unknown state: #{state}"
|
65
68
|
end
|
66
69
|
end
|
67
70
|
|
68
|
-
def R!
|
71
|
+
def R!(locker = nil)
|
69
72
|
case state
|
70
|
-
when CLOSED ; open(HDB::OREADER); @state = READABLE
|
73
|
+
when CLOSED ; open(HDB::OREADER, locker); @state = READABLE
|
71
74
|
when READABLE ; # NOP
|
72
75
|
when WRITABLE ; # NOP
|
73
76
|
else ; raise "unknown state: #{state}"
|
74
77
|
end
|
75
78
|
end
|
76
79
|
|
77
|
-
def W!
|
80
|
+
def W!(locker = nil)
|
78
81
|
case state
|
79
|
-
when CLOSED ; open(HDB::OCREAT | HDB::OWRITER); @state = WRITABLE
|
80
|
-
when READABLE ; C
|
82
|
+
when CLOSED ; open(HDB::OCREAT | HDB::OWRITER, locker); @state = WRITABLE
|
83
|
+
when READABLE ; C!(locker); W!(locker)
|
81
84
|
when WRITABLE ; # NOP
|
82
85
|
else ; raise "unknown state: #{state}"
|
83
86
|
end
|
84
87
|
end
|
85
88
|
|
86
|
-
def R(&block)
|
89
|
+
def R(locker = nil, &block)
|
87
90
|
case state
|
88
|
-
when CLOSED ; begin; R!(); yield; ensure; close; end
|
91
|
+
when CLOSED ; begin; R!(locker); yield; ensure; close(locker); end
|
89
92
|
when READABLE ; yield
|
90
93
|
when WRITABLE ; yield
|
91
94
|
else ; raise "unknown state: #{state}"
|
92
95
|
end
|
93
96
|
end
|
94
97
|
|
95
|
-
def W(&block)
|
98
|
+
def W(locker = nil, &block)
|
96
99
|
case state
|
97
|
-
when CLOSED ; begin; W!(); yield; ensure; close; end
|
100
|
+
when CLOSED ; begin; W!(locker); yield; ensure; close(locker); end
|
98
101
|
when READABLE ; raise "reopen from read to write is not permitted"
|
99
102
|
# TODO: close -> W -> close -> R ???
|
100
103
|
when WRITABLE ; yield
|
@@ -2,7 +2,10 @@ module Ccp
|
|
2
2
|
module Receivers
|
3
3
|
module Skippable
|
4
4
|
def execute(cmd)
|
5
|
-
|
5
|
+
if skip?(cmd)
|
6
|
+
notify_skip(cmd)
|
7
|
+
return false
|
8
|
+
end
|
6
9
|
super
|
7
10
|
end
|
8
11
|
|
@@ -11,6 +14,11 @@ module Ccp
|
|
11
14
|
key = "skip_%s" % cmd.class.name.underscore.gsub("/","_")
|
12
15
|
data.set?(key)
|
13
16
|
end
|
17
|
+
|
18
|
+
def notify_skip(cmd)
|
19
|
+
@logger ||= data.set?(:logger) ? data[:logger] : Logger.new(STDOUT)
|
20
|
+
@logger.debug Utils::Colorize.pink("[SKIP] #{cmd.class}")
|
21
|
+
end
|
14
22
|
end
|
15
23
|
end
|
16
24
|
end
|
data/lib/ccp/storage.rb
CHANGED
@@ -10,7 +10,7 @@ module Ccp
|
|
10
10
|
end
|
11
11
|
|
12
12
|
attr_reader :source, :kvs, :codec, :path
|
13
|
-
delegate :get, :set, :del, :keys, :read
|
13
|
+
delegate :get, :set, :del, :keys, :read, :to=>"@kvs"
|
14
14
|
|
15
15
|
def initialize(source, kvs, codec)
|
16
16
|
@source = source
|
@@ -43,20 +43,33 @@ module Ccp
|
|
43
43
|
)
|
44
44
|
end
|
45
45
|
|
46
|
+
def close
|
47
|
+
@tables.each_pair do |_,kvs|
|
48
|
+
kvs.close
|
49
|
+
end
|
50
|
+
@tables = {}
|
51
|
+
end
|
52
|
+
|
46
53
|
######################################################################
|
47
54
|
### kvs
|
48
55
|
|
49
|
-
def read
|
56
|
+
def read
|
50
57
|
if @path.directory?
|
51
58
|
tables
|
52
59
|
hash = {}
|
53
60
|
@tables.each_pair do |k, kvs|
|
54
|
-
hash[k] = kvs.read
|
61
|
+
hash[k] = kvs.read
|
55
62
|
end
|
56
63
|
return hash
|
57
64
|
else
|
58
|
-
return @kvs.read
|
65
|
+
return @kvs.read
|
59
66
|
end
|
60
67
|
end
|
68
|
+
|
69
|
+
# backward compat (until 0.3.6)
|
70
|
+
def read!
|
71
|
+
STDERR.puts "DEPRECATION WARNING: #{self.class}#read! will be removed in 0.3.6, use read instead"
|
72
|
+
read
|
73
|
+
end
|
61
74
|
end
|
62
75
|
end
|
data/lib/ccp/version.rb
CHANGED
data/spec/kvs/core_spec.rb
CHANGED
data/spec/kvs/kvs_spec.rb
CHANGED
@@ -72,13 +72,23 @@ Ccp::Kvs.each do |klass|
|
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
75
|
-
describe "#read
|
75
|
+
describe "#read" do
|
76
76
|
specify do
|
77
77
|
kvs.touch
|
78
78
|
kvs.codec! :json
|
79
79
|
kvs.set(:a, 1)
|
80
80
|
kvs.set(:b, ["x", 0])
|
81
|
-
kvs.read
|
81
|
+
kvs.read.should == {"a"=>1, "b"=>["x", 0]}
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
describe "#write" do
|
86
|
+
specify do
|
87
|
+
kvs.touch
|
88
|
+
kvs.codec! :json
|
89
|
+
kvs.write("a"=>1, "b"=>["x", 0])
|
90
|
+
kvs["a"].should == 1
|
91
|
+
kvs["b"].should == ["x", 0]
|
82
92
|
end
|
83
93
|
end
|
84
94
|
|
@@ -206,15 +206,25 @@ describe Ccp::Kvs::Tokyo::Cabinet do
|
|
206
206
|
end
|
207
207
|
|
208
208
|
######################################################################
|
209
|
-
### read
|
209
|
+
### read / write
|
210
210
|
|
211
|
-
describe "#read
|
211
|
+
describe "#read" do
|
212
212
|
specify do
|
213
213
|
put(:foo, 1)
|
214
214
|
put(:bar, 2)
|
215
215
|
|
216
216
|
kvs.R!
|
217
|
-
kvs.read
|
217
|
+
kvs.read.should == {"foo" => "1", "bar" => "2"}
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
describe "#write" do
|
222
|
+
specify do
|
223
|
+
kvs.W!
|
224
|
+
kvs.write("foo" => "1", "bar" => "2")
|
225
|
+
|
226
|
+
kvs.get("foo").should == "1"
|
227
|
+
kvs.get("bar").should == "2"
|
218
228
|
end
|
219
229
|
end
|
220
230
|
|
@@ -37,7 +37,7 @@ describe Ccp::Storage do
|
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
-
describe "#read
|
40
|
+
describe "#read" do
|
41
41
|
before { FileUtils.rm_rf(tmp_path) if tmp_path.directory? }
|
42
42
|
|
43
43
|
context "(file)" do
|
@@ -50,7 +50,7 @@ describe Ccp::Storage do
|
|
50
50
|
system("tchmgr put #{tch} b 0.1")
|
51
51
|
}
|
52
52
|
specify do
|
53
|
-
subject.read
|
53
|
+
subject.read.should == {"a" => [1, 2], "b" => 0.1}
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
@@ -66,13 +66,21 @@ describe Ccp::Storage do
|
|
66
66
|
system("tchmgr put #{tch}/b.json.tch y 0.1")
|
67
67
|
}
|
68
68
|
specify do
|
69
|
-
subject.read
|
69
|
+
subject.read.should == {
|
70
70
|
"a" => {"x" => [1, 2], "y" => []},
|
71
71
|
"b" => {"y" => 0.1},
|
72
72
|
}
|
73
73
|
end
|
74
74
|
end
|
75
|
+
end
|
76
|
+
|
77
|
+
describe "#close" do
|
78
|
+
before { FileUtils.rm_rf(tmp_path) if tmp_path.directory? }
|
75
79
|
|
80
|
+
let(:tch) { tmp_path + "foo.json.tch" }
|
81
|
+
subject { Ccp::Storage.new(tch, Ccp::Kvs::Tch, Ccp::Serializers::Json) }
|
82
|
+
|
83
|
+
it { should respond_to(:close) }
|
76
84
|
end
|
77
85
|
|
78
86
|
# it { should respond_to(:tables) }
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ccp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 25
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 3
|
9
|
-
-
|
10
|
-
version: 0.3.
|
9
|
+
- 5
|
10
|
+
version: 0.3.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- maiha
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2013-
|
18
|
+
date: 2013-10-29 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: activesupport
|