ccp 0.3.5 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/ccp.gemspec +10 -9
- data/lib/ccp/kvs.rb +9 -3
- data/lib/ccp/kvs/core.rb +0 -6
- data/lib/ccp/kvs/hash.rb +2 -0
- data/lib/ccp/kvs/kch.rb +33 -0
- data/lib/ccp/kvs/kyoto.rb +17 -0
- data/lib/ccp/kvs/kyoto/base.rb +60 -0
- data/lib/ccp/kvs/kyoto/cabinet.rb +137 -0
- data/lib/ccp/kvs/kyoto/state_machine.rb +142 -0
- data/lib/ccp/kvs/tch.rb +29 -13
- data/lib/ccp/kvs/tokyo/cabinet.rb +0 -5
- data/lib/ccp/storage.rb +0 -6
- data/lib/ccp/version.rb +1 -1
- data/spec/kvs/enum_spec.rb +131 -0
- data/spec/kvs/kvs_spec.rb +1 -0
- data/spec/kvs/kyoto/cabinet_spec.rb +282 -0
- data/spec/kvs/load_spec.rb +43 -0
- data/spec/kvs/usecase_spec.rb +33 -0
- data/spec/models.rb +1 -1
- data/spec/storage/loadable_spec.rb +7 -0
- data/spec/storage/usecase_spec.rb +36 -0
- metadata +160 -145
data/ccp.gemspec
CHANGED
@@ -20,18 +20,19 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.require_paths = ["lib"]
|
21
21
|
|
22
22
|
if RUBY_VERSION >= "1.9"
|
23
|
-
s.
|
23
|
+
s.add_runtime_dependency "activesupport"
|
24
24
|
else
|
25
|
-
s.
|
25
|
+
s.add_runtime_dependency "activesupport", "~> 3.2.0"
|
26
26
|
end
|
27
27
|
|
28
|
-
s.
|
29
|
-
s.
|
30
|
-
s.
|
31
|
-
s.
|
32
|
-
s.
|
33
|
-
s.
|
34
|
-
s.add_dependency "tokyocabinet", "~> 1.29.1"
|
28
|
+
s.add_runtime_dependency "typed", ">= 0.2.2"
|
29
|
+
s.add_runtime_dependency "must", ">= 0.3.0"
|
30
|
+
s.add_runtime_dependency "dsl_accessor", ">= 0.4.1"
|
31
|
+
s.add_runtime_dependency "json"
|
32
|
+
s.add_runtime_dependency "yajl-ruby"
|
33
|
+
s.add_runtime_dependency "msgpack", "> 0.4"
|
35
34
|
|
36
35
|
s.add_development_dependency "rspec"
|
36
|
+
s.add_development_dependency "tokyocabinet", "~> 1.29.1"
|
37
|
+
s.add_development_dependency "kyotocabinet-ruby", "~> 1.27.1"
|
37
38
|
end
|
data/lib/ccp/kvs.rb
CHANGED
@@ -10,6 +10,14 @@ module Ccp
|
|
10
10
|
|
11
11
|
DICTIONARY = {} # cache for (extname -> Kvs)
|
12
12
|
|
13
|
+
def self.load(path)
|
14
|
+
array = path.to_s.split(".")
|
15
|
+
kvs = Ccp::Kvs[array.pop].new(path)
|
16
|
+
codec = Ccp::Serializers[array.pop]
|
17
|
+
kvs.codec!(codec)
|
18
|
+
return kvs
|
19
|
+
end
|
20
|
+
|
13
21
|
include Enumerable
|
14
22
|
delegate :delete, :to=>"DICTIONARY"
|
15
23
|
|
@@ -39,8 +47,6 @@ module Ccp
|
|
39
47
|
end
|
40
48
|
|
41
49
|
require 'ccp/kvs/hash'
|
42
|
-
require 'ccp/kvs/tokyo'
|
43
50
|
require 'ccp/kvs/tch'
|
51
|
+
require 'ccp/kvs/kch'
|
44
52
|
|
45
|
-
Ccp::Kvs << Ccp::Kvs::Hash
|
46
|
-
Ccp::Kvs << Ccp::Kvs::Tch
|
data/lib/ccp/kvs/core.rb
CHANGED
@@ -35,12 +35,6 @@ module Ccp
|
|
35
35
|
# bulk operation
|
36
36
|
def read ; keys.inject({}){|h,k| h[k] = get(k); h } ; end
|
37
37
|
def write(h) ; h.each_pair{|k,v| set(k,v)} ; end
|
38
|
-
|
39
|
-
# backward compat (until 0.3.6)
|
40
|
-
def read!
|
41
|
-
STDERR.puts "DEPRECATION WARNING: #{self.class}#read! will be removed in 0.3.6, use read instead"
|
42
|
-
read
|
43
|
-
end
|
44
38
|
end
|
45
39
|
end
|
46
40
|
end
|
data/lib/ccp/kvs/hash.rb
CHANGED
data/lib/ccp/kvs/kch.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
begin
|
2
|
+
require 'kyotocabinet'
|
3
|
+
rescue LoadError
|
4
|
+
load_error = true
|
5
|
+
end
|
6
|
+
|
7
|
+
unless load_error
|
8
|
+
require 'ccp/kvs/kyoto'
|
9
|
+
|
10
|
+
module Ccp
|
11
|
+
module Kvs
|
12
|
+
class Kch < Kyoto::Cabinet
|
13
|
+
# core
|
14
|
+
def get(k) ; R{ super }; end
|
15
|
+
def set(k,v) ; W{ super }; end
|
16
|
+
def del(k) ; W{ super }; end
|
17
|
+
def count ; R{ super }; end
|
18
|
+
def read ; R{ super }; end
|
19
|
+
def write(h) ; W{ super }; end
|
20
|
+
|
21
|
+
# enum
|
22
|
+
def each(&b) ; R{ super }; end
|
23
|
+
def each_pair(&b) ; R{ super }; end
|
24
|
+
def each_key(&b) ; R{ super }; end
|
25
|
+
def keys ; R{ super }; end
|
26
|
+
def first_key ; R{ super }; end
|
27
|
+
def first ; R{ super }; end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Ccp::Kvs << Ccp::Kvs::Kch
|
33
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'kyotocabinet'
|
2
|
+
|
3
|
+
module Ccp
|
4
|
+
module Kvs
|
5
|
+
module Kyoto
|
6
|
+
Error = Class.new(Ccp::Kvs::Error)
|
7
|
+
Locked = Class.new(Error)
|
8
|
+
|
9
|
+
CONNECTIONS = {}
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
require 'ccp/kvs/kyoto/base'
|
15
|
+
require 'ccp/kvs/kyoto/state_machine'
|
16
|
+
require 'ccp/kvs/kyoto/cabinet'
|
17
|
+
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Ccp
|
2
|
+
module Kvs
|
3
|
+
module Kyoto
|
4
|
+
class Base
|
5
|
+
include Ccp::Kvs::Core
|
6
|
+
include KyotoCabinet
|
7
|
+
|
8
|
+
######################################################################
|
9
|
+
### info
|
10
|
+
|
11
|
+
def info
|
12
|
+
raise "not implemented yet"
|
13
|
+
end
|
14
|
+
|
15
|
+
######################################################################
|
16
|
+
### kvs
|
17
|
+
|
18
|
+
def path
|
19
|
+
file = @source.to_s.sub(/#.*$/, '') # parse "foo.tch#mode=r"
|
20
|
+
Pathname(file)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
# Check ecode and then raise. too boring... The library should implement this as atomic operation!
|
25
|
+
def atomic(&block)
|
26
|
+
raise NotImplementedError, "tc keep ecode until new erros occured"
|
27
|
+
|
28
|
+
if kyoto_error?
|
29
|
+
raise "tc already error before atomic: #{@db.ecode}"
|
30
|
+
end
|
31
|
+
v = block.call
|
32
|
+
kyoto_error! if kyoto_error?
|
33
|
+
return v
|
34
|
+
end
|
35
|
+
|
36
|
+
def kyoto_error!(label = nil)
|
37
|
+
raise Ccp::Kvs::Kyoto::Error, "%s%s (%s)" % [label, error_message, @source]
|
38
|
+
end
|
39
|
+
|
40
|
+
def kyoto_error?
|
41
|
+
@db.error.is_a?(KyotoCabinet::Error::XSUCCESS)
|
42
|
+
end
|
43
|
+
|
44
|
+
def threading_error?
|
45
|
+
false
|
46
|
+
end
|
47
|
+
|
48
|
+
def error_message
|
49
|
+
if @db
|
50
|
+
@db.error
|
51
|
+
else
|
52
|
+
'[Not Initialized]'
|
53
|
+
end
|
54
|
+
rescue Exception => e
|
55
|
+
"[BUG] #{e}"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
module Ccp
|
2
|
+
module Kvs
|
3
|
+
module Kyoto
|
4
|
+
class Cabinet < Base
|
5
|
+
include StateMachine
|
6
|
+
|
7
|
+
def initialize(source)
|
8
|
+
@source = source
|
9
|
+
@db = DB.new
|
10
|
+
end
|
11
|
+
|
12
|
+
######################################################################
|
13
|
+
### kvs
|
14
|
+
|
15
|
+
def get(k)
|
16
|
+
tryR("get")
|
17
|
+
v = @db[k.to_s]
|
18
|
+
if v
|
19
|
+
return decode(v)
|
20
|
+
else
|
21
|
+
if @db.error.is_a?(KyotoCabinet::Error::XNOREC)
|
22
|
+
return nil
|
23
|
+
else
|
24
|
+
kyoto_error!("get(%s): " % k)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def set(k,v)
|
30
|
+
tryW("set")
|
31
|
+
val = encode(v)
|
32
|
+
@db[k.to_s] = val or
|
33
|
+
kyoto_error!("set(%s): " % k)
|
34
|
+
end
|
35
|
+
|
36
|
+
def del(k)
|
37
|
+
tryW("del")
|
38
|
+
v = @db[k.to_s]
|
39
|
+
if v
|
40
|
+
if @db.delete(k.to_s)
|
41
|
+
return decode(v)
|
42
|
+
else
|
43
|
+
kyoto_error!("del(%s): " % k)
|
44
|
+
end
|
45
|
+
else
|
46
|
+
return nil
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def exist?(k)
|
51
|
+
tryR("exist?")
|
52
|
+
return !! @db[k.to_s] # TODO: fast access
|
53
|
+
end
|
54
|
+
|
55
|
+
def count
|
56
|
+
tryR("count")
|
57
|
+
return @db.count
|
58
|
+
end
|
59
|
+
|
60
|
+
######################################################################
|
61
|
+
### bulk operations (not DRY but fast)
|
62
|
+
|
63
|
+
def read
|
64
|
+
tryR("read")
|
65
|
+
hash = {}
|
66
|
+
@db.each do |k, v|
|
67
|
+
v or kyoto_error!("each(%s): " % k)
|
68
|
+
hash[k] = decode(v)
|
69
|
+
end
|
70
|
+
return hash
|
71
|
+
end
|
72
|
+
|
73
|
+
def write(h)
|
74
|
+
tryW("write")
|
75
|
+
h.each_pair do |k,v|
|
76
|
+
val = encode(v)
|
77
|
+
@db[k.to_s] = val or kyoto_error!("write(%s): " % k)
|
78
|
+
end
|
79
|
+
return h
|
80
|
+
end
|
81
|
+
|
82
|
+
######################################################################
|
83
|
+
### iterator
|
84
|
+
|
85
|
+
def each(&block)
|
86
|
+
each_pair(&block)
|
87
|
+
end
|
88
|
+
|
89
|
+
def each_pair(&block)
|
90
|
+
tryR("each_pair")
|
91
|
+
|
92
|
+
# TODO: Waste memory! But kc ignores exceptions in his each block.
|
93
|
+
array = []
|
94
|
+
@db.each{|k, v| array << [k, decode(v)]} or kyoto_error!("each_pair: ")
|
95
|
+
array.each do |a|
|
96
|
+
block.call(a[0], a[1])
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def each_key(&block)
|
101
|
+
tryR("each_key")
|
102
|
+
|
103
|
+
# TODO: Waste memory! But kc ignores exceptions in his each block.
|
104
|
+
array = []
|
105
|
+
@db.each_key{|k| array << k.first} or kyoto_error!("each_key: ")
|
106
|
+
array.each do |k|
|
107
|
+
block.call(k)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def keys
|
112
|
+
tryR("keys")
|
113
|
+
|
114
|
+
array = []
|
115
|
+
each_key do |key|
|
116
|
+
array << key
|
117
|
+
end
|
118
|
+
return array
|
119
|
+
end
|
120
|
+
|
121
|
+
def first_key
|
122
|
+
first.first
|
123
|
+
end
|
124
|
+
|
125
|
+
def first
|
126
|
+
tryR("first")
|
127
|
+
@db.cursor_process {|cur|
|
128
|
+
cur.jump
|
129
|
+
k, v = cur.get(true)
|
130
|
+
return [k, decode(v)]
|
131
|
+
}
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module Ccp
|
2
|
+
module Kvs
|
3
|
+
module Kyoto
|
4
|
+
module StateMachine
|
5
|
+
include KyotoCabinet
|
6
|
+
|
7
|
+
######################################################################
|
8
|
+
### state machine
|
9
|
+
|
10
|
+
CLOSED = 1
|
11
|
+
READABLE = 2
|
12
|
+
WRITABLE = 3
|
13
|
+
|
14
|
+
LOCKED_BY = proc{|c| Array(c).select{|i| i !~ %r{/ruby/[^/]+/gems/}}[0,5].join("\n") || c rescue c}
|
15
|
+
|
16
|
+
def state
|
17
|
+
@state || CLOSED
|
18
|
+
end
|
19
|
+
|
20
|
+
def locker_info
|
21
|
+
if CONNECTIONS[@source]
|
22
|
+
return LOCKED_BY[CONNECTIONS[@source]]
|
23
|
+
end
|
24
|
+
|
25
|
+
target = File.basename(@source)
|
26
|
+
CONNECTIONS.each_pair do |file, reason|
|
27
|
+
return LOCKED_BY[reason] if File.basename(file) == target
|
28
|
+
end
|
29
|
+
|
30
|
+
if CONNECTIONS.any?
|
31
|
+
return CONNECTIONS.inspect
|
32
|
+
else
|
33
|
+
return 'no brockers. maybe locked by other systems?'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def open(mode, locker = nil)
|
38
|
+
Pathname(@source.to_s).parent.mkpath
|
39
|
+
|
40
|
+
# open and mark filename for threading error
|
41
|
+
if @db.open(@source.to_s, mode)
|
42
|
+
locker ||= (caller rescue "???")
|
43
|
+
STDERR.puts "LOCK: #{@source} by [#{LOCKED_BY[locker]}]" if @debug
|
44
|
+
CONNECTIONS[@db.path.to_s] = locker
|
45
|
+
elsif threading_error?
|
46
|
+
raise Kyoto::Locked, "%s is locked by [%s]" % [@source, locker_info]
|
47
|
+
else
|
48
|
+
kyoto_error!("%s#open(%s,%s): " % [self.class, @source, mode])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def __close__(locker = nil)
|
53
|
+
@db.close
|
54
|
+
CONNECTIONS[@db.path] = nil
|
55
|
+
STDERR.puts "UNLOCK: #{@source} by [#{LOCKED_BY[locker || caller]}]" if @debug
|
56
|
+
end
|
57
|
+
|
58
|
+
def close(locker = nil)
|
59
|
+
C!(locker)
|
60
|
+
end
|
61
|
+
|
62
|
+
def C!(locker = nil)
|
63
|
+
case state
|
64
|
+
when CLOSED ; # NOP
|
65
|
+
when READABLE,
|
66
|
+
WRITABLE ; __close__(locker); @state = CLOSED
|
67
|
+
else ; raise "unknown state: #{state}"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def R!(locker = nil)
|
72
|
+
case state
|
73
|
+
when CLOSED ; open(DB::OREADER, locker); @state = READABLE
|
74
|
+
when READABLE ; # NOP
|
75
|
+
when WRITABLE ; # NOP
|
76
|
+
else ; raise "unknown state: #{state}"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def W!(locker = nil)
|
81
|
+
case state
|
82
|
+
when CLOSED ; open(DB::OCREATE | DB::OWRITER, locker); @state = WRITABLE
|
83
|
+
when READABLE ; C!(locker); W!(locker)
|
84
|
+
when WRITABLE ; # NOP
|
85
|
+
else ; raise "unknown state: #{state}"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def R(locker = nil, &block)
|
90
|
+
case state
|
91
|
+
when CLOSED ; begin; R!(locker); yield; ensure; close(locker); end
|
92
|
+
when READABLE ; yield
|
93
|
+
when WRITABLE ; yield
|
94
|
+
else ; raise "unknown state: #{state}"
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def W(locker = nil, &block)
|
99
|
+
case state
|
100
|
+
when CLOSED ; begin; W!(locker); yield; ensure; close(locker); end
|
101
|
+
when READABLE ; raise "reopen from read to write is not permitted"
|
102
|
+
# TODO: close -> W -> close -> R ???
|
103
|
+
when WRITABLE ; yield
|
104
|
+
else ; raise "unknown state: #{state}"
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def touch
|
109
|
+
W() {}
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
def isReadable
|
114
|
+
case state
|
115
|
+
when CLOSED ; false
|
116
|
+
when READABLE ; true
|
117
|
+
when WRITABLE ; true
|
118
|
+
else ; raise "unknown state: #{state}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def isWritable
|
123
|
+
case state
|
124
|
+
when CLOSED ; false
|
125
|
+
when READABLE ; false
|
126
|
+
when WRITABLE ; true
|
127
|
+
else ; raise "unknown state: #{state}"
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def tryR(op)
|
132
|
+
isReadable or raise NotAllowed, "use R! or R{tch.%s} (%s)" % [op, source]
|
133
|
+
end
|
134
|
+
|
135
|
+
def tryW(op)
|
136
|
+
isWritable or raise NotAllowed, "use W! or W{tch.%s} (%s)" % [op, source]
|
137
|
+
end
|
138
|
+
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|