moneta 1.2.0 → 1.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (166) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +418 -0
  3. data/.gitignore +2 -0
  4. data/.rubocop.yml +27 -9
  5. data/CHANGES +28 -0
  6. data/CONTRIBUTORS +4 -2
  7. data/Gemfile +12 -8
  8. data/README.md +20 -18
  9. data/feature_matrix.yaml +2 -11
  10. data/lib/moneta.rb +9 -9
  11. data/lib/moneta/adapters/client.rb +56 -19
  12. data/lib/moneta/adapters/couch.rb +5 -0
  13. data/lib/moneta/adapters/mongo.rb +264 -7
  14. data/lib/moneta/adapters/redis.rb +5 -1
  15. data/lib/moneta/adapters/sequel.rb +45 -464
  16. data/lib/moneta/adapters/sequel/mysql.rb +66 -0
  17. data/lib/moneta/adapters/sequel/postgres.rb +80 -0
  18. data/lib/moneta/adapters/sequel/postgres_hstore.rb +240 -0
  19. data/lib/moneta/adapters/sequel/sqlite.rb +57 -0
  20. data/lib/moneta/adapters/sqlite.rb +7 -7
  21. data/lib/moneta/builder.rb +2 -2
  22. data/lib/moneta/create_support.rb +21 -0
  23. data/lib/moneta/dbm_adapter.rb +31 -0
  24. data/lib/moneta/{mixins.rb → defaults.rb} +1 -302
  25. data/lib/moneta/each_key_support.rb +27 -0
  26. data/lib/moneta/expires_support.rb +60 -0
  27. data/lib/moneta/hash_adapter.rb +68 -0
  28. data/lib/moneta/increment_support.rb +16 -0
  29. data/lib/moneta/lock.rb +6 -1
  30. data/lib/moneta/nil_values.rb +35 -0
  31. data/lib/moneta/option_support.rb +51 -0
  32. data/lib/moneta/pool.rb +38 -6
  33. data/lib/moneta/proxy.rb +1 -1
  34. data/lib/moneta/server.rb +215 -61
  35. data/lib/moneta/shared.rb +13 -7
  36. data/lib/moneta/transformer.rb +50 -8
  37. data/lib/moneta/transformer/config.rb +59 -40
  38. data/lib/moneta/transformer/helper.rb +2 -2
  39. data/lib/moneta/transformer/helper/bson.rb +5 -15
  40. data/lib/moneta/version.rb +1 -1
  41. data/lib/rack/cache/moneta.rb +14 -15
  42. data/moneta.gemspec +14 -8
  43. data/script/benchmarks +7 -3
  44. data/script/contributors +11 -6
  45. data/script/start-couchdb +27 -0
  46. data/script/start-hbase +2 -2
  47. data/script/start-services +3 -3
  48. data/spec/active_support/cache_moneta_store_spec.rb +37 -39
  49. data/spec/features/concurrent_increment.rb +2 -3
  50. data/spec/features/create_expires.rb +15 -15
  51. data/spec/features/default_expires.rb +11 -12
  52. data/spec/features/expires.rb +215 -210
  53. data/spec/features/store.rb +3 -3
  54. data/spec/helper.rb +33 -41
  55. data/spec/moneta/adapters/activerecord/adapter_activerecord_existing_connection_spec.rb +3 -1
  56. data/spec/moneta/adapters/activerecord/adapter_activerecord_spec.rb +15 -7
  57. data/spec/moneta/adapters/activerecord/standard_activerecord_spec.rb +6 -3
  58. data/spec/moneta/adapters/activerecord/standard_activerecord_with_expires_spec.rb +6 -3
  59. data/spec/moneta/adapters/activesupportcache/adapter_activesupportcache_spec.rb +3 -3
  60. data/spec/moneta/adapters/activesupportcache/adapter_activesupportcache_with_default_expires_spec.rb +2 -2
  61. data/spec/moneta/adapters/cassandra/standard_cassandra_spec.rb +1 -1
  62. data/spec/moneta/adapters/client/adapter_client_spec.rb +6 -6
  63. data/spec/moneta/adapters/client/client_helper.rb +24 -0
  64. data/spec/moneta/adapters/client/standard_client_tcp_spec.rb +8 -8
  65. data/spec/moneta/adapters/client/standard_client_unix_spec.rb +23 -7
  66. data/spec/moneta/adapters/couch/adapter_couch_spec.rb +1 -1
  67. data/spec/moneta/adapters/couch/standard_couch_spec.rb +2 -2
  68. data/spec/moneta/adapters/couch/standard_couch_with_expires_spec.rb +2 -2
  69. data/spec/moneta/adapters/datamapper/adapter_datamapper_spec.rb +25 -8
  70. data/spec/moneta/adapters/datamapper/standard_datamapper_spec.rb +2 -2
  71. data/spec/moneta/adapters/datamapper/standard_datamapper_with_expires_spec.rb +2 -2
  72. data/spec/moneta/adapters/datamapper/standard_datamapper_with_repository_spec.rb +2 -2
  73. data/spec/moneta/adapters/daybreak/standard_daybreak_spec.rb +1 -1
  74. data/spec/moneta/adapters/daybreak/standard_daybreak_with_expires_spec.rb +1 -1
  75. data/spec/moneta/adapters/dbm/standard_dbm_spec.rb +1 -1
  76. data/spec/moneta/adapters/dbm/standard_dbm_with_expires_spec.rb +1 -1
  77. data/spec/moneta/adapters/file/standard_file_spec.rb +2 -2
  78. data/spec/moneta/adapters/file/standard_file_with_expires_spec.rb +1 -1
  79. data/spec/moneta/adapters/gdbm/standard_gdbm_spec.rb +1 -1
  80. data/spec/moneta/adapters/gdbm/standard_gdbm_with_expires_spec.rb +1 -1
  81. data/spec/moneta/adapters/kyotocabinet/adapter_kyotocabinet_spec.rb +1 -1
  82. data/spec/moneta/adapters/kyotocabinet/standard_kyotocabinet_spec.rb +2 -2
  83. data/spec/moneta/adapters/kyotocabinet/standard_kyotocabinet_with_expires_spec.rb +2 -2
  84. data/spec/moneta/adapters/leveldb/standard_leveldb_spec.rb +1 -1
  85. data/spec/moneta/adapters/leveldb/standard_leveldb_with_expires_spec.rb +1 -1
  86. data/spec/moneta/adapters/lmdb/standard_lmdb_spec.rb +1 -1
  87. data/spec/moneta/adapters/lmdb/standard_lmdb_with_expires_spec.rb +1 -1
  88. data/spec/moneta/adapters/lruhash/standard_lruhash_spec.rb +1 -1
  89. data/spec/moneta/adapters/lruhash/standard_lruhash_with_expires_spec.rb +1 -1
  90. data/spec/moneta/adapters/memcached/dalli/adapter_memcached_dalli_spec.rb +13 -3
  91. data/spec/moneta/adapters/memcached/native/adapter_memcached_native_spec.rb +13 -3
  92. data/spec/moneta/adapters/memory/standard_memory_spec.rb +1 -1
  93. data/spec/moneta/adapters/memory/standard_memory_with_compress_spec.rb +1 -1
  94. data/spec/moneta/adapters/memory/standard_memory_with_expires_spec.rb +1 -1
  95. data/spec/moneta/adapters/memory/standard_memory_with_json_key_serializer_spec.rb +1 -1
  96. data/spec/moneta/adapters/memory/standard_memory_with_json_serializer_spec.rb +1 -1
  97. data/spec/moneta/adapters/memory/standard_memory_with_json_value_serializer_spec.rb +2 -2
  98. data/spec/moneta/adapters/memory/standard_memory_with_prefix_spec.rb +39 -2
  99. data/spec/moneta/adapters/memory/standard_memory_with_snappy_compress_spec.rb +2 -2
  100. data/spec/moneta/adapters/mongo/adapter_mongo_spec.rb +32 -2
  101. data/spec/moneta/adapters/mongo/adapter_mongo_with_default_expires_spec.rb +6 -4
  102. data/spec/moneta/adapters/mongo/standard_mongo_spec.rb +2 -2
  103. data/spec/moneta/adapters/pstore/standard_pstore_spec.rb +1 -1
  104. data/spec/moneta/adapters/pstore/standard_pstore_with_expires_spec.rb +1 -1
  105. data/spec/moneta/adapters/redis/adapter_redis_spec.rb +13 -3
  106. data/spec/moneta/adapters/redis/standard_redis_spec.rb +9 -2
  107. data/spec/moneta/adapters/sdbm/standard_sdbm_spec.rb +1 -1
  108. data/spec/moneta/adapters/sdbm/standard_sdbm_with_expires_spec.rb +1 -1
  109. data/spec/moneta/adapters/sequel/adapter_sequel_spec.rb +11 -38
  110. data/spec/moneta/adapters/sequel/helper.rb +42 -0
  111. data/spec/moneta/adapters/sequel/standard_sequel_spec.rb +6 -12
  112. data/spec/moneta/adapters/sequel/standard_sequel_with_expires_spec.rb +9 -10
  113. data/spec/moneta/adapters/sqlite/adapter_sqlite_spec.rb +1 -1
  114. data/spec/moneta/adapters/sqlite/standard_sqlite_spec.rb +2 -2
  115. data/spec/moneta/adapters/sqlite/standard_sqlite_with_expires_spec.rb +2 -2
  116. data/spec/moneta/adapters/tdb/standard_tdb_spec.rb +1 -1
  117. data/spec/moneta/adapters/tdb/standard_tdb_with_expires_spec.rb +1 -1
  118. data/spec/moneta/adapters/tokyocabinet/standard_tokyocabinet_spec.rb +1 -1
  119. data/spec/moneta/adapters/tokyocabinet/standard_tokyocabinet_with_expires_spec.rb +1 -1
  120. data/spec/moneta/adapters/yaml/standard_yaml_spec.rb +1 -1
  121. data/spec/moneta/adapters/yaml/standard_yaml_with_expires_spec.rb +1 -1
  122. data/spec/moneta/builder_spec.rb +22 -0
  123. data/spec/moneta/proxies/expires/expires_file_spec.rb +1 -1
  124. data/spec/moneta/proxies/pool/pool_spec.rb +31 -3
  125. data/spec/moneta/proxies/shared/shared_tcp_spec.rb +14 -4
  126. data/spec/moneta/proxies/shared/shared_unix_spec.rb +14 -4
  127. data/spec/moneta/proxies/transformer/transformer_bencode_spec.rb +1 -1
  128. data/spec/moneta/proxies/transformer/transformer_bert_spec.rb +3 -3
  129. data/spec/moneta/proxies/transformer/transformer_bson_spec.rb +2 -2
  130. data/spec/moneta/proxies/transformer/transformer_json_spec.rb +1 -1
  131. data/spec/moneta/proxies/transformer/transformer_key_marshal_spec.rb +1 -1
  132. data/spec/moneta/proxies/transformer/transformer_key_yaml_spec.rb +1 -1
  133. data/spec/moneta/proxies/transformer/transformer_marshal_base64_spec.rb +1 -1
  134. data/spec/moneta/proxies/transformer/transformer_marshal_escape_spec.rb +8 -4
  135. data/spec/moneta/proxies/transformer/transformer_marshal_hex_spec.rb +1 -1
  136. data/spec/moneta/proxies/transformer/transformer_marshal_hmac_spec.rb +1 -1
  137. data/spec/moneta/proxies/transformer/transformer_marshal_prefix_base64_spec.rb +33 -0
  138. data/spec/moneta/proxies/transformer/transformer_marshal_prefix_spec.rb +1 -1
  139. data/spec/moneta/proxies/transformer/transformer_marshal_qp_spec.rb +1 -1
  140. data/spec/moneta/proxies/transformer/transformer_marshal_spec.rb +1 -1
  141. data/spec/moneta/proxies/transformer/transformer_marshal_urlsafe_base64_spec.rb +1 -1
  142. data/spec/moneta/proxies/transformer/transformer_marshal_uuencode_spec.rb +1 -1
  143. data/spec/moneta/proxies/transformer/transformer_msgpack_spec.rb +1 -1
  144. data/spec/moneta/proxies/transformer/transformer_ox_spec.rb +1 -1
  145. data/spec/moneta/proxies/transformer/transformer_php_spec.rb +1 -1
  146. data/spec/moneta/proxies/transformer/transformer_tnet_spec.rb +1 -1
  147. data/spec/moneta/proxies/transformer/transformer_yaml_spec.rb +2 -2
  148. data/spec/moneta/proxies/weak_each_key/weak_each_key_spec.rb +0 -2
  149. data/spec/restserver.rb +15 -0
  150. metadata +47 -66
  151. data/.travis.yml +0 -140
  152. data/lib/moneta/adapters/mongo/base.rb +0 -103
  153. data/lib/moneta/adapters/mongo/moped.rb +0 -163
  154. data/lib/moneta/adapters/mongo/official.rb +0 -156
  155. data/script/reconfigure-couchdb +0 -13
  156. data/spec/moneta/adapters/memcached/dalli/adapter_memcached_dalli_with_default_expires_spec.rb +0 -15
  157. data/spec/moneta/adapters/memcached/native/adapter_memcached_native_with_default_expires_spec.rb +0 -15
  158. data/spec/moneta/adapters/mongo/adapter_mongo_moped_spec.rb +0 -25
  159. data/spec/moneta/adapters/mongo/adapter_mongo_moped_with_default_expires_spec.rb +0 -12
  160. data/spec/moneta/adapters/mongo/adapter_mongo_official_spec.rb +0 -25
  161. data/spec/moneta/adapters/mongo/adapter_mongo_official_with_default_expires_spec.rb +0 -12
  162. data/spec/moneta/adapters/mongo/standard_mongo_moped_spec.rb +0 -7
  163. data/spec/moneta/adapters/mongo/standard_mongo_official_spec.rb +0 -7
  164. data/spec/moneta/adapters/redis/adapter_redis_with_default_expires_spec.rb +0 -10
  165. data/spec/moneta/proxies/proxy/proxy_redis_spec.rb +0 -13
  166. data/spec/quality_spec.rb +0 -51
@@ -1,12 +1,269 @@
1
+ require 'mongo'
2
+
1
3
  module Moneta
2
- # @api private
3
4
  module Adapters
4
- begin
5
- require 'moneta/adapters/mongo/official'
6
- Mongo = MongoOfficial
7
- rescue LoadError
8
- require 'moneta/adapters/mongo/moped'
9
- Mongo = MongoMoped
5
+ # MongoDB backend
6
+ #
7
+ # Supports expiration, documents will be automatically removed starting
8
+ # with mongodb >= 2.2 (see {http://docs.mongodb.org/manual/tutorial/expire-data/}).
9
+ #
10
+ # You can store hashes directly using this adapter.
11
+ #
12
+ # @example Store hashes
13
+ # db = Moneta::Adapters::MongoOfficial.new
14
+ # db['key'] = {a: 1, b: 2}
15
+ #
16
+ # @api public
17
+ class Mongo
18
+ include Defaults
19
+ include ExpiresSupport
20
+
21
+ supports :each_key, :create, :increment
22
+ attr_reader :backend
23
+
24
+ DEFAULT_PORT = 27017
25
+
26
+ # @param [Hash] options
27
+ # @option options [String] :collection ('moneta') MongoDB collection name
28
+ # @option options [String] :host ('127.0.0.1') MongoDB server host
29
+ # @option options [String] :user Username used to authenticate
30
+ # @option options [String] :password Password used to authenticate
31
+ # @option options [Integer] :port (MongoDB default port) MongoDB server port
32
+ # @option options [String] :database ('moneta') MongoDB database
33
+ # @option options [Integer] :expires Default expiration time
34
+ # @option options [String] :expires_field ('expiresAt') Document field to store expiration time
35
+ # @option options [String] :value_field ('value') Document field to store value
36
+ # @option options [String] :type_field ('type') Document field to store value type
37
+ # @option options [::Mongo::Client] :backend Use existing backend instance
38
+ # @option options Other options passed to `Mongo::MongoClient#new`
39
+ def initialize(options = {})
40
+ self.default_expires = options.delete(:expires)
41
+ @expires_field = options.delete(:expires_field) || 'expiresAt'
42
+ @value_field = options.delete(:value_field) || 'value'
43
+ @type_field = options.delete(:type_field) || 'type'
44
+
45
+ collection = options.delete(:collection) || 'moneta'
46
+
47
+ if options.key?(:db)
48
+ warn('Moneta::Adapters::Mongo - the :db option is deprecated and will be removed in a future version. Use :database instead')
49
+ end
50
+ database = options.delete(:database) || options.delete(:db) || 'moneta'
51
+ backend = options[:backend] ||
52
+ begin
53
+ host = options.delete(:host) || '127.0.0.1'
54
+ port = options.delete(:port) || DEFAULT_PORT
55
+ options[:logger] ||= ::Logger.new(STDERR).tap do |logger|
56
+ logger.level = ::Logger::ERROR
57
+ end
58
+ ::Mongo::Client.new(["#{host}:#{port}"], options)
59
+ end
60
+
61
+ @backend = backend.use(database)
62
+ @collection = @backend[collection]
63
+ if @backend.command(buildinfo: 1).documents.first['version'] >= '2.2'
64
+ @collection.indexes.create_one({ @expires_field => 1 }, expire_after: 0)
65
+ else
66
+ warn 'Moneta::Adapters::Mongo - You are using MongoDB version < 2.2, expired documents will not be deleted'
67
+ end
68
+ end
69
+
70
+ # (see Proxy#load)
71
+ def load(key, options = {})
72
+ view = @collection.find(:$and => [
73
+ { _id: to_binary(key) },
74
+ not_expired
75
+ ])
76
+
77
+ doc = view.limit(1).first
78
+
79
+ if doc
80
+ update_expiry(options, nil) do |expires|
81
+ view.update_one(:$set => { @expires_field => expires })
82
+ end
83
+
84
+ doc_to_value(doc)
85
+ end
86
+ end
87
+
88
+ # (see Proxy#store)
89
+ def store(key, value, options = {})
90
+ key = to_binary(key)
91
+ @collection.replace_one({ _id: key },
92
+ value_to_doc(key, value, options),
93
+ upsert: true)
94
+ value
95
+ end
96
+
97
+ # (see Proxy#each_key)
98
+ def each_key
99
+ return enum_for(:each_key) unless block_given?
100
+ @collection.find.each { |doc| yield from_binary(doc[:_id]) }
101
+ self
102
+ end
103
+
104
+ # (see Proxy#delete)
105
+ def delete(key, options = {})
106
+ key = to_binary(key)
107
+ if doc = @collection.find(_id: key).find_one_and_delete and
108
+ !doc[@expires_field] || doc[@expires_field] >= Time.now
109
+ doc_to_value(doc)
110
+ end
111
+ end
112
+
113
+ # (see Proxy#increment)
114
+ def increment(key, amount = 1, options = {})
115
+ @collection.find_one_and_update({ :$and => [{ _id: to_binary(key) }, not_expired] },
116
+ { :$inc => { @value_field => amount } },
117
+ return_document: :after,
118
+ upsert: true)[@value_field]
119
+ rescue ::Mongo::Error::OperationFailure
120
+ tries ||= 0
121
+ (tries += 1) < 3 ? retry : raise
122
+ end
123
+
124
+ # (see Proxy#create)
125
+ def create(key, value, options = {})
126
+ key = to_binary(key)
127
+ @collection.insert_one(value_to_doc(key, value, options))
128
+ true
129
+ rescue ::Mongo::Error::OperationFailure => ex
130
+ raise unless ex.message =~ /^E11000 / # duplicate key error
131
+ false
132
+ end
133
+
134
+ # (see Proxy#clear)
135
+ def clear(options = {})
136
+ @collection.delete_many
137
+ self
138
+ end
139
+
140
+ # (see Proxy#close)
141
+ def close
142
+ @backend.close
143
+ nil
144
+ end
145
+
146
+ # (see Proxy#slice)
147
+ def slice(*keys, **options)
148
+ view = @collection.find(:$and => [
149
+ { _id: { :$in => keys.map(&method(:to_binary)) } },
150
+ not_expired
151
+ ])
152
+ pairs = view.map { |doc| [from_binary(doc[:_id]), doc_to_value(doc)] }
153
+
154
+ update_expiry(options, nil) do |expires|
155
+ view.update_many(:$set => { @expires_field => expires })
156
+ end
157
+
158
+ pairs
159
+ end
160
+
161
+ # (see Proxy#merge!)
162
+ def merge!(pairs, options = {})
163
+ existing = Hash[slice(*pairs.map { |key, _| key })]
164
+ update_pairs, insert_pairs = pairs.partition { |key, _| existing.key?(key) }
165
+
166
+ @collection.insert_many(insert_pairs.map do |key, value|
167
+ value_to_doc(to_binary(key), value, options)
168
+ end)
169
+
170
+ update_pairs.each do |key, value|
171
+ value = yield(key, existing[key], value) if block_given?
172
+ binary = to_binary(key)
173
+ @collection.replace_one({ _id: binary }, value_to_doc(binary, value, options))
174
+ end
175
+
176
+ self
177
+ end
178
+
179
+ # (see Proxy#fetch_values)
180
+ def fetch_values(*keys, **options)
181
+ return values_at(*keys, **options) unless block_given?
182
+ hash = Hash[slice(*keys, **options)]
183
+ keys.map do |key|
184
+ if hash.key?(key)
185
+ hash[key]
186
+ else
187
+ yield key
188
+ end
189
+ end
190
+ end
191
+
192
+ # (see Proxy#values_at)
193
+ def values_at(*keys, **options)
194
+ hash = Hash[slice(*keys, **options)]
195
+ keys.map { |key| hash[key] }
196
+ end
197
+
198
+ private
199
+
200
+ def doc_to_value(doc)
201
+ case doc[@type_field]
202
+ when 'Hash'
203
+ doc = doc.dup
204
+ doc.delete('_id')
205
+ doc.delete(@type_field)
206
+ doc.delete(@expires_field)
207
+ doc
208
+ when 'Number'
209
+ doc[@value_field]
210
+ else
211
+ # In ruby_bson version 2 (and probably up), #to_s no longer returns the binary data
212
+ from_binary(doc[@value_field])
213
+ end
214
+ end
215
+
216
+ def value_to_doc(key, value, options)
217
+ case value
218
+ when Hash
219
+ value.merge('_id' => key,
220
+ @type_field => 'Hash',
221
+ # @expires_field must be a Time object (BSON date datatype)
222
+ @expires_field => expires_at(options) || nil)
223
+ when Float, Integer
224
+ { '_id' => key,
225
+ @type_field => 'Number',
226
+ @value_field => value,
227
+ # @expires_field must be a Time object (BSON date datatype)
228
+ @expires_field => expires_at(options) || nil }
229
+ when String
230
+ intvalue = value.to_i
231
+ { '_id' => key,
232
+ @type_field => 'String',
233
+ @value_field => intvalue.to_s == value ? intvalue : to_binary(value),
234
+ # @expires_field must be a Time object (BSON date datatype)
235
+ @expires_field => expires_at(options) || nil }
236
+ else
237
+ raise ArgumentError, "Invalid value type: #{value.class}"
238
+ end
239
+ end
240
+
241
+ # BSON will use String#force_encoding to make the string 8-bit
242
+ # ASCII. This could break unicode text so we should dup in this
243
+ # case, and it also fails with frozen strings.
244
+ def to_binary(str)
245
+ str = str.dup if str.frozen? || str.encoding != Encoding::ASCII_8BIT
246
+ ::BSON::Binary.new(str)
247
+ end
248
+
249
+ def from_binary(binary)
250
+ binary.is_a?(::BSON::Binary) ? binary.data : binary.to_s
251
+ end
252
+
253
+ def not_expired
254
+ {
255
+ :$or => [
256
+ { @expires_field => nil },
257
+ { @expires_field => { :$gte => Time.now } }
258
+ ]
259
+ }
260
+ end
261
+
262
+ def update_expiry(options, default)
263
+ if (expires = expires_at(options, default)) != nil
264
+ yield(expires || nil)
265
+ end
266
+ end
10
267
  end
11
268
  end
12
269
  end
@@ -26,7 +26,11 @@ module Moneta
26
26
  # number as a time to live in seconds.
27
27
  def key?(key, options = {})
28
28
  with_expiry_update(key, default: nil, **options) do
29
- @backend.exists(key)
29
+ if @backend.respond_to?(:exists?)
30
+ @backend.exists?(key)
31
+ else
32
+ @backend.exists(key)
33
+ end
30
34
  end
31
35
  end
32
36
 
@@ -7,6 +7,11 @@ module Moneta
7
7
  class Sequel
8
8
  include Defaults
9
9
 
10
+ autoload :MySQL, 'moneta/adapters/sequel/mysql'
11
+ autoload :Postgres, 'moneta/adapters/sequel/postgres'
12
+ autoload :PostgresHStore, 'moneta/adapters/sequel/postgres_hstore'
13
+ autoload :SQLite, 'moneta/adapters/sequel/sqlite'
14
+
10
15
  supports :create, :increment, :each_key
11
16
  attr_reader :backend, :key_column, :value_column
12
17
 
@@ -31,51 +36,20 @@ module Moneta
31
36
  # possible to specify a separate connection to use for `#each_key`. Use
32
37
  # in conjunction with Sequel's `:servers` option
33
38
  # @option options All other options passed to `Sequel#connect`
34
- def self.new(options = {})
39
+ def initialize(options = {})
35
40
  extensions = options.delete(:extensions)
36
41
  connection_validation_timeout = options.delete(:connection_validation_timeout)
37
42
  optimize = options.delete(:optimize)
38
- backend = options.delete(:backend) ||
39
- begin
40
- raise ArgumentError, 'Option :db is required' unless db = options.delete(:db)
41
- other_cols = [:table, :create_table, :key_column, :value_column, :hstore]
42
- ::Sequel.connect(db, options.reject { |k,| other_cols.member?(k) }).tap do |backend|
43
- if extensions
44
- raise ArgumentError, 'Option :extensions must be an Array' unless extensions.is_a?(Array)
45
- extensions.map(&:to_sym).each(&backend.method(:extension))
46
- end
47
-
48
- if connection_validation_timeout
49
- backend.pool.connection_validation_timeout = connection_validation_timeout
50
- end
51
- end
52
- end
43
+ @backend = options.delete(:backend) ||
44
+ connect(extensions: extensions, connection_validation_timeout: connection_validation_timeout, **options)
53
45
 
54
- instance =
55
- if optimize == nil || optimize
56
- case backend.database_type
57
- when :mysql
58
- MySQL.allocate
59
- when :postgres
60
- if options[:hstore]
61
- PostgresHStore.allocate
62
- elsif matches = backend.get(::Sequel[:version].function).match(/PostgreSQL (\d+)\.(\d+)/)
63
- # Our optimisations only work on Postgres 9.5+
64
- major, minor = matches[1..2].map(&:to_i)
65
- Postgres.allocate if major > 9 || (major == 9 && minor >= 5)
66
- end
67
- when :sqlite
68
- SQLite.allocate
69
- end
70
- end || allocate
71
-
72
- instance.instance_variable_set(:@backend, backend)
73
- instance.send(:initialize, options)
74
- instance
75
- end
46
+ if hstore = options.delete(:hstore)
47
+ @row = hstore.to_s
48
+ extend Sequel::PostgresHStore
49
+ elsif optimize == nil || optimize
50
+ add_optimizations
51
+ end
76
52
 
77
- # @api private
78
- def initialize(options)
79
53
  @table_name = (options.delete(:table) || :moneta).to_sym
80
54
  @key_column = options.delete(:key_column) || :k
81
55
  @value_column = options.delete(:value_column) || :v
@@ -227,6 +201,37 @@ module Moneta
227
201
 
228
202
  protected
229
203
 
204
+ # @api private
205
+ def connect(db:, extensions: nil, connection_validation_timeout: nil, **options)
206
+ other_cols = [:table, :create_table, :key_column, :value_column, :hstore]
207
+ ::Sequel.connect(db, options.reject { |k,| other_cols.member?(k) }).tap do |backend|
208
+ if extensions
209
+ raise ArgumentError, 'Option :extensions must be an Array' unless extensions.is_a?(Array)
210
+ extensions.map(&:to_sym).each(&backend.method(:extension))
211
+ end
212
+
213
+ if connection_validation_timeout
214
+ backend.pool.connection_validation_timeout = connection_validation_timeout
215
+ end
216
+ end
217
+ end
218
+
219
+ # @api private
220
+ def add_optimizations
221
+ case backend.database_type
222
+ when :mysql
223
+ extend Sequel::MySQL
224
+ when :postgres
225
+ if matches = backend.get(::Sequel[:version].function).match(/PostgreSQL (\d+)\.(\d+)/)
226
+ # Our optimisations only work on Postgres 9.5+
227
+ major, minor = matches[1..2].map(&:to_i)
228
+ extend Sequel::Postgres if major > 9 || (major == 9 && minor >= 5)
229
+ end
230
+ when :sqlite
231
+ extend Sequel::SQLite
232
+ end
233
+ end
234
+
230
235
  def blob(str)
231
236
  ::Sequel.blob(str) unless str == nil
232
237
  end
@@ -324,430 +329,6 @@ module Moneta
324
329
 
325
330
  # @api private
326
331
  class IncrementError < ::Sequel::DatabaseError; end
327
-
328
- # @api private
329
- class MySQL < Sequel
330
- def store(key, value, options = {})
331
- @store.call(key: key, value: blob(value))
332
- value
333
- end
334
-
335
- def increment(key, amount = 1, options = {})
336
- @backend.transaction do
337
- # this creates a row-level lock even if there is no existing row (a
338
- # "gap lock").
339
- if row = @load_for_update.call(key: key)
340
- # Integer() will raise an exception if the existing value cannot be parsed
341
- amount += Integer(row[value_column])
342
- @increment_update.call(key: key, value: amount)
343
- else
344
- @create.call(key: key, value: amount)
345
- end
346
- amount
347
- end
348
- rescue ::Sequel::SerializationFailure # Thrown on deadlock
349
- tries ||= 0
350
- (tries += 1) <= 3 ? retry : raise
351
- end
352
-
353
- def merge!(pairs, options = {}, &block)
354
- @backend.transaction do
355
- pairs = yield_merge_pairs(pairs, &block) if block_given?
356
- @table
357
- .on_duplicate_key_update
358
- .import([key_column, value_column], blob_pairs(pairs).to_a)
359
- end
360
-
361
- self
362
- end
363
-
364
- def each_key
365
- return super unless block_given? && @each_key_server && @table.respond_to?(:stream)
366
- # Order is not required when streaming
367
- @table.server(@each_key_server).select(key_column).paged_each do |row|
368
- yield row[key_column]
369
- end
370
- self
371
- end
372
-
373
- protected
374
-
375
- def prepare_store
376
- @store = @table
377
- .on_duplicate_key_update
378
- .prepare(:insert, statement_id(:store), key_column => :$key, value_column => :$value)
379
- end
380
-
381
- def prepare_increment
382
- @increment_update = @table
383
- .where(key_column => :$key)
384
- .prepare(:update, statement_id(:increment_update), value_column => :$value)
385
- super
386
- end
387
- end
388
-
389
- # @api private
390
- class Postgres < Sequel
391
- def store(key, value, options = {})
392
- @store.call(key: key, value: blob(value))
393
- value
394
- end
395
-
396
- def increment(key, amount = 1, options = {})
397
- result = @increment.call(key: key, value: blob(amount.to_s), amount: amount)
398
- if row = result.first
399
- row[value_column].to_i
400
- end
401
- end
402
-
403
- def delete(key, options = {})
404
- result = @delete.call(key: key)
405
- if row = result.first
406
- row[value_column]
407
- end
408
- end
409
-
410
- def merge!(pairs, options = {}, &block)
411
- @backend.transaction do
412
- pairs = yield_merge_pairs(pairs, &block) if block_given?
413
- @table
414
- .insert_conflict(target: key_column,
415
- update: { value_column => ::Sequel[:excluded][value_column] })
416
- .import([key_column, value_column], blob_pairs(pairs).to_a)
417
- end
418
-
419
- self
420
- end
421
-
422
- def each_key
423
- return super unless block_given? && !@each_key_server && @table.respond_to?(:use_cursor)
424
- # With a cursor, this will Just Work.
425
- @table.select(key_column).paged_each do |row|
426
- yield row[key_column]
427
- end
428
- self
429
- end
430
-
431
- protected
432
-
433
- def prepare_store
434
- @store = @table
435
- .insert_conflict(target: key_column,
436
- update: { value_column => ::Sequel[:excluded][value_column] })
437
- .prepare(:insert, statement_id(:store), key_column => :$key, value_column => :$value)
438
- end
439
-
440
- def prepare_increment
441
- update_expr = ::Sequel[:convert_to].function(
442
- (::Sequel[:convert_from].function(
443
- ::Sequel[@table_name][value_column],
444
- 'UTF8'
445
- ).cast(Integer) + :$amount).cast(String),
446
- 'UTF8'
447
- )
448
-
449
- @increment = @table
450
- .returning(value_column)
451
- .insert_conflict(target: key_column, update: { value_column => update_expr })
452
- .prepare(:insert, statement_id(:increment), key_column => :$key, value_column => :$value)
453
- end
454
-
455
- def prepare_delete
456
- @delete = @table
457
- .returning(value_column)
458
- .where(key_column => :$key)
459
- .prepare(:delete, statement_id(:delete))
460
- end
461
- end
462
-
463
- # @api private
464
- class PostgresHStore < Sequel
465
- def initialize(options)
466
- @row = options.delete(:hstore).to_s
467
- @backend.extension :pg_hstore
468
- ::Sequel.extension :pg_hstore_ops
469
- @backend.extension :pg_array
470
- super
471
- end
472
-
473
- def key?(key, options = {})
474
- if @key
475
- row = @key.call(row: @row, key: key) || false
476
- row && row[:present]
477
- else
478
- @key_pl.get(key)
479
- end
480
- end
481
-
482
- def store(key, value, options = {})
483
- @backend.transaction do
484
- create_row
485
- @store.call(row: @row, pair: ::Sequel.hstore(key => value))
486
- end
487
- value
488
- end
489
-
490
- def load(key, options = {})
491
- if row = @load.call(row: @row, key: key)
492
- row[:value]
493
- end
494
- end
495
-
496
- def delete(key, options = {})
497
- @backend.transaction do
498
- value = load(key, options)
499
- @delete.call(row: @row, key: key)
500
- value
501
- end
502
- end
503
-
504
- def increment(key, amount = 1, options = {})
505
- @backend.transaction do
506
- create_row
507
- if row = @increment.call(row: @row, key: key, amount: amount).first
508
- row[:value].to_i
509
- end
510
- end
511
- end
512
-
513
- def create(key, value, options = {})
514
- @backend.transaction do
515
- create_row
516
- 1 ==
517
- if @create
518
- @create.call(row: @row, key: key, pair: ::Sequel.hstore(key => value))
519
- else
520
- @table
521
- .where(key_column => @row)
522
- .exclude(::Sequel[value_column].hstore.key?(key))
523
- .update(value_column => ::Sequel[value_column].hstore.merge(key => value))
524
- end
525
- end
526
- end
527
-
528
- def clear(options = {})
529
- @clear.call(row: @row)
530
- self
531
- end
532
-
533
- def values_at(*keys, **options)
534
- if row = @values_at.call(row: @row, keys: ::Sequel.pg_array(keys))
535
- row[:values].to_a
536
- else
537
- []
538
- end
539
- end
540
-
541
- def slice(*keys, **options)
542
- if row = @slice.call(row: @row, keys: ::Sequel.pg_array(keys))
543
- row[:pairs].to_h
544
- else
545
- []
546
- end
547
- end
548
-
549
- def merge!(pairs, options = {}, &block)
550
- @backend.transaction do
551
- create_row
552
- pairs = yield_merge_pairs(pairs, &block) if block_given?
553
- hash = Hash === pairs ? pairs : Hash[pairs.to_a]
554
- @store.call(row: @row, pair: ::Sequel.hstore(hash))
555
- end
556
-
557
- self
558
- end
559
-
560
- def each_key
561
- return enum_for(:each_key) { @size.call(row: @row)[:size] } unless block_given?
562
-
563
- ds =
564
- if @each_key_server
565
- @table.server(@each_key_server)
566
- else
567
- @table
568
- end
569
- ds = ds.order(:skeys) unless @table.respond_to?(:use_cursor)
570
- ds.where(key_column => @row)
571
- .select(::Sequel[value_column].hstore.skeys)
572
- .paged_each do |row|
573
- yield row[:skeys]
574
- end
575
- self
576
- end
577
-
578
- protected
579
-
580
- def create_row
581
- @create_row.call(row: @row)
582
- end
583
-
584
- def create_table
585
- key_column = self.key_column
586
- value_column = self.value_column
587
-
588
- @backend.create_table?(@table_name) do
589
- column key_column, String, null: false, primary_key: true
590
- column value_column, :hstore
591
- index value_column, type: :gin
592
- end
593
- end
594
-
595
- def slice_for_update(pairs)
596
- keys = pairs.map { |k, _| k }.to_a
597
- if row = @slice_for_update.call(row: @row, keys: ::Sequel.pg_array(keys))
598
- row[:pairs].to_h
599
- else
600
- {}
601
- end
602
- end
603
-
604
- def prepare_statements
605
- super
606
- prepare_create_row
607
- prepare_clear
608
- prepare_values_at
609
- prepare_size
610
- end
611
-
612
- def prepare_create_row
613
- @create_row = @table
614
- .insert_ignore
615
- .prepare(:insert, statement_id(:hstore_create_row), key_column => :$row, value_column => '')
616
- end
617
-
618
- def prepare_clear
619
- @clear = @table.where(key_column => :$row).prepare(:update, statement_id(:hstore_clear), value_column => '')
620
- end
621
-
622
- def prepare_key
623
- if defined?(JRUBY_VERSION)
624
- @key_pl = ::Sequel::Dataset::PlaceholderLiteralizer.loader(@table) do |pl, ds|
625
- ds.where(key_column => @row).select(::Sequel[value_column].hstore.key?(pl.arg))
626
- end
627
- else
628
- @key = @table.where(key_column => :$row)
629
- .select(::Sequel[value_column].hstore.key?(:$key).as(:present))
630
- .prepare(:first, statement_id(:hstore_key))
631
- end
632
- end
633
-
634
- def prepare_store
635
- @store = @table
636
- .where(key_column => :$row)
637
- .prepare(:update, statement_id(:hstore_store), value_column => ::Sequel[value_column].hstore.merge(:$pair))
638
- end
639
-
640
- def prepare_increment
641
- pair = ::Sequel[:hstore]
642
- .function(:$key, (
643
- ::Sequel[:coalesce].function(::Sequel[value_column].hstore[:$key].cast(Integer), 0) +
644
- :$amount
645
- ).cast(String))
646
-
647
- @increment = @table
648
- .returning(::Sequel[value_column].hstore[:$key].as(:value))
649
- .where(key_column => :$row)
650
- .prepare(:update, statement_id(:hstore_increment), value_column => ::Sequel.join([value_column, pair]))
651
- end
652
-
653
- def prepare_load
654
- @load = @table.where(key_column => :$row)
655
- .select(::Sequel[value_column].hstore[:$key].as(:value))
656
- .prepare(:first, statement_id(:hstore_load))
657
- end
658
-
659
- def prepare_delete
660
- @delete = @table.where(key_column => :$row)
661
- .prepare(:update, statement_id(:hstore_delete), value_column => ::Sequel[value_column].hstore.delete(:$key))
662
- end
663
-
664
- def prepare_create
665
- # Under JRuby we can't use a prepared statement for queries involving
666
- # the hstore `?` (key?) operator. See
667
- # https://stackoverflow.com/questions/11940401/escaping-hstore-contains-operators-in-a-jdbc-prepared-statement
668
- return if defined?(JRUBY_VERSION)
669
- @create = @table
670
- .where(key_column => :$row)
671
- .exclude(::Sequel[value_column].hstore.key?(:$key))
672
- .prepare(:update, statement_id(:hstore_create), value_column => ::Sequel[value_column].hstore.merge(:$pair))
673
- end
674
-
675
- def prepare_values_at
676
- @values_at = @table
677
- .where(key_column => :$row)
678
- .select(::Sequel[value_column].hstore[::Sequel.cast(:$keys, :"text[]")].as(:values))
679
- .prepare(:first, statement_id(:hstore_values_at))
680
- end
681
-
682
- def prepare_slice
683
- slice = @table
684
- .where(key_column => :$row)
685
- .select(::Sequel[value_column].hstore.slice(:$keys).as(:pairs))
686
- @slice = slice.prepare(:first, statement_id(:hstore_slice))
687
- @slice_for_update = slice.for_update.prepare(:first, statement_id(:hstore_slice_for_update))
688
- end
689
-
690
- def prepare_size
691
- @size = @backend
692
- .from(@table.where(key_column => :$row)
693
- .select(::Sequel[value_column].hstore.each))
694
- .select { count.function.*.as(:size) }
695
- .prepare(:first, statement_id(:hstore_size))
696
- end
697
- end
698
-
699
- # @api private
700
- class SQLite < Sequel
701
- def initialize(options)
702
- @version = backend.get(::Sequel[:sqlite_version].function)
703
- # See https://sqlite.org/lang_UPSERT.html
704
- @can_upsert = ::Gem::Version.new(@version) >= ::Gem::Version.new('3.24.0')
705
- super
706
- end
707
-
708
- def store(key, value, options = {})
709
- @table.insert_conflict(:replace).insert(key_column => key, value_column => blob(value))
710
- value
711
- end
712
-
713
- def increment(key, amount = 1, options = {})
714
- return super unless @can_upsert
715
- @backend.transaction do
716
- @increment.call(key: key, value: blob(amount.to_s), amount: amount)
717
- Integer(load(key))
718
- end
719
- end
720
-
721
- def merge!(pairs, options = {}, &block)
722
- @backend.transaction do
723
- pairs = yield_merge_pairs(pairs, &block) if block_given?
724
- @table.insert_conflict(:replace).import([key_column, value_column], blob_pairs(pairs).to_a)
725
- end
726
-
727
- self
728
- end
729
-
730
- protected
731
-
732
- def prepare_store
733
- @store = @table
734
- .insert_conflict(:replace)
735
- .prepare(:insert, statement_id(:store), key_column => :$key, value_column => :$value)
736
- end
737
-
738
- def prepare_increment
739
- return super unless @can_upsert
740
- update_expr = (::Sequel[value_column].cast(Integer) + :$amount).cast(:blob)
741
- @increment = @table
742
- .insert_conflict(
743
- target: key_column,
744
- update: { value_column => update_expr },
745
- update_where: ::Sequel.|({ value_column => blob("0") },
746
- { ::Sequel.~(::Sequel[value_column].cast(Integer)) => 0 })
747
- )
748
- .prepare(:insert, statement_id(:increment), key_column => :$key, value_column => :$value)
749
- end
750
- end
751
332
  end
752
333
  end
753
334
  end