dynamoid 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +34 -0
- data/README.md +99 -8
- data/dynamoid.gemspec +1 -0
- data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb +80 -172
- data/lib/dynamoid/adapter_plugin/query.rb +144 -0
- data/lib/dynamoid/adapter_plugin/scan.rb +107 -0
- data/lib/dynamoid/components.rb +1 -1
- data/lib/dynamoid/config.rb +6 -1
- data/lib/dynamoid/criteria/chain.rb +19 -2
- data/lib/dynamoid/document.rb +32 -1
- data/lib/dynamoid/dumping.rb +128 -2
- data/lib/dynamoid/fields.rb +20 -18
- data/lib/dynamoid/finders.rb +15 -6
- data/lib/dynamoid/persistence.rb +61 -1
- data/lib/dynamoid/tasks/database.rake +1 -1
- data/lib/dynamoid/tasks/database.rb +1 -1
- data/lib/dynamoid/type_casting.rb +80 -0
- data/lib/dynamoid/undumping.rb +88 -10
- data/lib/dynamoid/version.rb +1 -1
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ec92a10397fa1be691eaebe98659952c7909ab97
|
4
|
+
data.tar.gz: 766016509f22d0ce3f4fe93ca68342ed6038d0b9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b0de822f2cda614a74a933c714072b97525a7b1a03444f2b12034327a8769d61e13220eb9d0b0f7ec15b2fbc2bf244574efe45a0660fac26cc118bb84fe848e
|
7
|
+
data.tar.gz: d6f3287d3741c21167b521d35771a4d794a9dff90047f9b46ebaa09373c8776389d4afa161c1430559b6fda4e0b9728f22a0a0007dbcdce507d8eedccc31018c
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,40 @@
|
|
6
6
|
|
7
7
|
## Fixes
|
8
8
|
|
9
|
+
---
|
10
|
+
|
11
|
+
|
12
|
+
|
13
|
+
# 3.1.0
|
14
|
+
|
15
|
+
## Improvements
|
16
|
+
* Feature: [#302](https://github.com/Dynamoid/dynamoid/pull/302) Add methods similar to `ActiveRecord::AttributeMethods::BeforeTypeCast`:
|
17
|
+
* method `attributes_before_type_cast`
|
18
|
+
* method `read_attribte_before_type_cast`
|
19
|
+
* methods `<name>_before_type_cast`
|
20
|
+
* Feature: [#303](https://github.com/Dynamoid/dynamoid/pull/303) Add `#update_attributes!` method
|
21
|
+
* Feature: [#304](https://github.com/Dynamoid/dynamoid/pull/304) Add `inheritance_field` option for `Document.table` method to specify column name for supporting STI and storing class name
|
22
|
+
* Feature: [#305](https://github.com/Dynamoid/dynamoid/pull/305) Add increment/decrement methods:
|
23
|
+
* `#increment`
|
24
|
+
* `#increment!`
|
25
|
+
* `#decrement`
|
26
|
+
* `#decrement!`
|
27
|
+
* `.inc`
|
28
|
+
* Feature: [#307](https://github.com/Dynamoid/dynamoid/pull/307) Allow to declare type of elements in `array`/`set` fields with `of` option. Only scalar types are supported as well as custom types
|
29
|
+
* Feature: [#312](https://github.com/Dynamoid/dynamoid/pull/312) Add Ability to specify network timeout connection settings (@lulu-ulul)
|
30
|
+
* Feature: [#313](https://github.com/Dynamoid/dynamoid/pull/313) Add support for backoff in scan and query (@bonty)
|
31
|
+
* Improvement: [#314](https://github.com/Dynamoid/dynamoid/pull/314) Re-implement `count` for `where`-chain query efficiently. So now `where(...).count` doesn't load all the documents, just statistics
|
32
|
+
|
33
|
+
## Fixes
|
34
|
+
* Bug: [#298](https://github.com/Dynamoid/dynamoid/pull/298) Fix `raw` field storing when value is a Hash with non-string keys
|
35
|
+
* Bug: [#299](https://github.com/Dynamoid/dynamoid/pull/299) Fix `raw` fields - skip empty strings and sets
|
36
|
+
* Bug: [#309](https://github.com/Dynamoid/dynamoid/pull/309) Fix loading of a document that contains not declared in model class fields
|
37
|
+
* Bug: [#310](https://github.com/Dynamoid/dynamoid/pull/310) Fix `Adapter#list_tables` method to return names of all tables, not just first page (@knovoselic)
|
38
|
+
* Bug: [#311](https://github.com/Dynamoid/dynamoid/pull/311) Fix `consistent_read` option of `.find` (@kokuyouwind)
|
39
|
+
* Bug: [#319](https://github.com/Dynamoid/dynamoid/pull/319) Repair consistent reading for `find_all`
|
40
|
+
* Bug: [#317](https://github.com/Dynamoid/dynamoid/pull/317) Fix `create_tables` rake task
|
41
|
+
|
42
|
+
|
9
43
|
|
10
44
|
# 3.0.0
|
11
45
|
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Dynamoid
|
2
2
|
|
3
|
-
You are viewing the README for version
|
3
|
+
You are viewing the README for version 3 of Dynamoid. See the [CHANGELOG](https://github.com/Dynamoid/Dynamoid/blob/master/CHANGELOG.md#200) for details on breaking changes since 1.3.x.
|
4
4
|
|
5
5
|
For version 1.3.x use the [1-3-stable branch](https://github.com/Dynamoid/Dynamoid/blob/1-3-stable/README.md).
|
6
6
|
|
@@ -20,9 +20,9 @@ But if you want a fast, scalable, simple, easy-to-use database (and a Gem that s
|
|
20
20
|
| gem name | dynamoid |
|
21
21
|
| license | MIT |
|
22
22
|
| download rank | [](https://rubygems.org/gems/dynamoid) |
|
23
|
-
| version | [](https://
|
24
|
-
| dependencies | [](https://badge.fury.io/rb/dynamoid) |
|
24
|
+
| dependencies | [](https://depfu.com) |
|
25
|
+
| code quality | [](https://codeclimate.com/github/Dynamoid/dynamoid) |
|
26
26
|
| continuous integration | [](https://travis-ci.org/Dynamoid/dynamoid) |
|
27
27
|
| test coverage | [](https://coveralls.io/github/Dynamoid/Dynamoid?branch=master) |
|
28
28
|
| triage helpers | [](https://www.codetriage.com/dynamoid/dynamoid) |
|
@@ -191,8 +191,16 @@ You have two options if you need to use a `datetime` field as a range key:
|
|
191
191
|
|
192
192
|
#### Note on set type
|
193
193
|
|
194
|
-
|
195
|
-
|
194
|
+
`Dynamoid`'s type `set` is stored as DynamoDB's Set attribute type.
|
195
|
+
DynamoDB supports only Set of strings, numbers and binary.
|
196
|
+
Moreover Set *must* contain elements of the same type only.
|
197
|
+
|
198
|
+
In order to use some other `Dynamoid`'s types you can specify `of` option
|
199
|
+
to declare the type of set elements.
|
200
|
+
|
201
|
+
As a result of that DynamoDB limitation, in Dynamoid only the following
|
202
|
+
scalar types are supported (note: does not support `boolean`):
|
203
|
+
`integer`, `number`, `date`, `datetime`, `serializable` and custom types.
|
196
204
|
|
197
205
|
```ruby
|
198
206
|
class Document
|
@@ -202,6 +210,38 @@ class Document
|
|
202
210
|
end
|
203
211
|
```
|
204
212
|
|
213
|
+
It's possible to specify field options like `store_as_string` for `datetime` field
|
214
|
+
or `serializer` for `serializable` field for `set` elements type:
|
215
|
+
|
216
|
+
```ruby
|
217
|
+
class Document
|
218
|
+
include DynamoId::Document
|
219
|
+
|
220
|
+
field :values, :set, of: { serialized: { serializer: JSON } }
|
221
|
+
field :dates, :set, of: { date: { store_as_string: true } }
|
222
|
+
field :datetimes, :set, of: { datetime: { store_as_string: false } }
|
223
|
+
end
|
224
|
+
```
|
225
|
+
|
226
|
+
DynamoDB doesn't allow empty strings in fields configured as `set`.
|
227
|
+
Abiding by this restriction, when `Dynamoid` saves a document it removes all empty strings in set fields.
|
228
|
+
|
229
|
+
#### Note on array type
|
230
|
+
|
231
|
+
`Dynamoid`'s type `array` is stored as DynamoDB's List attribute type.
|
232
|
+
It can contain elements of different types (in contrast to Set attribute type).
|
233
|
+
|
234
|
+
If you need to store in array field elements of `datetime`, `date`,
|
235
|
+
`serializable` or some custom type, which DynamoDB doesn't support
|
236
|
+
natively, you should specify element type with `of` option:
|
237
|
+
|
238
|
+
```ruby
|
239
|
+
class Document
|
240
|
+
include DynamoId::Document
|
241
|
+
|
242
|
+
field :dates, :array, of: :date
|
243
|
+
end
|
244
|
+
```
|
205
245
|
|
206
246
|
#### Magic Columns
|
207
247
|
|
@@ -399,6 +439,57 @@ animal.class
|
|
399
439
|
#=> Cat
|
400
440
|
```
|
401
441
|
|
442
|
+
If you already have DynamoDB tables and `type` field already exists and has its own semantic it leads to conflict.
|
443
|
+
It's possible to tell Dynamoid to use another field (even not existing)
|
444
|
+
instead of `type` one with `inheritance_field` table option:
|
445
|
+
|
446
|
+
```ruby
|
447
|
+
class Car
|
448
|
+
include Dynamoid::Document
|
449
|
+
table inheritance_field: :my_new_type
|
450
|
+
|
451
|
+
field :my_new_type
|
452
|
+
end
|
453
|
+
|
454
|
+
c = Car.create
|
455
|
+
c.my_new_type
|
456
|
+
#=> "Car"
|
457
|
+
```
|
458
|
+
|
459
|
+
### Type casting
|
460
|
+
|
461
|
+
Dynamid supports type casting and tryes to do it in the most convinient way.
|
462
|
+
Values for all fields (except custom type) are coerced to declared field types.
|
463
|
+
|
464
|
+
Some obvious rules are used, e.g.:
|
465
|
+
|
466
|
+
for boolean field:
|
467
|
+
```ruby
|
468
|
+
document.boolean_field = 'off'
|
469
|
+
# => false
|
470
|
+
document.boolean_field = 'false'
|
471
|
+
# => false
|
472
|
+
document.boolean_field = 'some string'
|
473
|
+
# => true
|
474
|
+
```
|
475
|
+
|
476
|
+
or for integer field:
|
477
|
+
```ruby
|
478
|
+
document.integer_field = 42.3
|
479
|
+
# => 42
|
480
|
+
document.integer_field = '42.3'
|
481
|
+
# => 42
|
482
|
+
document.integer_field = true
|
483
|
+
# => 1
|
484
|
+
```
|
485
|
+
|
486
|
+
If time zone isn't specified for `datetime` value - application time zone is used.
|
487
|
+
|
488
|
+
To access field value before type casting following method could be
|
489
|
+
used: `attributes_before_type_cast` and `read_attribute_before_type_cast`.
|
490
|
+
|
491
|
+
There is `<name>_before_type_cast` method for every field in a model as well.
|
492
|
+
|
402
493
|
## Usage
|
403
494
|
|
404
495
|
### Object Creation
|
@@ -724,7 +815,7 @@ Calls to `update` and `update!` also increment the `lock_version`, however they
|
|
724
815
|
### Backoff strategies
|
725
816
|
|
726
817
|
|
727
|
-
You can use several methods that run efficiently in batch mode like `.find_all` and `.import`.
|
818
|
+
You can use several methods that run efficiently in batch mode like `.find_all` and `.import`. It affects `Query` and `Scan` operations as well.
|
728
819
|
|
729
820
|
The backoff strategy will be used when, for any reason, some items could not be processed as part of a batch mode command.
|
730
821
|
Operations will be re-run to process these items.
|
@@ -786,7 +877,7 @@ module DynamoidReset
|
|
786
877
|
end
|
787
878
|
Dynamoid.adapter.tables.clear
|
788
879
|
# Recreate all tables to avoid unexpected errors
|
789
|
-
Dynamoid.included_models.each(
|
880
|
+
Dynamoid.included_models.each { |m| m.create_table(sync: true) }
|
790
881
|
end
|
791
882
|
end
|
792
883
|
|
data/dynamoid.gemspec
CHANGED
@@ -43,6 +43,7 @@ Gem::Specification.new do |spec|
|
|
43
43
|
spec.add_runtime_dependency 'activemodel', '>=4'
|
44
44
|
spec.add_runtime_dependency 'aws-sdk-dynamodb', '~> 1'
|
45
45
|
spec.add_runtime_dependency 'concurrent-ruby', '>= 1.0'
|
46
|
+
spec.add_runtime_dependency 'null-logger'
|
46
47
|
|
47
48
|
spec.add_development_dependency 'appraisal'
|
48
49
|
spec.add_development_dependency 'bundler'
|
@@ -1,5 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative 'query'
|
4
|
+
require_relative 'scan'
|
5
|
+
|
3
6
|
module Dynamoid
|
4
7
|
module AdapterPlugin
|
5
8
|
# The AwsSdkV3 adapter provides support for the aws-sdk version 2 for ruby.
|
@@ -49,6 +52,8 @@ module Dynamoid
|
|
49
52
|
}
|
50
53
|
BATCH_WRITE_ITEM_REQUESTS_LIMIT = 25
|
51
54
|
|
55
|
+
CONNECTION_CONFIG_OPTIONS = %i[endpoint region http_continue_timeout http_idle_timeout http_open_timeout http_read_timeout].freeze
|
56
|
+
|
52
57
|
attr_reader :table_cache
|
53
58
|
|
54
59
|
# Establish the connection to DynamoDB.
|
@@ -62,8 +67,8 @@ module Dynamoid
|
|
62
67
|
def connection_config
|
63
68
|
@connection_hash = {}
|
64
69
|
|
65
|
-
|
66
|
-
@connection_hash[
|
70
|
+
(Dynamoid::Config.settings.compact.keys & CONNECTION_CONFIG_OPTIONS).each do |option|
|
71
|
+
@connection_hash[option] = Dynamoid::Config.send(option)
|
67
72
|
end
|
68
73
|
if Dynamoid::Config.access_key?
|
69
74
|
@connection_hash[:access_key_id] = Dynamoid::Config.access_key
|
@@ -71,9 +76,6 @@ module Dynamoid
|
|
71
76
|
if Dynamoid::Config.secret_key?
|
72
77
|
@connection_hash[:secret_access_key] = Dynamoid::Config.secret_key
|
73
78
|
end
|
74
|
-
if Dynamoid::Config.region?
|
75
|
-
@connection_hash[:region] = Dynamoid::Config.region
|
76
|
-
end
|
77
79
|
|
78
80
|
# https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/logging.rb
|
79
81
|
# https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb
|
@@ -177,7 +179,7 @@ module Dynamoid
|
|
177
179
|
# @since 1.0.0
|
178
180
|
#
|
179
181
|
# @todo: Provide support for passing options to underlying batch_get_item
|
180
|
-
def batch_get_item(table_ids,
|
182
|
+
def batch_get_item(table_ids, options = {})
|
181
183
|
request_items = Hash.new { |h, k| h[k] = [] }
|
182
184
|
return request_items if table_ids.all? { |_k, v| v.blank? }
|
183
185
|
|
@@ -206,7 +208,8 @@ module Dynamoid
|
|
206
208
|
end
|
207
209
|
|
208
210
|
request_items[t] = {
|
209
|
-
keys: keys
|
211
|
+
keys: keys,
|
212
|
+
consistent_read: options[:consistent_read]
|
210
213
|
}
|
211
214
|
|
212
215
|
results = client.batch_get_item(
|
@@ -426,12 +429,16 @@ module Dynamoid
|
|
426
429
|
#
|
427
430
|
# @todo Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#get_item-instance_method
|
428
431
|
def get_item(table_name, key, options = {})
|
432
|
+
options = options.dup
|
429
433
|
options ||= {}
|
434
|
+
|
430
435
|
table = describe_table(table_name)
|
431
436
|
range_key = options.delete(:range_key)
|
437
|
+
consistent_read = options.delete(:consistent_read)
|
432
438
|
|
433
439
|
item = client.get_item(table_name: table_name,
|
434
|
-
key: key_stanza(table, key, range_key)
|
440
|
+
key: key_stanza(table, key, range_key),
|
441
|
+
consistent_read: consistent_read)[:item]
|
435
442
|
item ? result_item_to_hash(item) : nil
|
436
443
|
end
|
437
444
|
|
@@ -445,6 +452,8 @@ module Dynamoid
|
|
445
452
|
#
|
446
453
|
# @todo Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#update_item-instance_method
|
447
454
|
def update_item(table_name, key, options = {})
|
455
|
+
options = options.dup
|
456
|
+
|
448
457
|
range_key = options.delete(:range_key)
|
449
458
|
conditions = options.delete(:conditions)
|
450
459
|
table = describe_table(table_name)
|
@@ -467,10 +476,16 @@ module Dynamoid
|
|
467
476
|
# List all tables on DynamoDB.
|
468
477
|
#
|
469
478
|
# @since 1.0.0
|
470
|
-
#
|
471
|
-
# @todo Provide limit support http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#update_item-instance_method
|
472
479
|
def list_tables
|
473
|
-
|
480
|
+
[].tap do |result|
|
481
|
+
start_table_name = nil
|
482
|
+
loop do
|
483
|
+
result_page = client.list_tables exclusive_start_table_name: start_table_name
|
484
|
+
start_table_name = result_page.last_evaluated_table_name
|
485
|
+
result.concat result_page.table_names
|
486
|
+
break unless start_table_name
|
487
|
+
end
|
488
|
+
end
|
474
489
|
end
|
475
490
|
|
476
491
|
# Persists an item on DynamoDB.
|
@@ -503,191 +518,66 @@ module Dynamoid
|
|
503
518
|
# one range key to the hash.
|
504
519
|
#
|
505
520
|
# @param [String] table_name the name of the table
|
506
|
-
# @param [Hash]
|
507
|
-
# @option
|
508
|
-
# @option
|
509
|
-
# @option
|
510
|
-
# @option
|
511
|
-
# @option
|
512
|
-
# @option
|
521
|
+
# @param [Hash] options the options to query the table with
|
522
|
+
# @option options [String] :hash_value the value of the hash key to find
|
523
|
+
# @option options [Number, Number] :range_between find the range key within this range
|
524
|
+
# @option options [Number] :range_greater_than find range keys greater than this
|
525
|
+
# @option options [Number] :range_less_than find range keys less than this
|
526
|
+
# @option options [Number] :range_gte find range keys greater than or equal to this
|
527
|
+
# @option options [Number] :range_lte find range keys less than or equal to this
|
513
528
|
#
|
514
529
|
# @return [Enumerable] matching items
|
515
530
|
#
|
516
531
|
# @since 1.0.0
|
517
532
|
#
|
518
533
|
# @todo Provide support for various other options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#query-instance_method
|
519
|
-
def query(table_name,
|
534
|
+
def query(table_name, options = {})
|
520
535
|
table = describe_table(table_name)
|
521
|
-
hk = (opts[:hash_key].present? ? opts.delete(:hash_key) : table.hash_key).to_s
|
522
|
-
rng = (opts[:range_key].present? ? opts.delete(:range_key) : table.range_key).to_s
|
523
|
-
q = opts.slice(
|
524
|
-
:consistent_read,
|
525
|
-
:scan_index_forward,
|
526
|
-
:select,
|
527
|
-
:index_name
|
528
|
-
)
|
529
536
|
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
# Deal with various limits and batching
|
536
|
-
record_limit = opts.delete(:record_limit)
|
537
|
-
scan_limit = opts.delete(:scan_limit)
|
538
|
-
batch_size = opts.delete(:batch_size)
|
539
|
-
exclusive_start_key = opts.delete(:exclusive_start_key)
|
540
|
-
limit = [record_limit, scan_limit, batch_size].compact.min
|
541
|
-
|
542
|
-
key_conditions = {
|
543
|
-
hk => {
|
544
|
-
comparison_operator: EQ,
|
545
|
-
attribute_value_list: attribute_value_list(EQ, opts.delete(:hash_value).freeze)
|
546
|
-
}
|
547
|
-
}
|
548
|
-
|
549
|
-
opts.each_pair do |k, _v|
|
550
|
-
next unless (op = RANGE_MAP[k])
|
551
|
-
key_conditions[rng] = {
|
552
|
-
comparison_operator: op,
|
553
|
-
attribute_value_list: attribute_value_list(op, opts.delete(k).freeze)
|
554
|
-
}
|
555
|
-
end
|
556
|
-
|
557
|
-
query_filter = {}
|
558
|
-
opts.reject { |k, _| k.in? RANGE_MAP.keys }.each do |attr, hash|
|
559
|
-
query_filter[attr] = {
|
560
|
-
comparison_operator: FIELD_MAP[hash.keys[0]],
|
561
|
-
attribute_value_list: attribute_value_list(FIELD_MAP[hash.keys[0]], hash.values[0].freeze)
|
562
|
-
}
|
537
|
+
Enumerator.new do |yielder|
|
538
|
+
Query.new(client, table, options).call.each do |page|
|
539
|
+
page.items.each { |row| yielder << result_item_to_hash(row) }
|
540
|
+
end
|
563
541
|
end
|
542
|
+
end
|
564
543
|
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
q[:key_conditions] = key_conditions
|
569
|
-
q[:query_filter] = query_filter
|
570
|
-
|
571
|
-
Enumerator.new do |y|
|
572
|
-
record_count = 0
|
573
|
-
scan_count = 0
|
574
|
-
loop do
|
575
|
-
# Adjust the limit down if the remaining record and/or scan limit are
|
576
|
-
# lower to obey limits. We can assume the difference won't be
|
577
|
-
# negative due to break statements below but choose smaller limit
|
578
|
-
# which is why we have 2 separate if statements.
|
579
|
-
# NOTE: Adjusting based on record_limit can cause many HTTP requests
|
580
|
-
# being made. We may want to change this behavior, but it affects
|
581
|
-
# filtering on data with potentially large gaps.
|
582
|
-
# Example:
|
583
|
-
# User.where('created_at.gte' => 1.day.ago).record_limit(1000)
|
584
|
-
# Records 1-999 User's that fit criteria
|
585
|
-
# Records 1000-2000 Users's that do not fit criteria
|
586
|
-
# Record 2001 fits criteria
|
587
|
-
# The underlying implementation will have 1 page for records 1-999
|
588
|
-
# then will request with limit 1 for records 1000-2000 (making 1000
|
589
|
-
# requests of limit 1) until hit record 2001.
|
590
|
-
if q[:limit] && record_limit && record_limit - record_count < q[:limit]
|
591
|
-
q[:limit] = record_limit - record_count
|
592
|
-
end
|
593
|
-
if q[:limit] && scan_limit && scan_limit - scan_count < q[:limit]
|
594
|
-
q[:limit] = scan_limit - scan_count
|
595
|
-
end
|
596
|
-
|
597
|
-
results = client.query(q)
|
598
|
-
results.items.each { |row| y << result_item_to_hash(row) }
|
599
|
-
|
600
|
-
record_count += results.items.size
|
601
|
-
break if record_limit && record_count >= record_limit
|
602
|
-
|
603
|
-
scan_count += results.scanned_count
|
604
|
-
break if scan_limit && scan_count >= scan_limit
|
544
|
+
def query_count(table_name, options = {})
|
545
|
+
table = describe_table(table_name)
|
546
|
+
options[:select] = 'COUNT'
|
605
547
|
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
break
|
610
|
-
end
|
611
|
-
end
|
612
|
-
end
|
548
|
+
Query.new(client, table, options).call
|
549
|
+
.map(&:count)
|
550
|
+
.reduce(:+)
|
613
551
|
end
|
614
552
|
|
615
553
|
# Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on
|
616
554
|
# the DynamoDB servers.
|
617
555
|
#
|
618
556
|
# @param [String] table_name the name of the table
|
619
|
-
# @param [Hash]
|
557
|
+
# @param [Hash] conditions a hash of attributes: matching records will be returned by the scan
|
620
558
|
#
|
621
559
|
# @return [Enumerable] matching items
|
622
560
|
#
|
623
561
|
# @since 1.0.0
|
624
562
|
#
|
625
563
|
# @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method
|
626
|
-
def scan(table_name,
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
scan_limit = select_opts.delete(:scan_limit)
|
633
|
-
batch_size = select_opts.delete(:batch_size)
|
634
|
-
exclusive_start_key = select_opts.delete(:exclusive_start_key)
|
635
|
-
request_limit = [record_limit, scan_limit, batch_size].compact.min
|
636
|
-
request[:limit] = request_limit if request_limit
|
637
|
-
request[:exclusive_start_key] = exclusive_start_key if exclusive_start_key
|
638
|
-
|
639
|
-
if scan_hash.present?
|
640
|
-
request[:scan_filter] = scan_hash.reduce({}) do |memo, (attr, cond)|
|
641
|
-
memo.merge(attr.to_s => {
|
642
|
-
comparison_operator: FIELD_MAP[cond.keys[0]],
|
643
|
-
attribute_value_list: attribute_value_list(FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
|
644
|
-
})
|
564
|
+
def scan(table_name, conditions = {}, options = {})
|
565
|
+
table = describe_table(table_name)
|
566
|
+
|
567
|
+
Enumerator.new do |yielder|
|
568
|
+
Scan.new(client, table, conditions, options).call.each do |page|
|
569
|
+
page.items.each { |row| yielder << result_item_to_hash(row) }
|
645
570
|
end
|
646
571
|
end
|
572
|
+
end
|
647
573
|
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
loop do
|
652
|
-
# Adjust the limit down if the remaining record and/or scan limit are
|
653
|
-
# lower to obey limits. We can assume the difference won't be
|
654
|
-
# negative due to break statements below but choose smaller limit
|
655
|
-
# which is why we have 2 separate if statements.
|
656
|
-
# NOTE: Adjusting based on record_limit can cause many HTTP requests
|
657
|
-
# being made. We may want to change this behavior, but it affects
|
658
|
-
# filtering on data with potentially large gaps.
|
659
|
-
# Example:
|
660
|
-
# User.where('created_at.gte' => 1.day.ago).record_limit(1000)
|
661
|
-
# Records 1-999 User's that fit criteria
|
662
|
-
# Records 1000-2000 Users's that do not fit criteria
|
663
|
-
# Record 2001 fits criteria
|
664
|
-
# The underlying implementation will have 1 page for records 1-999
|
665
|
-
# then will request with limit 1 for records 1000-2000 (making 1000
|
666
|
-
# requests of limit 1) until hit record 2001.
|
667
|
-
if request[:limit] && record_limit && record_limit - record_count < request[:limit]
|
668
|
-
request[:limit] = record_limit - record_count
|
669
|
-
end
|
670
|
-
if request[:limit] && scan_limit && scan_limit - scan_count < request[:limit]
|
671
|
-
request[:limit] = scan_limit - scan_count
|
672
|
-
end
|
673
|
-
|
674
|
-
results = client.scan(request)
|
675
|
-
results.items.each { |row| y << result_item_to_hash(row) }
|
676
|
-
|
677
|
-
record_count += results.items.size
|
678
|
-
break if record_limit && record_count >= record_limit
|
679
|
-
|
680
|
-
scan_count += results.scanned_count
|
681
|
-
break if scan_limit && scan_count >= scan_limit
|
574
|
+
def scan_count(table_name, conditions = {}, options = {})
|
575
|
+
table = describe_table(table_name)
|
576
|
+
options[:select] = 'COUNT'
|
682
577
|
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
else
|
687
|
-
break
|
688
|
-
end
|
689
|
-
end
|
690
|
-
end
|
578
|
+
Scan.new(client, table, conditions, options).call
|
579
|
+
.map(&:count)
|
580
|
+
.reduce(:+)
|
691
581
|
end
|
692
582
|
|
693
583
|
#
|
@@ -968,6 +858,10 @@ module Dynamoid
|
|
968
858
|
# @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
|
969
859
|
# @params [Object] value: scalar value or array/set
|
970
860
|
def attribute_value_list(operator, value)
|
861
|
+
self.class.attribute_value_list(operator, value)
|
862
|
+
end
|
863
|
+
|
864
|
+
def self.attribute_value_list(operator, value)
|
971
865
|
# For BETWEEN and IN operators we should keep value as is (it should be already an array)
|
972
866
|
# For all the other operators we wrap the value with array
|
973
867
|
if %w[BETWEEN IN].include?(operator)
|
@@ -1018,6 +912,10 @@ module Dynamoid
|
|
1018
912
|
def item_count
|
1019
913
|
schema[:item_count]
|
1020
914
|
end
|
915
|
+
|
916
|
+
def name
|
917
|
+
schema[:table_name]
|
918
|
+
end
|
1021
919
|
end
|
1022
920
|
|
1023
921
|
#
|
@@ -1043,7 +941,7 @@ module Dynamoid
|
|
1043
941
|
# add. values must be a Set, Array, or Numeric
|
1044
942
|
#
|
1045
943
|
def add(values)
|
1046
|
-
@additions.merge!(values)
|
944
|
+
@additions.merge!(sanitize_attributes(values))
|
1047
945
|
end
|
1048
946
|
|
1049
947
|
#
|
@@ -1053,14 +951,14 @@ module Dynamoid
|
|
1053
951
|
# to remove
|
1054
952
|
#
|
1055
953
|
def delete(values)
|
1056
|
-
@deletions.merge!(values)
|
954
|
+
@deletions.merge!(sanitize_attributes(values))
|
1057
955
|
end
|
1058
956
|
|
1059
957
|
#
|
1060
958
|
# Replaces the values of one or more attributes
|
1061
959
|
#
|
1062
960
|
def set(values)
|
1063
|
-
@updates.merge!(values)
|
961
|
+
@updates.merge!(sanitize_attributes(values))
|
1064
962
|
end
|
1065
963
|
|
1066
964
|
#
|
@@ -1091,6 +989,14 @@ module Dynamoid
|
|
1091
989
|
ret
|
1092
990
|
end
|
1093
991
|
|
992
|
+
private
|
993
|
+
|
994
|
+
def sanitize_attributes(attributes)
|
995
|
+
attributes.transform_values do |v|
|
996
|
+
v.is_a?(Hash) ? v.stringify_keys : v
|
997
|
+
end
|
998
|
+
end
|
999
|
+
|
1094
1000
|
ADD = 'ADD'
|
1095
1001
|
DELETE = 'DELETE'
|
1096
1002
|
PUT = 'PUT'
|
@@ -1099,6 +1005,8 @@ module Dynamoid
|
|
1099
1005
|
def sanitize_item(attributes)
|
1100
1006
|
attributes.reject do |_k, v|
|
1101
1007
|
v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
|
1008
|
+
end.transform_values do |v|
|
1009
|
+
v.is_a?(Hash) ? v.stringify_keys : v
|
1102
1010
|
end
|
1103
1011
|
end
|
1104
1012
|
end
|