trifle-stats 2.4.1 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +20 -2
- data/lib/trifle/stats/driver/mysql.rb +33 -15
- data/lib/trifle/stats/driver/postgres.rb +66 -37
- data/lib/trifle/stats/driver/redis.rb +37 -30
- data/lib/trifle/stats/driver/sqlite.rb +66 -35
- data/lib/trifle/stats/nocturnal.rb +2 -2
- data/lib/trifle/stats/transponder/expression.rb +117 -0
- data/lib/trifle/stats/transponder/expression_engine.rb +356 -0
- data/lib/trifle/stats/version.rb +1 -1
- data/lib/trifle/stats.rb +2 -10
- metadata +4 -13
- data/lib/trifle/stats/nocturnal_legacy.rb +0 -161
- data/lib/trifle/stats/transponder/add.rb +0 -30
- data/lib/trifle/stats/transponder/divide.rb +0 -30
- data/lib/trifle/stats/transponder/max.rb +0 -31
- data/lib/trifle/stats/transponder/mean.rb +0 -31
- data/lib/trifle/stats/transponder/min.rb +0 -31
- data/lib/trifle/stats/transponder/multiply.rb +0 -30
- data/lib/trifle/stats/transponder/ratio.rb +0 -30
- data/lib/trifle/stats/transponder/standard_deviation.rb +0 -34
- data/lib/trifle/stats/transponder/subtract.rb +0 -30
- data/lib/trifle/stats/transponder/sum.rb +0 -28
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '014813aa6a45f6fece54f7fc332272c85023a37236cd3b4eeb270f3f00fed3a0'
|
|
4
|
+
data.tar.gz: f83d0bfecc64f03cd3160e3762eb09a12d76ea8c9cda78c8108ee063bd8caf50
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 71ecb7d927d16ec3f29fec1f57624b8f72bb482841a7a6c0d948ddf8ddc42b0d8b88b858f12436a330189a0731ba41d9514332cfc25e4721c3b7387f11d1c809
|
|
7
|
+
data.tar.gz: da222c50726b1241f2fbfe86c21553c1fea4393c80e53c065e94ea4895fbf755fb6f1e59d15b61b3d1219d114fb444b1bc4eb935b21e9970a4b704b0348a23fc
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -57,6 +57,25 @@ Trifle::Stats.values(
|
|
|
57
57
|
#=> { at: [Mon, Tue, Wed, ...], values: [{ "count" => 12, "revenue" => 598_80, ... }, ...] }
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
+
### 5. Process with Series
|
|
61
|
+
|
|
62
|
+
```ruby
|
|
63
|
+
series = Trifle::Stats.series(
|
|
64
|
+
key: 'orders',
|
|
65
|
+
from: 1.week.ago,
|
|
66
|
+
to: Time.now,
|
|
67
|
+
granularity: :day
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
series.transpond.expression(
|
|
71
|
+
paths: ['revenue', 'count'],
|
|
72
|
+
expression: 'a / b',
|
|
73
|
+
response: 'avg_order'
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
series.aggregate.sum(path: 'count')
|
|
77
|
+
```
|
|
78
|
+
|
|
60
79
|
## Drivers
|
|
61
80
|
|
|
62
81
|
| Driver | Backend | Best for |
|
|
@@ -67,13 +86,12 @@ Trifle::Stats.values(
|
|
|
67
86
|
| **MySQL** | JSON column | MySQL shops |
|
|
68
87
|
| **SQLite** | JSON1 extension | Single-server apps, dev/test |
|
|
69
88
|
| **Process** | In-memory | Testing |
|
|
70
|
-
| **Dummy** | No-op | Disabled analytics |
|
|
71
89
|
|
|
72
90
|
## Features
|
|
73
91
|
|
|
74
92
|
- **Dynamic time granularities.** Use any interval like `1m`, `10m`, `1h`, `6h`, `1d`, `1w`, `1mo`, `1q`, `1y`.
|
|
75
93
|
- **Nested value hierarchies.** Track dimensional breakdowns in a single call.
|
|
76
|
-
- **Series operations.** Aggregators
|
|
94
|
+
- **Series operations.** Aggregators, the expression transponder, and formatters.
|
|
77
95
|
- **Buffered writes.** Queue metrics in-memory before flushing to reduce write load.
|
|
78
96
|
- **Driver flexibility.** Switch backends without changing application code.
|
|
79
97
|
|
|
@@ -95,10 +95,9 @@ module Trifle
|
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
def get(keys:)
|
|
98
|
-
keys.map
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
end
|
|
98
|
+
identifiers = keys.map { |key| identifier_for(key) }
|
|
99
|
+
data = get_all(identifiers: identifiers)
|
|
100
|
+
identifiers.map { |identifier| self.class.unpack(hash: data.fetch(lookup_key_for(identifier), {})) }
|
|
102
101
|
end
|
|
103
102
|
|
|
104
103
|
def ping(key:, values:)
|
|
@@ -157,23 +156,42 @@ module Trifle
|
|
|
157
156
|
execute_prepared(connection, query, args)
|
|
158
157
|
end
|
|
159
158
|
|
|
160
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
161
|
-
def
|
|
162
|
-
|
|
159
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
160
|
+
def get_all(identifiers:)
|
|
161
|
+
columns = identifiers.first.keys
|
|
162
|
+
# Select `at` as a plain string to avoid client time zone casting
|
|
163
|
+
columns_sql = columns.map do |column|
|
|
164
|
+
if column == :at
|
|
165
|
+
"DATE_FORMAT(`at`, '%Y-%m-%d %H:%i:%s.%f') AS at"
|
|
166
|
+
else
|
|
167
|
+
self.class.quote_identifier(column)
|
|
168
|
+
end
|
|
169
|
+
end.join(', ')
|
|
170
|
+
conditions = identifiers.map do |identifier|
|
|
171
|
+
"(#{identifier.keys.map { |column| "#{self.class.quote_identifier(column)} = ?" }.join(' AND ')})"
|
|
172
|
+
end.join(' OR ')
|
|
173
|
+
|
|
163
174
|
query = <<~SQL
|
|
164
|
-
SELECT CAST(`data` AS CHAR) AS data
|
|
175
|
+
SELECT #{columns_sql}, CAST(`data` AS CHAR) AS data
|
|
165
176
|
FROM #{self.class.quote_identifier(table_name)}
|
|
166
177
|
WHERE #{conditions}
|
|
167
|
-
LIMIT 1
|
|
168
178
|
SQL
|
|
169
|
-
|
|
170
|
-
return {} if packed_data.nil? || packed_data.empty?
|
|
179
|
+
args = identifiers.flat_map { |identifier| query_values(identifier) }
|
|
171
180
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
181
|
+
execute_prepared(client, query, args).each_with_object({}) do |row, o|
|
|
182
|
+
identifier = columns.to_h { |column| [column, row[column.to_s]] }
|
|
183
|
+
o[lookup_key_for(identifier)] = JSON.parse(row['data'])
|
|
184
|
+
rescue JSON::ParserError
|
|
185
|
+
nil
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
189
|
+
|
|
190
|
+
def lookup_key_for(identifier)
|
|
191
|
+
identifier.map do |_column, value|
|
|
192
|
+
value.is_a?(Time) || value.is_a?(DateTime) ? format_time_value(value) : value.to_s
|
|
193
|
+
end.join('|')
|
|
175
194
|
end
|
|
176
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
177
195
|
|
|
178
196
|
def inc_query(identifier:, data:)
|
|
179
197
|
upsert_query(
|
|
@@ -61,22 +61,26 @@ module Trifle
|
|
|
61
61
|
client.transaction do |c|
|
|
62
62
|
keys.map do |key|
|
|
63
63
|
identifier = identifier_for(key)
|
|
64
|
-
|
|
64
|
+
query, params = inc_query(identifier: identifier, data: data)
|
|
65
|
+
c.exec_params(query, params)
|
|
65
66
|
track_system_data(c, key, count, tracking_key)
|
|
66
67
|
end
|
|
67
68
|
end
|
|
68
69
|
end
|
|
69
70
|
|
|
70
|
-
def inc_query(identifier:, data:)
|
|
71
|
+
def inc_query(identifier:, data:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
71
72
|
columns = identifier.keys.join(', ')
|
|
72
|
-
|
|
73
|
-
|
|
73
|
+
placeholders = identifier.each_with_index.map { |_, i| "$#{i + 1}" }.join(', ')
|
|
74
|
+
expression = data.inject("to_jsonb(#{table_name}.data)") do |o, (k, v)|
|
|
75
|
+
path = escape_string(k.to_s)
|
|
76
|
+
"jsonb_set(#{o}, '{#{path}}', (COALESCE(#{table_name}.data->>'#{path}', '0')::numeric + #{numeric_value(k, v)})::text::jsonb)" # rubocop:disable Layout/LineLength
|
|
77
|
+
end
|
|
74
78
|
|
|
75
|
-
<<-SQL
|
|
76
|
-
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{
|
|
77
|
-
ON CONFLICT (#{
|
|
78
|
-
#{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (COALESCE(#{table_name}.data->>'#{k}', '0')::int + #{v})::text::jsonb)" }};
|
|
79
|
+
query = <<-SQL
|
|
80
|
+
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{placeholders}, $#{identifier.size + 1})
|
|
81
|
+
ON CONFLICT (#{columns}) DO UPDATE SET data = #{expression};
|
|
79
82
|
SQL
|
|
83
|
+
[query, query_params(identifier) + [data.to_json]]
|
|
80
84
|
end
|
|
81
85
|
|
|
82
86
|
def set(keys:, values:, count: 1, tracking_key: nil)
|
|
@@ -84,7 +88,8 @@ module Trifle
|
|
|
84
88
|
client.transaction do |c|
|
|
85
89
|
keys.map do |key|
|
|
86
90
|
identifier = identifier_for(key)
|
|
87
|
-
|
|
91
|
+
query, params = set_query(identifier: identifier, data: data)
|
|
92
|
+
c.exec_params(query, params)
|
|
88
93
|
track_system_data(c, key, count, tracking_key)
|
|
89
94
|
end
|
|
90
95
|
end
|
|
@@ -94,21 +99,24 @@ module Trifle
|
|
|
94
99
|
return unless @system_tracking
|
|
95
100
|
|
|
96
101
|
system_data = system_data_for(key: key, count: count, tracking_key: tracking_key)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
)
|
|
102
|
+
query, params = inc_query(identifier: system_identifier_for(key: key), data: system_data)
|
|
103
|
+
connection.exec_params(query, params)
|
|
100
104
|
end
|
|
101
105
|
|
|
102
|
-
def set_query(identifier:, data:)
|
|
106
|
+
def set_query(identifier:, data:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
103
107
|
columns = identifier.keys.join(', ')
|
|
104
|
-
|
|
105
|
-
|
|
108
|
+
placeholders = identifier.each_with_index.map { |_, i| "$#{i + 1}" }.join(', ')
|
|
109
|
+
params = query_params(identifier) + [data.to_json]
|
|
110
|
+
expression = data.inject("to_jsonb(#{table_name}.data)") do |o, (k, v)|
|
|
111
|
+
params << v.to_json
|
|
112
|
+
"jsonb_set(#{o}, '{#{escape_string(k.to_s)}}', $#{params.size}::jsonb)"
|
|
113
|
+
end
|
|
106
114
|
|
|
107
|
-
<<-SQL
|
|
108
|
-
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{
|
|
109
|
-
ON CONFLICT (#{
|
|
110
|
-
#{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', '#{v.to_json}'::jsonb)" }}
|
|
115
|
+
query = <<-SQL
|
|
116
|
+
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{placeholders}, $#{identifier.size + 1})
|
|
117
|
+
ON CONFLICT (#{columns}) DO UPDATE SET data = #{expression}
|
|
111
118
|
SQL
|
|
119
|
+
[query, params]
|
|
112
120
|
end
|
|
113
121
|
|
|
114
122
|
def get(keys:)
|
|
@@ -118,7 +126,8 @@ module Trifle
|
|
|
118
126
|
end
|
|
119
127
|
|
|
120
128
|
def get_all(identifiers:) # rubocop:disable Metrics/AbcSize
|
|
121
|
-
|
|
129
|
+
query, params = get_query(identifiers: identifiers)
|
|
130
|
+
results = client.exec_params(query, params).to_a
|
|
122
131
|
sample = identifiers.first
|
|
123
132
|
|
|
124
133
|
results.each_with_object(Hash.new({})) do |r, o|
|
|
@@ -130,37 +139,44 @@ module Trifle
|
|
|
130
139
|
end
|
|
131
140
|
end
|
|
132
141
|
|
|
133
|
-
def get_query(identifiers:)
|
|
142
|
+
def get_query(identifiers:) # rubocop:disable Metrics/MethodLength
|
|
143
|
+
params = []
|
|
134
144
|
conditions = identifiers.map do |identifier|
|
|
135
|
-
identifier.map
|
|
145
|
+
identifier.map do |k, v|
|
|
146
|
+
params << query_param(v)
|
|
147
|
+
"#{k} = $#{params.size}"
|
|
148
|
+
end.join(' AND ')
|
|
136
149
|
end.join(' OR ')
|
|
137
150
|
|
|
138
|
-
<<-SQL
|
|
151
|
+
query = <<-SQL
|
|
139
152
|
SELECT * FROM #{table_name} WHERE #{conditions};
|
|
140
153
|
SQL
|
|
154
|
+
[query, params]
|
|
141
155
|
end
|
|
142
156
|
|
|
143
157
|
def ping(key:, values:)
|
|
144
158
|
return [] if @joined_identifier
|
|
145
159
|
|
|
146
160
|
data = self.class.pack(hash: { data: values, at: key.at })
|
|
147
|
-
|
|
161
|
+
query, params = ping_query(key: key.key, at: key.at, data: data)
|
|
148
162
|
client.transaction do |c|
|
|
149
|
-
c.
|
|
163
|
+
c.exec_params(query, params)
|
|
150
164
|
end
|
|
151
165
|
end
|
|
152
166
|
|
|
153
167
|
def ping_query(key:, at:, data:)
|
|
154
|
-
<<-SQL
|
|
155
|
-
INSERT INTO #{ping_table_name} (key, at, data) VALUES (
|
|
156
|
-
ON CONFLICT (key) DO UPDATE SET at =
|
|
168
|
+
query = <<-SQL
|
|
169
|
+
INSERT INTO #{ping_table_name} (key, at, data) VALUES ($1, $2, $3)
|
|
170
|
+
ON CONFLICT (key) DO UPDATE SET at = $2, data = $3::jsonb;
|
|
157
171
|
SQL
|
|
172
|
+
[query, [key.to_s, at.iso8601, data.to_json]]
|
|
158
173
|
end
|
|
159
174
|
|
|
160
175
|
def scan(key:)
|
|
161
176
|
return [] if @joined_identifier
|
|
162
177
|
|
|
163
|
-
|
|
178
|
+
query, params = scan_query(key: key.key)
|
|
179
|
+
result = client.exec_params(query, params).to_a.first
|
|
164
180
|
return [] if result.nil?
|
|
165
181
|
|
|
166
182
|
[Time.parse(result['at']), self.class.unpack(hash: JSON.parse(result['data']))]
|
|
@@ -169,9 +185,10 @@ module Trifle
|
|
|
169
185
|
end
|
|
170
186
|
|
|
171
187
|
def scan_query(key:)
|
|
172
|
-
<<-SQL
|
|
173
|
-
SELECT at, data FROM #{ping_table_name} WHERE key =
|
|
188
|
+
query = <<-SQL
|
|
189
|
+
SELECT at, data FROM #{ping_table_name} WHERE key = $1 ORDER BY at DESC LIMIT 1;
|
|
174
190
|
SQL
|
|
191
|
+
[query, [key.to_s]]
|
|
175
192
|
end
|
|
176
193
|
|
|
177
194
|
def self.normalize_joined_identifier(value)
|
|
@@ -185,19 +202,31 @@ module Trifle
|
|
|
185
202
|
|
|
186
203
|
private
|
|
187
204
|
|
|
188
|
-
def
|
|
205
|
+
def query_params(identifier)
|
|
206
|
+
identifier.values.map { |value| query_param(value) }
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def query_param(value)
|
|
189
210
|
case value
|
|
190
|
-
when String
|
|
191
|
-
"'#{value}'"
|
|
192
211
|
when Time
|
|
193
|
-
|
|
212
|
+
value.utc.strftime('%Y-%m-%d %H:%M:%S+00')
|
|
194
213
|
when Integer, Float
|
|
195
|
-
value
|
|
214
|
+
value
|
|
196
215
|
else
|
|
197
|
-
|
|
216
|
+
value.to_s
|
|
198
217
|
end
|
|
199
218
|
end
|
|
200
219
|
|
|
220
|
+
def numeric_value(key, value)
|
|
221
|
+
return value.to_s if value.is_a?(Numeric)
|
|
222
|
+
|
|
223
|
+
raise ArgumentError, "increment requires numeric value for key #{key.inspect}"
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def escape_string(value)
|
|
227
|
+
value.gsub("'", "''")
|
|
228
|
+
end
|
|
229
|
+
|
|
201
230
|
def build_map_key(data)
|
|
202
231
|
return data[:key] if @joined_identifier == :full
|
|
203
232
|
return "#{data[:key]}::#{data[:at]}" if @joined_identifier == :partial
|
|
@@ -31,47 +31,43 @@ module Trifle
|
|
|
31
31
|
self.class.pack(hash: { count: count, keys: { tracking_key => count } })
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
def inc(keys:, values:, count: 1, tracking_key: nil) # rubocop:disable Metrics/
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
system_data_for(key: key, count: count, tracking_key: tracking_key).each do |k, c|
|
|
46
|
-
client.hincrby(skey, k, c)
|
|
34
|
+
def inc(keys:, values:, count: 1, tracking_key: nil) # rubocop:disable Metrics/MethodLength
|
|
35
|
+
packed = self.class.pack(hash: values)
|
|
36
|
+
client.pipelined do |pipeline|
|
|
37
|
+
keys.each do |key|
|
|
38
|
+
key.prefix = prefix
|
|
39
|
+
pkey = key.join(separator)
|
|
40
|
+
|
|
41
|
+
packed.each do |k, c|
|
|
42
|
+
pipeline.hincrby(pkey, k, c)
|
|
43
|
+
end
|
|
44
|
+
track_system_data(pipeline, key, count, tracking_key)
|
|
47
45
|
end
|
|
48
46
|
end
|
|
49
47
|
end
|
|
50
48
|
|
|
51
49
|
def set(keys:, values:, count: 1, tracking_key: nil)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
system_data_for(key: key, count: count, tracking_key: tracking_key).each do |k, c|
|
|
61
|
-
client.hincrby(skey, k, c)
|
|
50
|
+
packed = self.class.pack(hash: values)
|
|
51
|
+
client.pipelined do |pipeline|
|
|
52
|
+
keys.each do |key|
|
|
53
|
+
key.prefix = prefix
|
|
54
|
+
pkey = key.join(separator)
|
|
55
|
+
|
|
56
|
+
pipeline.hmset(pkey, *packed)
|
|
57
|
+
track_system_data(pipeline, key, count, tracking_key)
|
|
62
58
|
end
|
|
63
59
|
end
|
|
64
60
|
end
|
|
65
61
|
|
|
66
62
|
def get(keys:)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
hash: client.hgetall(pkey)
|
|
73
|
-
)
|
|
63
|
+
results = client.pipelined do |pipeline|
|
|
64
|
+
keys.each do |key|
|
|
65
|
+
key.prefix = prefix
|
|
66
|
+
pipeline.hgetall(key.join(separator))
|
|
67
|
+
end
|
|
74
68
|
end
|
|
69
|
+
|
|
70
|
+
results.map { |hash| self.class.unpack(hash: hash) }
|
|
75
71
|
end
|
|
76
72
|
|
|
77
73
|
def ping(*)
|
|
@@ -81,6 +77,17 @@ module Trifle
|
|
|
81
77
|
def scan(*)
|
|
82
78
|
[]
|
|
83
79
|
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def track_system_data(pipeline, key, count, tracking_key)
|
|
84
|
+
return unless @system_tracking
|
|
85
|
+
|
|
86
|
+
skey = system_join_for(key: key)
|
|
87
|
+
system_data_for(key: key, count: count, tracking_key: tracking_key).each do |k, c|
|
|
88
|
+
pipeline.hincrby(skey, k, c)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
84
91
|
end
|
|
85
92
|
end
|
|
86
93
|
end
|
|
@@ -70,16 +70,19 @@ module Trifle
|
|
|
70
70
|
end
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
-
def inc_query(identifier:, data:)
|
|
73
|
+
def inc_query(identifier:, data:) # rubocop:disable Metrics/MethodLength
|
|
74
74
|
columns = identifier.keys.join(', ')
|
|
75
|
-
|
|
76
|
-
|
|
75
|
+
placeholders = (['?'] * identifier.size).join(', ')
|
|
76
|
+
expression = data.inject('data') do |o, (k, v)|
|
|
77
|
+
path = json_path_for(k)
|
|
78
|
+
"json_set(#{o}, '#{path}', IFNULL(json_extract(data, '#{path}'), 0) + #{numeric_value(k, v)})"
|
|
79
|
+
end
|
|
77
80
|
|
|
78
|
-
<<-SQL
|
|
79
|
-
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{
|
|
80
|
-
ON CONFLICT (#{
|
|
81
|
-
#{data.inject('data') { |o, (k, v)| "json_set(#{o}, '$.#{k}', IFNULL(json_extract(data, '$.#{k}'), 0) + #{v})" }};
|
|
81
|
+
query = <<-SQL
|
|
82
|
+
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{placeholders}, json(?))
|
|
83
|
+
ON CONFLICT (#{columns}) DO UPDATE SET data = #{expression};
|
|
82
84
|
SQL
|
|
85
|
+
[query, query_params(identifier) + [data.to_json]]
|
|
83
86
|
end
|
|
84
87
|
|
|
85
88
|
def set(keys:, values:, count: 1, tracking_key: nil)
|
|
@@ -104,16 +107,20 @@ module Trifle
|
|
|
104
107
|
)
|
|
105
108
|
end
|
|
106
109
|
|
|
107
|
-
def set_query(identifier:, data:)
|
|
110
|
+
def set_query(identifier:, data:) # rubocop:disable Metrics/MethodLength
|
|
108
111
|
columns = identifier.keys.join(', ')
|
|
109
|
-
|
|
110
|
-
|
|
112
|
+
placeholders = (['?'] * identifier.size).join(', ')
|
|
113
|
+
params = query_params(identifier) + [data.to_json]
|
|
114
|
+
expression = data.inject('data') do |o, (k, v)|
|
|
115
|
+
params << v.to_json
|
|
116
|
+
"json_set(#{o}, '#{json_path_for(k)}', json(?))"
|
|
117
|
+
end
|
|
111
118
|
|
|
112
|
-
<<-SQL
|
|
113
|
-
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{
|
|
114
|
-
ON CONFLICT (#{
|
|
115
|
-
#{data.inject('data') { |o, (k, v)| "json_set(#{o}, '$.#{k}', #{v.to_json})" }};
|
|
119
|
+
query = <<-SQL
|
|
120
|
+
INSERT INTO #{table_name} (#{columns}, data) VALUES (#{placeholders}, json(?))
|
|
121
|
+
ON CONFLICT (#{columns}) DO UPDATE SET data = #{expression};
|
|
116
122
|
SQL
|
|
123
|
+
[query, params]
|
|
117
124
|
end
|
|
118
125
|
|
|
119
126
|
def get(keys:)
|
|
@@ -123,7 +130,8 @@ module Trifle
|
|
|
123
130
|
end
|
|
124
131
|
|
|
125
132
|
def get_all(identifiers:) # rubocop:disable Metrics/AbcSize
|
|
126
|
-
|
|
133
|
+
query, params = get_query(identifiers: identifiers)
|
|
134
|
+
results = client.execute(query, params).to_a
|
|
127
135
|
sample = identifiers.first
|
|
128
136
|
|
|
129
137
|
results.each_with_object(Hash.new({})) do |r, o|
|
|
@@ -136,38 +144,42 @@ module Trifle
|
|
|
136
144
|
end
|
|
137
145
|
|
|
138
146
|
def get_query(identifiers:)
|
|
147
|
+
params = []
|
|
139
148
|
conditions = identifiers.map do |identifier|
|
|
140
|
-
identifier.map { |k, v| build_field_condition(k, v) }.join(' AND ')
|
|
149
|
+
identifier.map { |k, v| build_field_condition(k, v, params) }.join(' AND ')
|
|
141
150
|
end.join(' OR ')
|
|
142
151
|
|
|
143
|
-
<<-SQL
|
|
152
|
+
query = <<-SQL
|
|
144
153
|
SELECT * FROM #{table_name} WHERE #{conditions};
|
|
145
154
|
SQL
|
|
155
|
+
[query, params]
|
|
146
156
|
end
|
|
147
157
|
|
|
148
158
|
def ping(key:, values:)
|
|
149
159
|
return [] if @joined_identifier
|
|
150
160
|
|
|
151
161
|
data = self.class.pack(hash: { data: values, at: key.at })
|
|
152
|
-
|
|
162
|
+
query, params = ping_query(key: key.key, at: key.at, data: data)
|
|
153
163
|
client.transaction do |c|
|
|
154
|
-
c.execute(
|
|
164
|
+
c.execute(query, params) # TODO: should use batch_data_operations
|
|
155
165
|
end
|
|
156
166
|
end
|
|
157
167
|
|
|
158
168
|
def ping_query(key:, at:, data:)
|
|
159
169
|
at_formatted = format_time_value(at)
|
|
160
170
|
|
|
161
|
-
<<-SQL
|
|
162
|
-
INSERT INTO #{ping_table_name} (key, at, data) VALUES (
|
|
163
|
-
ON CONFLICT (key) DO UPDATE SET at =
|
|
171
|
+
query = <<-SQL
|
|
172
|
+
INSERT INTO #{ping_table_name} (key, at, data) VALUES (?, ?, json(?))
|
|
173
|
+
ON CONFLICT (key) DO UPDATE SET at = ?, data = json(?);
|
|
164
174
|
SQL
|
|
175
|
+
[query, [key.to_s, at_formatted, data.to_json, at_formatted, data.to_json]]
|
|
165
176
|
end
|
|
166
177
|
|
|
167
178
|
def scan(key:)
|
|
168
179
|
return [] if @joined_identifier
|
|
169
180
|
|
|
170
|
-
|
|
181
|
+
query, params = scan_query(key: key.key)
|
|
182
|
+
result = client.execute(query, params).first
|
|
171
183
|
return [] if result.nil?
|
|
172
184
|
|
|
173
185
|
# SQLite returns columns in order: key, at, data
|
|
@@ -177,9 +189,10 @@ module Trifle
|
|
|
177
189
|
end
|
|
178
190
|
|
|
179
191
|
def scan_query(key:)
|
|
180
|
-
<<-SQL
|
|
181
|
-
SELECT key, at, data FROM #{ping_table_name} WHERE key =
|
|
192
|
+
query = <<-SQL
|
|
193
|
+
SELECT key, at, data FROM #{ping_table_name} WHERE key = ? ORDER BY at DESC LIMIT 1;
|
|
182
194
|
SQL
|
|
195
|
+
[query, [key.to_s]]
|
|
183
196
|
end
|
|
184
197
|
|
|
185
198
|
def self.normalize_joined_identifier(value)
|
|
@@ -193,13 +206,17 @@ module Trifle
|
|
|
193
206
|
|
|
194
207
|
private
|
|
195
208
|
|
|
196
|
-
def build_field_condition(key, value)
|
|
197
|
-
|
|
209
|
+
def build_field_condition(key, value, params)
|
|
210
|
+
unless key == :at
|
|
211
|
+
params << query_param(value)
|
|
212
|
+
return "#{key} = ?"
|
|
213
|
+
end
|
|
198
214
|
|
|
199
215
|
formatted = format_time_value(value)
|
|
200
|
-
|
|
216
|
+
params << formatted
|
|
217
|
+
params << formatted.sub('Z', '.000000Z')
|
|
201
218
|
|
|
202
|
-
|
|
219
|
+
'(at = ? OR at = ?)'
|
|
203
220
|
end
|
|
204
221
|
|
|
205
222
|
# Batch data operations to avoid SQLite parser stack overflow
|
|
@@ -209,16 +226,30 @@ module Trifle
|
|
|
209
226
|
batch_size = 10
|
|
210
227
|
data.each_slice(batch_size) do |batch|
|
|
211
228
|
batch_data = batch.to_h
|
|
212
|
-
query = send("#{operation}_query", identifier: identifier, data: batch_data)
|
|
213
|
-
connection.execute(query)
|
|
229
|
+
query, params = send("#{operation}_query", identifier: identifier, data: batch_data)
|
|
230
|
+
connection.execute(query, params)
|
|
214
231
|
end
|
|
215
232
|
end
|
|
216
233
|
|
|
217
|
-
def
|
|
218
|
-
|
|
219
|
-
|
|
234
|
+
def query_params(identifier)
|
|
235
|
+
identifier.values.map { |value| query_param(value) }
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def query_param(value)
|
|
239
|
+
return format_time_value(value) if value.is_a?(Time) || value.is_a?(DateTime)
|
|
240
|
+
return value if value.is_a?(Integer) || value.is_a?(Float)
|
|
241
|
+
|
|
242
|
+
value.to_s
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def numeric_value(key, value)
|
|
246
|
+
return value.to_s if value.is_a?(Numeric)
|
|
247
|
+
|
|
248
|
+
raise ArgumentError, "increment requires numeric value for key #{key.inspect}"
|
|
249
|
+
end
|
|
220
250
|
|
|
221
|
-
|
|
251
|
+
def json_path_for(key)
|
|
252
|
+
"$.#{key.to_s.gsub("'", "''")}"
|
|
222
253
|
end
|
|
223
254
|
|
|
224
255
|
def format_time_value(value)
|
|
@@ -87,9 +87,9 @@ module Trifle
|
|
|
87
87
|
rescue ArgumentError
|
|
88
88
|
# Handle edge cases like Feb 29 in non-leap years
|
|
89
89
|
Time.new(
|
|
90
|
-
time.year +
|
|
90
|
+
time.year + offset,
|
|
91
91
|
time.month,
|
|
92
|
-
[time.day, days_in_month(time.year +
|
|
92
|
+
[time.day, days_in_month(time.year + offset, time.month)].min,
|
|
93
93
|
time.hour,
|
|
94
94
|
time.min,
|
|
95
95
|
time.sec,
|