rollups 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +3 -0
- data/LICENSE.txt +21 -0
- data/README.md +386 -0
- data/lib/generators/rollups_generator.rb +26 -0
- data/lib/generators/templates/dimensions.rb.tt +12 -0
- data/lib/generators/templates/standard.rb.tt +11 -0
- data/lib/rollup.rb +98 -0
- data/lib/rollup/aggregator.rb +178 -0
- data/lib/rollup/model.rb +10 -0
- data/lib/rollup/utils.rb +75 -0
- data/lib/rollup/version.rb +5 -0
- data/lib/rollups.rb +16 -0
- metadata +180 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ac85a23af0477ab8ce7abed983995b06a935f82c2836fc0a63ff1fe4b150b9d7
|
4
|
+
data.tar.gz: bb60e3c7c87e22e68a4ba94652e011031292ce879d60824f745d7f808055684c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: abee36f374c42202b8a13919ff863aef511af7c28258eb5f657b0b00de4d8f9d329d53d1f58768377372eae428c09b7f223fbcdeb2b7a43a87339fbe3d9f3e9e
|
7
|
+
data.tar.gz: 452951f5d6a8fda80288650b7061e0e17ab9bfd76a2638a7f529b4919ec2933fb725b07e6f85392665fd4e44b69790eed1d69b88341122aa8338640e4b45c868
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2020 Andrew Kane
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,386 @@
|
|
1
|
+
# Rollup
|
2
|
+
|
3
|
+
:fire: Rollup time-series data in Rails
|
4
|
+
|
5
|
+
Works great with [Ahoy](https://github.com/ankane/ahoy) and [Searchjoy](https://github.com/ankane/searchjoy)
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application’s Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'rollups'
|
13
|
+
```
|
14
|
+
|
15
|
+
For Rails < 6, also add:
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
gem 'activerecord-import'
|
19
|
+
```
|
20
|
+
|
21
|
+
And run:
|
22
|
+
|
23
|
+
```sh
|
24
|
+
bundle install
|
25
|
+
rails generate rollups
|
26
|
+
rails db:migrate
|
27
|
+
```
|
28
|
+
|
29
|
+
## Contents
|
30
|
+
|
31
|
+
- [Getting Started](#getting-started)
|
32
|
+
- [Creating Rollups](#creating-rollups)
|
33
|
+
- [Querying Rollups](#querying-rollups)
|
34
|
+
- [Other Topics](#other-topics)
|
35
|
+
- [Examples](#examples)
|
36
|
+
|
37
|
+
## Getting Started
|
38
|
+
|
39
|
+
Store the number of users created by day in the `rollups` table
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
User.rollup("New users")
|
43
|
+
```
|
44
|
+
|
45
|
+
Get the series
|
46
|
+
|
47
|
+
```ruby
|
48
|
+
Rollup.series("New users")
|
49
|
+
# {
|
50
|
+
# Sat, 24 May 2020 => 50,
|
51
|
+
# Sun, 25 May 2020 => 100,
|
52
|
+
# Mon, 26 May 2020 => 34
|
53
|
+
# }
|
54
|
+
```
|
55
|
+
|
56
|
+
Use a rake task or background job to create rollups on a regular basis. Don’t worry too much about naming - you can [rename](#naming) later if needed.
|
57
|
+
|
58
|
+
## Creating Rollups
|
59
|
+
|
60
|
+
### Time Column
|
61
|
+
|
62
|
+
Specify the time column - `created_at` by default
|
63
|
+
|
64
|
+
```ruby
|
65
|
+
User.rollup("New users", column: :joined_at)
|
66
|
+
```
|
67
|
+
|
68
|
+
Change the default column for a model
|
69
|
+
|
70
|
+
```ruby
|
71
|
+
class User < ApplicationRecord
|
72
|
+
self.rollup_column = :joined_at
|
73
|
+
end
|
74
|
+
```
|
75
|
+
|
76
|
+
### Time Intervals
|
77
|
+
|
78
|
+
Specify the interval - `day` by default
|
79
|
+
|
80
|
+
```ruby
|
81
|
+
User.rollup("New users", interval: "week")
|
82
|
+
```
|
83
|
+
|
84
|
+
And when querying
|
85
|
+
|
86
|
+
```ruby
|
87
|
+
Rollup.series("New users", interval: "week")
|
88
|
+
```
|
89
|
+
|
90
|
+
Supported intervals are:
|
91
|
+
|
92
|
+
- hour
|
93
|
+
- day
|
94
|
+
- week
|
95
|
+
- month
|
96
|
+
- quarter
|
97
|
+
- year
|
98
|
+
|
99
|
+
Or any number of minutes or seconds:
|
100
|
+
|
101
|
+
- 1m, 5m, 15m
|
102
|
+
- 1s, 30s, 90s
|
103
|
+
|
104
|
+
Weeks start on Sunday by default. Change this with:
|
105
|
+
|
106
|
+
```ruby
|
107
|
+
Rollup.week_start = :monday
|
108
|
+
```
|
109
|
+
|
110
|
+
### Time Zones
|
111
|
+
|
112
|
+
The default time zone is `Time.zone`. Change this with:
|
113
|
+
|
114
|
+
```ruby
|
115
|
+
Rollup.time_zone = "Pacific Time (US & Canada)"
|
116
|
+
```
|
117
|
+
|
118
|
+
or
|
119
|
+
|
120
|
+
```ruby
|
121
|
+
User.rollup("New users", time_zone: "Pacific Time (US & Canada)")
|
122
|
+
```
|
123
|
+
|
124
|
+
Time zone objects also work. To see a list of available time zones in Rails, run `rake time:zones:all`.
|
125
|
+
|
126
|
+
See [date storage](#date-storage) for how dates are stored.
|
127
|
+
|
128
|
+
### Calculations
|
129
|
+
|
130
|
+
Rollups use `count` by default. For other calculations, use:
|
131
|
+
|
132
|
+
```ruby
|
133
|
+
Order.rollup("Revenue") { |r| r.sum(:revenue) }
|
134
|
+
```
|
135
|
+
|
136
|
+
Works with `count`, `sum`, `minimum`, `maximum`, and `average`. For `median` and `percentile`, check out [ActiveMedian](https://github.com/ankane/active_median).
|
137
|
+
|
138
|
+
### Dimensions
|
139
|
+
|
140
|
+
*PostgreSQL only*
|
141
|
+
|
142
|
+
Create rollups with dimensions
|
143
|
+
|
144
|
+
```ruby
|
145
|
+
Order.group(:platform).rollup("Orders by platform")
|
146
|
+
```
|
147
|
+
|
148
|
+
Works with multiple groups as well
|
149
|
+
|
150
|
+
```ruby
|
151
|
+
Order.group(:platform, :channel).rollup("Orders by platform and channel")
|
152
|
+
```
|
153
|
+
|
154
|
+
Dimension names are determined by the `group` clause. To set manually, use:
|
155
|
+
|
156
|
+
```ruby
|
157
|
+
Order.group(:channel).rollup("Orders by source", dimension_names: ["source"])
|
158
|
+
```
|
159
|
+
|
160
|
+
See how to [query dimensions](#multiple-series).
|
161
|
+
|
162
|
+
### Updating Data
|
163
|
+
|
164
|
+
When you run a rollup for the first time, the entire series is calculated. When you run it again, newer data is added.
|
165
|
+
|
166
|
+
By default, the latest interval stored for a series is recalculated, since it was likely calculated before the interval completed. Earlier intervals aren’t recalculated since the source rows may have been deleted (this also improves performance).
|
167
|
+
|
168
|
+
To recalculate the last few intervals, use:
|
169
|
+
|
170
|
+
```ruby
|
171
|
+
User.rollup("New users", last: 3)
|
172
|
+
```
|
173
|
+
|
174
|
+
To only store data for completed intervals, use:
|
175
|
+
|
176
|
+
```ruby
|
177
|
+
User.rollup("New users", current: false)
|
178
|
+
```
|
179
|
+
|
180
|
+
To clear and recalculate the entire series, use:
|
181
|
+
|
182
|
+
```ruby
|
183
|
+
User.rollup("New users", clear: true)
|
184
|
+
```
|
185
|
+
|
186
|
+
To delete a series, use:
|
187
|
+
|
188
|
+
```ruby
|
189
|
+
Rollup.where(name: "New users", interval: "day").delete_all
|
190
|
+
```
|
191
|
+
|
192
|
+
## Querying Rollups
|
193
|
+
|
194
|
+
### Single Series
|
195
|
+
|
196
|
+
Get a series
|
197
|
+
|
198
|
+
```ruby
|
199
|
+
Rollup.series("New users")
|
200
|
+
```
|
201
|
+
|
202
|
+
Specify the interval if it’s not day
|
203
|
+
|
204
|
+
```ruby
|
205
|
+
Rollup.series("New users", interval: "week")
|
206
|
+
```
|
207
|
+
|
208
|
+
If a series has dimensions, they must match exactly as well
|
209
|
+
|
210
|
+
```ruby
|
211
|
+
Rollup.series("Orders by platform and channel", dimensions: {platform: "Web", channel: "Search"})
|
212
|
+
```
|
213
|
+
|
214
|
+
### Multiple Series
|
215
|
+
|
216
|
+
*PostgreSQL only*
|
217
|
+
|
218
|
+
Get multiple series grouped by dimensions
|
219
|
+
|
220
|
+
```ruby
|
221
|
+
Rollup.multi_series("Orders by platform")
|
222
|
+
```
|
223
|
+
|
224
|
+
Specify the interval if it’s not day
|
225
|
+
|
226
|
+
```ruby
|
227
|
+
Rollup.multi_series("Orders by platform", interval: "week")
|
228
|
+
```
|
229
|
+
|
230
|
+
Filter by dimensions
|
231
|
+
|
232
|
+
```ruby
|
233
|
+
Rollup.where_dimensions(platform: "Web").multi_series("Orders by platform and channel")
|
234
|
+
```
|
235
|
+
|
236
|
+
### Raw Data
|
237
|
+
|
238
|
+
Uses the `Rollup` model to query the data directly
|
239
|
+
|
240
|
+
```ruby
|
241
|
+
Rollup.where(name: "New users", interval: "day")
|
242
|
+
```
|
243
|
+
|
244
|
+
### List
|
245
|
+
|
246
|
+
List names and intervals
|
247
|
+
|
248
|
+
```ruby
|
249
|
+
Rollup.list
|
250
|
+
```
|
251
|
+
|
252
|
+
### Charts
|
253
|
+
|
254
|
+
Rollup works great with [Chartkick](https://github.com/ankane/chartkick)
|
255
|
+
|
256
|
+
```erb
|
257
|
+
<%= line_chart Rollup.series("New users") %>
|
258
|
+
```
|
259
|
+
|
260
|
+
For multiple series, set a `name` for each series before charting
|
261
|
+
|
262
|
+
```ruby
|
263
|
+
series = Rollup.multi_series("Orders by platform")
|
264
|
+
series.each do |s|
|
265
|
+
s[:name] = s[:dimensions]["platform"]
|
266
|
+
end
|
267
|
+
```
|
268
|
+
|
269
|
+
## Other Topics
|
270
|
+
|
271
|
+
### Naming
|
272
|
+
|
273
|
+
Use any naming convention you prefer. Some ideas are:
|
274
|
+
|
275
|
+
- Human - `New users`
|
276
|
+
- Underscore - `new_users`
|
277
|
+
- Dots - `new_users.count`
|
278
|
+
|
279
|
+
Rename with:
|
280
|
+
|
281
|
+
```ruby
|
282
|
+
Rollup.rename("Old name", "New name")
|
283
|
+
```
|
284
|
+
|
285
|
+
### Date Storage
|
286
|
+
|
287
|
+
Rollup stores both dates and times in the `time` column depending on the interval. For date intervals (day, week, etc), it stores `00:00:00` for the time part. Cast the `time` column to a date when querying in SQL to get the correct value.
|
288
|
+
|
289
|
+
- PostgreSQL: `time::date`
|
290
|
+
- MySQL: `CAST(time AS date)`
|
291
|
+
- SQLite: `date(time)`
|
292
|
+
|
293
|
+
## Examples
|
294
|
+
|
295
|
+
- [Ahoy](#ahoy)
|
296
|
+
- [Searchjoy](#searchjoy)
|
297
|
+
|
298
|
+
### Ahoy
|
299
|
+
|
300
|
+
Set the default rollup column for your models
|
301
|
+
|
302
|
+
```ruby
|
303
|
+
class Ahoy::Visit < ApplicationRecord
|
304
|
+
self.rollup_column = :started_at
|
305
|
+
end
|
306
|
+
```
|
307
|
+
|
308
|
+
and
|
309
|
+
|
310
|
+
```ruby
|
311
|
+
class Ahoy::Event < ApplicationRecord
|
312
|
+
self.rollup_column = :time
|
313
|
+
end
|
314
|
+
```
|
315
|
+
|
316
|
+
Hourly visits
|
317
|
+
|
318
|
+
```ruby
|
319
|
+
Ahoy::Visit.rollup("Visits", interval: "hour")
|
320
|
+
```
|
321
|
+
|
322
|
+
Visits by browser
|
323
|
+
|
324
|
+
```ruby
|
325
|
+
Ahoy::Visit.group(:browser).rollup("Visits by browser")
|
326
|
+
```
|
327
|
+
|
328
|
+
Unique homepage views
|
329
|
+
|
330
|
+
```ruby
|
331
|
+
Ahoy::Event.where(name: "Viewed homepage").joins(:visit).rollup("Homepage views") { |r| r.distinct.count(:visitor_token) }
|
332
|
+
```
|
333
|
+
|
334
|
+
Product views
|
335
|
+
|
336
|
+
```ruby
|
337
|
+
Ahoy::Event.where(name: "Viewed product").group_prop(:product_id).rollup("Product views")
|
338
|
+
```
|
339
|
+
|
340
|
+
### Searchjoy
|
341
|
+
|
342
|
+
Daily searches
|
343
|
+
|
344
|
+
```ruby
|
345
|
+
Searchjoy::Search.rollup("Searches")
|
346
|
+
```
|
347
|
+
|
348
|
+
Searches by query
|
349
|
+
|
350
|
+
```ruby
|
351
|
+
Searchjoy::Search.group(:normalized_query).rollup("Searches by query", dimension_names: ["query"])
|
352
|
+
```
|
353
|
+
|
354
|
+
Conversion rate
|
355
|
+
|
356
|
+
```ruby
|
357
|
+
Searchjoy::Search.rollup("Search conversion rate") { |r| r.average("(converted_at IS NOT NULL)::int") }
|
358
|
+
```
|
359
|
+
|
360
|
+
## History
|
361
|
+
|
362
|
+
View the [changelog](https://github.com/ankane/rollup/blob/master/CHANGELOG.md)
|
363
|
+
|
364
|
+
## Contributing
|
365
|
+
|
366
|
+
Everyone is encouraged to help improve this project. Here are a few ways you can help:
|
367
|
+
|
368
|
+
- [Report bugs](https://github.com/ankane/rollup/issues)
|
369
|
+
- Fix bugs and [submit pull requests](https://github.com/ankane/rollup/pulls)
|
370
|
+
- Write, clarify, or fix documentation
|
371
|
+
- Suggest or add new features
|
372
|
+
|
373
|
+
To get started with development:
|
374
|
+
|
375
|
+
```sh
|
376
|
+
git clone https://github.com/ankane/rollup.git
|
377
|
+
cd rollup
|
378
|
+
bundle install
|
379
|
+
|
380
|
+
# create databases
|
381
|
+
createdb rollup_test
|
382
|
+
mysqladmin create rollup_test
|
383
|
+
|
384
|
+
# run tests
|
385
|
+
bundle exec rake test
|
386
|
+
```
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require "rails/generators/active_record"
|
2
|
+
|
3
|
+
# use rollups instead of rollup:install to avoid
|
4
|
+
# class Rollup < ActiveRecord::Base
|
5
|
+
# also works out nicely since it's the gem name
|
6
|
+
class RollupsGenerator < Rails::Generators::Base
|
7
|
+
include ActiveRecord::Generators::Migration
|
8
|
+
source_root File.join(__dir__, "templates")
|
9
|
+
|
10
|
+
def copy_templates
|
11
|
+
migration_template migration_source, "db/migrate/create_rollups.rb", migration_version: migration_version
|
12
|
+
end
|
13
|
+
|
14
|
+
def migration_source
|
15
|
+
case ActiveRecord::Base.connection_config[:adapter].to_s
|
16
|
+
when /postg/i
|
17
|
+
"dimensions.rb"
|
18
|
+
else
|
19
|
+
"standard.rb"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def migration_version
|
24
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
+
def change
|
3
|
+
create_table :rollups do |t|
|
4
|
+
t.string :name, null: false
|
5
|
+
t.string :interval, null: false
|
6
|
+
t.datetime :time, null: false
|
7
|
+
t.jsonb :dimensions, null: false, default: {}
|
8
|
+
t.float :value
|
9
|
+
end
|
10
|
+
add_index :rollups, [:name, :interval, :time, :dimensions], unique: true
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
+
def change
|
3
|
+
create_table :rollups do |t|
|
4
|
+
t.string :name, null: false
|
5
|
+
t.string :interval, null: false
|
6
|
+
t.datetime :time, null: false
|
7
|
+
t.float :value
|
8
|
+
end
|
9
|
+
add_index :rollups, [:name, :interval, :time], unique: true
|
10
|
+
end
|
11
|
+
end
|
data/lib/rollup.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
class Rollup < ActiveRecord::Base
|
2
|
+
validates :name, presence: true
|
3
|
+
validates :interval, presence: true
|
4
|
+
validates :time, presence: true
|
5
|
+
|
6
|
+
class << self
|
7
|
+
attr_accessor :week_start
|
8
|
+
attr_writer :time_zone
|
9
|
+
end
|
10
|
+
self.week_start = :sunday
|
11
|
+
|
12
|
+
class << self
|
13
|
+
# do not memoize so Time.zone can change
|
14
|
+
def time_zone
|
15
|
+
(defined?(@time_zone) && @time_zone) || Time.zone || "Etc/UTC"
|
16
|
+
end
|
17
|
+
|
18
|
+
def series(name, interval: "day", dimensions: {})
|
19
|
+
Utils.check_dimensions if dimensions.any?
|
20
|
+
|
21
|
+
relation = where(name: name, interval: interval)
|
22
|
+
relation = relation.where(dimensions: dimensions) if Utils.dimensions_supported?
|
23
|
+
|
24
|
+
# use select_all due to incorrect casting with pluck
|
25
|
+
sql = relation.order(:time).select(Utils.time_sql(interval), :value).to_sql
|
26
|
+
result = connection.select_all(sql).rows
|
27
|
+
|
28
|
+
Utils.make_series(result, interval)
|
29
|
+
end
|
30
|
+
|
31
|
+
def multi_series(name, interval: "day")
|
32
|
+
Utils.check_dimensions
|
33
|
+
|
34
|
+
relation = where(name: name, interval: interval)
|
35
|
+
|
36
|
+
# use select_all to reduce allocations
|
37
|
+
sql = relation.order(:time).select(Utils.time_sql(interval), :value, :dimensions).to_sql
|
38
|
+
result = connection.select_all(sql).rows
|
39
|
+
|
40
|
+
result.group_by { |r| JSON.parse(r[2]) }.map do |dimensions, rollups|
|
41
|
+
{dimensions: dimensions, data: Utils.make_series(rollups, interval)}
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def where_dimensions(dimensions)
|
46
|
+
Utils.check_dimensions
|
47
|
+
|
48
|
+
relation = self
|
49
|
+
dimensions.each do |k, v|
|
50
|
+
k = k.to_s
|
51
|
+
relation =
|
52
|
+
if v.nil?
|
53
|
+
relation.where("dimensions ->> ? IS NULL", k)
|
54
|
+
elsif v.is_a?(Array)
|
55
|
+
relation.where("dimensions ->> ? IN (?)", k, v.map { |vi| vi.as_json.to_s })
|
56
|
+
else
|
57
|
+
relation.where("dimensions ->> ? = ?", k, v.as_json.to_s)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
relation
|
61
|
+
end
|
62
|
+
|
63
|
+
def list
|
64
|
+
select(:name, :interval).distinct.order(:name, :interval).map do |r|
|
65
|
+
{name: r.name, interval: r.interval}
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# TODO maybe use in_batches
|
70
|
+
def rename(old_name, new_name)
|
71
|
+
where(name: old_name).update_all(name: new_name)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# feels cleaner than overriding _read_attribute
|
76
|
+
def inspect
|
77
|
+
if Utils.date_interval?(interval)
|
78
|
+
previous_time = time_before_type_cast
|
79
|
+
previous_time = previous_time.to_s(:db) if previous_time.is_a?(Time)
|
80
|
+
previous_time = previous_time + " 00:00:00" if previous_time.length == 10
|
81
|
+
super.sub("time: \"#{previous_time}\"", "time: \"#{time.to_s(:db)}\"")
|
82
|
+
else
|
83
|
+
super
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def time
|
88
|
+
if Utils.date_interval?(interval) && !time_before_type_cast.nil?
|
89
|
+
if time_before_type_cast.is_a?(Time)
|
90
|
+
time_before_type_cast.utc.to_date
|
91
|
+
else
|
92
|
+
Date.parse(time_before_type_cast.to_s)
|
93
|
+
end
|
94
|
+
else
|
95
|
+
super
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
class Rollup
|
2
|
+
class Aggregator
|
3
|
+
def initialize(klass)
|
4
|
+
@klass = klass # or relation
|
5
|
+
end
|
6
|
+
|
7
|
+
def rollup(name, column: nil, interval: "day", dimension_names: nil, time_zone: nil, current: true, last: nil, clear: false, &block)
|
8
|
+
raise "Name can't be blank" if name.blank?
|
9
|
+
|
10
|
+
column ||= @klass.rollup_column || :created_at
|
11
|
+
validate_column(column)
|
12
|
+
|
13
|
+
relation = perform_group(name, column: column, interval: interval, time_zone: time_zone, current: current, last: last, clear: clear)
|
14
|
+
result = perform_calculation(relation, &block)
|
15
|
+
|
16
|
+
dimension_names = set_dimension_names(dimension_names, relation)
|
17
|
+
records = prepare_result(result, name, dimension_names, interval)
|
18
|
+
|
19
|
+
maybe_clear(clear, name, interval) do
|
20
|
+
save_records(records) if records.any?
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# basic version of Active Record disallow_raw_sql!
|
25
|
+
# symbol = column (safe), Arel node = SQL (safe), other = untrusted
|
26
|
+
# no need to quote/resolve column here, as Groupdate handles it
|
27
|
+
# TODO remove this method when gem depends on Groupdate 6+
|
28
|
+
def validate_column(column)
|
29
|
+
# matches table.column and column
|
30
|
+
unless column.is_a?(Symbol) || column.is_a?(Arel::Nodes::SqlLiteral) || /\A\w+(\.\w+)?\z/i.match(column.to_s)
|
31
|
+
raise "Non-attribute argument: #{column}. Use Arel.sql() for known-safe values"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def perform_group(name, column:, interval:, time_zone:, current:, last:, clear:)
|
36
|
+
raise ArgumentError, "Cannot use last and clear together" if last && clear
|
37
|
+
|
38
|
+
gd_options = {
|
39
|
+
current: current
|
40
|
+
}
|
41
|
+
|
42
|
+
# make sure Groupdate global options aren't applied
|
43
|
+
gd_options[:time_zone] = time_zone || Rollup.time_zone
|
44
|
+
gd_options[:week_start] = Rollup.week_start if interval.to_s == "week"
|
45
|
+
gd_options[:day_start] = 0 if Utils.date_interval?(interval)
|
46
|
+
|
47
|
+
if last
|
48
|
+
gd_options[:last] = last
|
49
|
+
elsif !clear
|
50
|
+
# if no rollups, compute all intervals
|
51
|
+
# if rollups, recompute last interval
|
52
|
+
max_time = Rollup.where(name: name, interval: interval).maximum(Utils.time_sql(interval))
|
53
|
+
if max_time
|
54
|
+
# aligns perfectly if time zone doesn't change
|
55
|
+
# if time zone does change, there are other problems besides this
|
56
|
+
gd_options[:range] = max_time..
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# intervals are stored as given
|
61
|
+
# we don't normalize intervals (i.e. change 60s -> 1m)
|
62
|
+
case interval.to_s
|
63
|
+
when "hour", "day", "week", "month", "quarter", "year"
|
64
|
+
@klass.group_by_period(interval, column, **gd_options)
|
65
|
+
when /\A\d+s\z/
|
66
|
+
@klass.group_by_second(column, n: interval.to_i, **gd_options)
|
67
|
+
when /\A\d+m\z/
|
68
|
+
@klass.group_by_minute(column, n: interval.to_i, **gd_options)
|
69
|
+
else
|
70
|
+
raise ArgumentError, "Invalid interval: #{interval}"
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def set_dimension_names(dimension_names, relation)
|
75
|
+
groups = relation.group_values[0..-2]
|
76
|
+
|
77
|
+
if dimension_names
|
78
|
+
Utils.check_dimensions
|
79
|
+
if dimension_names.size != groups.size
|
80
|
+
raise ArgumentError, "Expected dimension_names to be size #{groups.size}, not #{dimension_names.size}"
|
81
|
+
end
|
82
|
+
dimension_names
|
83
|
+
else
|
84
|
+
Utils.check_dimensions if groups.any?
|
85
|
+
groups.map { |group| determine_dimension_name(group) }
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def determine_dimension_name(group)
|
90
|
+
# split by ., ->>, and -> and remove whitespace
|
91
|
+
value = group.to_s.split(/\s*((\.)|(->>)|(->))\s*/).last
|
92
|
+
|
93
|
+
# removing starting and ending quotes
|
94
|
+
# for simplicity, they don't need to be the same
|
95
|
+
value = value[1..-2] if value.match(/\A["'`].+["'`]\z/)
|
96
|
+
|
97
|
+
unless value.match(/\A\w+\z/)
|
98
|
+
raise "Cannot determine dimension name: #{group}. Use the dimension_names option"
|
99
|
+
end
|
100
|
+
|
101
|
+
value
|
102
|
+
end
|
103
|
+
|
104
|
+
# calculation can mutate relation, but that's fine
|
105
|
+
def perform_calculation(relation, &block)
|
106
|
+
if block_given?
|
107
|
+
yield(relation)
|
108
|
+
else
|
109
|
+
relation.count
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def prepare_result(result, name, dimension_names, interval)
|
114
|
+
raise "Expected calculation to return Hash, not #{result.class.name}" unless result.is_a?(Hash)
|
115
|
+
|
116
|
+
time_class = Utils.date_interval?(interval) ? Date : Time
|
117
|
+
dimensions_supported = Utils.dimensions_supported?
|
118
|
+
expected_key_size = dimension_names.size + 1
|
119
|
+
|
120
|
+
result.map do |key, value|
|
121
|
+
dimensions = {}
|
122
|
+
if dimensions_supported && dimension_names.any?
|
123
|
+
unless key.is_a?(Array) && key.size == expected_key_size
|
124
|
+
raise "Expected result key to be Array with size #{expected_key_size}"
|
125
|
+
end
|
126
|
+
time = key[-1]
|
127
|
+
# may be able to support dimensions in SQLite by sorting dimension names
|
128
|
+
dimension_names.each_with_index do |dn, i|
|
129
|
+
dimensions[dn] = key[i]
|
130
|
+
end
|
131
|
+
else
|
132
|
+
time = key
|
133
|
+
end
|
134
|
+
|
135
|
+
raise "Expected time to be #{time_class.name}, not #{time.class.name}" unless time.is_a?(time_class)
|
136
|
+
raise "Expected value to be Numeric or nil, not #{value.class.name}" unless value.is_a?(Numeric) || value.nil?
|
137
|
+
|
138
|
+
record = {
|
139
|
+
name: name,
|
140
|
+
interval: interval,
|
141
|
+
time: time,
|
142
|
+
value: value
|
143
|
+
}
|
144
|
+
record[:dimensions] = dimensions if dimensions_supported
|
145
|
+
record
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def maybe_clear(clear, name, interval)
|
150
|
+
if clear
|
151
|
+
Rollup.transaction do
|
152
|
+
Rollup.where(name: name, interval: interval).delete_all
|
153
|
+
yield
|
154
|
+
end
|
155
|
+
else
|
156
|
+
yield
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def save_records(records)
|
161
|
+
# order must match unique index
|
162
|
+
# consider using index name instead
|
163
|
+
conflict_target = [:name, :interval, :time]
|
164
|
+
conflict_target << :dimensions if Utils.dimensions_supported?
|
165
|
+
|
166
|
+
if ActiveRecord::VERSION::MAJOR >= 6
|
167
|
+
options = Utils.mysql? ? {} : {unique_by: conflict_target}
|
168
|
+
Rollup.upsert_all(records, **options)
|
169
|
+
else
|
170
|
+
update = Utils.mysql? ? [:value] : {columns: [:value], conflict_target: conflict_target}
|
171
|
+
Rollup.import(records,
|
172
|
+
on_duplicate_key_update: update,
|
173
|
+
validate: false
|
174
|
+
)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
data/lib/rollup/model.rb
ADDED
data/lib/rollup/utils.rb
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
class Rollup
|
2
|
+
module Utils
|
3
|
+
DATE_INTERVALS = %w(day week month quarter year)
|
4
|
+
|
5
|
+
class << self
|
6
|
+
def time_sql(interval)
|
7
|
+
if date_interval?(interval)
|
8
|
+
if postgresql?
|
9
|
+
"rollups.time::date"
|
10
|
+
elsif sqlite?
|
11
|
+
"date(rollups.time)"
|
12
|
+
else
|
13
|
+
"CAST(rollups.time AS date)"
|
14
|
+
end
|
15
|
+
else
|
16
|
+
:time
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def date_interval?(interval)
|
21
|
+
DATE_INTERVALS.include?(interval.to_s)
|
22
|
+
end
|
23
|
+
|
24
|
+
def dimensions_supported?
|
25
|
+
unless defined?(@dimensions_supported)
|
26
|
+
@dimensions_supported = postgresql? && Rollup.column_names.include?("dimensions")
|
27
|
+
end
|
28
|
+
@dimensions_supported
|
29
|
+
end
|
30
|
+
|
31
|
+
def check_dimensions
|
32
|
+
raise "Dimensions not supported" unless dimensions_supported?
|
33
|
+
end
|
34
|
+
|
35
|
+
def adapter_name
|
36
|
+
Rollup.connection.adapter_name
|
37
|
+
end
|
38
|
+
|
39
|
+
def postgresql?
|
40
|
+
adapter_name =~ /postg/i
|
41
|
+
end
|
42
|
+
|
43
|
+
def mysql?
|
44
|
+
adapter_name =~ /mysql/i
|
45
|
+
end
|
46
|
+
|
47
|
+
def sqlite?
|
48
|
+
adapter_name =~ /sqlite/i
|
49
|
+
end
|
50
|
+
|
51
|
+
def make_series(result, interval)
|
52
|
+
series = {}
|
53
|
+
if Utils.date_interval?(interval)
|
54
|
+
result.each do |row|
|
55
|
+
series[row[0].to_date] = row[1]
|
56
|
+
end
|
57
|
+
else
|
58
|
+
time_zone = Rollup.time_zone
|
59
|
+
if result.any? && result[0][0].is_a?(Time)
|
60
|
+
result.each do |row|
|
61
|
+
series[row[0].in_time_zone(time_zone)] = row[1]
|
62
|
+
end
|
63
|
+
else
|
64
|
+
utc = ActiveSupport::TimeZone["Etc/UTC"]
|
65
|
+
result.each do |row|
|
66
|
+
# row can be time or string
|
67
|
+
series[utc.parse(row[0]).in_time_zone(time_zone)] = row[1]
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
series
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
data/lib/rollups.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# dependencies
|
2
|
+
require "active_support"
|
3
|
+
require "groupdate"
|
4
|
+
|
5
|
+
ActiveSupport.on_load(:active_record) do
|
6
|
+
# must come first
|
7
|
+
require "rollup"
|
8
|
+
|
9
|
+
require "rollup/model"
|
10
|
+
extend Rollup::Model
|
11
|
+
|
12
|
+
# modules
|
13
|
+
require "rollup/aggregator"
|
14
|
+
require "rollup/utils"
|
15
|
+
require "rollup/version"
|
16
|
+
end
|
metadata
ADDED
@@ -0,0 +1,180 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rollups
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Andrew Kane
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-09-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '5.1'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '5.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: groupdate
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '5.2'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '5.2'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: minitest
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '5'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '5'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: activerecord
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pg
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: mysql2
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: sqlite3
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description:
|
140
|
+
email: andrew@chartkick.com
|
141
|
+
executables: []
|
142
|
+
extensions: []
|
143
|
+
extra_rdoc_files: []
|
144
|
+
files:
|
145
|
+
- CHANGELOG.md
|
146
|
+
- LICENSE.txt
|
147
|
+
- README.md
|
148
|
+
- lib/generators/rollups_generator.rb
|
149
|
+
- lib/generators/templates/dimensions.rb.tt
|
150
|
+
- lib/generators/templates/standard.rb.tt
|
151
|
+
- lib/rollup.rb
|
152
|
+
- lib/rollup/aggregator.rb
|
153
|
+
- lib/rollup/model.rb
|
154
|
+
- lib/rollup/utils.rb
|
155
|
+
- lib/rollup/version.rb
|
156
|
+
- lib/rollups.rb
|
157
|
+
homepage: https://github.com/ankane/rollup
|
158
|
+
licenses:
|
159
|
+
- MIT
|
160
|
+
metadata: {}
|
161
|
+
post_install_message:
|
162
|
+
rdoc_options: []
|
163
|
+
require_paths:
|
164
|
+
- lib
|
165
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - ">="
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '2.6'
|
170
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
171
|
+
requirements:
|
172
|
+
- - ">="
|
173
|
+
- !ruby/object:Gem::Version
|
174
|
+
version: '0'
|
175
|
+
requirements: []
|
176
|
+
rubygems_version: 3.1.2
|
177
|
+
signing_key:
|
178
|
+
specification_version: 4
|
179
|
+
summary: Rollup time-series data in Rails
|
180
|
+
test_files: []
|