logidze 0.11.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +79 -4
- data/LICENSE.txt +1 -1
- data/README.md +305 -102
- data/lib/generators/logidze/fx_helper.rb +17 -0
- data/lib/generators/logidze/inject_sql.rb +18 -0
- data/lib/generators/logidze/install/USAGE +6 -1
- data/lib/generators/logidze/install/functions/logidze_capture_exception.sql +23 -0
- data/lib/generators/logidze/install/functions/logidze_compact_history.sql +38 -0
- data/lib/generators/logidze/install/functions/logidze_filter_keys.sql +27 -0
- data/lib/generators/logidze/install/functions/logidze_logger.sql +203 -0
- data/lib/generators/logidze/install/functions/logidze_snapshot.sql +33 -0
- data/lib/generators/logidze/install/functions/logidze_version.sql +21 -0
- data/lib/generators/logidze/install/install_generator.rb +43 -1
- data/lib/generators/logidze/install/templates/hstore.rb.erb +1 -1
- data/lib/generators/logidze/install/templates/migration.rb.erb +19 -232
- data/lib/generators/logidze/install/templates/migration_fx.rb.erb +41 -0
- data/lib/generators/logidze/model/model_generator.rb +53 -13
- data/lib/generators/logidze/model/templates/migration.rb.erb +57 -36
- data/lib/generators/logidze/model/triggers/logidze.sql +6 -0
- data/lib/logidze.rb +37 -14
- data/lib/logidze/engine.rb +9 -0
- data/lib/logidze/has_logidze.rb +1 -1
- data/lib/logidze/history.rb +2 -11
- data/lib/logidze/ignore_log_data.rb +1 -3
- data/lib/logidze/meta.rb +43 -16
- data/lib/logidze/model.rb +51 -44
- data/lib/logidze/utils/check_pending.rb +57 -0
- data/lib/logidze/utils/function_definitions.rb +49 -0
- data/lib/logidze/utils/pending_migration_error.rb +25 -0
- data/lib/logidze/version.rb +1 -1
- metadata +69 -77
- data/.gitattributes +0 -3
- data/.github/ISSUE_TEMPLATE.md +0 -20
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -29
- data/.gitignore +0 -40
- data/.rubocop.yml +0 -55
- data/.travis.yml +0 -42
- data/Gemfile +0 -15
- data/Rakefile +0 -28
- data/assets/pg_log_data_chart.png +0 -0
- data/bench/performance/README.md +0 -109
- data/bench/performance/diff_bench.rb +0 -38
- data/bench/performance/insert_bench.rb +0 -22
- data/bench/performance/memory_profile.rb +0 -56
- data/bench/performance/setup.rb +0 -315
- data/bench/performance/update_bench.rb +0 -38
- data/bench/triggers/Makefile +0 -56
- data/bench/triggers/Readme.md +0 -58
- data/bench/triggers/bench.sql +0 -6
- data/bench/triggers/hstore_trigger_setup.sql +0 -38
- data/bench/triggers/jsonb_minus_2_setup.sql +0 -47
- data/bench/triggers/jsonb_minus_setup.sql +0 -49
- data/bench/triggers/keys2_trigger_setup.sql +0 -44
- data/bench/triggers/keys_trigger_setup.sql +0 -50
- data/bin/console +0 -8
- data/bin/setup +0 -9
- data/gemfiles/rails42.gemfile +0 -6
- data/gemfiles/rails5.gemfile +0 -6
- data/gemfiles/rails52.gemfile +0 -6
- data/gemfiles/rails6.gemfile +0 -6
- data/gemfiles/railsmaster.gemfile +0 -7
- data/lib/logidze/ignore_log_data/association.rb +0 -11
- data/lib/logidze/ignore_log_data/ignored_columns.rb +0 -46
- data/lib/logidze/migration.rb +0 -20
- data/logidze.gemspec +0 -41
data/bench/performance/setup.rb
DELETED
@@ -1,315 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
begin
|
4
|
-
require "bundler/inline"
|
5
|
-
rescue LoadError => e
|
6
|
-
warn "Bundler version 1.10 or later is required. Please update your Bundler"
|
7
|
-
raise e
|
8
|
-
end
|
9
|
-
|
10
|
-
gemfile(true) do
|
11
|
-
source "https://rubygems.org"
|
12
|
-
gem "activerecord", "~>4.2"
|
13
|
-
gem "pg"
|
14
|
-
gem "paper_trail", "~>4.2", require: false
|
15
|
-
gem "pry-byebug"
|
16
|
-
gem "faker"
|
17
|
-
gem "benchmark-ips"
|
18
|
-
gem "memory_profiler"
|
19
|
-
end
|
20
|
-
|
21
|
-
DB_NAME = ENV["DB_NAME"] || "logidze_query_bench"
|
22
|
-
|
23
|
-
begin
|
24
|
-
system("createdb #{DB_NAME}")
|
25
|
-
rescue
|
26
|
-
$stdout.puts "DB already exists"
|
27
|
-
end
|
28
|
-
|
29
|
-
$LOAD_PATH.unshift File.expand_path("../../../lib", __FILE__)
|
30
|
-
|
31
|
-
require "active_record"
|
32
|
-
require "logger"
|
33
|
-
require "logidze"
|
34
|
-
|
35
|
-
ActiveRecord::Base.send :include, Logidze::HasLogidze
|
36
|
-
|
37
|
-
ActiveRecord::Base.establish_connection(adapter: "postgresql", database: DB_NAME)
|
38
|
-
|
39
|
-
at_exit do
|
40
|
-
ActiveRecord::Base.connection.disconnect!
|
41
|
-
end
|
42
|
-
|
43
|
-
require "paper_trail"
|
44
|
-
|
45
|
-
module LogidzeBench
|
46
|
-
module_function
|
47
|
-
|
48
|
-
def setup_db
|
49
|
-
ActiveRecord::Schema.define do
|
50
|
-
# PaperTrail setup
|
51
|
-
create_table :versions, force: true do |t|
|
52
|
-
t.string :item_type, null: false
|
53
|
-
t.integer :item_id, null: false
|
54
|
-
t.string :event, null: false
|
55
|
-
t.string :whodunnit
|
56
|
-
t.text :object
|
57
|
-
t.jsonb :object_changes
|
58
|
-
|
59
|
-
t.datetime :created_at
|
60
|
-
end
|
61
|
-
|
62
|
-
add_index :versions, [:item_type, :item_id]
|
63
|
-
|
64
|
-
# Logidze setup
|
65
|
-
enable_extension :hstore
|
66
|
-
|
67
|
-
execute <<~SQL
|
68
|
-
DO $$
|
69
|
-
BEGIN
|
70
|
-
EXECUTE 'ALTER DATABASE ' || current_database() || ' SET logidze.disabled TO off';
|
71
|
-
END;
|
72
|
-
$$
|
73
|
-
LANGUAGE plpgsql;
|
74
|
-
SQL
|
75
|
-
|
76
|
-
execute <<~SQL
|
77
|
-
CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
|
78
|
-
DECLARE
|
79
|
-
changes jsonb;
|
80
|
-
new_v integer;
|
81
|
-
ts bigint;
|
82
|
-
size integer;
|
83
|
-
history_limit integer;
|
84
|
-
current_version integer;
|
85
|
-
merged jsonb;
|
86
|
-
iterator integer;
|
87
|
-
item record;
|
88
|
-
BEGIN
|
89
|
-
ts := (extract(epoch from now()) * 1000)::bigint;
|
90
|
-
|
91
|
-
IF TG_OP = 'INSERT' THEN
|
92
|
-
changes := to_jsonb(NEW.*) - 'log_data';
|
93
|
-
new_v := 1;
|
94
|
-
|
95
|
-
NEW.log_data := json_build_object(
|
96
|
-
'v',
|
97
|
-
1,
|
98
|
-
'h',
|
99
|
-
jsonb_build_array(
|
100
|
-
jsonb_build_object(
|
101
|
-
'ts',
|
102
|
-
ts,
|
103
|
-
'v',
|
104
|
-
new_v,
|
105
|
-
'c',
|
106
|
-
changes
|
107
|
-
)
|
108
|
-
)
|
109
|
-
);
|
110
|
-
ELSIF TG_OP = 'UPDATE' THEN
|
111
|
-
history_limit := TG_ARGV[0];
|
112
|
-
current_version := (NEW.log_data->>'v')::int;
|
113
|
-
|
114
|
-
IF NEW = OLD THEN
|
115
|
-
RETURN NEW;
|
116
|
-
END IF;
|
117
|
-
|
118
|
-
IF current_version < (NEW.log_data#>>'{h,-1,v}')::int THEN
|
119
|
-
iterator := 0;
|
120
|
-
FOR item in SELECT * FROM jsonb_array_elements(NEW.log_data->'h')
|
121
|
-
LOOP
|
122
|
-
IF (item.value->>'v')::int > current_version THEN
|
123
|
-
NEW.log_data := jsonb_set(
|
124
|
-
NEW.log_data,
|
125
|
-
'{h}',
|
126
|
-
(NEW.log_data->'h') - iterator
|
127
|
-
);
|
128
|
-
END IF;
|
129
|
-
iterator := iterator + 1;
|
130
|
-
END LOOP;
|
131
|
-
END IF;
|
132
|
-
|
133
|
-
changes := hstore_to_jsonb_loose(
|
134
|
-
hstore(NEW.*) - hstore(OLD.*)
|
135
|
-
) - 'log_data';
|
136
|
-
|
137
|
-
new_v := (NEW.log_data#>>'{h,-1,v}')::int + 1;
|
138
|
-
|
139
|
-
size := jsonb_array_length(NEW.log_data->'h');
|
140
|
-
|
141
|
-
NEW.log_data := jsonb_set(
|
142
|
-
NEW.log_data,
|
143
|
-
ARRAY['h', size::text],
|
144
|
-
jsonb_build_object(
|
145
|
-
'ts',
|
146
|
-
ts,
|
147
|
-
'v',
|
148
|
-
new_v,
|
149
|
-
'c',
|
150
|
-
changes
|
151
|
-
),
|
152
|
-
true
|
153
|
-
);
|
154
|
-
|
155
|
-
NEW.log_data := jsonb_set(
|
156
|
-
NEW.log_data,
|
157
|
-
'{v}',
|
158
|
-
to_jsonb(new_v)
|
159
|
-
);
|
160
|
-
|
161
|
-
IF history_limit IS NOT NULL AND history_limit = size THEN
|
162
|
-
merged := jsonb_build_object(
|
163
|
-
'ts',
|
164
|
-
NEW.log_data#>'{h,1,ts}',
|
165
|
-
'v',
|
166
|
-
NEW.log_data#>'{h,1,v}',
|
167
|
-
'c',
|
168
|
-
(NEW.log_data#>'{h,0,c}') || (NEW.log_data#>'{h,1,c}')
|
169
|
-
);
|
170
|
-
|
171
|
-
NEW.log_data := jsonb_set(
|
172
|
-
NEW.log_data,
|
173
|
-
'{h}',
|
174
|
-
jsonb_set(
|
175
|
-
NEW.log_data->'h',
|
176
|
-
'{1}',
|
177
|
-
merged
|
178
|
-
) - 0
|
179
|
-
);
|
180
|
-
END IF;
|
181
|
-
END IF;
|
182
|
-
|
183
|
-
return NEW;
|
184
|
-
END;
|
185
|
-
$body$
|
186
|
-
LANGUAGE plpgsql;
|
187
|
-
SQL
|
188
|
-
|
189
|
-
create_table :users, force: true do |t|
|
190
|
-
t.string :email
|
191
|
-
t.integer :position
|
192
|
-
t.string :name
|
193
|
-
t.text :bio
|
194
|
-
t.integer :age
|
195
|
-
t.timestamps
|
196
|
-
end
|
197
|
-
|
198
|
-
create_table :logidze_users, force: true do |t|
|
199
|
-
t.string :email
|
200
|
-
t.integer :position
|
201
|
-
t.string :name
|
202
|
-
t.text :bio
|
203
|
-
t.integer :age
|
204
|
-
t.jsonb :log_data, default: "{}", null: false
|
205
|
-
t.timestamps
|
206
|
-
end
|
207
|
-
|
208
|
-
execute <<~SQL
|
209
|
-
CREATE TRIGGER logidze_on_logidze_users
|
210
|
-
BEFORE UPDATE OR INSERT ON logidze_users FOR EACH ROW
|
211
|
-
WHEN (current_setting('logidze.disabled') <> 'on')
|
212
|
-
EXECUTE PROCEDURE logidze_logger();
|
213
|
-
SQL
|
214
|
-
end
|
215
|
-
end
|
216
|
-
|
217
|
-
module_function
|
218
|
-
|
219
|
-
def populate(n = 1_000)
|
220
|
-
n.times do
|
221
|
-
params = fake_params
|
222
|
-
User.create!(params)
|
223
|
-
LogidzeUser.create!(params)
|
224
|
-
end
|
225
|
-
end
|
226
|
-
|
227
|
-
module_function
|
228
|
-
|
229
|
-
def cleanup
|
230
|
-
LogidzeUser.delete_all
|
231
|
-
User.delete_all
|
232
|
-
PaperTrail::Version.delete_all
|
233
|
-
end
|
234
|
-
|
235
|
-
module_function
|
236
|
-
|
237
|
-
def generate_versions(num = 1)
|
238
|
-
num.times do
|
239
|
-
User.find_each do |u|
|
240
|
-
u.update!(fake_params(sample: true))
|
241
|
-
end
|
242
|
-
|
243
|
-
LogidzeUser.find_each do |u|
|
244
|
-
u.update!(fake_params(sample: true))
|
245
|
-
end
|
246
|
-
|
247
|
-
# make at least 1 second between versions
|
248
|
-
sleep 1
|
249
|
-
end
|
250
|
-
Time.now
|
251
|
-
end
|
252
|
-
|
253
|
-
module_function
|
254
|
-
|
255
|
-
def fake_params(sample: false)
|
256
|
-
params = {
|
257
|
-
email: Faker::Internet.email,
|
258
|
-
position: Faker::Number.number(3),
|
259
|
-
name: Faker::Name.name,
|
260
|
-
age: Faker::Number.number(2),
|
261
|
-
bio: Faker::Lorem.paragraph
|
262
|
-
}
|
263
|
-
|
264
|
-
return params.slice(%i[email position name age bio].sample) if sample
|
265
|
-
params
|
266
|
-
end
|
267
|
-
end
|
268
|
-
|
269
|
-
module ARandom
|
270
|
-
def random(num = 1)
|
271
|
-
rel = order("random()")
|
272
|
-
num == 1 ? rel.first : rel.limit(num)
|
273
|
-
end
|
274
|
-
end
|
275
|
-
|
276
|
-
class User < ActiveRecord::Base
|
277
|
-
extend ARandom
|
278
|
-
has_paper_trail
|
279
|
-
|
280
|
-
def self.diff_from(ts)
|
281
|
-
includes(:versions).map { |u| {"id" => u.id, "changes" => u.diff_from(ts)} }
|
282
|
-
end
|
283
|
-
|
284
|
-
def self.diff_from_joined(ts)
|
285
|
-
eager_load(:versions).map { |u| {"id" => u.id, "changes" => u.diff_from(ts)} }
|
286
|
-
end
|
287
|
-
|
288
|
-
def diff_from(ts)
|
289
|
-
changes = {}
|
290
|
-
versions.each do |v|
|
291
|
-
next if v.created_at < ts
|
292
|
-
merge_changeset(changes, v.changeset)
|
293
|
-
end
|
294
|
-
changes
|
295
|
-
end
|
296
|
-
|
297
|
-
private
|
298
|
-
|
299
|
-
def merge_changeset(acc, data)
|
300
|
-
data.each do |k, v|
|
301
|
-
unless acc.key?(k)
|
302
|
-
acc[k] = {"old" => v[0]}
|
303
|
-
end
|
304
|
-
acc[k]["new"] = v[1]
|
305
|
-
end
|
306
|
-
end
|
307
|
-
end
|
308
|
-
|
309
|
-
class LogidzeUser < ActiveRecord::Base
|
310
|
-
extend ARandom
|
311
|
-
has_logidze
|
312
|
-
end
|
313
|
-
|
314
|
-
# Run migration only if neccessary
|
315
|
-
LogidzeBench.setup_db if ENV["FORCE"].present? || !ActiveRecord::Base.connection.tables.include?("logidze_users")
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "benchmark/ips"
|
4
|
-
require "./setup"
|
5
|
-
|
6
|
-
params = {
|
7
|
-
age: Faker::Number.number(2),
|
8
|
-
email: Faker::Internet.email
|
9
|
-
}
|
10
|
-
|
11
|
-
params2 = {
|
12
|
-
email: Faker::Internet.email,
|
13
|
-
position: Faker::Number.number(3),
|
14
|
-
name: Faker::Name.name,
|
15
|
-
age: Faker::Number.number(2),
|
16
|
-
bio: Faker::Lorem.paragraph
|
17
|
-
}
|
18
|
-
|
19
|
-
LogidzeBench.cleanup
|
20
|
-
LogidzeBench.populate
|
21
|
-
|
22
|
-
Benchmark.ips do |x|
|
23
|
-
x.report("PT UPDATE #1") do
|
24
|
-
User.random.update!(params)
|
25
|
-
end
|
26
|
-
|
27
|
-
x.report("Logidze UPDATE #1") do
|
28
|
-
LogidzeUser.random.update!(params)
|
29
|
-
end
|
30
|
-
|
31
|
-
x.report("PT UPDATE #2") do
|
32
|
-
User.random.update!(params2)
|
33
|
-
end
|
34
|
-
|
35
|
-
x.report("Logidze UPDATE #2") do
|
36
|
-
LogidzeUser.random.update!(params2)
|
37
|
-
end
|
38
|
-
end
|
data/bench/triggers/Makefile
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
# Database name
|
2
|
-
ifndef DB
|
3
|
-
DB = logidze_bench
|
4
|
-
endif
|
5
|
-
|
6
|
-
# Transactions count
|
7
|
-
ifndef T
|
8
|
-
T = 10000
|
9
|
-
endif
|
10
|
-
|
11
|
-
all: plain hstore jsonb jsonb2 keys keys2
|
12
|
-
|
13
|
-
setup:
|
14
|
-
createdb $(DB) -w
|
15
|
-
psql -q -d $(DB) -c 'CREATE EXTENSION IF NOT EXISTS hstore;'
|
16
|
-
|
17
|
-
plain:
|
18
|
-
$(info )
|
19
|
-
$(info ====== [START] Update without triggers ======)
|
20
|
-
pgbench -i -q $(DB)
|
21
|
-
pgbench -f bench.sql -t $(T) -r $(DB)
|
22
|
-
|
23
|
-
hstore:
|
24
|
-
$(info )
|
25
|
-
$(info ====== [START] Update with hstore-based triggers ======)
|
26
|
-
pgbench -i -q $(DB)
|
27
|
-
psql -q -d $(DB) -f hstore_trigger_setup.sql
|
28
|
-
pgbench -f bench.sql -t $(T) -r $(DB)
|
29
|
-
|
30
|
-
jsonb:
|
31
|
-
$(info )
|
32
|
-
$(info ====== [START] Update with jsonb-minus triggers ======)
|
33
|
-
pgbench -i -q $(DB)
|
34
|
-
psql -q -d $(DB) -f jsonb_minus_setup.sql
|
35
|
-
pgbench -f bench.sql -t $(T) -r $(DB)
|
36
|
-
|
37
|
-
jsonb2:
|
38
|
-
$(info )
|
39
|
-
$(info ====== [START] Update with jsonb-minus triggers ======)
|
40
|
-
pgbench -i -q $(DB)
|
41
|
-
psql -q -d $(DB) -f jsonb_minus_2_setup.sql
|
42
|
-
pgbench -f bench.sql -t $(T) -r $(DB)
|
43
|
-
|
44
|
-
keys:
|
45
|
-
$(info )
|
46
|
-
$(info ====== [START] Update with loop thru keys triggers (v1) ======)
|
47
|
-
pgbench -i -q $(DB)
|
48
|
-
psql -q -d $(DB) -f keys_trigger_setup.sql
|
49
|
-
pgbench -f bench.sql -t $(T) -r $(DB)
|
50
|
-
|
51
|
-
keys2:
|
52
|
-
$(info )
|
53
|
-
$(info ====== [START] Update with loop thru keys triggers (v2) ======)
|
54
|
-
pgbench -i -q $(DB)
|
55
|
-
psql -q -d $(DB) -f keys2_trigger_setup.sql
|
56
|
-
pgbench -f bench.sql -t $(T) -r $(DB)
|
data/bench/triggers/Readme.md
DELETED
@@ -1,58 +0,0 @@
|
|
1
|
-
# Triggers benchmarks
|
2
|
-
|
3
|
-
This benchmark uses standard _pg\_bench_ table `pgbench_accounts`.
|
4
|
-
We consider several approaches for calculating records diff: one uses _hstore_ extension, two uses jsonb functions and two others iterate through record fields.
|
5
|
-
|
6
|
-
# Usage
|
7
|
-
|
8
|
-
Create database:
|
9
|
-
|
10
|
-
```sh
|
11
|
-
make setup
|
12
|
-
```
|
13
|
-
|
14
|
-
You can provide database name by `DB` variable (defaults to "logidze_bench").
|
15
|
-
|
16
|
-
Run all benchmarks:
|
17
|
-
|
18
|
-
```sh
|
19
|
-
make
|
20
|
-
```
|
21
|
-
|
22
|
-
or separate benchmark:
|
23
|
-
|
24
|
-
```sh
|
25
|
-
make hstore
|
26
|
-
|
27
|
-
make jsonb
|
28
|
-
|
29
|
-
make jsonb2
|
30
|
-
|
31
|
-
make keys
|
32
|
-
|
33
|
-
make keys2
|
34
|
-
|
35
|
-
# Raw update, no triggers
|
36
|
-
make plain
|
37
|
-
```
|
38
|
-
|
39
|
-
You can specify the number of transactions by `T` variable (defaults to 10000):
|
40
|
-
|
41
|
-
```sh
|
42
|
-
make T=1000000
|
43
|
-
```
|
44
|
-
|
45
|
-
# Results
|
46
|
-
|
47
|
-
The benchmark shows that hstore variant is the most efficient (running on MacPro 2013, 2.4 GHz Core i5, 4GB, SSD, 1 million transactions per test):
|
48
|
-
|
49
|
-
|Mode | TPS | Statement latency (ms) |
|
50
|
-
|--------|------|------------------------|
|
51
|
-
| plain | 3628 | 0.113 |
|
52
|
-
| hstore | 3015 | 0.168 |
|
53
|
-
| jsonb | 1647 | 0.363 |
|
54
|
-
| jsonb2 | 1674 | 0.354 |
|
55
|
-
| keys | 2355 | 0.219 |
|
56
|
-
| keys2 | 2542 | 0.210 |
|
57
|
-
|
58
|
-
_Logidze_ uses hstore variant.
|