logidze 0.1.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.hound.yml +3 -0
- data/.travis.yml +2 -4
- data/CHANGELOG.md +6 -0
- data/Gemfile +2 -2
- data/README.md +16 -15
- data/bench/performance/README.md +109 -0
- data/bench/performance/diff_bench.rb +36 -0
- data/bench/performance/insert_bench.rb +20 -0
- data/bench/performance/memory_profile.rb +53 -0
- data/bench/performance/setup.rb +308 -0
- data/bench/performance/update_bench.rb +36 -0
- data/bench/{Makefile → triggers/Makefile} +0 -0
- data/bench/{Readme.md → triggers/Readme.md} +11 -11
- data/bench/{bench.sql → triggers/bench.sql} +0 -0
- data/bench/{hstore_trigger_setup.sql → triggers/hstore_trigger_setup.sql} +0 -0
- data/bench/{jsonb_minus_2_setup.sql → triggers/jsonb_minus_2_setup.sql} +0 -0
- data/bench/{jsonb_minus_setup.sql → triggers/jsonb_minus_setup.sql} +0 -0
- data/bench/{keys2_trigger_setup.sql → triggers/keys2_trigger_setup.sql} +0 -0
- data/bench/{keys_trigger_setup.sql → triggers/keys_trigger_setup.sql} +0 -0
- data/circle.yml +1 -3
- data/gemfiles/rails42.gemfile +1 -1
- data/gemfiles/rails5.gemfile +2 -1
- data/lib/logidze/history.rb +7 -28
- data/lib/logidze/history/type.rb +40 -0
- data/lib/logidze/history/version.rb +30 -0
- data/lib/logidze/model.rb +7 -1
- data/lib/logidze/version.rb +1 -1
- data/logidze.gemspec +1 -2
- metadata +21 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 437f0707fbb215fbfb5655156b0f2f8450f2aa20
|
4
|
+
data.tar.gz: 533a010cbc9d240c5c70d565621ace20d571579a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 419c21e03ba9196ff236bb7d8ea1bc897eaa1e7b8573bff1e1196008113641975341712e2f55d0ea8abb887820c18e0f32b654ee673ea379ef263cdae63ce875
|
7
|
+
data.tar.gz: 8a885b62f97e4492001ef11a1961556faa0810d976f95fdb44dcb3071a5506622edc798f87aa6dd8fb27e40b46d605dd6adced7aa16e19278d2feb40f20beeb8
|
data/.hound.yml
ADDED
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -3,10 +3,10 @@ source 'https://rubygems.org'
|
|
3
3
|
# Specify your gem's dependencies in logidze.gemspec
|
4
4
|
gemspec
|
5
5
|
|
6
|
-
local_gemfile =
|
6
|
+
local_gemfile = "#{File.dirname(__FILE__)}/Gemfile.local"
|
7
7
|
|
8
8
|
if File.exist?(local_gemfile)
|
9
9
|
eval(File.read(local_gemfile)) # rubocop:disable Lint/Eval
|
10
10
|
else
|
11
|
-
gem 'activerecord', '~>
|
11
|
+
gem 'activerecord', '~> 5.0.0'
|
12
12
|
end
|
data/README.md
CHANGED
@@ -2,23 +2,24 @@
|
|
2
2
|
|
3
3
|
# Logidze
|
4
4
|
|
5
|
-
Logidze provides tools for logging DB records changes.
|
6
|
-
**This is not [audited](https://github.com/collectiveidea/audited) or [paper_trail](https://github.com/airblade/paper_trail) alternative!**
|
5
|
+
Logidze provides tools for logging DB records changes. Just like [audited](https://github.com/collectiveidea/audited) and [paper_trail](https://github.com/airblade/paper_trail) do (but [faster](bench/performance)).
|
7
6
|
|
8
|
-
Logidze allows you to create DB-level log (using triggers) and gives you an API to browse this log.
|
9
|
-
|
7
|
+
Logidze allows you to create a DB-level log (using triggers) and gives you an API to browse this log.
|
8
|
+
The log is stored with the record itself in JSONB column. No additional tables required.
|
10
9
|
Currently, only PostgreSQL 9.5+ is supported.
|
11
10
|
|
11
|
+
[Read the story behind Logidze](https://evilmartians.com/chronicles/introducing-logidze?utm_source=logidze)
|
12
|
+
|
12
13
|
Other requirements:
|
13
|
-
- Ruby ~>2.3;
|
14
|
-
- Rails
|
14
|
+
- Ruby ~> 2.3;
|
15
|
+
- Rails >= 4.2 (Yes, Rails 5 is supported!).
|
15
16
|
|
16
17
|
<a href="https://evilmartians.com/">
|
17
18
|
<img src="https://evilmartians.com/badges/sponsored-by-evil-martians.svg" alt="Sponsored by Evil Martians" width="236" height="54"></a>
|
18
19
|
|
19
20
|
## Installation
|
20
21
|
|
21
|
-
1. Add Logidze your application's Gemfile:
|
22
|
+
1. Add Logidze to your application's Gemfile:
|
22
23
|
|
23
24
|
```ruby
|
24
25
|
gem 'logidze'
|
@@ -48,7 +49,7 @@ rake db:migrate
|
|
48
49
|
You can provide `limit` option to `generate` to limit the size of the log (by default it's unlimited):
|
49
50
|
|
50
51
|
```ruby
|
51
|
-
rails generate logidze:
|
52
|
+
rails generate logidze:model Post --limit=10
|
52
53
|
```
|
53
54
|
|
54
55
|
This also adds `has_logidze` line to your model, which adds methods for working with logs.
|
@@ -68,7 +69,7 @@ post.log_version #=> 3
|
|
68
69
|
# Show log size (number of versions)
|
69
70
|
post.log_size #=> 3
|
70
71
|
|
71
|
-
# Get copy of a record at a given time
|
72
|
+
# Get copy of a record at a given time
|
72
73
|
old_post = post.at(2.days.ago)
|
73
74
|
|
74
75
|
# or revert the record itself to the previous state (without committing to DB)
|
@@ -134,12 +135,12 @@ The `log_data` column has the following format:
|
|
134
135
|
```js
|
135
136
|
{
|
136
137
|
"v": 2, // current record version,
|
137
|
-
"h": // list of changes
|
138
|
+
"h": // list of changes
|
138
139
|
[
|
139
140
|
{
|
140
141
|
"v": 1, // change number
|
141
142
|
"ts": 1460805759352, // change timestamp in milliseconds
|
142
|
-
"c": {
|
143
|
+
"c": {
|
143
144
|
"attr": "new value", // updated fields with new values
|
144
145
|
"attr2": "new value"
|
145
146
|
}
|
@@ -148,7 +149,7 @@ The `log_data` column has the following format:
|
|
148
149
|
}
|
149
150
|
```
|
150
151
|
|
151
|
-
If you
|
152
|
+
If you specify the limit in the trigger definition then log size will not exceed the specified size. When a new change occurs, and there is no more room for it, the two oldest changes will be merged.
|
152
153
|
|
153
154
|
## Development
|
154
155
|
|
@@ -161,9 +162,9 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/palkan
|
|
161
162
|
|
162
163
|
## TODO
|
163
164
|
|
164
|
-
- Exclude columns from log
|
165
|
-
- Enhance
|
166
|
-
- Other DB adapters
|
165
|
+
- Exclude columns from the log.
|
166
|
+
- Enhance update_all to support mass-logging.
|
167
|
+
- Other DB adapters.
|
167
168
|
|
168
169
|
## License
|
169
170
|
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# Performance benchmarks: PaperTail vs. Logidze
|
2
|
+
|
3
|
+
We want to compare Logidze with the most popular versioning library for Rails – PaperTrail.
|
4
|
+
|
5
|
+
|
6
|
+
## Insert ([source](insert_bench.rb))
|
7
|
+
|
8
|
+
```
|
9
|
+
PaperTrail INSERT 213.148 (± 8.9%) i/s - 1.060k in 5.018504s
|
10
|
+
Logidze INSERT 613.387 (±16.3%) i/s - 2.970k in 5.036127s
|
11
|
+
```
|
12
|
+
|
13
|
+
|
14
|
+
## Update ([source](update_bench.rb))
|
15
|
+
|
16
|
+
When changeset has 2 fields:
|
17
|
+
|
18
|
+
```
|
19
|
+
PaperTrail UPDATE #1 256.651 (±26.5%) i/s - 1.206k in 5.002300s
|
20
|
+
Logidze UPDATE #1 356.932 (±12.6%) i/s - 1.764k in 5.030560s
|
21
|
+
```
|
22
|
+
|
23
|
+
When changeset has 5 fields:
|
24
|
+
|
25
|
+
```
|
26
|
+
PaperTrail UPDATE #2 246.281 (±24.0%) i/s - 1.168k in 5.008234s
|
27
|
+
Logidze UPDATE #2 331.942 (±16.6%) i/s - 1.593k in 5.028135s
|
28
|
+
```
|
29
|
+
|
30
|
+
## Getting diff ([source](diff_bench.rb))
|
31
|
+
|
32
|
+
PaperTrail doesn't have built-in method to calculate diff between not adjacent versions.
|
33
|
+
We add `diff_from(ts)` and `diff_from_joined(ts)` (which uses SQL JOIN) methods to calculate diff from specified version using changesets.
|
34
|
+
|
35
|
+
When each record has 10 versions:
|
36
|
+
|
37
|
+
```
|
38
|
+
PT DIFF 20.874 (± 4.8%) i/s - 106.000 in 5.091402s
|
39
|
+
PT (join) DIFF 20.619 (± 4.8%) i/s - 104.000 in 5.070160s
|
40
|
+
Logidze DIFF 109.482 (±24.7%) i/s - 500.000 in 5.103534s
|
41
|
+
```
|
42
|
+
|
43
|
+
When each record has 100 versions:
|
44
|
+
|
45
|
+
```
|
46
|
+
PT DIFF 2.998 (± 0.0%) i/s - 15.000 in 5.019494s
|
47
|
+
PT (join) DIFF 3.193 (± 0.0%) i/s - 16.000 in 5.030155s
|
48
|
+
Logidze DIFF 19.627 (±25.5%) i/s - 88.000 in 5.035555s
|
49
|
+
```
|
50
|
+
|
51
|
+
And, finally, when each record has 1000 versions:
|
52
|
+
|
53
|
+
```
|
54
|
+
PT DIFF 0.270 (± 0.0%) i/s - 17.000 in 63.038374s
|
55
|
+
PT (join) DIFF 0.235 (± 0.0%) i/s - 14.000 in 60.350886s
|
56
|
+
Logidze DIFF 2.022 (± 0.0%) i/s - 120.000 in 60.142965s
|
57
|
+
```
|
58
|
+
|
59
|
+
## Select memory usage ([source](memory_profile.rb))
|
60
|
+
|
61
|
+
Logidze loads more data (because it stores log in-place). But how much more?
|
62
|
+
We consider two cases for PaperTrail: when we want to calculate diff (and thus loading versions) and when we don't need any history related data.
|
63
|
+
|
64
|
+
When each record has 10 versions:
|
65
|
+
|
66
|
+
```
|
67
|
+
PT records
|
68
|
+
Total Allocated: 27.8 KB
|
69
|
+
Total Retained: 16.59 KB
|
70
|
+
Retained memory (per record): 2.14 KB
|
71
|
+
|
72
|
+
PT with versions
|
73
|
+
Total Allocated: 228.01 KB
|
74
|
+
Total Retained: 170.78 KB
|
75
|
+
Retained memory (per record): 143.13 KB
|
76
|
+
|
77
|
+
Logidze records
|
78
|
+
Total Allocated: 46.45 KB
|
79
|
+
Total Retained: 34.73 KB
|
80
|
+
Retained memory (per record): 4.11 KB
|
81
|
+
```
|
82
|
+
|
83
|
+
When each record has 100 versions:
|
84
|
+
|
85
|
+
```
|
86
|
+
PT with versions
|
87
|
+
Total Allocated: 1.92 MB
|
88
|
+
Total Retained: 1.56 MB
|
89
|
+
Retained memory (per record): 1.53 MB
|
90
|
+
|
91
|
+
Logidze records
|
92
|
+
Total Allocated: 162.48 KB
|
93
|
+
Total Retained: 150.76 KB
|
94
|
+
Retained memory (per record): 15.4 KB
|
95
|
+
```
|
96
|
+
|
97
|
+
When each record has 1000 versions:
|
98
|
+
|
99
|
+
```
|
100
|
+
PT with versions
|
101
|
+
Total Allocated: 18.23 MB
|
102
|
+
Total Retained: 14.86 MB
|
103
|
+
Retained memory (per record): 14.83 MB
|
104
|
+
|
105
|
+
Logidze records
|
106
|
+
Total Allocated: 1.32 MB
|
107
|
+
Total Retained: 1.31 MB
|
108
|
+
Retained memory (per record): 131.59 KB
|
109
|
+
```
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'benchmark/ips'
|
2
|
+
require './setup'
|
3
|
+
|
4
|
+
# How many records do you want?
|
5
|
+
N = (ENV['N'] || '100').to_i
|
6
|
+
|
7
|
+
# How many version each record has?
|
8
|
+
V = (ENV['V'] || '10').to_i
|
9
|
+
|
10
|
+
# Benchmark run time
|
11
|
+
BM_TIME = (ENV['BM_TIME'] || 5).to_i
|
12
|
+
|
13
|
+
BM_WARMUP = [(BM_TIME / 10), 2].max
|
14
|
+
|
15
|
+
LogidzeBench.cleanup
|
16
|
+
LogidzeBench.populate(N)
|
17
|
+
|
18
|
+
ts1 = LogidzeBench.generate_versions(V/2)
|
19
|
+
|
20
|
+
LogidzeBench.generate_versions(V/2)
|
21
|
+
|
22
|
+
Benchmark.ips do |x|
|
23
|
+
x.config(time: BM_TIME, warmup: BM_WARMUP)
|
24
|
+
|
25
|
+
x.report('PT DIFF') do
|
26
|
+
User.random(N/2).diff_from(ts1)
|
27
|
+
end
|
28
|
+
|
29
|
+
x.report('PT (join) DIFF') do
|
30
|
+
User.random(N/2).diff_from_joined(ts1)
|
31
|
+
end
|
32
|
+
|
33
|
+
x.report('Logidze DIFF') do
|
34
|
+
LogidzeUser.random(N/2).diff_from(ts1)
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'benchmark/ips'
|
2
|
+
require './setup'
|
3
|
+
|
4
|
+
params = {
|
5
|
+
email: Faker::Internet.email,
|
6
|
+
position: Faker::Number.number(3),
|
7
|
+
name: Faker::Name.name,
|
8
|
+
age: Faker::Number.number(2),
|
9
|
+
bio: Faker::Lorem.paragraph
|
10
|
+
}
|
11
|
+
|
12
|
+
Benchmark.ips do |x|
|
13
|
+
x.report('PaperTrail INSERT') do
|
14
|
+
User.create!(params)
|
15
|
+
end
|
16
|
+
|
17
|
+
x.report('Logidze INSERT') do
|
18
|
+
LogidzeUser.create!(params)
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require './setup'
|
2
|
+
require 'active_support/core_ext'
|
3
|
+
require 'memory_profiler'
|
4
|
+
|
5
|
+
# How many records do you want?
|
6
|
+
N = (ENV['N'] || '10').to_i
|
7
|
+
|
8
|
+
# How many version each record has?
|
9
|
+
V = (ENV['V'] || '10').to_i
|
10
|
+
|
11
|
+
LogidzeBench.cleanup
|
12
|
+
LogidzeBench.populate(N)
|
13
|
+
LogidzeBench.generate_versions(V)
|
14
|
+
|
15
|
+
module MemoryReport
|
16
|
+
KILO_BYTE = 1024
|
17
|
+
MEGA_BYTE = 1024 * 1024
|
18
|
+
|
19
|
+
module_function
|
20
|
+
def call(msg, relation)
|
21
|
+
buffer = nil
|
22
|
+
delta = N / 10
|
23
|
+
r0 = MemoryProfiler.report do
|
24
|
+
buffer = relation.random(N - delta).to_a
|
25
|
+
end
|
26
|
+
|
27
|
+
buffer = nil
|
28
|
+
r1 = MemoryProfiler.report do
|
29
|
+
buffer = relation.to_a
|
30
|
+
end
|
31
|
+
|
32
|
+
$stdout.puts msg
|
33
|
+
$stdout.puts "Total Allocated:\t\t\t\t#{to_human_size(r1.total_allocated_memsize)}"
|
34
|
+
$stdout.puts "Total Retained:\t\t\t\t\t#{to_human_size(r1.total_retained_memsize)}"
|
35
|
+
$stdout.puts "Retained_memsize memory (per record):\t\t#{to_human_size((r1.total_retained_memsize - r0.total_retained_memsize) / delta)}"
|
36
|
+
end
|
37
|
+
|
38
|
+
module_function
|
39
|
+
def to_human_size(size)
|
40
|
+
if size > MEGA_BYTE
|
41
|
+
"#{(size.to_f / MEGA_BYTE).round(2)} MB"
|
42
|
+
elsif size > KILO_BYTE
|
43
|
+
"#{(size.to_f / KILO_BYTE).round(2)} KB"
|
44
|
+
else
|
45
|
+
"#{size} B"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
MemoryReport.("PT records", User.all)
|
52
|
+
MemoryReport.("PT with versions", User.joins(:versions).all)
|
53
|
+
MemoryReport.("Logidze records", LogidzeUser.all)
|
@@ -0,0 +1,308 @@
|
|
1
|
+
begin
|
2
|
+
require 'bundler/inline'
|
3
|
+
rescue LoadError => e
|
4
|
+
$stderr.puts 'Bundler version 1.10 or later is required. Please update your Bundler'
|
5
|
+
raise e
|
6
|
+
end
|
7
|
+
|
8
|
+
gemfile(true) do
|
9
|
+
source 'https://rubygems.org'
|
10
|
+
gem 'activerecord', '~>4.2'
|
11
|
+
gem 'pg'
|
12
|
+
gem 'paper_trail', '~>4.2', require: false
|
13
|
+
gem 'pry-byebug'
|
14
|
+
gem 'faker'
|
15
|
+
gem 'benchmark-ips'
|
16
|
+
gem 'memory_profiler'
|
17
|
+
end
|
18
|
+
|
19
|
+
DB_NAME = ENV['DB_NAME'] || 'logidze_query_bench'
|
20
|
+
|
21
|
+
begin
|
22
|
+
system("createdb #{DB_NAME}")
|
23
|
+
rescue
|
24
|
+
$stdout.puts "DB already exists"
|
25
|
+
end
|
26
|
+
|
27
|
+
$LOAD_PATH.unshift File.expand_path('../../../lib', __FILE__)
|
28
|
+
|
29
|
+
require 'active_record'
|
30
|
+
require 'logger'
|
31
|
+
require 'logidze'
|
32
|
+
|
33
|
+
ActiveRecord::Base.send :include, Logidze::HasLogidze
|
34
|
+
|
35
|
+
ActiveRecord::Base.establish_connection(adapter: 'postgresql', database: DB_NAME)
|
36
|
+
|
37
|
+
at_exit do
|
38
|
+
ActiveRecord::Base.connection.disconnect!
|
39
|
+
end
|
40
|
+
|
41
|
+
require 'paper_trail'
|
42
|
+
|
43
|
+
module LogidzeBench
|
44
|
+
module_function
|
45
|
+
def setup_db
|
46
|
+
ActiveRecord::Schema.define do
|
47
|
+
# PaperTrail setup
|
48
|
+
create_table :versions, force: true do |t|
|
49
|
+
t.string :item_type, null: false
|
50
|
+
t.integer :item_id, null: false
|
51
|
+
t.string :event, null: false
|
52
|
+
t.string :whodunnit
|
53
|
+
t.text :object
|
54
|
+
t.jsonb :object_changes
|
55
|
+
|
56
|
+
t.datetime :created_at
|
57
|
+
end
|
58
|
+
|
59
|
+
add_index :versions, [:item_type, :item_id]
|
60
|
+
|
61
|
+
# Logidze setup
|
62
|
+
enable_extension :hstore
|
63
|
+
|
64
|
+
execute <<~SQL
|
65
|
+
DO $$
|
66
|
+
BEGIN
|
67
|
+
EXECUTE 'ALTER DATABASE ' || current_database() || ' SET logidze.disabled TO off';
|
68
|
+
END;
|
69
|
+
$$
|
70
|
+
LANGUAGE plpgsql;
|
71
|
+
SQL
|
72
|
+
|
73
|
+
execute <<~SQL
|
74
|
+
CREATE OR REPLACE FUNCTION logidze_logger() RETURNS TRIGGER AS $body$
|
75
|
+
DECLARE
|
76
|
+
changes jsonb;
|
77
|
+
new_v integer;
|
78
|
+
ts bigint;
|
79
|
+
size integer;
|
80
|
+
history_limit integer;
|
81
|
+
current_version integer;
|
82
|
+
merged jsonb;
|
83
|
+
iterator integer;
|
84
|
+
item record;
|
85
|
+
BEGIN
|
86
|
+
ts := (extract(epoch from now()) * 1000)::bigint;
|
87
|
+
|
88
|
+
IF TG_OP = 'INSERT' THEN
|
89
|
+
changes := to_jsonb(NEW.*) - 'log_data';
|
90
|
+
new_v := 1;
|
91
|
+
|
92
|
+
NEW.log_data := json_build_object(
|
93
|
+
'v',
|
94
|
+
1,
|
95
|
+
'h',
|
96
|
+
jsonb_build_array(
|
97
|
+
jsonb_build_object(
|
98
|
+
'ts',
|
99
|
+
ts,
|
100
|
+
'v',
|
101
|
+
new_v,
|
102
|
+
'c',
|
103
|
+
changes
|
104
|
+
)
|
105
|
+
)
|
106
|
+
);
|
107
|
+
ELSIF TG_OP = 'UPDATE' THEN
|
108
|
+
history_limit := TG_ARGV[0];
|
109
|
+
current_version := (NEW.log_data->>'v')::int;
|
110
|
+
|
111
|
+
IF NEW = OLD THEN
|
112
|
+
RETURN NEW;
|
113
|
+
END IF;
|
114
|
+
|
115
|
+
IF current_version < (NEW.log_data#>>'{h,-1,v}')::int THEN
|
116
|
+
iterator := 0;
|
117
|
+
FOR item in SELECT * FROM jsonb_array_elements(NEW.log_data->'h')
|
118
|
+
LOOP
|
119
|
+
IF (item.value->>'v')::int > current_version THEN
|
120
|
+
NEW.log_data := jsonb_set(
|
121
|
+
NEW.log_data,
|
122
|
+
'{h}',
|
123
|
+
(NEW.log_data->'h') - iterator
|
124
|
+
);
|
125
|
+
END IF;
|
126
|
+
iterator := iterator + 1;
|
127
|
+
END LOOP;
|
128
|
+
END IF;
|
129
|
+
|
130
|
+
changes := hstore_to_jsonb_loose(
|
131
|
+
hstore(NEW.*) - hstore(OLD.*)
|
132
|
+
) - 'log_data';
|
133
|
+
|
134
|
+
new_v := (NEW.log_data#>>'{h,-1,v}')::int + 1;
|
135
|
+
|
136
|
+
size := jsonb_array_length(NEW.log_data->'h');
|
137
|
+
|
138
|
+
NEW.log_data := jsonb_set(
|
139
|
+
NEW.log_data,
|
140
|
+
ARRAY['h', size::text],
|
141
|
+
jsonb_build_object(
|
142
|
+
'ts',
|
143
|
+
ts,
|
144
|
+
'v',
|
145
|
+
new_v,
|
146
|
+
'c',
|
147
|
+
changes
|
148
|
+
),
|
149
|
+
true
|
150
|
+
);
|
151
|
+
|
152
|
+
NEW.log_data := jsonb_set(
|
153
|
+
NEW.log_data,
|
154
|
+
'{v}',
|
155
|
+
to_jsonb(new_v)
|
156
|
+
);
|
157
|
+
|
158
|
+
IF history_limit IS NOT NULL AND history_limit = size THEN
|
159
|
+
merged := jsonb_build_object(
|
160
|
+
'ts',
|
161
|
+
NEW.log_data#>'{h,1,ts}',
|
162
|
+
'v',
|
163
|
+
NEW.log_data#>'{h,1,v}',
|
164
|
+
'c',
|
165
|
+
(NEW.log_data#>'{h,0,c}') || (NEW.log_data#>'{h,1,c}')
|
166
|
+
);
|
167
|
+
|
168
|
+
NEW.log_data := jsonb_set(
|
169
|
+
NEW.log_data,
|
170
|
+
'{h}',
|
171
|
+
jsonb_set(
|
172
|
+
NEW.log_data->'h',
|
173
|
+
'{1}',
|
174
|
+
merged
|
175
|
+
) - 0
|
176
|
+
);
|
177
|
+
END IF;
|
178
|
+
END IF;
|
179
|
+
|
180
|
+
return NEW;
|
181
|
+
END;
|
182
|
+
$body$
|
183
|
+
LANGUAGE plpgsql;
|
184
|
+
SQL
|
185
|
+
|
186
|
+
create_table :users, force: true do |t|
|
187
|
+
t.string :email
|
188
|
+
t.integer :position
|
189
|
+
t.string :name
|
190
|
+
t.text :bio
|
191
|
+
t.integer :age
|
192
|
+
t.timestamps
|
193
|
+
end
|
194
|
+
|
195
|
+
create_table :logidze_users, force: true do |t|
|
196
|
+
t.string :email
|
197
|
+
t.integer :position
|
198
|
+
t.string :name
|
199
|
+
t.text :bio
|
200
|
+
t.integer :age
|
201
|
+
t.jsonb :log_data, default: '{}', null: false
|
202
|
+
t.timestamps
|
203
|
+
end
|
204
|
+
|
205
|
+
execute <<~SQL
|
206
|
+
CREATE TRIGGER logidze_on_logidze_users
|
207
|
+
BEFORE UPDATE OR INSERT ON logidze_users FOR EACH ROW
|
208
|
+
WHEN (current_setting('logidze.disabled') <> 'on')
|
209
|
+
EXECUTE PROCEDURE logidze_logger();
|
210
|
+
SQL
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
module_function
|
215
|
+
def populate(n = 1_000)
|
216
|
+
n.times do
|
217
|
+
params = fake_params
|
218
|
+
User.create!(params)
|
219
|
+
LogidzeUser.create!(params)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
module_function
|
224
|
+
def cleanup
|
225
|
+
LogidzeUser.delete_all
|
226
|
+
User.delete_all
|
227
|
+
PaperTrail::Version.delete_all
|
228
|
+
end
|
229
|
+
|
230
|
+
module_function
|
231
|
+
def generate_versions(num = 1)
|
232
|
+
num.times do
|
233
|
+
User.find_each do |u|
|
234
|
+
u.update!(fake_params(sample: true))
|
235
|
+
end
|
236
|
+
|
237
|
+
LogidzeUser.find_each do |u|
|
238
|
+
u.update!(fake_params(sample: true))
|
239
|
+
end
|
240
|
+
|
241
|
+
# make at least 1 second between versions
|
242
|
+
sleep 1
|
243
|
+
end
|
244
|
+
Time.now
|
245
|
+
end
|
246
|
+
|
247
|
+
module_function
|
248
|
+
def fake_params(sample: false)
|
249
|
+
params = {
|
250
|
+
email: Faker::Internet.email,
|
251
|
+
position: Faker::Number.number(3),
|
252
|
+
name: Faker::Name.name,
|
253
|
+
age: Faker::Number.number(2),
|
254
|
+
bio: Faker::Lorem.paragraph
|
255
|
+
}
|
256
|
+
|
257
|
+
return params.slice(%i(email position name age bio).sample) if sample
|
258
|
+
params
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
module ARandom
|
263
|
+
def random(num = 1)
|
264
|
+
rel = order('random()')
|
265
|
+
num == 1 ? rel.first : rel.limit(num)
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
class User < ActiveRecord::Base
|
270
|
+
extend ARandom
|
271
|
+
has_paper_trail
|
272
|
+
|
273
|
+
def self.diff_from(ts)
|
274
|
+
includes(:versions).map { |u| { 'id' => u.id, 'changes' => u.diff_from(ts) } }
|
275
|
+
end
|
276
|
+
|
277
|
+
def self.diff_from_joined(ts)
|
278
|
+
eager_load(:versions).map { |u| { 'id' => u.id, 'changes' => u.diff_from(ts) } }
|
279
|
+
end
|
280
|
+
|
281
|
+
def diff_from(ts)
|
282
|
+
changes = {}
|
283
|
+
versions.each do |v|
|
284
|
+
next if v.created_at < ts
|
285
|
+
merge_changeset(changes, v.changeset)
|
286
|
+
end
|
287
|
+
changes
|
288
|
+
end
|
289
|
+
|
290
|
+
private
|
291
|
+
|
292
|
+
def merge_changeset(acc, data)
|
293
|
+
data.each do |k,v|
|
294
|
+
unless acc.key?(k)
|
295
|
+
acc[k] = { 'old' => v[0] }
|
296
|
+
end
|
297
|
+
acc[k]['new'] = v[1]
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
301
|
+
|
302
|
+
class LogidzeUser < ActiveRecord::Base
|
303
|
+
extend ARandom
|
304
|
+
has_logidze
|
305
|
+
end
|
306
|
+
|
307
|
+
# Run migration only if neccessary
|
308
|
+
LogidzeBench.setup_db if ENV['FORCE'].present? || !ActiveRecord::Base.connection.tables.include?('logidze_users')
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'benchmark/ips'
|
2
|
+
require './setup'
|
3
|
+
|
4
|
+
params = {
|
5
|
+
age: Faker::Number.number(2),
|
6
|
+
email: Faker::Internet.email
|
7
|
+
}
|
8
|
+
|
9
|
+
params2 = {
|
10
|
+
email: Faker::Internet.email,
|
11
|
+
position: Faker::Number.number(3),
|
12
|
+
name: Faker::Name.name,
|
13
|
+
age: Faker::Number.number(2),
|
14
|
+
bio: Faker::Lorem.paragraph
|
15
|
+
}
|
16
|
+
|
17
|
+
LogidzeBench.cleanup
|
18
|
+
LogidzeBench.populate
|
19
|
+
|
20
|
+
Benchmark.ips do |x|
|
21
|
+
x.report('PT UPDATE #1') do
|
22
|
+
User.random.update!(params)
|
23
|
+
end
|
24
|
+
|
25
|
+
x.report('Logidze UPDATE #1') do
|
26
|
+
LogidzeUser.random.update!(params)
|
27
|
+
end
|
28
|
+
|
29
|
+
x.report('PT UPDATE #2') do
|
30
|
+
User.random.update!(params2)
|
31
|
+
end
|
32
|
+
|
33
|
+
x.report('Logidze UPDATE #2') do
|
34
|
+
LogidzeUser.random.update!(params2)
|
35
|
+
end
|
36
|
+
end
|
File without changes
|
@@ -11,7 +11,7 @@ Create database:
|
|
11
11
|
make setup
|
12
12
|
```
|
13
13
|
|
14
|
-
You can provide database name
|
14
|
+
You can provide database name by `DB` variable (defaults to "logidze_bench").
|
15
15
|
|
16
16
|
Run all benchmarks:
|
17
17
|
|
@@ -32,7 +32,7 @@ make keys
|
|
32
32
|
|
33
33
|
make keys2
|
34
34
|
|
35
|
-
#
|
35
|
+
# Raw update, no triggers
|
36
36
|
make plain
|
37
37
|
```
|
38
38
|
|
@@ -44,15 +44,15 @@ make T=1000000
|
|
44
44
|
|
45
45
|
# Results
|
46
46
|
|
47
|
-
The benchmark shows that hstore
|
47
|
+
The benchmark shows that hstore variant is the most efficient (running on MacPro 2013, 2.4 GHz Core i5, 4GB, SSD, 1 million transactions per test):
|
48
48
|
|
49
49
|
|Mode | TPS | Statement latency (ms) |
|
50
50
|
|--------|------|------------------------|
|
51
|
-
| plain |
|
52
|
-
| hstore |
|
53
|
-
| jsonb |
|
54
|
-
| jsonb2 |
|
55
|
-
| keys |
|
56
|
-
| keys2 |
|
57
|
-
|
58
|
-
_Logidze_ uses
|
51
|
+
| plain | 3628 | 0.113 |
|
52
|
+
| hstore | 3015 | 0.168 |
|
53
|
+
| jsonb | 1647 | 0.363 |
|
54
|
+
| jsonb2 | 1674 | 0.354 |
|
55
|
+
| keys | 2355 | 0.219 |
|
56
|
+
| keys2 | 2542 | 0.210 |
|
57
|
+
|
58
|
+
_Logidze_ uses hstore variant.
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
data/circle.yml
CHANGED
@@ -3,13 +3,11 @@ machine:
|
|
3
3
|
version: 2.3.0
|
4
4
|
|
5
5
|
environment:
|
6
|
-
DATABASE_URL: postgres://postgres@127.0.0.1:
|
6
|
+
DATABASE_URL: postgres://postgres@127.0.0.1:5432/test_database
|
7
7
|
|
8
8
|
dependencies:
|
9
9
|
pre:
|
10
10
|
- gem install bundler -v 1.11.2
|
11
|
-
- sudo service postgresql stop 9.4
|
12
|
-
- sudo cp -v /etc/postgresql/9.{4,5}/main/pg_hba.conf && sudo service postgresql restart 9.5
|
13
11
|
|
14
12
|
database:
|
15
13
|
override:
|
data/gemfiles/rails42.gemfile
CHANGED
data/gemfiles/rails5.gemfile
CHANGED
data/lib/logidze/history.rb
CHANGED
@@ -1,38 +1,19 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
module Logidze
|
3
|
-
# Log data
|
3
|
+
# Log data wrapper
|
4
4
|
class History
|
5
|
+
require 'logidze/history/version'
|
6
|
+
|
5
7
|
# History key
|
6
8
|
HISTORY = 'h'
|
7
9
|
# Version key
|
8
10
|
VERSION = 'v'
|
9
11
|
|
10
|
-
|
11
|
-
class Version
|
12
|
-
# Timestamp key
|
13
|
-
TS = 'ts'
|
14
|
-
# Changes key
|
15
|
-
CHANGES = 'c'
|
16
|
-
|
17
|
-
attr_reader :data
|
18
|
-
|
19
|
-
def initialize(data)
|
20
|
-
@data = data
|
21
|
-
end
|
22
|
-
|
23
|
-
def version
|
24
|
-
data.fetch(VERSION)
|
25
|
-
end
|
26
|
-
|
27
|
-
def changes
|
28
|
-
data.fetch(CHANGES)
|
29
|
-
end
|
12
|
+
attr_reader :data
|
30
13
|
|
31
|
-
|
32
|
-
data.fetch(TS)
|
33
|
-
end
|
34
|
-
end
|
14
|
+
delegate :size, to: :versions
|
35
15
|
|
16
|
+
### Rails 4 ###
|
36
17
|
def self.dump(object)
|
37
18
|
ActiveSupport::JSON.encode(object)
|
38
19
|
end
|
@@ -41,9 +22,7 @@ module Logidze
|
|
41
22
|
new(json) unless json.nil?
|
42
23
|
end
|
43
24
|
|
44
|
-
|
45
|
-
|
46
|
-
delegate :size, to: :versions
|
25
|
+
### Rails 4 ###
|
47
26
|
|
48
27
|
def initialize(data)
|
49
28
|
@data = data
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'active_model/type/value'
|
3
|
+
|
4
|
+
module Logidze
|
5
|
+
class History
|
6
|
+
# Type for converting JSONB to and from History
|
7
|
+
class Type < ActiveModel::Type::Value
|
8
|
+
def type
|
9
|
+
:jsonb
|
10
|
+
end
|
11
|
+
|
12
|
+
# rubocop:disable Style/RescueModifier
|
13
|
+
def cast_value(value)
|
14
|
+
case value
|
15
|
+
when String
|
16
|
+
decoded = ::ActiveSupport::JSON.decode(value) rescue nil
|
17
|
+
History.new(decoded) if decoded.present?
|
18
|
+
when Hash
|
19
|
+
History.new(value)
|
20
|
+
when History
|
21
|
+
value
|
22
|
+
end
|
23
|
+
end
|
24
|
+
# rubocop:enable Style/RescueModifier
|
25
|
+
|
26
|
+
def serialize(value)
|
27
|
+
case value
|
28
|
+
when Hash, History
|
29
|
+
::ActiveSupport::JSON.encode(value)
|
30
|
+
else
|
31
|
+
super
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def changed_in_place?(raw_old_value, new_value)
|
36
|
+
cast_value(raw_old_value) != new_value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Logidze
|
3
|
+
class History
|
4
|
+
# Represents one log item
|
5
|
+
class Version
|
6
|
+
# Timestamp key
|
7
|
+
TS = 'ts'
|
8
|
+
# Changes key
|
9
|
+
CHANGES = 'c'
|
10
|
+
|
11
|
+
attr_reader :data
|
12
|
+
|
13
|
+
def initialize(data)
|
14
|
+
@data = data
|
15
|
+
end
|
16
|
+
|
17
|
+
def version
|
18
|
+
data.fetch(VERSION)
|
19
|
+
end
|
20
|
+
|
21
|
+
def changes
|
22
|
+
data.fetch(CHANGES)
|
23
|
+
end
|
24
|
+
|
25
|
+
def time
|
26
|
+
data.fetch(TS)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/logidze/model.rb
CHANGED
@@ -4,10 +4,16 @@ require 'active_support'
|
|
4
4
|
module Logidze
|
5
5
|
# Extends model with methods to browse history
|
6
6
|
module Model
|
7
|
+
require 'logidze/history/type' if Rails::VERSION::MAJOR >= 5
|
8
|
+
|
7
9
|
extend ActiveSupport::Concern
|
8
10
|
|
9
11
|
included do
|
10
|
-
|
12
|
+
if Rails::VERSION::MAJOR < 5
|
13
|
+
serialize :log_data, Logidze::History
|
14
|
+
else
|
15
|
+
attribute :log_data, Logidze::History::Type.new
|
16
|
+
end
|
11
17
|
|
12
18
|
delegate :version, :size, to: :log_data, prefix: "log"
|
13
19
|
end
|
data/lib/logidze/version.rb
CHANGED
data/logidze.gemspec
CHANGED
@@ -17,12 +17,11 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
|
-
spec.add_dependency "rails", ">= 4.2
|
20
|
+
spec.add_dependency "rails", ">= 4.2"
|
21
21
|
|
22
22
|
spec.add_development_dependency "pg", "~>0.18"
|
23
23
|
spec.add_development_dependency "bundler", "~> 1.11"
|
24
24
|
spec.add_development_dependency "rake", "~> 10.0"
|
25
|
-
spec.add_development_dependency "rspec", ">= 3.4"
|
26
25
|
spec.add_development_dependency "rspec-rails", ">= 3.4"
|
27
26
|
spec.add_development_dependency "database_cleaner", "~> 1.5"
|
28
27
|
spec.add_development_dependency "simplecov", ">= 0.3.8"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logidze
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- palkan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 4.2
|
19
|
+
version: '4.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 4.2
|
26
|
+
version: '4.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: pg
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,20 +66,6 @@ dependencies:
|
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '10.0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: rspec
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '3.4'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '3.4'
|
83
69
|
- !ruby/object:Gem::Dependency
|
84
70
|
name: rspec-rails
|
85
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -158,6 +144,7 @@ extensions: []
|
|
158
144
|
extra_rdoc_files: []
|
159
145
|
files:
|
160
146
|
- ".gitignore"
|
147
|
+
- ".hound.yml"
|
161
148
|
- ".rspec"
|
162
149
|
- ".rubocop.yml"
|
163
150
|
- ".travis.yml"
|
@@ -166,14 +153,20 @@ files:
|
|
166
153
|
- LICENSE.txt
|
167
154
|
- README.md
|
168
155
|
- Rakefile
|
169
|
-
- bench/
|
170
|
-
- bench/
|
171
|
-
- bench/
|
172
|
-
- bench/
|
173
|
-
- bench/
|
174
|
-
- bench/
|
175
|
-
- bench/
|
176
|
-
- bench/
|
156
|
+
- bench/performance/README.md
|
157
|
+
- bench/performance/diff_bench.rb
|
158
|
+
- bench/performance/insert_bench.rb
|
159
|
+
- bench/performance/memory_profile.rb
|
160
|
+
- bench/performance/setup.rb
|
161
|
+
- bench/performance/update_bench.rb
|
162
|
+
- bench/triggers/Makefile
|
163
|
+
- bench/triggers/Readme.md
|
164
|
+
- bench/triggers/bench.sql
|
165
|
+
- bench/triggers/hstore_trigger_setup.sql
|
166
|
+
- bench/triggers/jsonb_minus_2_setup.sql
|
167
|
+
- bench/triggers/jsonb_minus_setup.sql
|
168
|
+
- bench/triggers/keys2_trigger_setup.sql
|
169
|
+
- bench/triggers/keys_trigger_setup.sql
|
177
170
|
- bin/console
|
178
171
|
- bin/setup
|
179
172
|
- circle.yml
|
@@ -190,6 +183,8 @@ files:
|
|
190
183
|
- lib/logidze/engine.rb
|
191
184
|
- lib/logidze/has_logidze.rb
|
192
185
|
- lib/logidze/history.rb
|
186
|
+
- lib/logidze/history/type.rb
|
187
|
+
- lib/logidze/history/version.rb
|
193
188
|
- lib/logidze/model.rb
|
194
189
|
- lib/logidze/version.rb
|
195
190
|
- logidze.gemspec
|