surus 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -82
- data/bench/benchmark_helper.rb +0 -9
- data/bench/database_structure.sql +0 -18
- data/bench/hstore_find.rb +7 -7
- data/bench/json_generation.rb +1 -1
- data/lib/surus.rb +0 -6
- data/lib/surus/array/scope.rb +1 -1
- data/lib/surus/hstore/serializer.rb +23 -47
- data/lib/surus/json/belongs_to_scope_builder.rb +3 -3
- data/lib/surus/json/has_and_belongs_to_many_scope_builder.rb +4 -5
- data/lib/surus/json/has_many_scope_builder.rb +3 -4
- data/lib/surus/json/model.rb +1 -1
- data/lib/surus/version.rb +1 -1
- data/spec/array/scope_spec.rb +6 -6
- data/spec/hstore/scope_spec.rb +21 -21
- data/spec/spec_helper.rb +6 -20
- data/surus.gemspec +1 -1
- metadata +5 -24
- data/bench/array_create.rb +0 -46
- data/bench/array_find.rb +0 -79
- data/bench/array_serialize.rb +0 -34
- data/bench/hstore_create.rb +0 -57
- data/bench/hstore_serialize.rb +0 -42
- data/lib/surus/array/decimal_serializer.rb +0 -28
- data/lib/surus/array/float_serializer.rb +0 -28
- data/lib/surus/array/integer_serializer.rb +0 -28
- data/lib/surus/array/text_serializer.rb +0 -47
- data/lib/surus/hstore/connection_adapters.rb +0 -46
- data/lib/surus/json/connection_adapters.rb +0 -46
- data/spec/array/decimal_serializer_spec.rb +0 -23
- data/spec/array/float_serializer_spec.rb +0 -23
- data/spec/array/integer_serializer_spec.rb +0 -22
- data/spec/array/text_serializer_spec.rb +0 -29
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0a513dd97a43e25dbe91de60dee39323f42b410a
|
4
|
+
data.tar.gz: 092f7b1ea714785b14f59f68088f33101ae57244
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 111bcea66d6e02e09a8b7314e3768399dd55422ecb403305461bdd4d4f16fcc62f97181d41e6cbf7551fdd4772201bbcdc9d7f5b9a77c2f3ce0482efe0a64d3c
|
7
|
+
data.tar.gz: e4e759ac1c896166afae2e26033667b30f88f578ba1a9dc3c3caef6374252b3a19cc1491d397dd1dc11ea383e45d15246e779f97eacee2219124eed47ff52473
|
data/README.md
CHANGED
@@ -3,11 +3,11 @@ Surus
|
|
3
3
|
|
4
4
|
# Description
|
5
5
|
|
6
|
-
Surus
|
7
|
-
|
6
|
+
Surus adds PostgreSQL specific functionality to ActiveRecord. It adds
|
7
|
+
helper methods for searching PostgreSQL arrays and hstores.
|
8
8
|
It also can control PostgreSQL synchronous commit behavior. By relaxing
|
9
9
|
PostgreSQL's durability guarantee, transaction commit rate can be increased by
|
10
|
-
50% or more. It also directly generate JSON in PostgreSQL which can be
|
10
|
+
50% or more. It can also directly generate JSON in PostgreSQL which can be
|
11
11
|
substantially faster than converting ActiveRecord objects to JSON.
|
12
12
|
|
13
13
|
# Installation
|
@@ -18,33 +18,13 @@ Or add to your Gemfile.
|
|
18
18
|
|
19
19
|
gem 'surus'
|
20
20
|
|
21
|
-
|
21
|
+
## Rails 3
|
22
22
|
|
23
|
-
This version of Surus
|
24
|
-
compatible with Rails 4. Checkout the rails4 branch if you are using Rails 4.
|
23
|
+
This version of Surus only works on Rails 4. Use the 0.4 line for Rails 3
|
25
24
|
|
26
|
-
|
27
|
-
|
28
|
-
Hashes can be serialized to an hstore column. hstore is a PostgreSQL key/value
|
29
|
-
type that can be indexed for fast searching.
|
30
|
-
|
31
|
-
class User < ActiveRecord::Base
|
32
|
-
serialize :properties, Surus::Hstore::Serializer.new
|
33
|
-
end
|
34
|
-
|
35
|
-
User.create :properties => { :favorite_color => "green", :results_per_page => 20 }
|
36
|
-
User.create :properties => { :favorite_colors => ["green", "blue", "red"] }
|
37
|
-
|
38
|
-
Even though the underlying hstore can only use strings for keys and values
|
39
|
-
(and NULL for values) Surus can successfully maintain type for integers,
|
40
|
-
floats, bigdecimals, dates, and any value that YAML can serialize. It does
|
41
|
-
this by storing an extra key value pair (or two) to maintain type information.
|
25
|
+
gem 'surus', '~> 0.4.2'
|
42
26
|
|
43
|
-
|
44
|
-
nested data structures can be serialized to an hstore. In other words, any
|
45
|
-
hash that can be serialized with the normal Rails YAML serialization can be
|
46
|
-
serialized with Surus. But you can get the benefits of PostgreSQL indexing
|
47
|
-
on the top level keys and values for free.
|
27
|
+
# Hstore
|
48
28
|
|
49
29
|
Hstores can be searched with helper scopes.
|
50
30
|
|
@@ -58,33 +38,18 @@ Hstore is a PostgreSQL extension. You can generate a migration to install it.
|
|
58
38
|
rails g surus:hstore:install
|
59
39
|
rake db:migrate
|
60
40
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
add_column :users, :properties, :hstore
|
66
|
-
end
|
67
|
-
end
|
41
|
+
Even though the underlying hstore can only use strings for keys and values
|
42
|
+
(and NULL for values) Surus can successfully maintain type for integers,
|
43
|
+
floats, bigdecimals, dates, and any value that YAML can serialize. It does
|
44
|
+
this by storing an extra key value pair (or two) to maintain type information.
|
68
45
|
|
69
|
-
|
46
|
+
Because it falls back to YAML serialization for complex types, this means that
|
47
|
+
nested data structures can be serialized to an hstore. In other words, any
|
48
|
+
hash that can be serialized with the normal Rails YAML serialization can be
|
49
|
+
serialized with Surus.
|
70
50
|
|
71
51
|
# Array
|
72
52
|
|
73
|
-
Ruby arrays can be serialized to PostgreSQL arrays. Surus includes support
|
74
|
-
for text, integer, float, and decimal arrays.
|
75
|
-
|
76
|
-
class User < ActiveRecord::Base
|
77
|
-
|
78
|
-
serialize :favorite_integers, Surus::Array::IntegerSerializer.new
|
79
|
-
serialize :favorite_floats, Surus::Array::FloatSerializer.new
|
80
|
-
serialize :favorite_decimals, Surus::Array::DecimalSerializer.new
|
81
|
-
end
|
82
|
-
|
83
|
-
User.create :permissions => %w{ read_notes write_notes, manage_topics },
|
84
|
-
:favorite_integers => [1, 2, 3],
|
85
|
-
:favorite_floats => [1.3, 2.2, 3.1],
|
86
|
-
:favorite_decimals => [BigDecimal("3.14"), BigDecimal("4.23"]
|
87
|
-
|
88
53
|
Arrays can be searched with helper scopes.
|
89
54
|
|
90
55
|
User.array_has(:permissions, "admin")
|
@@ -131,40 +96,8 @@ the Rails `to_json` interface.
|
|
131
96
|
User.all_json(columns: [:id, :name, :email], include: {posts: {columns: [:id, :subject]}})
|
132
97
|
Post.all_json(include: [:forum, :post])
|
133
98
|
|
134
|
-
You can use the json type in migrations.
|
135
|
-
|
136
|
-
class AddPropertiesToUsers < ActiveRecord::Migration
|
137
|
-
def change
|
138
|
-
add_column :users, :properties, :json
|
139
|
-
end
|
140
|
-
end
|
141
|
-
|
142
|
-
|
143
99
|
# Benchmarks
|
144
100
|
|
145
|
-
Using PostgreSQL's hstore enables searches to be done quickly in the database.
|
146
|
-
|
147
|
-
jack@moya:~/work/surus$ ruby -I lib -I bench bench/hstore_find.rb
|
148
|
-
Skipping EAV test. Use -e to enable (VERY SLOW!)
|
149
|
-
Skipping YAML test. Use -y to enable (VERY SLOW!)
|
150
|
-
Creating Surus test data... Done.
|
151
|
-
|
152
|
-
2000 records with 5 string key/value pairs
|
153
|
-
Finding all by inclusion of a key 200 times
|
154
|
-
user system total real
|
155
|
-
Surus 0.120000 0.030000 0.150000 ( 0.356240)
|
156
|
-
|
157
|
-
Arrays are also searchable.
|
158
|
-
|
159
|
-
jack@moya:~/work/surus$ ruby -I lib -I bench bench/array_find.rb
|
160
|
-
Skipping YAML test. Use -y to enable (VERY SLOW!)
|
161
|
-
Creating Surus test data... Done.
|
162
|
-
|
163
|
-
2000 records with 10 element arrays
|
164
|
-
Finding all where array includes value 200 times
|
165
|
-
user system total real
|
166
|
-
Surus 0.120000 0.040000 0.160000 ( 0.531735)
|
167
|
-
|
168
101
|
JSON generation is with all_json and find_json is substantially faster than to_json.
|
169
102
|
|
170
103
|
jack@hk-47~/dev/surus$ ruby -I lib -I bench bench/json_generation.rb
|
data/bench/benchmark_helper.rb
CHANGED
@@ -40,14 +40,6 @@ class EavDetailRecord < ActiveRecord::Base
|
|
40
40
|
belongs_to :eav_master_record
|
41
41
|
end
|
42
42
|
|
43
|
-
class YamlArrayRecord < ActiveRecord::Base
|
44
|
-
serialize :names
|
45
|
-
end
|
46
|
-
|
47
|
-
class SurusTextArrayRecord < ActiveRecord::Base
|
48
|
-
serialize :names, Surus::Array::TextSerializer.new
|
49
|
-
end
|
50
|
-
|
51
43
|
class WideRecord < ActiveRecord::Base
|
52
44
|
end
|
53
45
|
|
@@ -77,7 +69,6 @@ def clean_database
|
|
77
69
|
SurusKeyValueRecord.delete_all
|
78
70
|
EavDetailRecord.delete_all
|
79
71
|
EavMasterRecord.delete_all
|
80
|
-
YamlArrayRecord.delete_all
|
81
72
|
WideRecord.delete_all
|
82
73
|
NarrowRecord.delete_all
|
83
74
|
Post.destroy_all # destroy instead of delete so it removes join records in posts_tags
|
@@ -38,24 +38,6 @@ CREATE UNIQUE INDEX ON eav_detail_records(eav_master_record_id, "key");
|
|
38
38
|
CREATE INDEX ON eav_detail_records ("value");
|
39
39
|
|
40
40
|
|
41
|
-
DROP TABLE IF EXISTS yaml_array_records;
|
42
|
-
|
43
|
-
CREATE TABLE yaml_array_records(
|
44
|
-
id serial PRIMARY KEY,
|
45
|
-
names text
|
46
|
-
);
|
47
|
-
|
48
|
-
|
49
|
-
DROP TABLE IF EXISTS surus_text_array_records;
|
50
|
-
|
51
|
-
CREATE TABLE surus_text_array_records(
|
52
|
-
id serial PRIMARY KEY,
|
53
|
-
names text[]
|
54
|
-
);
|
55
|
-
|
56
|
-
CREATE INDEX ON surus_text_array_records USING GIN (names);
|
57
|
-
|
58
|
-
|
59
41
|
|
60
42
|
DROP TABLE IF EXISTS wide_records;
|
61
43
|
|
data/bench/hstore_find.rb
CHANGED
@@ -6,7 +6,7 @@ optparse = OptionParser.new do |opts|
|
|
6
6
|
opts.on '-r NUM', '--records NUM', Integer, 'Number of records to create' do |n|
|
7
7
|
options[:records] = n
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
options[:pairs] = 5
|
11
11
|
opts.on '-p NUM', '--pairs NUM', Integer, 'Number of key/value pairs' do |n|
|
12
12
|
options[:pairs] = n
|
@@ -16,7 +16,7 @@ optparse = OptionParser.new do |opts|
|
|
16
16
|
opts.on '-e', '--eav', 'Include EAV in benchmark (VERY SLOW!)' do
|
17
17
|
options[:eav] = true
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
options[:yaml] = false
|
21
21
|
opts.on '-y', '--yaml', 'Include YAML in benchmark (VERY SLOW!)' do
|
22
22
|
options[:yaml] = true
|
@@ -84,22 +84,22 @@ Benchmark.bm(8) do |x|
|
|
84
84
|
EavMasterRecord
|
85
85
|
.includes(:eav_detail_records)
|
86
86
|
.where("EXISTS(SELECT 1 FROM eav_detail_records WHERE eav_master_records.id=eav_detail_records.eav_master_record_id AND key=?)", key_to_find)
|
87
|
-
.
|
87
|
+
.to_a
|
88
88
|
end
|
89
|
-
end
|
89
|
+
end
|
90
90
|
end
|
91
91
|
|
92
92
|
x.report("Surus") do
|
93
93
|
keys_to_find.each do |key_to_find|
|
94
|
-
SurusKeyValueRecord.hstore_has_key(:properties, key_to_find).
|
94
|
+
SurusKeyValueRecord.hstore_has_key(:properties, key_to_find).to_a
|
95
95
|
end
|
96
96
|
end
|
97
97
|
|
98
98
|
if yaml
|
99
99
|
x.report("YAML") do
|
100
100
|
keys_to_find.each do |key_to_find|
|
101
|
-
YamlKeyValueRecord.
|
101
|
+
YamlKeyValueRecord.to_a.select { |r| r.properties.key?(key_to_find) }
|
102
102
|
end
|
103
|
-
end
|
103
|
+
end
|
104
104
|
end
|
105
105
|
end
|
data/bench/json_generation.rb
CHANGED
@@ -55,7 +55,7 @@ Benchmark.bm(55) do |x|
|
|
55
55
|
|
56
56
|
x.report("to_json: 50 records with 3 associations #{num_long_iterations} times") do
|
57
57
|
num_long_iterations.times do
|
58
|
-
Post.includes(:author, :forum).
|
58
|
+
Post.includes(:author, :forum).to_a.to_json include: [:author, :forum, :tags]
|
59
59
|
end
|
60
60
|
end
|
61
61
|
end
|
data/lib/surus.rb
CHANGED
@@ -2,15 +2,9 @@ require 'active_record'
|
|
2
2
|
require 'surus/version'
|
3
3
|
require 'surus/hstore/serializer'
|
4
4
|
require 'surus/hstore/scope'
|
5
|
-
require 'surus/hstore/connection_adapters'
|
6
|
-
require 'surus/array/text_serializer'
|
7
|
-
require 'surus/array/integer_serializer'
|
8
|
-
require 'surus/array/float_serializer'
|
9
|
-
require 'surus/array/decimal_serializer'
|
10
5
|
require 'surus/array/scope'
|
11
6
|
require 'surus/synchronous_commit/connection'
|
12
7
|
require 'surus/synchronous_commit/model'
|
13
|
-
require 'surus/json/connection_adapters'
|
14
8
|
require 'surus/json/query'
|
15
9
|
require 'surus/json/row_query'
|
16
10
|
require 'surus/json/array_agg_query'
|
data/lib/surus/array/scope.rb
CHANGED
@@ -1,71 +1,47 @@
|
|
1
1
|
module Surus
|
2
2
|
module Hstore
|
3
3
|
class Serializer
|
4
|
-
|
5
|
-
|
6
|
-
((?:[^"\\]|\\.)*)
|
7
|
-
"
|
8
|
-
=>
|
9
|
-
(
|
10
|
-
"
|
11
|
-
(?:[^"\\]|\\.)*
|
12
|
-
"
|
13
|
-
|(NULL)
|
14
|
-
)
|
15
|
-
}x
|
16
|
-
|
17
|
-
def load(string)
|
18
|
-
return unless string
|
19
|
-
stringified_hash = string.scan(KEY_VALUE_REGEX).each_with_object({}) do |key_value, hash|
|
20
|
-
key, value = key_value
|
21
|
-
key = unescape(key)
|
22
|
-
value = if value == "NULL"
|
23
|
-
nil
|
24
|
-
else
|
25
|
-
unescape(value[1..-2])
|
26
|
-
end
|
27
|
-
|
28
|
-
hash[key] = value
|
29
|
-
end
|
4
|
+
def load(stringified_hash)
|
5
|
+
return unless stringified_hash
|
30
6
|
|
31
7
|
key_types = stringified_hash.delete "__key_types"
|
32
8
|
key_types = YAML.load key_types if key_types
|
33
9
|
value_types = stringified_hash.delete "__value_types"
|
34
10
|
value_types = YAML.load value_types if value_types
|
35
|
-
|
11
|
+
|
36
12
|
return stringified_hash unless key_types || value_types
|
37
|
-
|
13
|
+
|
38
14
|
stringified_hash.each_with_object({}) do |key_value, hash|
|
39
15
|
string_key, string_value = key_value
|
40
|
-
|
41
|
-
key = if key_types && key_types.key?(string_key)
|
16
|
+
|
17
|
+
key = if key_types && key_types.key?(string_key)
|
42
18
|
typecast(string_key, key_types[string_key])
|
43
19
|
else
|
44
20
|
string_key
|
45
21
|
end
|
46
|
-
|
22
|
+
|
47
23
|
value = if value_types && value_types.key?(string_key)
|
48
24
|
typecast(string_value, value_types[string_key])
|
49
25
|
else
|
50
26
|
string_value
|
51
27
|
end
|
52
|
-
|
28
|
+
|
53
29
|
hash[key] = value
|
54
|
-
end
|
30
|
+
end
|
55
31
|
end
|
56
|
-
|
32
|
+
|
57
33
|
def dump(hash)
|
58
34
|
return unless hash
|
59
|
-
|
35
|
+
|
60
36
|
key_types = {}
|
61
37
|
value_types = {}
|
62
|
-
|
38
|
+
|
63
39
|
stringified_hash = hash.each_with_object({}) do |key_value, stringified_hash|
|
64
40
|
key_string, key_type = stringify(key_value[0])
|
65
41
|
value_string, value_type = stringify(key_value[1])
|
66
|
-
|
42
|
+
|
67
43
|
stringified_hash[key_string] = value_string
|
68
|
-
|
44
|
+
|
69
45
|
key_types[key_string] = key_type unless key_type == "String"
|
70
46
|
value_types[key_string] = value_type unless value_type == "String"
|
71
47
|
end
|
@@ -76,34 +52,34 @@ module Surus
|
|
76
52
|
# the mess for us.
|
77
53
|
stringified_hash["__key_types"] = YAML.dump(key_types) if key_types.present?
|
78
54
|
stringified_hash["__value_types"] = YAML.dump(value_types) if value_types.present?
|
79
|
-
|
55
|
+
|
80
56
|
stringified_hash.map do |key, value|
|
81
57
|
"#{format_key(key)}=>#{format_value(value)}"
|
82
58
|
end.join(", ")
|
83
59
|
end
|
84
|
-
|
60
|
+
|
85
61
|
def format_key(key)
|
86
62
|
%Q("#{escape(key)}")
|
87
63
|
end
|
88
|
-
|
64
|
+
|
89
65
|
def format_value(value)
|
90
66
|
value ? %Q("#{escape(value)}") : "NULL"
|
91
67
|
end
|
92
|
-
|
68
|
+
|
93
69
|
# Escape a value for use as a key or value in an hstore
|
94
70
|
def escape(value)
|
95
71
|
value
|
96
72
|
.gsub('\\', '\\\\\\')
|
97
73
|
.gsub('"', '\\"')
|
98
74
|
end
|
99
|
-
|
75
|
+
|
100
76
|
# Unescape a value from a key or value in an hstore
|
101
77
|
def unescape(value)
|
102
78
|
value
|
103
79
|
.gsub('\\\\', '\\')
|
104
80
|
.gsub('\\"', '"')
|
105
81
|
end
|
106
|
-
|
82
|
+
|
107
83
|
# Returns an array of value as a string and value type
|
108
84
|
def stringify(value)
|
109
85
|
if value.kind_of?(String)
|
@@ -126,9 +102,9 @@ module Surus
|
|
126
102
|
[nil, "String"] # we don't actually stringify nil because format_value special cases nil
|
127
103
|
else
|
128
104
|
[YAML.dump(value), "YAML"]
|
129
|
-
end
|
105
|
+
end
|
130
106
|
end
|
131
|
-
|
107
|
+
|
132
108
|
def typecast(value, type)
|
133
109
|
case type
|
134
110
|
when "Symbol"
|
@@ -152,5 +128,5 @@ module Surus
|
|
152
128
|
end
|
153
129
|
end
|
154
130
|
end
|
155
|
-
end
|
131
|
+
end
|
156
132
|
end
|
@@ -2,11 +2,11 @@ module Surus
|
|
2
2
|
module JSON
|
3
3
|
class BelongsToScopeBuilder < AssociationScopeBuilder
|
4
4
|
def scope
|
5
|
-
|
5
|
+
s = association
|
6
6
|
.klass
|
7
7
|
.where("#{quote_column_name association.active_record_primary_key}=#{quote_column_name association.foreign_key}")
|
8
|
-
|
9
|
-
|
8
|
+
s = s.instance_eval(&association.scope) if association.scope
|
9
|
+
s
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
@@ -2,17 +2,16 @@ module Surus
|
|
2
2
|
module JSON
|
3
3
|
class HasAndBelongsToManyScopeBuilder < AssociationScopeBuilder
|
4
4
|
def scope
|
5
|
-
|
5
|
+
s = association
|
6
6
|
.klass
|
7
7
|
.joins("JOIN #{join_table} ON #{join_table}.#{association_foreign_key}=#{association_table}.#{association_primary_key}")
|
8
8
|
.where("#{outside_class.quoted_table_name}.#{association_primary_key}=#{join_table}.#{foreign_key}")
|
9
|
-
|
10
|
-
|
11
|
-
association_scope
|
9
|
+
s = s.instance_eval(&association.scope) if association.scope
|
10
|
+
s
|
12
11
|
end
|
13
12
|
|
14
13
|
def join_table
|
15
|
-
connection.quote_table_name association.
|
14
|
+
connection.quote_table_name association.join_table
|
16
15
|
end
|
17
16
|
|
18
17
|
def association_foreign_key
|