janko 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +2 -0
- data/Guardfile +7 -0
- data/LICENSE.txt +202 -0
- data/README.md +186 -0
- data/Rakefile +14 -0
- data/assets/insert-performance-graph.png +0 -0
- data/assets/merge-performance-graph.png +0 -0
- data/config/environment.rb +7 -0
- data/janko.gemspec +32 -0
- data/lib/janko.rb +5 -0
- data/lib/janko/bulk_merge.rb +57 -0
- data/lib/janko/column_list.rb +193 -0
- data/lib/janko/connection.rb +107 -0
- data/lib/janko/constants.rb +13 -0
- data/lib/janko/copy_importer.rb +36 -0
- data/lib/janko/flag.rb +26 -0
- data/lib/janko/import.rb +80 -0
- data/lib/janko/insert_importer.rb +32 -0
- data/lib/janko/merge.rb +167 -0
- data/lib/janko/merge_result.rb +37 -0
- data/lib/janko/single_merge.rb +33 -0
- data/lib/janko/tagged_column.rb +133 -0
- data/lib/janko/upsert.rb +193 -0
- data/lib/janko/version.rb +3 -0
- data/spec/column_list_spec.rb +164 -0
- data/spec/connection_spec.rb +27 -0
- data/spec/flag_spec.rb +16 -0
- data/spec/import_spec.rb +112 -0
- data/spec/merge_spec.rb +400 -0
- data/spec/spec_helper.rb +32 -0
- data/spec/tagged_column_spec.rb +56 -0
- metadata +261 -0
data/lib/janko/flag.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
module Janko
|
2
|
+
class Flag
|
3
|
+
attr_reader :value
|
4
|
+
|
5
|
+
def initialize(value)
|
6
|
+
@value = value
|
7
|
+
end
|
8
|
+
|
9
|
+
def |(other)
|
10
|
+
self.class.new(value | other.value)
|
11
|
+
end
|
12
|
+
|
13
|
+
def eql?(other)
|
14
|
+
return unless other.is_a?(self.class)
|
15
|
+
(value & other.value) != 0
|
16
|
+
end
|
17
|
+
|
18
|
+
def ==(other)
|
19
|
+
eql?(other)
|
20
|
+
end
|
21
|
+
|
22
|
+
def ===(other)
|
23
|
+
eql?(other)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/janko/import.rb
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
require "agrippa/mutable_hash"
|
2
|
+
require "janko/connection"
|
3
|
+
require "janko/column_list"
|
4
|
+
require "janko/copy_importer"
|
5
|
+
require "janko/insert_importer"
|
6
|
+
require "janko/constants"
|
7
|
+
|
8
|
+
# http://starfighter.ngrok.io/
|
9
|
+
|
10
|
+
# delegate :start, :stop, :push, to: :delegate
|
11
|
+
# set(table:, columns:)
|
12
|
+
#
|
13
|
+
# connect(connection)
|
14
|
+
# builder => self
|
15
|
+
|
16
|
+
module Janko
|
17
|
+
class Import
|
18
|
+
include Agrippa::MutableHash
|
19
|
+
|
20
|
+
state_reader :connection
|
21
|
+
|
22
|
+
state_writer :table, prefix: false
|
23
|
+
|
24
|
+
def default_state
|
25
|
+
{ columns: Janko::ALL, importer: Janko::CopyImporter }
|
26
|
+
end
|
27
|
+
|
28
|
+
def connect(connection)
|
29
|
+
@state[:connection] = Connection.build(connection)
|
30
|
+
self
|
31
|
+
end
|
32
|
+
|
33
|
+
def use(importer)
|
34
|
+
@state[:importer] = importer
|
35
|
+
self
|
36
|
+
end
|
37
|
+
|
38
|
+
def columns(*columns)
|
39
|
+
@state[:columns] = columns.flatten
|
40
|
+
self
|
41
|
+
end
|
42
|
+
|
43
|
+
def start
|
44
|
+
@state[:started] = true
|
45
|
+
delegate.start
|
46
|
+
self
|
47
|
+
end
|
48
|
+
|
49
|
+
def push(values)
|
50
|
+
raise("Call #start before #push") unless @state[:started]
|
51
|
+
delegate.push(values)
|
52
|
+
self
|
53
|
+
end
|
54
|
+
|
55
|
+
def stop
|
56
|
+
raise("Call #start before #stop") unless @state[:started]
|
57
|
+
delegate.stop
|
58
|
+
@state[:started] = false
|
59
|
+
self
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def preserve_state_if_started
|
65
|
+
return(self) unless @state[:started]
|
66
|
+
raise("Call #stop before changing import options.")
|
67
|
+
end
|
68
|
+
|
69
|
+
def delegate
|
70
|
+
@delegate ||= @state[:importer].new(delegate_options)
|
71
|
+
end
|
72
|
+
|
73
|
+
def delegate_options
|
74
|
+
raise("No table specified.") unless @state[:table]
|
75
|
+
column_list = ColumnList.build(@state[:columns])
|
76
|
+
raise("No columns specified.") if column_list.empty?
|
77
|
+
@state.merge(columns: column_list)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require "agrippa/mutable"
|
2
|
+
|
3
|
+
module Janko
|
4
|
+
class InsertImporter
|
5
|
+
include Agrippa::Mutable
|
6
|
+
|
7
|
+
state_reader :connection, :table, :columns
|
8
|
+
|
9
|
+
def start
|
10
|
+
query = sprintf("INSERT INTO %s(%s) VALUES(%s)", table,
|
11
|
+
columns.to_list, columns.to_binds)
|
12
|
+
connection.prepare(statement_name, query)
|
13
|
+
self
|
14
|
+
end
|
15
|
+
|
16
|
+
def push(values)
|
17
|
+
connection.exec_prepared(statement_name, columns.pack(values))
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
connection.exec("DEALLOCATE \"#{statement_name}\"")
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def statement_name
|
29
|
+
@statement_name ||= "import-#{SecureRandom.hex(8)}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/lib/janko/merge.rb
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
require "agrippa/mutable_hash"
|
2
|
+
require "agrippa/delegation"
|
3
|
+
require "janko/connection"
|
4
|
+
require "janko/column_list"
|
5
|
+
require "janko/single_merge"
|
6
|
+
require "janko/bulk_merge"
|
7
|
+
|
8
|
+
module Janko
|
9
|
+
class Merge
|
10
|
+
include Agrippa::MutableHash
|
11
|
+
|
12
|
+
include Agrippa::Delegation
|
13
|
+
|
14
|
+
state_writer :table, :locking, :transaction, :collector
|
15
|
+
|
16
|
+
state_reader :table, :connection
|
17
|
+
|
18
|
+
delegate :exec, to: "connection"
|
19
|
+
|
20
|
+
delegate :result, to: "delegate"
|
21
|
+
|
22
|
+
def default_state
|
23
|
+
{ strategy: Janko::BulkMerge, connection: Connection.default }
|
24
|
+
end
|
25
|
+
|
26
|
+
def connect(connection)
|
27
|
+
@state[:connection] = Connection.build(connection)
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def use(strategy)
|
32
|
+
@state[:strategy] = strategy
|
33
|
+
self
|
34
|
+
end
|
35
|
+
|
36
|
+
def returning(returning)
|
37
|
+
returning = returning.to_s
|
38
|
+
raise("Merge can return inserted, updated, all, or none.") \
|
39
|
+
unless %w(inserted updated all none).include?(returning)
|
40
|
+
chain(returning: returning)
|
41
|
+
end
|
42
|
+
|
43
|
+
def key(*list)
|
44
|
+
preserve_state_if_started
|
45
|
+
columns.tag("key", *list)
|
46
|
+
self
|
47
|
+
end
|
48
|
+
|
49
|
+
def update(*list)
|
50
|
+
preserve_state_if_started
|
51
|
+
columns.tag("update", *list)
|
52
|
+
self
|
53
|
+
end
|
54
|
+
|
55
|
+
def insert(*list)
|
56
|
+
preserve_state_if_started
|
57
|
+
columns.tag("insert", *list)
|
58
|
+
self
|
59
|
+
end
|
60
|
+
|
61
|
+
def select(*list)
|
62
|
+
preserve_state_if_started
|
63
|
+
columns.tag("select", *list)
|
64
|
+
self
|
65
|
+
end
|
66
|
+
|
67
|
+
def alter(*list, &block)
|
68
|
+
preserve_state_if_started
|
69
|
+
columns.alter(*list, &block)
|
70
|
+
self
|
71
|
+
end
|
72
|
+
|
73
|
+
def start
|
74
|
+
@state[:started] = true
|
75
|
+
reset_delegate
|
76
|
+
begin_transaction and lock_table
|
77
|
+
rollback_on_error { delegate.start }
|
78
|
+
self
|
79
|
+
end
|
80
|
+
|
81
|
+
def push(*args)
|
82
|
+
raise("Call #start before #push") unless @state[:started]
|
83
|
+
rollback_on_error { delegate.push(*args) }
|
84
|
+
self
|
85
|
+
end
|
86
|
+
|
87
|
+
def stop
|
88
|
+
raise("Call #start before #stop") unless @state[:started]
|
89
|
+
rollback_on_error { delegate.stop }
|
90
|
+
@state[:started] = false
|
91
|
+
self
|
92
|
+
end
|
93
|
+
|
94
|
+
def columns
|
95
|
+
@state[:columns] ||= begin
|
96
|
+
raise("Connect before setting merge parameters.") \
|
97
|
+
unless connected?
|
98
|
+
default_column_list
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def connected?
|
105
|
+
@state.has_key?(:connection)
|
106
|
+
end
|
107
|
+
|
108
|
+
def default_column_list
|
109
|
+
ColumnList.new(parent: self)
|
110
|
+
.tag(:key, :id)
|
111
|
+
.tag(:select, Janko::ALL)
|
112
|
+
.tag(:insert, Janko::ALL).untag(:insert, :id)
|
113
|
+
.tag(:update, Janko::ALL).untag(:update, :id)
|
114
|
+
end
|
115
|
+
|
116
|
+
def rollback_on_error
|
117
|
+
begin
|
118
|
+
yield
|
119
|
+
rescue
|
120
|
+
raise
|
121
|
+
ensure
|
122
|
+
commit_or_rollback_transaction
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def begin_transaction
|
127
|
+
return(self) if connection.in_transaction?
|
128
|
+
return(self) if (@state[:transaction] == false)
|
129
|
+
@state[:our_transaction] == true
|
130
|
+
exec("BEGIN")
|
131
|
+
self
|
132
|
+
end
|
133
|
+
|
134
|
+
def commit_or_rollback_transaction
|
135
|
+
return(self) unless connection.in_transaction?
|
136
|
+
return(self) unless @state.delete(:our_transaction)
|
137
|
+
exec(connection.failed? ? "ROLLBACK" : "COMMIT")
|
138
|
+
self
|
139
|
+
end
|
140
|
+
|
141
|
+
def lock_table
|
142
|
+
return(self) unless connection.in_transaction?
|
143
|
+
return(self) if (@state[:locking] == false)
|
144
|
+
exec("LOCK TABLE #{table} IN SHARE ROW EXCLUSIVE MODE;")
|
145
|
+
self
|
146
|
+
end
|
147
|
+
|
148
|
+
def preserve_state_if_started
|
149
|
+
return(self) unless @state[:started]
|
150
|
+
raise("Call #stop before changing import options.")
|
151
|
+
end
|
152
|
+
|
153
|
+
def delegate
|
154
|
+
@delegate ||= begin
|
155
|
+
strategy_class = @state[:strategy]
|
156
|
+
strategy_class || raise("Set strategy before merging.")
|
157
|
+
strategy_class.new(@state.slice(:connection, :table,
|
158
|
+
:returning, :collector).merge(columns: columns))
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def reset_delegate
|
163
|
+
@delegate = nil
|
164
|
+
self
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Janko
|
2
|
+
class MergeResult
|
3
|
+
include Enumerable
|
4
|
+
|
5
|
+
attr_reader :count
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@tuples = Hash.new { |h, k| h[k] = [] }
|
9
|
+
@count = 0
|
10
|
+
end
|
11
|
+
|
12
|
+
def push(tag, tuple)
|
13
|
+
@tuples[tag.to_s].push(tuple)
|
14
|
+
@count += 1
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def inserted
|
19
|
+
@tuples["inserted"]
|
20
|
+
end
|
21
|
+
|
22
|
+
def updated
|
23
|
+
@tuples["updated"]
|
24
|
+
end
|
25
|
+
|
26
|
+
def clear
|
27
|
+
@tuples.clear
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def each(&block)
|
32
|
+
inserted.each(&block)
|
33
|
+
updated.each(&block)
|
34
|
+
self
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require "janko/merge_result"
|
2
|
+
require "janko/upsert"
|
3
|
+
|
4
|
+
module Janko
|
5
|
+
class SingleMerge
|
6
|
+
attr_reader :upsert
|
7
|
+
|
8
|
+
def initialize(options = {})
|
9
|
+
@upsert = Upsert.new(options)
|
10
|
+
@options = options
|
11
|
+
end
|
12
|
+
|
13
|
+
def start
|
14
|
+
upsert.result.clear
|
15
|
+
upsert.prepare if @options[:use_prepared_query]
|
16
|
+
self
|
17
|
+
end
|
18
|
+
|
19
|
+
def push(*values)
|
20
|
+
upsert.push(*values)
|
21
|
+
self
|
22
|
+
end
|
23
|
+
|
24
|
+
def stop
|
25
|
+
upsert.cleanup
|
26
|
+
self
|
27
|
+
end
|
28
|
+
|
29
|
+
def result
|
30
|
+
upsert.result
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
require "janko/constants"
|
2
|
+
require "agrippa/state"
|
3
|
+
require "agrippa/mutable"
|
4
|
+
require "agrippa/delegation"
|
5
|
+
|
6
|
+
module Janko
|
7
|
+
include Constants
|
8
|
+
|
9
|
+
class TaggedColumn
|
10
|
+
include Agrippa::Delegation
|
11
|
+
|
12
|
+
include Agrippa::Mutable
|
13
|
+
|
14
|
+
state_reader :tags, :name, :parent
|
15
|
+
|
16
|
+
state_writer :wrap, :default, :on_update, prefix: false
|
17
|
+
|
18
|
+
delegate :table, :connection, to: :parent
|
19
|
+
|
20
|
+
def default_state
|
21
|
+
{ tags: {} }
|
22
|
+
end
|
23
|
+
|
24
|
+
def set(updates)
|
25
|
+
chain(updates)
|
26
|
+
end
|
27
|
+
|
28
|
+
def tag(tag)
|
29
|
+
tag = tag.to_s
|
30
|
+
tags.merge!(tag => true) unless (tag == "")
|
31
|
+
self
|
32
|
+
end
|
33
|
+
|
34
|
+
def untag(tag)
|
35
|
+
tag = tag.to_s
|
36
|
+
tags.reject! { |k| k == tag } unless (tag == "")
|
37
|
+
self
|
38
|
+
end
|
39
|
+
|
40
|
+
def has_tag?(tag)
|
41
|
+
tags.has_key?(tag.to_s)
|
42
|
+
end
|
43
|
+
|
44
|
+
def tagged?
|
45
|
+
not tags.empty?
|
46
|
+
end
|
47
|
+
|
48
|
+
def to_s
|
49
|
+
tags.keys.join(" ")
|
50
|
+
end
|
51
|
+
|
52
|
+
def type
|
53
|
+
connection.column_type(table, name)
|
54
|
+
end
|
55
|
+
|
56
|
+
# FIXME: Quoting
|
57
|
+
def quoted(prefix = nil)
|
58
|
+
prefix.nil? ? "\"#{name}\"" : "\"#{prefix}\".\"#{name}\""
|
59
|
+
end
|
60
|
+
|
61
|
+
def to_condition(left, right)
|
62
|
+
"#{quoted(left)} = #{quoted(right)}"
|
63
|
+
end
|
64
|
+
|
65
|
+
def to_setter(left, right)
|
66
|
+
"#{quoted} = #{maybe_on_update(left, right)}"
|
67
|
+
end
|
68
|
+
|
69
|
+
def to_value(prefix = nil)
|
70
|
+
maybe_wrap(nil, prefix)
|
71
|
+
end
|
72
|
+
|
73
|
+
def to_bind(position)
|
74
|
+
"$#{position}"
|
75
|
+
end
|
76
|
+
|
77
|
+
def to_typecast_bind(position)
|
78
|
+
"$#{position}::#{type}"
|
79
|
+
end
|
80
|
+
|
81
|
+
def inspect
|
82
|
+
children = "(#{tags.keys.join(" ")})"
|
83
|
+
"#<#{self.class}:0x#{self.__id__.to_s(16)} #{name}#{children}>"
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
def maybe_wrap(left, right)
|
89
|
+
inner = maybe_default(left, right)
|
90
|
+
return(inner) unless @wrap
|
91
|
+
@wrap.gsub(/\$NEW/i) { inner }
|
92
|
+
end
|
93
|
+
|
94
|
+
def maybe_on_update(left, right)
|
95
|
+
inner = maybe_wrap(left, right)
|
96
|
+
return(inner) unless @on_update
|
97
|
+
output = Agrippa::State.new(@on_update, :gsub)
|
98
|
+
output.gsub(/\$NEW/i) { inner }
|
99
|
+
output.gsub(/\$OLD/i) { quoted(left) }
|
100
|
+
output._value
|
101
|
+
end
|
102
|
+
|
103
|
+
def maybe_default(left, right)
|
104
|
+
values = [ quoted(right) ]
|
105
|
+
values.push(keep_existing_value(left))
|
106
|
+
values.push(column_default_value)
|
107
|
+
values.compact!
|
108
|
+
return(values.first) if (values.length == 1)
|
109
|
+
"COALESCE(#{values.join(", ")})"
|
110
|
+
end
|
111
|
+
|
112
|
+
def keep_existing_value(prefix)
|
113
|
+
return if prefix.nil?
|
114
|
+
return unless (value = @default)
|
115
|
+
return unless (value == Janko::KEEP)
|
116
|
+
quoted(prefix)
|
117
|
+
end
|
118
|
+
|
119
|
+
def column_default_value
|
120
|
+
@column_default_value ||= begin
|
121
|
+
return unless (value = @default)
|
122
|
+
return(value) unless value.is_a?(Flag)
|
123
|
+
return unless (value == Janko::DEFAULT)
|
124
|
+
connection.column_default(table, name)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def flagged(value, flag)
|
129
|
+
return(false) unless value.is_a?(Fixnum)
|
130
|
+
(value & flag) == value
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|