taps 0.2.26 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +8 -2
- data/Rakefile +20 -15
- data/VERSION.yml +3 -3
- data/bin/schema +13 -5
- data/bin/taps +3 -10
- data/lib/taps/cli.rb +157 -37
- data/lib/taps/config.rb +14 -3
- data/lib/taps/data_stream.rb +299 -0
- data/lib/taps/db_session.rb +4 -46
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +537 -0
- data/lib/taps/schema.rb +48 -66
- data/lib/taps/server.rb +76 -50
- data/lib/taps/utils.rb +20 -19
- data/spec/base.rb +3 -1
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +32 -0
- data/spec/server_spec.rb +1 -1
- data/spec/utils_spec.rb +3 -13
- metadata +41 -50
- data/lib/taps/adapter_hacks.rb +0 -21
- data/lib/taps/adapter_hacks/invalid_binary_limit.rb +0 -13
- data/lib/taps/adapter_hacks/invalid_text_limit.rb +0 -13
- data/lib/taps/adapter_hacks/mysql_invalid_primary_key.rb +0 -17
- data/lib/taps/adapter_hacks/non_rails_schema_dump.rb +0 -15
- data/lib/taps/client_session.rb +0 -304
- data/spec/client_session_spec.rb +0 -88
- data/spec/schema_spec.rb +0 -45
data/lib/taps/schema.rb
CHANGED
@@ -1,93 +1,75 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
3
|
-
require '
|
4
|
-
require 'uri'
|
1
|
+
require 'sequel'
|
2
|
+
require 'sequel/extensions/schema_dumper'
|
3
|
+
require 'sequel/extensions/migration'
|
5
4
|
|
6
|
-
require
|
5
|
+
require 'json'
|
7
6
|
|
8
7
|
module Taps
|
9
8
|
module Schema
|
10
9
|
extend self
|
11
10
|
|
12
|
-
def
|
13
|
-
|
14
|
-
|
15
|
-
adapter = 'postgresql' if adapter == 'postgres'
|
16
|
-
adapter = 'sqlite3' if adapter == 'sqlite'
|
17
|
-
config = {
|
18
|
-
'adapter' => adapter,
|
19
|
-
'database' => uri.path.blank? ? uri.host : uri.path.split('/')[1],
|
20
|
-
'username' => uri.user,
|
21
|
-
'password' => uri.password,
|
22
|
-
'host' => uri.host,
|
23
|
-
}
|
24
|
-
config = sqlite_config(url) if config['adapter'] == 'sqlite3'
|
25
|
-
config
|
11
|
+
def dump(database_url)
|
12
|
+
db = Sequel.connect(database_url)
|
13
|
+
db.dump_schema_migration(:indexes => false)
|
26
14
|
end
|
27
15
|
|
28
|
-
def
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
16
|
+
def dump_table(database_url, table)
|
17
|
+
Sequel.connect(database_url) do |db|
|
18
|
+
<<END_MIG
|
19
|
+
Class.new(Sequel::Migration) do
|
20
|
+
def up
|
21
|
+
#{db.dump_table_schema(table, :indexes => false)}
|
33
22
|
end
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
c = ActiveRecord::Base.establish_connection(config)
|
38
|
-
Taps::AdapterHacks.load(config['adapter'])
|
39
|
-
c
|
23
|
+
end
|
24
|
+
END_MIG
|
25
|
+
end
|
40
26
|
end
|
41
27
|
|
42
|
-
def
|
43
|
-
|
44
|
-
|
45
|
-
stream = StringIO.new
|
46
|
-
ActiveRecord::SchemaDumper.ignore_tables = []
|
47
|
-
ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
|
48
|
-
stream.string
|
28
|
+
def indexes(database_url)
|
29
|
+
db = Sequel.connect(database_url)
|
30
|
+
db.dump_indexes_migration
|
49
31
|
end
|
50
32
|
|
51
|
-
def
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
33
|
+
def indexes_individual(database_url)
|
34
|
+
idxs = {}
|
35
|
+
Sequel.connect(database_url) do |db|
|
36
|
+
tables = db.tables
|
37
|
+
tables.each do |table|
|
38
|
+
idxs[table] = db.send(:dump_table_indexes, table, :add_index, {}).split("\n")
|
56
39
|
end
|
57
|
-
|
58
|
-
end.join("\n")
|
59
|
-
end
|
40
|
+
end
|
60
41
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
42
|
+
idxs.each do |table, indexes|
|
43
|
+
idxs[table] = indexes.map do |idx|
|
44
|
+
<<END_MIG
|
45
|
+
Class.new(Sequel::Migration) do
|
46
|
+
def up
|
47
|
+
#{idx}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
END_MIG
|
51
|
+
end
|
52
|
+
end
|
53
|
+
idxs.to_json
|
66
54
|
end
|
67
55
|
|
68
56
|
def load(database_url, schema)
|
69
|
-
|
70
|
-
|
57
|
+
Sequel.connect(database_url) do |db|
|
58
|
+
eval(schema).apply(db, :up)
|
59
|
+
end
|
71
60
|
end
|
72
61
|
|
73
62
|
def load_indexes(database_url, indexes)
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
ActiveRecord::Schema.define do
|
78
|
-
#{indexes}
|
79
|
-
end
|
80
|
-
EORUBY
|
81
|
-
eval(schema)
|
63
|
+
Sequel.connect(database_url) do |db|
|
64
|
+
eval(indexes).apply(db, :up)
|
65
|
+
end
|
82
66
|
end
|
83
67
|
|
84
68
|
def reset_db_sequences(database_url)
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
ActiveRecord::Base.connection.reset_pk_sequence!(table)
|
90
|
-
end
|
69
|
+
db = Sequel.connect(database_url)
|
70
|
+
return unless db.respond_to?(:reset_primary_key_sequence)
|
71
|
+
db.tables.each do |table|
|
72
|
+
db.reset_primary_key_sequence(table)
|
91
73
|
end
|
92
74
|
end
|
93
75
|
end
|
data/lib/taps/server.rb
CHANGED
@@ -1,17 +1,20 @@
|
|
1
1
|
require 'sinatra/base'
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
2
|
+
require 'taps/config'
|
3
|
+
require 'taps/utils'
|
4
|
+
require 'taps/db_session'
|
5
|
+
require 'taps/data_stream'
|
5
6
|
|
6
7
|
module Taps
|
7
|
-
class Server < Sinatra::
|
8
|
+
class Server < Sinatra::Base
|
8
9
|
use Rack::Auth::Basic do |login, password|
|
9
10
|
login == Taps::Config.login && password == Taps::Config.password
|
10
11
|
end
|
11
12
|
|
13
|
+
use Rack::Deflater unless ENV['NO_DEFLATE']
|
14
|
+
|
12
15
|
error do
|
13
16
|
e = request.env['sinatra.error']
|
14
|
-
"Taps Server Error: #{e}"
|
17
|
+
"Taps Server Error: #{e}\n#{e.backtrace}"
|
15
18
|
end
|
16
19
|
|
17
20
|
before do
|
@@ -27,38 +30,46 @@ class Server < Sinatra::Default
|
|
27
30
|
|
28
31
|
post '/sessions' do
|
29
32
|
key = rand(9999999999).to_s
|
30
|
-
|
33
|
+
|
34
|
+
if ENV['NO_DEFAULT_DATABASE_URL']
|
35
|
+
database_url = request.body.string
|
36
|
+
else
|
37
|
+
database_url = Taps::Config.database_url || request.body.string
|
38
|
+
end
|
31
39
|
|
32
40
|
DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
|
33
41
|
|
34
42
|
"/sessions/#{key}"
|
35
43
|
end
|
36
44
|
|
37
|
-
post '/sessions/:key/
|
45
|
+
post '/sessions/:key/push/table' do
|
38
46
|
session = DbSession.filter(:key => params[:key]).first
|
39
47
|
halt 404 unless session
|
40
48
|
|
41
|
-
|
42
|
-
halt 412 unless Taps::Utils.valid_data?(gzip_data, request.env['HTTP_TAPS_CHECKSUM'])
|
49
|
+
json = DataStream.parse_json(params[:json])
|
43
50
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
51
|
+
size = 0
|
52
|
+
session.conn do |db|
|
53
|
+
begin
|
54
|
+
stream = DataStream.factory(db, json[:state])
|
55
|
+
size = stream.fetch_remote_in_server(params)
|
56
|
+
rescue Taps::DataStream::CorruptedData
|
57
|
+
halt 412
|
58
|
+
end
|
59
|
+
end
|
49
60
|
|
50
|
-
|
61
|
+
# TODO: return the stream's state with the size
|
62
|
+
size.to_s
|
51
63
|
end
|
52
64
|
|
53
|
-
post '/sessions/:key/reset_sequences' do
|
65
|
+
post '/sessions/:key/push/reset_sequences' do
|
54
66
|
session = DbSession.filter(:key => params[:key]).first
|
55
67
|
halt 404 unless session
|
56
68
|
|
57
|
-
schema_app = File.dirname(__FILE__) + '/../../bin/schema'
|
58
69
|
Taps::Utils.schema_bin(:reset_db_sequences, session.database_url)
|
59
70
|
end
|
60
71
|
|
61
|
-
post '/sessions/:key/schema' do
|
72
|
+
post '/sessions/:key/push/schema' do
|
62
73
|
session = DbSession.filter(:key => params[:key]).first
|
63
74
|
halt 404 unless session
|
64
75
|
|
@@ -66,7 +77,7 @@ class Server < Sinatra::Default
|
|
66
77
|
Taps::Utils.load_schema(session.database_url, schema_data)
|
67
78
|
end
|
68
79
|
|
69
|
-
post '/sessions/:key/indexes' do
|
80
|
+
post '/sessions/:key/push/indexes' do
|
70
81
|
session = DbSession.filter(:key => params[:key]).first
|
71
82
|
halt 404 unless session
|
72
83
|
|
@@ -74,63 +85,78 @@ class Server < Sinatra::Default
|
|
74
85
|
Taps::Utils.load_indexes(session.database_url, index_data)
|
75
86
|
end
|
76
87
|
|
77
|
-
|
88
|
+
post '/sessions/:key/pull/schema' do
|
78
89
|
session = DbSession.filter(:key => params[:key]).first
|
79
90
|
halt 404 unless session
|
80
91
|
|
81
|
-
|
82
|
-
Taps::Utils.schema_bin(:dump, session.database_url)
|
92
|
+
Taps::Utils.schema_bin(:dump_table, session.database_url, params[:table_name])
|
83
93
|
end
|
84
94
|
|
85
|
-
get '/sessions/:key/indexes' do
|
95
|
+
get '/sessions/:key/pull/indexes' do
|
86
96
|
session = DbSession.filter(:key => params[:key]).first
|
87
97
|
halt 404 unless session
|
88
98
|
|
89
|
-
|
90
|
-
Taps::Utils.schema_bin(:
|
99
|
+
content_type 'application/json'
|
100
|
+
Taps::Utils.schema_bin(:indexes_individual, session.database_url)
|
91
101
|
end
|
92
102
|
|
93
|
-
get '/sessions/:key/
|
103
|
+
get '/sessions/:key/pull/table_names' do
|
94
104
|
session = DbSession.filter(:key => params[:key]).first
|
95
105
|
halt 404 unless session
|
96
106
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
tables_with_counts = tables.inject({}) do |accum, table|
|
101
|
-
accum[table] = db[table].count
|
102
|
-
accum
|
107
|
+
tables = []
|
108
|
+
session.conn do |db|
|
109
|
+
tables = db.tables
|
103
110
|
end
|
104
111
|
|
105
|
-
|
112
|
+
content_type 'application/json'
|
113
|
+
tables.to_json
|
114
|
+
end
|
115
|
+
|
116
|
+
post '/sessions/:key/pull/table_count' do
|
117
|
+
session = DbSession.filter(:key => params[:key]).first
|
118
|
+
halt 404 unless session
|
119
|
+
|
120
|
+
count = 0
|
121
|
+
session.conn do |db|
|
122
|
+
count = db[ params[:table].to_sym ].count
|
123
|
+
end
|
124
|
+
count.to_s
|
106
125
|
end
|
107
126
|
|
108
|
-
|
127
|
+
post '/sessions/:key/pull/table' do
|
109
128
|
session = DbSession.filter(:key => params[:key]).first
|
110
129
|
halt 404 unless session
|
111
130
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
131
|
+
encoded_data = nil
|
132
|
+
stream = nil
|
133
|
+
|
134
|
+
session.conn do |db|
|
135
|
+
state = JSON.parse(params[:state]).symbolize_keys
|
136
|
+
stream = Taps::DataStream.factory(db, state)
|
137
|
+
encoded_data = stream.fetch.first
|
138
|
+
end
|
139
|
+
|
140
|
+
checksum = Taps::Utils.checksum(encoded_data).to_s
|
141
|
+
json = { :checksum => checksum, :state => stream.to_hash }.to_json
|
142
|
+
|
143
|
+
content, content_type_value = Taps::Multipart.create do |r|
|
144
|
+
r.attach :name => :encoded_data,
|
145
|
+
:payload => encoded_data,
|
146
|
+
:content_type => 'application/octet-stream'
|
147
|
+
r.attach :name => :json,
|
148
|
+
:payload => json,
|
149
|
+
:content_type => 'application/json'
|
150
|
+
end
|
151
|
+
|
152
|
+
content_type content_type_value
|
153
|
+
content
|
127
154
|
end
|
128
155
|
|
129
156
|
delete '/sessions/:key' do
|
130
157
|
session = DbSession.filter(:key => params[:key]).first
|
131
158
|
halt 404 unless session
|
132
159
|
|
133
|
-
session.disconnect
|
134
160
|
session.destroy
|
135
161
|
|
136
162
|
"ok"
|
data/lib/taps/utils.rb
CHANGED
@@ -8,7 +8,9 @@ module Utils
|
|
8
8
|
extend self
|
9
9
|
|
10
10
|
def windows?
|
11
|
-
|
11
|
+
return @windows if defined?(@windows)
|
12
|
+
require 'rbconfig'
|
13
|
+
@windows = !!(::Config::CONFIG['host_os'] =~ /mswin|mingw/)
|
12
14
|
end
|
13
15
|
|
14
16
|
def bin(cmd)
|
@@ -24,24 +26,18 @@ module Utils
|
|
24
26
|
Zlib.crc32(data) == crc32.to_i
|
25
27
|
end
|
26
28
|
|
27
|
-
def
|
28
|
-
|
29
|
-
gz = Zlib::GzipWriter.new(io)
|
30
|
-
gz.write data
|
31
|
-
gz.close
|
32
|
-
io.string
|
29
|
+
def base64encode(data)
|
30
|
+
[data].pack("m")
|
33
31
|
end
|
34
32
|
|
35
|
-
def
|
36
|
-
|
37
|
-
gz = Zlib::GzipReader.new(io)
|
38
|
-
data = gz.read
|
39
|
-
gz.close
|
40
|
-
data
|
33
|
+
def base64decode(data)
|
34
|
+
data.unpack("m").first
|
41
35
|
end
|
42
36
|
|
43
|
-
def format_data(data,
|
37
|
+
def format_data(data, opts={})
|
44
38
|
return {} if data.size == 0
|
39
|
+
string_columns = opts[:string_columns] || []
|
40
|
+
|
45
41
|
header = data[0].keys
|
46
42
|
only_data = data.collect do |row|
|
47
43
|
row = blobs_to_string(row, string_columns)
|
@@ -54,7 +50,7 @@ module Utils
|
|
54
50
|
# this is not true for other databases so we must check if the field is
|
55
51
|
# actually text and manually convert it back to a string
|
56
52
|
def incorrect_blobs(db, table)
|
57
|
-
return []
|
53
|
+
return [] if (db.url =~ /mysql:\/\//).nil?
|
58
54
|
|
59
55
|
columns = []
|
60
56
|
db.schema(table).each do |data|
|
@@ -81,7 +77,7 @@ module Utils
|
|
81
77
|
t1 = Time.now
|
82
78
|
time_in_db = yield chunksize
|
83
79
|
time_in_db = time_in_db.to_f rescue 0
|
84
|
-
rescue Errno::EPIPE, RestClient::RequestFailed
|
80
|
+
rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout
|
85
81
|
retries += 1
|
86
82
|
raise if retries > 2
|
87
83
|
|
@@ -126,21 +122,26 @@ module Utils
|
|
126
122
|
end
|
127
123
|
|
128
124
|
def schema_bin(*args)
|
129
|
-
`#{File.dirname(__FILE__)}/../../bin/#{bin('schema')} #{args.join(' ')}`
|
125
|
+
`#{File.dirname(__FILE__)}/../../bin/#{bin('schema')} #{args.map { |a| "'#{a}'" }.join(' ')}`
|
130
126
|
end
|
131
127
|
|
132
128
|
def primary_key(db, table)
|
133
129
|
if db.respond_to?(:primary_key)
|
134
130
|
db.primary_key(table)
|
135
131
|
else
|
136
|
-
db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first }
|
132
|
+
db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first.to_sym }
|
137
133
|
end
|
138
134
|
end
|
139
135
|
|
136
|
+
def single_integer_primary_key(db, table)
|
137
|
+
keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
|
138
|
+
not keys.nil? and keys.size == 1
|
139
|
+
end
|
140
|
+
|
140
141
|
def order_by(db, table)
|
141
142
|
pkey = primary_key(db, table)
|
142
143
|
if pkey
|
143
|
-
[pkey.to_sym]
|
144
|
+
pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
|
144
145
|
else
|
145
146
|
db[table].columns
|
146
147
|
end
|
data/spec/base.rb
CHANGED
@@ -4,6 +4,8 @@ require 'mocha'
|
|
4
4
|
require 'rack/test'
|
5
5
|
require 'tempfile'
|
6
6
|
|
7
|
+
$:.unshift File.dirname(__FILE__) + "/../lib"
|
8
|
+
|
7
9
|
class Bacon::Context
|
8
10
|
include Mocha::Standalone
|
9
11
|
include Rack::Test::Methods
|
@@ -19,6 +21,6 @@ class Bacon::Context
|
|
19
21
|
end
|
20
22
|
end
|
21
23
|
|
22
|
-
require
|
24
|
+
require 'taps/config'
|
23
25
|
Taps::Config.taps_database_url = "sqlite://#{Tempfile.new('test.db').path}"
|
24
26
|
Sequel.connect(Taps::Config.taps_database_url)
|