taps-taps 0.3.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.rdoc +51 -0
- data/VERSION.yml +5 -0
- data/bin/schema +54 -0
- data/bin/schema.cmd +6 -0
- data/bin/taps +6 -0
- data/lib/taps/chunksize.rb +52 -0
- data/lib/taps/cli.rb +196 -0
- data/lib/taps/config.rb +32 -0
- data/lib/taps/data_stream.rb +343 -0
- data/lib/taps/db_session.rb +20 -0
- data/lib/taps/errors.rb +15 -0
- data/lib/taps/log.rb +15 -0
- data/lib/taps/monkey.rb +21 -0
- data/lib/taps/multipart.rb +73 -0
- data/lib/taps/operation.rb +577 -0
- data/lib/taps/progress_bar.rb +236 -0
- data/lib/taps/schema.rb +82 -0
- data/lib/taps/server.rb +210 -0
- data/lib/taps/utils.rb +182 -0
- data/lib/taps/version.rb +18 -0
- data/lib/vendor/okjson.rb +555 -0
- data/spec/base.rb +26 -0
- data/spec/chunksize_spec.rb +41 -0
- data/spec/cli_spec.rb +16 -0
- data/spec/data_stream_spec.rb +23 -0
- data/spec/operation_spec.rb +42 -0
- data/spec/server_spec.rb +40 -0
- data/spec/utils_spec.rb +30 -0
- metadata +234 -0
@@ -0,0 +1,236 @@
|
|
1
|
+
#
|
2
|
+
# Ruby/ProgressBar - a text progress bar library
|
3
|
+
#
|
4
|
+
# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
|
5
|
+
# All rights reserved.
|
6
|
+
# This is free software with ABSOLUTELY NO WARRANTY.
|
7
|
+
#
|
8
|
+
# You can redistribute it and/or modify it under the terms
|
9
|
+
# of Ruby's license.
|
10
|
+
#
|
11
|
+
|
12
|
+
class ProgressBar
|
13
|
+
VERSION = "0.9"
|
14
|
+
|
15
|
+
def initialize (title, total, out = STDERR)
|
16
|
+
@title = title
|
17
|
+
@total = total
|
18
|
+
@out = out
|
19
|
+
@terminal_width = 80
|
20
|
+
@bar_mark = "="
|
21
|
+
@current = 0
|
22
|
+
@previous = 0
|
23
|
+
@finished_p = false
|
24
|
+
@start_time = Time.now
|
25
|
+
@previous_time = @start_time
|
26
|
+
@title_width = 14
|
27
|
+
@format = "%-#{@title_width}s %3d%% %s %s"
|
28
|
+
@format_arguments = [:title, :percentage, :bar, :stat]
|
29
|
+
clear
|
30
|
+
show
|
31
|
+
end
|
32
|
+
attr_reader :title
|
33
|
+
attr_reader :current
|
34
|
+
attr_reader :total
|
35
|
+
attr_accessor :start_time
|
36
|
+
|
37
|
+
private
|
38
|
+
def fmt_bar
|
39
|
+
bar_width = do_percentage * @terminal_width / 100
|
40
|
+
sprintf("|%s%s|",
|
41
|
+
@bar_mark * bar_width,
|
42
|
+
" " * (@terminal_width - bar_width))
|
43
|
+
end
|
44
|
+
|
45
|
+
def fmt_percentage
|
46
|
+
do_percentage
|
47
|
+
end
|
48
|
+
|
49
|
+
def fmt_stat
|
50
|
+
if @finished_p then elapsed else eta end
|
51
|
+
end
|
52
|
+
|
53
|
+
def fmt_stat_for_file_transfer
|
54
|
+
if @finished_p then
|
55
|
+
sprintf("%s %s %s", bytes, transfer_rate, elapsed)
|
56
|
+
else
|
57
|
+
sprintf("%s %s %s", bytes, transfer_rate, eta)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def fmt_title
|
62
|
+
@title[0,(@title_width - 1)] + ":"
|
63
|
+
end
|
64
|
+
|
65
|
+
def convert_bytes (bytes)
|
66
|
+
if bytes < 1024
|
67
|
+
sprintf("%6dB", bytes)
|
68
|
+
elsif bytes < 1024 * 1000 # 1000kb
|
69
|
+
sprintf("%5.1fKB", bytes.to_f / 1024)
|
70
|
+
elsif bytes < 1024 * 1024 * 1000 # 1000mb
|
71
|
+
sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
|
72
|
+
else
|
73
|
+
sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def transfer_rate
|
78
|
+
bytes_per_second = @current.to_f / (Time.now - @start_time)
|
79
|
+
sprintf("%s/s", convert_bytes(bytes_per_second))
|
80
|
+
end
|
81
|
+
|
82
|
+
def bytes
|
83
|
+
convert_bytes(@current)
|
84
|
+
end
|
85
|
+
|
86
|
+
def format_time (t)
|
87
|
+
t = t.to_i
|
88
|
+
sec = t % 60
|
89
|
+
min = (t / 60) % 60
|
90
|
+
hour = t / 3600
|
91
|
+
sprintf("%02d:%02d:%02d", hour, min, sec);
|
92
|
+
end
|
93
|
+
|
94
|
+
# ETA stands for Estimated Time of Arrival.
|
95
|
+
def eta
|
96
|
+
if @current == 0
|
97
|
+
"ETA: --:--:--"
|
98
|
+
else
|
99
|
+
elapsed = Time.now - @start_time
|
100
|
+
eta = elapsed * @total / @current - elapsed;
|
101
|
+
sprintf("ETA: %s", format_time(eta))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def elapsed
|
106
|
+
elapsed = Time.now - @start_time
|
107
|
+
sprintf("Time: %s", format_time(elapsed))
|
108
|
+
end
|
109
|
+
|
110
|
+
def eol
|
111
|
+
if @finished_p then "\n" else "\r" end
|
112
|
+
end
|
113
|
+
|
114
|
+
def do_percentage
|
115
|
+
if @total.zero?
|
116
|
+
100
|
117
|
+
else
|
118
|
+
@current * 100 / @total
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def get_width
|
123
|
+
# FIXME: I don't know how portable it is.
|
124
|
+
default_width = 80
|
125
|
+
begin
|
126
|
+
tiocgwinsz = 0x5413
|
127
|
+
data = [0, 0, 0, 0].pack("SSSS")
|
128
|
+
if @out.ioctl(tiocgwinsz, data) >= 0 then
|
129
|
+
rows, cols, xpixels, ypixels = data.unpack("SSSS")
|
130
|
+
if cols > 0 then cols else default_width end
|
131
|
+
else
|
132
|
+
default_width
|
133
|
+
end
|
134
|
+
rescue Exception
|
135
|
+
default_width
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def show
|
140
|
+
arguments = @format_arguments.map {|method|
|
141
|
+
method = sprintf("fmt_%s", method)
|
142
|
+
send(method)
|
143
|
+
}
|
144
|
+
line = sprintf(@format, *arguments)
|
145
|
+
|
146
|
+
width = get_width
|
147
|
+
if line.length == width - 1
|
148
|
+
@out.print(line + eol)
|
149
|
+
@out.flush
|
150
|
+
elsif line.length >= width
|
151
|
+
@terminal_width = [@terminal_width - (line.length - width + 1), 0].max
|
152
|
+
if @terminal_width == 0 then @out.print(line + eol) else show end
|
153
|
+
else # line.length < width - 1
|
154
|
+
@terminal_width += width - line.length + 1
|
155
|
+
show
|
156
|
+
end
|
157
|
+
@previous_time = Time.now
|
158
|
+
end
|
159
|
+
|
160
|
+
def show_if_needed
|
161
|
+
if @total.zero?
|
162
|
+
cur_percentage = 100
|
163
|
+
prev_percentage = 0
|
164
|
+
else
|
165
|
+
cur_percentage = (@current * 100 / @total).to_i
|
166
|
+
prev_percentage = (@previous * 100 / @total).to_i
|
167
|
+
end
|
168
|
+
|
169
|
+
# Use "!=" instead of ">" to support negative changes
|
170
|
+
if cur_percentage != prev_percentage ||
|
171
|
+
Time.now - @previous_time >= 1 || @finished_p
|
172
|
+
show
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
public
|
177
|
+
def clear
|
178
|
+
@out.print "\r"
|
179
|
+
@out.print(" " * (get_width - 1))
|
180
|
+
@out.print "\r"
|
181
|
+
end
|
182
|
+
|
183
|
+
def finish
|
184
|
+
@current = @total
|
185
|
+
@finished_p = true
|
186
|
+
show
|
187
|
+
end
|
188
|
+
|
189
|
+
def finished?
|
190
|
+
@finished_p
|
191
|
+
end
|
192
|
+
|
193
|
+
def file_transfer_mode
|
194
|
+
@format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
|
195
|
+
end
|
196
|
+
|
197
|
+
def format= (format)
|
198
|
+
@format = format
|
199
|
+
end
|
200
|
+
|
201
|
+
def format_arguments= (arguments)
|
202
|
+
@format_arguments = arguments
|
203
|
+
end
|
204
|
+
|
205
|
+
def halt
|
206
|
+
@finished_p = true
|
207
|
+
show
|
208
|
+
end
|
209
|
+
|
210
|
+
def inc (step = 1)
|
211
|
+
@current += step
|
212
|
+
@current = @total if @current > @total
|
213
|
+
show_if_needed
|
214
|
+
@previous = @current
|
215
|
+
end
|
216
|
+
|
217
|
+
def set (count)
|
218
|
+
if count < 0 || count > @total
|
219
|
+
raise "invalid count: #{count} (total: #{@total})"
|
220
|
+
end
|
221
|
+
@current = count
|
222
|
+
show_if_needed
|
223
|
+
@previous = @current
|
224
|
+
end
|
225
|
+
|
226
|
+
def inspect
|
227
|
+
"#<ProgressBar:#{@current}/#{@total}>"
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
class ReversedProgressBar < ProgressBar
|
232
|
+
def do_percentage
|
233
|
+
100 - super
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
data/lib/taps/schema.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
require 'sequel/extensions/schema_dumper'
|
3
|
+
require 'sequel/extensions/migration'
|
4
|
+
require 'vendor/okjson'
|
5
|
+
|
6
|
+
module Taps
|
7
|
+
module Schema
|
8
|
+
extend self
|
9
|
+
|
10
|
+
def dump(database_url)
|
11
|
+
db = Sequel.connect(database_url)
|
12
|
+
db.dump_schema_migration(:indexes => false)
|
13
|
+
end
|
14
|
+
|
15
|
+
def dump_table(database_url, table)
|
16
|
+
table = table.to_sym
|
17
|
+
Sequel.connect(database_url) do |db|
|
18
|
+
<<END_MIG
|
19
|
+
Class.new(Sequel::Migration) do
|
20
|
+
def up
|
21
|
+
#{db.dump_table_schema(table.identifier, :indexes => false)}
|
22
|
+
end
|
23
|
+
|
24
|
+
def down
|
25
|
+
drop_table("#{table}") if @db.table_exists?("#{table}")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
END_MIG
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def indexes(database_url)
|
33
|
+
db = Sequel.connect(database_url)
|
34
|
+
db.dump_indexes_migration
|
35
|
+
end
|
36
|
+
|
37
|
+
def indexes_individual(database_url)
|
38
|
+
idxs = {}
|
39
|
+
Sequel.connect(database_url) do |db|
|
40
|
+
tables = db.tables
|
41
|
+
tables.each do |table|
|
42
|
+
idxs[table] = db.send(:dump_table_indexes, table, :add_index, {}).split("\n")
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
idxs.each do |table, indexes|
|
47
|
+
idxs[table] = indexes.map do |idx|
|
48
|
+
<<END_MIG
|
49
|
+
Class.new(Sequel::Migration) do
|
50
|
+
def up
|
51
|
+
#{idx}
|
52
|
+
end
|
53
|
+
end
|
54
|
+
END_MIG
|
55
|
+
end
|
56
|
+
end
|
57
|
+
OkJson.encode(idxs)
|
58
|
+
end
|
59
|
+
|
60
|
+
def load(database_url, schema)
|
61
|
+
Sequel.connect(database_url) do |db|
|
62
|
+
klass = eval(schema)
|
63
|
+
klass.apply(db, :down)
|
64
|
+
klass.apply(db, :up)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def load_indexes(database_url, indexes)
|
69
|
+
Sequel.connect(database_url) do |db|
|
70
|
+
eval(indexes).apply(db, :up)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def reset_db_sequences(database_url)
|
75
|
+
db = Sequel.connect(database_url)
|
76
|
+
return unless db.respond_to?(:reset_primary_key_sequence)
|
77
|
+
db.tables.each do |table|
|
78
|
+
db.reset_primary_key_sequence(table)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/taps/server.rb
ADDED
@@ -0,0 +1,210 @@
|
|
1
|
+
require 'sinatra/base'
|
2
|
+
require 'taps/config'
|
3
|
+
require 'taps/utils'
|
4
|
+
require 'taps/db_session'
|
5
|
+
require 'taps/data_stream'
|
6
|
+
|
7
|
+
module Taps
|
8
|
+
class Server < Sinatra::Base
|
9
|
+
use Rack::Auth::Basic do |login, password|
|
10
|
+
login == Taps::Config.login && password == Taps::Config.password
|
11
|
+
end
|
12
|
+
|
13
|
+
use Rack::Deflater unless ENV['NO_DEFLATE']
|
14
|
+
|
15
|
+
set :raise_errors => false
|
16
|
+
set :show_exceptions => false
|
17
|
+
|
18
|
+
error do
|
19
|
+
e = request.env['sinatra.error']
|
20
|
+
puts "ERROR: #{e.class}: #{e.message}"
|
21
|
+
begin
|
22
|
+
require 'hoptoad_notifier'
|
23
|
+
HoptoadNotifier.configure do |config|
|
24
|
+
config.api_key = ENV["HOPTOAD_API_KEY"]
|
25
|
+
end
|
26
|
+
HoptoadNotifier.notify(e)
|
27
|
+
puts " notified Hoptoad"
|
28
|
+
rescue LoadError
|
29
|
+
puts "An error occurred but Hoptoad was not notified. To use Hoptoad, please"
|
30
|
+
puts "install the 'hoptoad_notifier' gem and set ENV[\"HOPTOAD_API_KEY\"]"
|
31
|
+
end
|
32
|
+
if e.kind_of?(Taps::BaseError)
|
33
|
+
content_type "application/json"
|
34
|
+
halt 412, ::OkJson.encode({ 'error_class' => e.class.to_s, 'error_message' => e.message, 'error_backtrace' => e.backtrace.join("\n") })
|
35
|
+
else
|
36
|
+
"Taps Server Error: #{e}\n#{e.backtrace}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
before do
|
41
|
+
unless request.path_info == '/health'
|
42
|
+
major, minor, patch = request.env['HTTP_TAPS_VERSION'].split('.') rescue []
|
43
|
+
unless "#{major}.#{minor}" == Taps.compatible_version && patch.to_i >= 23
|
44
|
+
halt 417, "Taps >= v#{Taps.compatible_version}.22 is required for this server"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
get '/health' do
|
50
|
+
content_type 'application/json'
|
51
|
+
::OkJson.encode({ :ok => true })
|
52
|
+
end
|
53
|
+
|
54
|
+
get '/' do
|
55
|
+
"hello"
|
56
|
+
end
|
57
|
+
|
58
|
+
post '/sessions' do
|
59
|
+
key = rand(9999999999).to_s
|
60
|
+
|
61
|
+
if ENV['NO_DEFAULT_DATABASE_URL']
|
62
|
+
database_url = request.body.string
|
63
|
+
else
|
64
|
+
database_url = Taps::Config.database_url || request.body.string
|
65
|
+
end
|
66
|
+
|
67
|
+
DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
|
68
|
+
|
69
|
+
"/sessions/#{key}"
|
70
|
+
end
|
71
|
+
|
72
|
+
post '/sessions/:key/push/verify_stream' do
|
73
|
+
session = DbSession.filter(:key => params[:key]).first
|
74
|
+
halt 404 unless session
|
75
|
+
|
76
|
+
state = DataStream.parse_json(params[:state])
|
77
|
+
stream = nil
|
78
|
+
|
79
|
+
size = 0
|
80
|
+
session.conn do |db|
|
81
|
+
Taps::Utils.server_error_handling do
|
82
|
+
stream = DataStream.factory(db, state)
|
83
|
+
stream.verify_stream
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
content_type 'application/json'
|
88
|
+
::OkJson.encode({ :state => stream.to_hash })
|
89
|
+
end
|
90
|
+
|
91
|
+
post '/sessions/:key/push/table' do
|
92
|
+
session = DbSession.filter(:key => params[:key]).first
|
93
|
+
halt 404 unless session
|
94
|
+
|
95
|
+
json = DataStream.parse_json(params[:json])
|
96
|
+
|
97
|
+
size = 0
|
98
|
+
session.conn do |db|
|
99
|
+
Taps::Utils.server_error_handling do
|
100
|
+
stream = DataStream.factory(db, json[:state])
|
101
|
+
size = stream.fetch_remote_in_server(params)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# TODO: return the stream's state with the size
|
106
|
+
size.to_s
|
107
|
+
end
|
108
|
+
|
109
|
+
post '/sessions/:key/push/reset_sequences' do
|
110
|
+
session = DbSession.filter(:key => params[:key]).first
|
111
|
+
halt 404 unless session
|
112
|
+
|
113
|
+
Taps::Utils.schema_bin(:reset_db_sequences, session.database_url)
|
114
|
+
end
|
115
|
+
|
116
|
+
post '/sessions/:key/push/schema' do
|
117
|
+
session = DbSession.filter(:key => params[:key]).first
|
118
|
+
halt 404 unless session
|
119
|
+
|
120
|
+
schema_data = request.body.read
|
121
|
+
Taps::Utils.load_schema(session.database_url, schema_data)
|
122
|
+
end
|
123
|
+
|
124
|
+
post '/sessions/:key/push/indexes' do
|
125
|
+
session = DbSession.filter(:key => params[:key]).first
|
126
|
+
halt 404 unless session
|
127
|
+
|
128
|
+
index_data = request.body.read
|
129
|
+
Taps::Utils.load_indexes(session.database_url, index_data)
|
130
|
+
end
|
131
|
+
|
132
|
+
post '/sessions/:key/pull/schema' do
|
133
|
+
session = DbSession.filter(:key => params[:key]).first
|
134
|
+
halt 404 unless session
|
135
|
+
|
136
|
+
Taps::Utils.schema_bin(:dump_table, session.database_url, params[:table_name])
|
137
|
+
end
|
138
|
+
|
139
|
+
get '/sessions/:key/pull/indexes' do
|
140
|
+
session = DbSession.filter(:key => params[:key]).first
|
141
|
+
halt 404 unless session
|
142
|
+
|
143
|
+
content_type 'application/json'
|
144
|
+
Taps::Utils.schema_bin(:indexes_individual, session.database_url)
|
145
|
+
end
|
146
|
+
|
147
|
+
get '/sessions/:key/pull/table_names' do
|
148
|
+
session = DbSession.filter(:key => params[:key]).first
|
149
|
+
halt 404 unless session
|
150
|
+
|
151
|
+
tables = []
|
152
|
+
session.conn do |db|
|
153
|
+
tables = db.tables
|
154
|
+
end
|
155
|
+
|
156
|
+
content_type 'application/json'
|
157
|
+
::OkJson.encode(tables)
|
158
|
+
end
|
159
|
+
|
160
|
+
post '/sessions/:key/pull/table_count' do
|
161
|
+
session = DbSession.filter(:key => params[:key]).first
|
162
|
+
halt 404 unless session
|
163
|
+
|
164
|
+
count = 0
|
165
|
+
session.conn do |db|
|
166
|
+
count = db[ params[:table].to_sym.identifier ].count
|
167
|
+
end
|
168
|
+
count.to_s
|
169
|
+
end
|
170
|
+
|
171
|
+
post '/sessions/:key/pull/table' do
|
172
|
+
session = DbSession.filter(:key => params[:key]).first
|
173
|
+
halt 404 unless session
|
174
|
+
|
175
|
+
encoded_data = nil
|
176
|
+
stream = nil
|
177
|
+
|
178
|
+
session.conn do |db|
|
179
|
+
state = ::OkJson.decode(params[:state]).symbolize_keys
|
180
|
+
stream = Taps::DataStream.factory(db, state)
|
181
|
+
encoded_data = stream.fetch.first
|
182
|
+
end
|
183
|
+
|
184
|
+
checksum = Taps::Utils.checksum(encoded_data).to_s
|
185
|
+
json = ::OkJson.encode({ :checksum => checksum, :state => stream.to_hash })
|
186
|
+
|
187
|
+
content, content_type_value = Taps::Multipart.create do |r|
|
188
|
+
r.attach :name => :encoded_data,
|
189
|
+
:payload => encoded_data,
|
190
|
+
:content_type => 'application/octet-stream'
|
191
|
+
r.attach :name => :json,
|
192
|
+
:payload => json,
|
193
|
+
:content_type => 'application/json'
|
194
|
+
end
|
195
|
+
|
196
|
+
content_type content_type_value
|
197
|
+
content
|
198
|
+
end
|
199
|
+
|
200
|
+
delete '/sessions/:key' do
|
201
|
+
session = DbSession.filter(:key => params[:key]).first
|
202
|
+
halt 404 unless session
|
203
|
+
|
204
|
+
session.destroy
|
205
|
+
|
206
|
+
"ok"
|
207
|
+
end
|
208
|
+
|
209
|
+
end
|
210
|
+
end
|