tapsoob 0.2.11-java → 0.3.2-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/data_stream.rb +103 -0
- data/lib/tapsoob/cli/root.rb +130 -0
- data/lib/tapsoob/cli/schema.rb +73 -0
- data/lib/tapsoob/cli.rb +6 -180
- data/lib/tapsoob/data_stream.rb +6 -4
- data/lib/tapsoob/log.rb +2 -2
- data/lib/tapsoob/operation.rb +27 -20
- data/lib/tapsoob/schema.rb +1 -1
- data/lib/tapsoob/utils.rb +1 -1
- data/lib/tapsoob/version.rb +1 -1
- data/tapsoob.gemspec +1 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 206ac871453b7e907832ffe47570f0824df8c30d02145f7e7fe94f619aa82591
|
4
|
+
data.tar.gz: 6de6aa4512184890f177bba0af43766b904d44951a1e55b01359b49a9bea6c51
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ea3ed70e1dcd34d5a15efeb6d45167ba575fc7fdab99930b7b13a3cf89a7afe910d91fb6423332efdd3fe26db935c88ab9de81deb593e010dfaf9b91ebb14c28
|
7
|
+
data.tar.gz: a2da2fc215587c9d4442dcae961b8b90635089d5a07672b995163bcdb0703c3ed333f4b396622dc0ff5394d058c0f7c146ccb8761194ab80f570c67169009da3
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'pathname'
|
3
|
+
require 'thor'
|
4
|
+
require 'sequel'
|
5
|
+
|
6
|
+
require_relative '../data_stream.rb'
|
7
|
+
require_relative '../log'
|
8
|
+
require_relative '../operation'
|
9
|
+
|
10
|
+
module Tapsoob
|
11
|
+
module CLI
|
12
|
+
class DataStream < Thor
|
13
|
+
desc "pull DATABASE_URL [DUMP_PATH]", "Pull data from a database."
|
14
|
+
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
15
|
+
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
16
|
+
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
17
|
+
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
18
|
+
option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
|
19
|
+
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
20
|
+
def pull(database_url, dump_path = nil)
|
21
|
+
op = Tapsoob::Operation.factory(:pull, database_url, dump_path, parse_opts(options))
|
22
|
+
op.pull_data
|
23
|
+
end
|
24
|
+
|
25
|
+
desc "push DATABASE_URL [DUMP_PATH]", "Push data to a database."
|
26
|
+
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
27
|
+
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
28
|
+
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
29
|
+
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
30
|
+
option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
|
31
|
+
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
32
|
+
def push(database_url, dump_path = nil)
|
33
|
+
# instantiate stuff
|
34
|
+
data = []
|
35
|
+
opts = parse_opts(options)
|
36
|
+
|
37
|
+
# read data from dump_path or from STDIN
|
38
|
+
if dump_path && Dir.exists?(dump_path)
|
39
|
+
files = Dir[Pathname.new(dump_path).join("*.json")]
|
40
|
+
files.each { |file| data << JSON.parse(File.read(file)) }
|
41
|
+
else
|
42
|
+
STDIN.each_line { |line| data << JSON.parse(line, symbolize_names: true) }
|
43
|
+
end
|
44
|
+
|
45
|
+
# import data
|
46
|
+
data.each do |table|
|
47
|
+
stream = Tapsoob::DataStream.factory(db(database_url, opts), {
|
48
|
+
table_name: table[:table_name],
|
49
|
+
chunksize: opts[:default_chunksize],
|
50
|
+
debug: opts[:debug]
|
51
|
+
})
|
52
|
+
|
53
|
+
begin
|
54
|
+
stream.import_rows(table)
|
55
|
+
rescue Exception => e
|
56
|
+
stream.log.debug e.message
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
def parse_opts(options)
|
63
|
+
# Default options
|
64
|
+
opts = {
|
65
|
+
progress: options[:progress],
|
66
|
+
debug: options[:debug]
|
67
|
+
}
|
68
|
+
|
69
|
+
# Default chunksize
|
70
|
+
if options[:chunksize]
|
71
|
+
opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
|
72
|
+
end
|
73
|
+
|
74
|
+
# Regex filter
|
75
|
+
opts[:table_filter] = options[:filter] if options[:filter]
|
76
|
+
|
77
|
+
# Table filter
|
78
|
+
if options[:tables]
|
79
|
+
r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
|
80
|
+
opts[:table_filter] = "#{r_tables}"
|
81
|
+
end
|
82
|
+
|
83
|
+
# Exclude tables
|
84
|
+
opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
|
85
|
+
|
86
|
+
opts
|
87
|
+
end
|
88
|
+
|
89
|
+
def db(database_url, opts = {})
|
90
|
+
@db ||= Sequel.connect(database_url)
|
91
|
+
@db.loggers << Tapsoob.log if opts[:debug]
|
92
|
+
|
93
|
+
# Set parameters
|
94
|
+
if @db.uri =~ /oracle/i
|
95
|
+
@db << "ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"
|
96
|
+
@db << "ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS:FF6'"
|
97
|
+
end
|
98
|
+
|
99
|
+
@db
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,130 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
require_relative '../config'
|
5
|
+
require_relative '../log'
|
6
|
+
require_relative '../operation'
|
7
|
+
require_relative '../version'
|
8
|
+
|
9
|
+
module Tapsoob
|
10
|
+
module CLI
|
11
|
+
class Root < Thor
|
12
|
+
desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
|
13
|
+
option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
|
14
|
+
option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
|
15
|
+
option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
|
16
|
+
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
17
|
+
option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
|
18
|
+
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
19
|
+
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
20
|
+
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
21
|
+
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
22
|
+
def pull(dump_path, database_url)
|
23
|
+
opts = parse_opts(options)
|
24
|
+
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
25
|
+
if opts[:resume_filename]
|
26
|
+
clientresumexfer(:pull, dump_path, database_url, opts)
|
27
|
+
else
|
28
|
+
clientxfer(:pull, dump_path, database_url, opts)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
|
33
|
+
option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
|
34
|
+
option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
|
35
|
+
option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
|
36
|
+
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
37
|
+
option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
|
38
|
+
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
39
|
+
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
40
|
+
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
41
|
+
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
42
|
+
def push(dump_path, database_url)
|
43
|
+
opts = parse_opts(options)
|
44
|
+
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
45
|
+
if opts[:resume_filename]
|
46
|
+
clientresumexfer(:push, dump_path, database_url, opts)
|
47
|
+
else
|
48
|
+
clientxfer(:push, dump_path, database_url, opts)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
desc "version", "Show tapsoob version"
|
53
|
+
def version
|
54
|
+
puts Tapsoob::VERSION.dup
|
55
|
+
end
|
56
|
+
|
57
|
+
desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
|
58
|
+
subcommand "schema", Schema
|
59
|
+
|
60
|
+
desc "data SUBCOMMAND ...ARGS", "Pull/Push data with internal Tapsoob classes"
|
61
|
+
subcommand "data", DataStream
|
62
|
+
|
63
|
+
private
|
64
|
+
def parse_opts(options)
|
65
|
+
# Default options
|
66
|
+
opts = {
|
67
|
+
skip_schema: options[:"skip-schema"],
|
68
|
+
indexes_first: options[:"indexes_first"],
|
69
|
+
disable_compression: options[:"disable-compression"],
|
70
|
+
debug: options[:debug]
|
71
|
+
}
|
72
|
+
|
73
|
+
# Resume
|
74
|
+
if options[:resume]
|
75
|
+
if File.exists?(options[:resume])
|
76
|
+
opts[:resume_file] = options[:resume]
|
77
|
+
else
|
78
|
+
raise "Unable to find resume file."
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# Default chunksize
|
83
|
+
if options[:chunksize]
|
84
|
+
opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
|
85
|
+
end
|
86
|
+
|
87
|
+
# Regex filter
|
88
|
+
opts[:table_filter] = options[:filter] if options[:filter]
|
89
|
+
|
90
|
+
# Table filter
|
91
|
+
if options[:tables]
|
92
|
+
r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
|
93
|
+
opts[:table_filter] = "#{r_tables}"
|
94
|
+
end
|
95
|
+
|
96
|
+
# Exclude tables
|
97
|
+
opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
|
98
|
+
|
99
|
+
opts
|
100
|
+
end
|
101
|
+
|
102
|
+
def clientxfer(method, dump_path, database_url, opts)
|
103
|
+
Tapsoob::Config.verify_database_url(database_url)
|
104
|
+
|
105
|
+
FileUtils.mkpath "#{dump_path}/schemas"
|
106
|
+
FileUtils.mkpath "#{dump_path}/data"
|
107
|
+
FileUtils.mkpath "#{dump_path}/indexes"
|
108
|
+
|
109
|
+
Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
|
110
|
+
end
|
111
|
+
|
112
|
+
def clientresumexfer(method, dump_path, database_url, opts)
|
113
|
+
session = JSON.parse(File.read(opts.delete(:resume_filename)))
|
114
|
+
session.symbolize_recursively!
|
115
|
+
|
116
|
+
dump_path = dump_path || session.delete(:dump_path)
|
117
|
+
|
118
|
+
require 'taps/operation'
|
119
|
+
|
120
|
+
newsession = session.merge({
|
121
|
+
:default_chunksize => opts[:default_chunksize],
|
122
|
+
:disable_compression => opts[:disable_compression],
|
123
|
+
:resume => true
|
124
|
+
})
|
125
|
+
|
126
|
+
Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'sequel'
|
3
|
+
|
4
|
+
require_relative '../schema'
|
5
|
+
|
6
|
+
module Tapsoob
|
7
|
+
module CLI
|
8
|
+
class Schema < Thor
|
9
|
+
desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
|
10
|
+
def console(database_url)
|
11
|
+
$db = Sequel.connect(database_url)
|
12
|
+
require 'irb'
|
13
|
+
require 'irb/completion'
|
14
|
+
IRB.start
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "dump DATABASE_URL", "Dump a database using a database URL"
|
18
|
+
def dump(database_url)
|
19
|
+
puts Tapsoob::Schema.dump(database_url)
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
|
23
|
+
def dump_table(database_url, table)
|
24
|
+
puts Tapsoob::Schema.dump_table(database_url, table)
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
|
28
|
+
def indexes(database_url)
|
29
|
+
puts Tapsoob::Schema.indexes(database_url)
|
30
|
+
end
|
31
|
+
|
32
|
+
desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
|
33
|
+
def indexes_individual(database_url)
|
34
|
+
puts Tapsoob::Schema.indexes_individual(database_url)
|
35
|
+
end
|
36
|
+
|
37
|
+
desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
|
38
|
+
def reset_db_sequences(database_url)
|
39
|
+
Tapsoob::Schema.reset_db_sequences(database_url)
|
40
|
+
end
|
41
|
+
|
42
|
+
desc "load DATABASE_URL [FILENAME]", "Load a database schema from a file to a database using a database URL"
|
43
|
+
def load(database_url, filename = nil)
|
44
|
+
schema = if filename && File.exists?(filename)
|
45
|
+
File.read(filename)
|
46
|
+
else
|
47
|
+
STDIN.read
|
48
|
+
end
|
49
|
+
|
50
|
+
begin
|
51
|
+
Tapsoob::Schema.load(database_url, schema)
|
52
|
+
rescue Exception => e
|
53
|
+
throw e
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
desc "load_indexes DATABASE_URL [FILENAME]", "Load indexes from a file to a database using a database URL"
|
58
|
+
def load_indexes(database_url, filename = nil)
|
59
|
+
indexes = if filename && File.exists?(filename)
|
60
|
+
File.read(filename)
|
61
|
+
else
|
62
|
+
STDIN.read
|
63
|
+
end
|
64
|
+
|
65
|
+
begin
|
66
|
+
Tapsoob::Schema.load_indexes(database_url, indexes)
|
67
|
+
rescue Exception => e
|
68
|
+
throw e
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
data/lib/tapsoob/cli.rb
CHANGED
@@ -1,17 +1,8 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
require 'fileutils'
|
5
|
-
require 'sequel'
|
6
|
-
require 'tempfile'
|
7
1
|
require 'thor'
|
2
|
+
require 'tempfile'
|
8
3
|
|
9
4
|
# tapsoob deps
|
10
|
-
|
11
|
-
require 'tapsoob/log'
|
12
|
-
require 'tapsoob/operation'
|
13
|
-
require 'tapsoob/schema'
|
14
|
-
require 'tapsoob/version'
|
5
|
+
require_relative 'config'
|
15
6
|
|
16
7
|
Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
|
17
8
|
# this is dirty but it solves a weird problem where the tempfile disappears mid-process
|
@@ -21,172 +12,7 @@ Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
|
|
21
12
|
"sqlite://#{$__taps_database.path}"
|
22
13
|
end
|
23
14
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
def console(database_url)
|
29
|
-
$db = Sequel.connect(database_url)
|
30
|
-
require 'irb'
|
31
|
-
require 'irb/completion'
|
32
|
-
IRB.start
|
33
|
-
end
|
34
|
-
|
35
|
-
desc "dump DATABASE_URL", "Dump a database using a database URL"
|
36
|
-
def dump(database_url)
|
37
|
-
puts Tapsoob::Schema.dump(database_url)
|
38
|
-
end
|
39
|
-
|
40
|
-
desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
|
41
|
-
def dump_table(database_url, table)
|
42
|
-
puts Tapsoob::Schema.dump_table(database_url, table)
|
43
|
-
end
|
44
|
-
|
45
|
-
desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
|
46
|
-
def indexes(database_url)
|
47
|
-
puts Tapsoob::Schema.indexes(database_url)
|
48
|
-
end
|
49
|
-
|
50
|
-
desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
|
51
|
-
def indexes_individual(database_url)
|
52
|
-
puts Tapsoob::Schema.indexes_individual(database_url)
|
53
|
-
end
|
54
|
-
|
55
|
-
desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
|
56
|
-
def reset_db_sequences(database_url)
|
57
|
-
Tapsoob::Schema.reset_db_sequences(database_url)
|
58
|
-
end
|
59
|
-
|
60
|
-
desc "load DATABASE_URL FILENAME", "Load a database schema from a file to a database using a database URL"
|
61
|
-
def load(database_url, filename)
|
62
|
-
schema = File.read(filename) rescue help
|
63
|
-
Tapsoob::Schema.load(database_url, schema)
|
64
|
-
end
|
65
|
-
|
66
|
-
desc "load_indexes DATABASE_URL FILENAME", "Load indexes from a file to a database using a database URL"
|
67
|
-
def load_indexes(database_url, filename)
|
68
|
-
indexes = File.read(filename) rescue help
|
69
|
-
Tapsoob::Schema.load_indexes(database_url, indexes)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
class Root < Thor
|
74
|
-
desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
|
75
|
-
option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
|
76
|
-
option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
|
77
|
-
option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
|
78
|
-
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
79
|
-
option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
|
80
|
-
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
81
|
-
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
82
|
-
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
83
|
-
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
84
|
-
def pull(dump_path, database_url)
|
85
|
-
opts = parse_opts(options)
|
86
|
-
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
87
|
-
if opts[:resume_filename]
|
88
|
-
clientresumexfer(:pull, dump_path, database_url, opts)
|
89
|
-
else
|
90
|
-
clientxfer(:pull, dump_path, database_url, opts)
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
|
95
|
-
option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
|
96
|
-
option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
|
97
|
-
option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
|
98
|
-
option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
|
99
|
-
option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
|
100
|
-
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
101
|
-
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
102
|
-
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
103
|
-
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
104
|
-
def push(dump_path, database_url)
|
105
|
-
opts = parse_opts(options)
|
106
|
-
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
107
|
-
if opts[:resume_filename]
|
108
|
-
clientresumexfer(:push, dump_path, database_url, opts)
|
109
|
-
else
|
110
|
-
clientxfer(:push, dump_path, database_url, opts)
|
111
|
-
end
|
112
|
-
end
|
113
|
-
|
114
|
-
desc "version", "Show tapsoob version"
|
115
|
-
def version
|
116
|
-
puts Tapsoob::VERSION.dup
|
117
|
-
end
|
118
|
-
|
119
|
-
desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
|
120
|
-
subcommand "schema", Schema
|
121
|
-
|
122
|
-
private
|
123
|
-
def parse_opts(options)
|
124
|
-
# Default options
|
125
|
-
opts = {
|
126
|
-
skip_schema: options[:"skip-schema"],
|
127
|
-
indexes_first: options[:"indexes_first"],
|
128
|
-
disable_compression: options[:"disable-compression"],
|
129
|
-
debug: options[:debug]
|
130
|
-
}
|
131
|
-
|
132
|
-
# Resume
|
133
|
-
if options[:resume]
|
134
|
-
if File.exists?(options[:resume])
|
135
|
-
opts[:resume_file] = options[:resume]
|
136
|
-
else
|
137
|
-
raise "Unable to find resume file."
|
138
|
-
end
|
139
|
-
end
|
140
|
-
|
141
|
-
# Default chunksize
|
142
|
-
if options[:chunksize]
|
143
|
-
opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
|
144
|
-
end
|
145
|
-
|
146
|
-
# Regex filter
|
147
|
-
opts[:table_filter] = options[:filter] if options[:filter]
|
148
|
-
|
149
|
-
# Table filter
|
150
|
-
if options[:tables]
|
151
|
-
r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
|
152
|
-
opts[:table_filter] = "#{r_tables}"
|
153
|
-
end
|
154
|
-
|
155
|
-
# Exclude tables
|
156
|
-
opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
|
157
|
-
|
158
|
-
opts
|
159
|
-
end
|
160
|
-
|
161
|
-
def clientxfer(method, dump_path, database_url, opts)
|
162
|
-
Tapsoob::Config.verify_database_url(database_url)
|
163
|
-
|
164
|
-
FileUtils.mkpath "#{dump_path}/schemas"
|
165
|
-
FileUtils.mkpath "#{dump_path}/data"
|
166
|
-
FileUtils.mkpath "#{dump_path}/indexes"
|
167
|
-
|
168
|
-
require 'tapsoob/operation'
|
169
|
-
|
170
|
-
Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
|
171
|
-
end
|
172
|
-
|
173
|
-
def clientresumexfer(method, dump_path, database_url, opts)
|
174
|
-
session = JSON.parse(File.read(opts.delete(:resume_filename)))
|
175
|
-
session.symbolize_recursively!
|
176
|
-
|
177
|
-
dump_path = dump_path || session.delete(:dump_path)
|
178
|
-
|
179
|
-
require 'taps/operation'
|
180
|
-
|
181
|
-
newsession = session.merge({
|
182
|
-
:default_chunksize => opts[:default_chunksize],
|
183
|
-
:disable_compression => opts[:disable_compression],
|
184
|
-
:resume => true
|
185
|
-
})
|
186
|
-
|
187
|
-
Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
|
188
|
-
end
|
189
|
-
end
|
190
|
-
end
|
191
|
-
end
|
192
|
-
|
15
|
+
# CLI modules
|
16
|
+
require_relative 'cli/schema'
|
17
|
+
require_relative 'cli/data_stream'
|
18
|
+
require_relative 'cli/root'
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -21,6 +21,7 @@ module Tapsoob
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def log
|
24
|
+
Tapsoob.log.level = Logger::DEBUG if state[:debug]
|
24
25
|
Tapsoob.log
|
25
26
|
end
|
26
27
|
|
@@ -87,8 +88,9 @@ module Tapsoob
|
|
87
88
|
ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
|
88
89
|
log.debug "DataStream#fetch_file"
|
89
90
|
rows = {
|
90
|
-
:
|
91
|
-
:
|
91
|
+
:table_name => ds["table_name"],
|
92
|
+
:header => ds["header"],
|
93
|
+
:data => ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]
|
92
94
|
}
|
93
95
|
update_chunksize_stats
|
94
96
|
rows
|
@@ -141,7 +143,7 @@ module Tapsoob
|
|
141
143
|
@complete
|
142
144
|
end
|
143
145
|
|
144
|
-
def fetch_database
|
146
|
+
def fetch_database
|
145
147
|
params = fetch_from_database
|
146
148
|
encoded_data = params[:encoded_data]
|
147
149
|
json = params[:json]
|
@@ -154,7 +156,7 @@ module Tapsoob
|
|
154
156
|
state.merge!(json[:state].merge(:chunksize => state[:chunksize]))
|
155
157
|
|
156
158
|
unless @complete
|
157
|
-
|
159
|
+
yield rows if block_given?
|
158
160
|
state[:offset] += rows[:data].size
|
159
161
|
rows[:data].size
|
160
162
|
else
|
data/lib/tapsoob/log.rb
CHANGED
data/lib/tapsoob/operation.rb
CHANGED
@@ -10,7 +10,7 @@ module Tapsoob
|
|
10
10
|
class Operation
|
11
11
|
attr_reader :database_url, :dump_path, :opts
|
12
12
|
|
13
|
-
def initialize(database_url, dump_path, opts={})
|
13
|
+
def initialize(database_url, dump_path = nil, opts={})
|
14
14
|
@database_url = database_url
|
15
15
|
@dump_path = dump_path
|
16
16
|
@opts = opts
|
@@ -55,12 +55,13 @@ module Tapsoob
|
|
55
55
|
end
|
56
56
|
|
57
57
|
def log
|
58
|
+
Tapsoob.log.level = Logger::DEBUG if opts[:debug]
|
58
59
|
Tapsoob.log
|
59
60
|
end
|
60
61
|
|
61
62
|
def store_session
|
62
63
|
file = "#{file_prefix}_#{Time.now.strftime("%Y%m%d%H%M")}.dat"
|
63
|
-
|
64
|
+
log.info "\nSaving session to #{file}..."
|
64
65
|
File.open(file, 'w') do |f|
|
65
66
|
f.write(JSON.generate(to_hash))
|
66
67
|
end
|
@@ -174,7 +175,7 @@ module Tapsoob
|
|
174
175
|
end
|
175
176
|
|
176
177
|
def pull_schema
|
177
|
-
|
178
|
+
log.info "Receiving schema"
|
178
179
|
|
179
180
|
progress = ProgressBar.new('Schema', tables.size)
|
180
181
|
tables.each do |table_name, count|
|
@@ -188,12 +189,12 @@ module Tapsoob
|
|
188
189
|
end
|
189
190
|
|
190
191
|
def pull_data
|
191
|
-
|
192
|
+
log.info "Receiving data"
|
192
193
|
|
193
|
-
|
194
|
+
log.info "#{tables.size} tables, #{format_number(record_count)} records"
|
194
195
|
|
195
196
|
tables.each do |table_name, count|
|
196
|
-
progress = ProgressBar.new(table_name.to_s, count)
|
197
|
+
progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
|
197
198
|
stream = Tapsoob::DataStream.factory(db, {
|
198
199
|
:chunksize => default_chunksize,
|
199
200
|
:table_name => table_name
|
@@ -207,9 +208,9 @@ module Tapsoob
|
|
207
208
|
|
208
209
|
table_name = stream_state[:table_name]
|
209
210
|
record_count = tables[table_name.to_s]
|
210
|
-
|
211
|
+
log.info "Resuming #{table_name}, #{format_number(record_count)} records"
|
211
212
|
|
212
|
-
progress = ProgressBar.new(table_name.to_s, record_count)
|
213
|
+
progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, record_count) : nil)
|
213
214
|
stream = Tapsoob::DataStream.factory(db, stream_state)
|
214
215
|
pull_data_from_table(stream, progress)
|
215
216
|
end
|
@@ -219,19 +220,25 @@ module Tapsoob
|
|
219
220
|
begin
|
220
221
|
exit 0 if exiting?
|
221
222
|
|
222
|
-
size = stream.fetch_database
|
223
|
+
size = stream.fetch_database do |rows|
|
224
|
+
if dump_path.nil?
|
225
|
+
puts JSON.generate(rows)
|
226
|
+
else
|
227
|
+
Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
|
228
|
+
end
|
229
|
+
end
|
223
230
|
break if stream.complete?
|
224
|
-
progress.inc(size)
|
231
|
+
progress.inc(size) if progress && !exiting?
|
225
232
|
stream.error = false
|
226
233
|
self.stream_state = stream.to_hash
|
227
234
|
rescue Tapsoob::CorruptedData => e
|
228
|
-
|
235
|
+
log.info "Corrupted Data Received #{e.message}, retrying..."
|
229
236
|
stream.error = true
|
230
237
|
next
|
231
238
|
end
|
232
239
|
end
|
233
240
|
|
234
|
-
progress.finish
|
241
|
+
progress.finish if progress
|
235
242
|
completed_tables << stream.table_name.to_s
|
236
243
|
self.stream_state = {}
|
237
244
|
end
|
@@ -280,7 +287,7 @@ module Tapsoob
|
|
280
287
|
end
|
281
288
|
|
282
289
|
def pull_indexes
|
283
|
-
|
290
|
+
log.info "Receiving indexes"
|
284
291
|
|
285
292
|
raw_idxs = Tapsoob::Utils.schema_bin(:indexes_individual, database_url)
|
286
293
|
idxs = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
|
@@ -298,7 +305,7 @@ module Tapsoob
|
|
298
305
|
end
|
299
306
|
|
300
307
|
def pull_reset_sequences
|
301
|
-
|
308
|
+
log.info "Resetting sequences"
|
302
309
|
|
303
310
|
output = Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
|
304
311
|
puts output if output
|
@@ -337,7 +344,7 @@ module Tapsoob
|
|
337
344
|
|
338
345
|
return unless idxs.size > 0
|
339
346
|
|
340
|
-
|
347
|
+
log.info "Sending indexes"
|
341
348
|
|
342
349
|
apply_table_filter(idxs).each do |table, indexes|
|
343
350
|
next unless indexes.size > 0
|
@@ -351,7 +358,7 @@ module Tapsoob
|
|
351
358
|
end
|
352
359
|
|
353
360
|
def push_schema
|
354
|
-
|
361
|
+
log.info "Sending schema"
|
355
362
|
|
356
363
|
progress = ProgressBar.new('Schema', tables.size)
|
357
364
|
tables.each do |table, count|
|
@@ -363,7 +370,7 @@ module Tapsoob
|
|
363
370
|
end
|
364
371
|
|
365
372
|
def push_reset_sequences
|
366
|
-
|
373
|
+
log.info "Resetting sequences"
|
367
374
|
|
368
375
|
Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
|
369
376
|
end
|
@@ -373,16 +380,16 @@ module Tapsoob
|
|
373
380
|
|
374
381
|
table_name = stream_state[:table_name]
|
375
382
|
record_count = tables[table_name.to_s]
|
376
|
-
|
383
|
+
log.info "Resuming #{table_name}, #{format_number(record_count)} records"
|
377
384
|
progress = ProgressBar.new(table_name.to_s, record_count)
|
378
385
|
stream = Tapsoob::DataStream.factory(db, stream_state)
|
379
386
|
push_data_from_file(stream, progress)
|
380
387
|
end
|
381
388
|
|
382
389
|
def push_data
|
383
|
-
|
390
|
+
log.info "Sending data"
|
384
391
|
|
385
|
-
|
392
|
+
log.info "#{tables.size} tables, #{format_number(record_count)} records"
|
386
393
|
|
387
394
|
tables.each do |table_name, count|
|
388
395
|
next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
|
data/lib/tapsoob/schema.rb
CHANGED
data/lib/tapsoob/utils.rb
CHANGED
data/lib/tapsoob/version.rb
CHANGED
data/tapsoob.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tapsoob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2
|
4
|
+
version: 0.3.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Félix Bellanger
|
@@ -9,14 +9,14 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-06-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 5.
|
19
|
+
version: 5.21.0
|
20
20
|
name: sequel
|
21
21
|
prerelease: false
|
22
22
|
type: :runtime
|
@@ -24,7 +24,7 @@ dependencies:
|
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: 5.
|
27
|
+
version: 5.21.0
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
@@ -98,6 +98,9 @@ files:
|
|
98
98
|
- lib/tapsoob.rb
|
99
99
|
- lib/tapsoob/chunksize.rb
|
100
100
|
- lib/tapsoob/cli.rb
|
101
|
+
- lib/tapsoob/cli/data_stream.rb
|
102
|
+
- lib/tapsoob/cli/root.rb
|
103
|
+
- lib/tapsoob/cli/schema.rb
|
101
104
|
- lib/tapsoob/config.rb
|
102
105
|
- lib/tapsoob/data_stream.rb
|
103
106
|
- lib/tapsoob/errors.rb
|