tapsoob 0.2.11-java → 0.3.2-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8fc725d2a78102584abbc1cc8da65fd9253b7c2bd9c1c61d70bdc8185e0522a0
4
- data.tar.gz: ff02473ed72b7a6a0130124e2d8cc81a322db9f1661aec191564b3b60ce902f7
3
+ metadata.gz: 206ac871453b7e907832ffe47570f0824df8c30d02145f7e7fe94f619aa82591
4
+ data.tar.gz: 6de6aa4512184890f177bba0af43766b904d44951a1e55b01359b49a9bea6c51
5
5
  SHA512:
6
- metadata.gz: 9fe99651b48dc352b775e10d2100abee97cc7c070e7d8278879ced60102aa5ad28b6002e5efad6f2818a76e50dca958a5e6f79182cba43a083fb5aadcf46f691
7
- data.tar.gz: 2df8296b5774071cfe6ba3e3b43bc67ef1085de30531ef131d6c9322a13c6faf5e6fec07752c9a83a9f6a1ed571c7430b7e938e22f5268cf17871c9413b96efc
6
+ metadata.gz: ea3ed70e1dcd34d5a15efeb6d45167ba575fc7fdab99930b7b13a3cf89a7afe910d91fb6423332efdd3fe26db935c88ab9de81deb593e010dfaf9b91ebb14c28
7
+ data.tar.gz: a2da2fc215587c9d4442dcae961b8b90635089d5a07672b995163bcdb0703c3ed333f4b396622dc0ff5394d058c0f7c146ccb8761194ab80f570c67169009da3
@@ -0,0 +1,103 @@
1
+ require 'json'
2
+ require 'pathname'
3
+ require 'thor'
4
+ require 'sequel'
5
+
6
+ require_relative '../data_stream.rb'
7
+ require_relative '../log'
8
+ require_relative '../operation'
9
+
10
+ module Tapsoob
11
+ module CLI
12
+ class DataStream < Thor
13
+ desc "pull DATABASE_URL [DUMP_PATH]", "Pull data from a database."
14
+ option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
15
+ option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
16
+ option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
17
+ option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
18
+ option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
19
+ option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
20
+ def pull(database_url, dump_path = nil)
21
+ op = Tapsoob::Operation.factory(:pull, database_url, dump_path, parse_opts(options))
22
+ op.pull_data
23
+ end
24
+
25
+ desc "push DATABASE_URL [DUMP_PATH]", "Push data to a database."
26
+ option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
27
+ option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
28
+ option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
29
+ option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
30
+ option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
31
+ option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
32
+ def push(database_url, dump_path = nil)
33
+ # instantiate stuff
34
+ data = []
35
+ opts = parse_opts(options)
36
+
37
+ # read data from dump_path or from STDIN
38
+ if dump_path && Dir.exists?(dump_path)
39
+ files = Dir[Pathname.new(dump_path).join("*.json")]
40
+ files.each { |file| data << JSON.parse(File.read(file)) }
41
+ else
42
+ STDIN.each_line { |line| data << JSON.parse(line, symbolize_names: true) }
43
+ end
44
+
45
+ # import data
46
+ data.each do |table|
47
+ stream = Tapsoob::DataStream.factory(db(database_url, opts), {
48
+ table_name: table[:table_name],
49
+ chunksize: opts[:default_chunksize],
50
+ debug: opts[:debug]
51
+ })
52
+
53
+ begin
54
+ stream.import_rows(table)
55
+ rescue Exception => e
56
+ stream.log.debug e.message
57
+ end
58
+ end
59
+ end
60
+
61
+ private
62
+ def parse_opts(options)
63
+ # Default options
64
+ opts = {
65
+ progress: options[:progress],
66
+ debug: options[:debug]
67
+ }
68
+
69
+ # Default chunksize
70
+ if options[:chunksize]
71
+ opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
72
+ end
73
+
74
+ # Regex filter
75
+ opts[:table_filter] = options[:filter] if options[:filter]
76
+
77
+ # Table filter
78
+ if options[:tables]
79
+ r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
80
+ opts[:table_filter] = "#{r_tables}"
81
+ end
82
+
83
+ # Exclude tables
84
+ opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
85
+
86
+ opts
87
+ end
88
+
89
+ def db(database_url, opts = {})
90
+ @db ||= Sequel.connect(database_url)
91
+ @db.loggers << Tapsoob.log if opts[:debug]
92
+
93
+ # Set parameters
94
+ if @db.uri =~ /oracle/i
95
+ @db << "ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"
96
+ @db << "ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS:FF6'"
97
+ end
98
+
99
+ @db
100
+ end
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,130 @@
1
+ require 'thor'
2
+ require 'fileutils'
3
+
4
+ require_relative '../config'
5
+ require_relative '../log'
6
+ require_relative '../operation'
7
+ require_relative '../version'
8
+
9
+ module Tapsoob
10
+ module CLI
11
+ class Root < Thor
12
+ desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
13
+ option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
14
+ option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
15
+ option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
16
+ option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
17
+ option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
18
+ option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
19
+ option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
20
+ option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
21
+ option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
22
+ def pull(dump_path, database_url)
23
+ opts = parse_opts(options)
24
+ Tapsoob.log.level = Logger::DEBUG if opts[:debug]
25
+ if opts[:resume_filename]
26
+ clientresumexfer(:pull, dump_path, database_url, opts)
27
+ else
28
+ clientxfer(:pull, dump_path, database_url, opts)
29
+ end
30
+ end
31
+
32
+ desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
33
+ option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
34
+ option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
35
+ option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
36
+ option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
37
+ option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
38
+ option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
39
+ option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
40
+ option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
41
+ option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
42
+ def push(dump_path, database_url)
43
+ opts = parse_opts(options)
44
+ Tapsoob.log.level = Logger::DEBUG if opts[:debug]
45
+ if opts[:resume_filename]
46
+ clientresumexfer(:push, dump_path, database_url, opts)
47
+ else
48
+ clientxfer(:push, dump_path, database_url, opts)
49
+ end
50
+ end
51
+
52
+ desc "version", "Show tapsoob version"
53
+ def version
54
+ puts Tapsoob::VERSION.dup
55
+ end
56
+
57
+ desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
58
+ subcommand "schema", Schema
59
+
60
+ desc "data SUBCOMMAND ...ARGS", "Pull/Push data with internal Tapsoob classes"
61
+ subcommand "data", DataStream
62
+
63
+ private
64
+ def parse_opts(options)
65
+ # Default options
66
+ opts = {
67
+ skip_schema: options[:"skip-schema"],
68
+ indexes_first: options[:"indexes_first"],
69
+ disable_compression: options[:"disable-compression"],
70
+ debug: options[:debug]
71
+ }
72
+
73
+ # Resume
74
+ if options[:resume]
75
+ if File.exists?(options[:resume])
76
+ opts[:resume_file] = options[:resume]
77
+ else
78
+ raise "Unable to find resume file."
79
+ end
80
+ end
81
+
82
+ # Default chunksize
83
+ if options[:chunksize]
84
+ opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
85
+ end
86
+
87
+ # Regex filter
88
+ opts[:table_filter] = options[:filter] if options[:filter]
89
+
90
+ # Table filter
91
+ if options[:tables]
92
+ r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
93
+ opts[:table_filter] = "#{r_tables}"
94
+ end
95
+
96
+ # Exclude tables
97
+ opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
98
+
99
+ opts
100
+ end
101
+
102
+ def clientxfer(method, dump_path, database_url, opts)
103
+ Tapsoob::Config.verify_database_url(database_url)
104
+
105
+ FileUtils.mkpath "#{dump_path}/schemas"
106
+ FileUtils.mkpath "#{dump_path}/data"
107
+ FileUtils.mkpath "#{dump_path}/indexes"
108
+
109
+ Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
110
+ end
111
+
112
+ def clientresumexfer(method, dump_path, database_url, opts)
113
+ session = JSON.parse(File.read(opts.delete(:resume_filename)))
114
+ session.symbolize_recursively!
115
+
116
+ dump_path = dump_path || session.delete(:dump_path)
117
+
118
+ require 'taps/operation'
119
+
120
+ newsession = session.merge({
121
+ :default_chunksize => opts[:default_chunksize],
122
+ :disable_compression => opts[:disable_compression],
123
+ :resume => true
124
+ })
125
+
126
+ Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
127
+ end
128
+ end
129
+ end
130
+ end
@@ -0,0 +1,73 @@
1
+ require 'thor'
2
+ require 'sequel'
3
+
4
+ require_relative '../schema'
5
+
6
+ module Tapsoob
7
+ module CLI
8
+ class Schema < Thor
9
+ desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
10
+ def console(database_url)
11
+ $db = Sequel.connect(database_url)
12
+ require 'irb'
13
+ require 'irb/completion'
14
+ IRB.start
15
+ end
16
+
17
+ desc "dump DATABASE_URL", "Dump a database using a database URL"
18
+ def dump(database_url)
19
+ puts Tapsoob::Schema.dump(database_url)
20
+ end
21
+
22
+ desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
23
+ def dump_table(database_url, table)
24
+ puts Tapsoob::Schema.dump_table(database_url, table)
25
+ end
26
+
27
+ desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
28
+ def indexes(database_url)
29
+ puts Tapsoob::Schema.indexes(database_url)
30
+ end
31
+
32
+ desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
33
+ def indexes_individual(database_url)
34
+ puts Tapsoob::Schema.indexes_individual(database_url)
35
+ end
36
+
37
+ desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
38
+ def reset_db_sequences(database_url)
39
+ Tapsoob::Schema.reset_db_sequences(database_url)
40
+ end
41
+
42
+ desc "load DATABASE_URL [FILENAME]", "Load a database schema from a file to a database using a database URL"
43
+ def load(database_url, filename = nil)
44
+ schema = if filename && File.exists?(filename)
45
+ File.read(filename)
46
+ else
47
+ STDIN.read
48
+ end
49
+
50
+ begin
51
+ Tapsoob::Schema.load(database_url, schema)
52
+ rescue Exception => e
53
+ throw e
54
+ end
55
+ end
56
+
57
+ desc "load_indexes DATABASE_URL [FILENAME]", "Load indexes from a file to a database using a database URL"
58
+ def load_indexes(database_url, filename = nil)
59
+ indexes = if filename && File.exists?(filename)
60
+ File.read(filename)
61
+ else
62
+ STDIN.read
63
+ end
64
+
65
+ begin
66
+ Tapsoob::Schema.load_indexes(database_url, indexes)
67
+ rescue Exception => e
68
+ throw e
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
data/lib/tapsoob/cli.rb CHANGED
@@ -1,17 +1,8 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'rubygems'
4
- require 'fileutils'
5
- require 'sequel'
6
- require 'tempfile'
7
1
  require 'thor'
2
+ require 'tempfile'
8
3
 
9
4
  # tapsoob deps
10
- require 'tapsoob/config'
11
- require 'tapsoob/log'
12
- require 'tapsoob/operation'
13
- require 'tapsoob/schema'
14
- require 'tapsoob/version'
5
+ require_relative 'config'
15
6
 
16
7
  Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
17
8
  # this is dirty but it solves a weird problem where the tempfile disappears mid-process
@@ -21,172 +12,7 @@ Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
21
12
  "sqlite://#{$__taps_database.path}"
22
13
  end
23
14
 
24
- module Tapsoob
25
- module CLI
26
- class Schema < Thor
27
- desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
28
- def console(database_url)
29
- $db = Sequel.connect(database_url)
30
- require 'irb'
31
- require 'irb/completion'
32
- IRB.start
33
- end
34
-
35
- desc "dump DATABASE_URL", "Dump a database using a database URL"
36
- def dump(database_url)
37
- puts Tapsoob::Schema.dump(database_url)
38
- end
39
-
40
- desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
41
- def dump_table(database_url, table)
42
- puts Tapsoob::Schema.dump_table(database_url, table)
43
- end
44
-
45
- desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
46
- def indexes(database_url)
47
- puts Tapsoob::Schema.indexes(database_url)
48
- end
49
-
50
- desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
51
- def indexes_individual(database_url)
52
- puts Tapsoob::Schema.indexes_individual(database_url)
53
- end
54
-
55
- desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
56
- def reset_db_sequences(database_url)
57
- Tapsoob::Schema.reset_db_sequences(database_url)
58
- end
59
-
60
- desc "load DATABASE_URL FILENAME", "Load a database schema from a file to a database using a database URL"
61
- def load(database_url, filename)
62
- schema = File.read(filename) rescue help
63
- Tapsoob::Schema.load(database_url, schema)
64
- end
65
-
66
- desc "load_indexes DATABASE_URL FILENAME", "Load indexes from a file to a database using a database URL"
67
- def load_indexes(database_url, filename)
68
- indexes = File.read(filename) rescue help
69
- Tapsoob::Schema.load_indexes(database_url, indexes)
70
- end
71
- end
72
-
73
- class Root < Thor
74
- desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
75
- option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
76
- option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
77
- option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
78
- option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
79
- option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
80
- option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
81
- option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
82
- option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
83
- option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
84
- def pull(dump_path, database_url)
85
- opts = parse_opts(options)
86
- Tapsoob.log.level = Logger::DEBUG if opts[:debug]
87
- if opts[:resume_filename]
88
- clientresumexfer(:pull, dump_path, database_url, opts)
89
- else
90
- clientxfer(:pull, dump_path, database_url, opts)
91
- end
92
- end
93
-
94
- desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
95
- option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
96
- option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
97
- option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
98
- option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
99
- option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
100
- option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
101
- option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
102
- option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
103
- option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
104
- def push(dump_path, database_url)
105
- opts = parse_opts(options)
106
- Tapsoob.log.level = Logger::DEBUG if opts[:debug]
107
- if opts[:resume_filename]
108
- clientresumexfer(:push, dump_path, database_url, opts)
109
- else
110
- clientxfer(:push, dump_path, database_url, opts)
111
- end
112
- end
113
-
114
- desc "version", "Show tapsoob version"
115
- def version
116
- puts Tapsoob::VERSION.dup
117
- end
118
-
119
- desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
120
- subcommand "schema", Schema
121
-
122
- private
123
- def parse_opts(options)
124
- # Default options
125
- opts = {
126
- skip_schema: options[:"skip-schema"],
127
- indexes_first: options[:"indexes_first"],
128
- disable_compression: options[:"disable-compression"],
129
- debug: options[:debug]
130
- }
131
-
132
- # Resume
133
- if options[:resume]
134
- if File.exists?(options[:resume])
135
- opts[:resume_file] = options[:resume]
136
- else
137
- raise "Unable to find resume file."
138
- end
139
- end
140
-
141
- # Default chunksize
142
- if options[:chunksize]
143
- opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
144
- end
145
-
146
- # Regex filter
147
- opts[:table_filter] = options[:filter] if options[:filter]
148
-
149
- # Table filter
150
- if options[:tables]
151
- r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
152
- opts[:table_filter] = "#{r_tables}"
153
- end
154
-
155
- # Exclude tables
156
- opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
157
-
158
- opts
159
- end
160
-
161
- def clientxfer(method, dump_path, database_url, opts)
162
- Tapsoob::Config.verify_database_url(database_url)
163
-
164
- FileUtils.mkpath "#{dump_path}/schemas"
165
- FileUtils.mkpath "#{dump_path}/data"
166
- FileUtils.mkpath "#{dump_path}/indexes"
167
-
168
- require 'tapsoob/operation'
169
-
170
- Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
171
- end
172
-
173
- def clientresumexfer(method, dump_path, database_url, opts)
174
- session = JSON.parse(File.read(opts.delete(:resume_filename)))
175
- session.symbolize_recursively!
176
-
177
- dump_path = dump_path || session.delete(:dump_path)
178
-
179
- require 'taps/operation'
180
-
181
- newsession = session.merge({
182
- :default_chunksize => opts[:default_chunksize],
183
- :disable_compression => opts[:disable_compression],
184
- :resume => true
185
- })
186
-
187
- Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
188
- end
189
- end
190
- end
191
- end
192
-
15
+ # CLI modules
16
+ require_relative 'cli/schema'
17
+ require_relative 'cli/data_stream'
18
+ require_relative 'cli/root'
@@ -21,6 +21,7 @@ module Tapsoob
21
21
  end
22
22
 
23
23
  def log
24
+ Tapsoob.log.level = Logger::DEBUG if state[:debug]
24
25
  Tapsoob.log
25
26
  end
26
27
 
@@ -87,8 +88,9 @@ module Tapsoob
87
88
  ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
88
89
  log.debug "DataStream#fetch_file"
89
90
  rows = {
90
- :header => ds["header"],
91
- :data => ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]
91
+ :table_name => ds["table_name"],
92
+ :header => ds["header"],
93
+ :data => ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]
92
94
  }
93
95
  update_chunksize_stats
94
96
  rows
@@ -141,7 +143,7 @@ module Tapsoob
141
143
  @complete
142
144
  end
143
145
 
144
- def fetch_database(dump_path)
146
+ def fetch_database
145
147
  params = fetch_from_database
146
148
  encoded_data = params[:encoded_data]
147
149
  json = params[:json]
@@ -154,7 +156,7 @@ module Tapsoob
154
156
  state.merge!(json[:state].merge(:chunksize => state[:chunksize]))
155
157
 
156
158
  unless @complete
157
- Tapsoob::Utils.export_rows(dump_path, table_name, rows)
159
+ yield rows if block_given?
158
160
  state[:offset] += rows[:data].size
159
161
  rows[:data].size
160
162
  else
data/lib/tapsoob/log.rb CHANGED
@@ -7,8 +7,8 @@ module Tapsoob
7
7
  def self.log
8
8
  @@log ||= begin
9
9
  require 'logger'
10
- log = Logger.new($stderr)
11
- log.level = Logger::ERROR
10
+ log = Logger.new(STDERR)
11
+ log.level = Logger::INFO
12
12
  log.datetime_format = "%Y-%m-%d %H:%M:%S"
13
13
  log
14
14
  end
@@ -10,7 +10,7 @@ module Tapsoob
10
10
  class Operation
11
11
  attr_reader :database_url, :dump_path, :opts
12
12
 
13
- def initialize(database_url, dump_path, opts={})
13
+ def initialize(database_url, dump_path = nil, opts={})
14
14
  @database_url = database_url
15
15
  @dump_path = dump_path
16
16
  @opts = opts
@@ -55,12 +55,13 @@ module Tapsoob
55
55
  end
56
56
 
57
57
  def log
58
+ Tapsoob.log.level = Logger::DEBUG if opts[:debug]
58
59
  Tapsoob.log
59
60
  end
60
61
 
61
62
  def store_session
62
63
  file = "#{file_prefix}_#{Time.now.strftime("%Y%m%d%H%M")}.dat"
63
- puts "\nSaving session to #{file}..."
64
+ log.info "\nSaving session to #{file}..."
64
65
  File.open(file, 'w') do |f|
65
66
  f.write(JSON.generate(to_hash))
66
67
  end
@@ -174,7 +175,7 @@ module Tapsoob
174
175
  end
175
176
 
176
177
  def pull_schema
177
- puts "Receiving schema"
178
+ log.info "Receiving schema"
178
179
 
179
180
  progress = ProgressBar.new('Schema', tables.size)
180
181
  tables.each do |table_name, count|
@@ -188,12 +189,12 @@ module Tapsoob
188
189
  end
189
190
 
190
191
  def pull_data
191
- puts "Receiving data"
192
+ log.info "Receiving data"
192
193
 
193
- puts "#{tables.size} tables, #{format_number(record_count)} records"
194
+ log.info "#{tables.size} tables, #{format_number(record_count)} records"
194
195
 
195
196
  tables.each do |table_name, count|
196
- progress = ProgressBar.new(table_name.to_s, count)
197
+ progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
197
198
  stream = Tapsoob::DataStream.factory(db, {
198
199
  :chunksize => default_chunksize,
199
200
  :table_name => table_name
@@ -207,9 +208,9 @@ module Tapsoob
207
208
 
208
209
  table_name = stream_state[:table_name]
209
210
  record_count = tables[table_name.to_s]
210
- puts "Resuming #{table_name}, #{format_number(record_count)} records"
211
+ log.info "Resuming #{table_name}, #{format_number(record_count)} records"
211
212
 
212
- progress = ProgressBar.new(table_name.to_s, record_count)
213
+ progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, record_count) : nil)
213
214
  stream = Tapsoob::DataStream.factory(db, stream_state)
214
215
  pull_data_from_table(stream, progress)
215
216
  end
@@ -219,19 +220,25 @@ module Tapsoob
219
220
  begin
220
221
  exit 0 if exiting?
221
222
 
222
- size = stream.fetch_database(dump_path)
223
+ size = stream.fetch_database do |rows|
224
+ if dump_path.nil?
225
+ puts JSON.generate(rows)
226
+ else
227
+ Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
228
+ end
229
+ end
223
230
  break if stream.complete?
224
- progress.inc(size) unless exiting?
231
+ progress.inc(size) if progress && !exiting?
225
232
  stream.error = false
226
233
  self.stream_state = stream.to_hash
227
234
  rescue Tapsoob::CorruptedData => e
228
- puts "Corrupted Data Received #{e.message}, retrying..."
235
+ log.info "Corrupted Data Received #{e.message}, retrying..."
229
236
  stream.error = true
230
237
  next
231
238
  end
232
239
  end
233
240
 
234
- progress.finish
241
+ progress.finish if progress
235
242
  completed_tables << stream.table_name.to_s
236
243
  self.stream_state = {}
237
244
  end
@@ -280,7 +287,7 @@ module Tapsoob
280
287
  end
281
288
 
282
289
  def pull_indexes
283
- puts "Receiving indexes"
290
+ log.info "Receiving indexes"
284
291
 
285
292
  raw_idxs = Tapsoob::Utils.schema_bin(:indexes_individual, database_url)
286
293
  idxs = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
@@ -298,7 +305,7 @@ module Tapsoob
298
305
  end
299
306
 
300
307
  def pull_reset_sequences
301
- puts "Resetting sequences"
308
+ log.info "Resetting sequences"
302
309
 
303
310
  output = Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
304
311
  puts output if output
@@ -337,7 +344,7 @@ module Tapsoob
337
344
 
338
345
  return unless idxs.size > 0
339
346
 
340
- puts "Sending indexes"
347
+ log.info "Sending indexes"
341
348
 
342
349
  apply_table_filter(idxs).each do |table, indexes|
343
350
  next unless indexes.size > 0
@@ -351,7 +358,7 @@ module Tapsoob
351
358
  end
352
359
 
353
360
  def push_schema
354
- puts "Sending schema"
361
+ log.info "Sending schema"
355
362
 
356
363
  progress = ProgressBar.new('Schema', tables.size)
357
364
  tables.each do |table, count|
@@ -363,7 +370,7 @@ module Tapsoob
363
370
  end
364
371
 
365
372
  def push_reset_sequences
366
- puts "Resetting sequences"
373
+ log.info "Resetting sequences"
367
374
 
368
375
  Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
369
376
  end
@@ -373,16 +380,16 @@ module Tapsoob
373
380
 
374
381
  table_name = stream_state[:table_name]
375
382
  record_count = tables[table_name.to_s]
376
- puts "Resuming #{table_name}, #{format_number(record_count)} records"
383
+ log.info "Resuming #{table_name}, #{format_number(record_count)} records"
377
384
  progress = ProgressBar.new(table_name.to_s, record_count)
378
385
  stream = Tapsoob::DataStream.factory(db, stream_state)
379
386
  push_data_from_file(stream, progress)
380
387
  end
381
388
 
382
389
  def push_data
383
- puts "Sending data"
390
+ log.info "Sending data"
384
391
 
385
- puts "#{tables.size} tables, #{format_number(record_count)} records"
392
+ log.info "#{tables.size} tables, #{format_number(record_count)} records"
386
393
 
387
394
  tables.each do |table_name, count|
388
395
  next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
@@ -67,7 +67,7 @@ END_MIG
67
67
  Sequel.connect(database_url) do |db|
68
68
  db.extension :schema_dumper
69
69
  klass = eval(schema)
70
- klass.apply(db, :down)
70
+ #klass.apply(db, :down)
71
71
  klass.apply(db, :up)
72
72
  end
73
73
  end
data/lib/tapsoob/utils.rb CHANGED
@@ -70,7 +70,7 @@ Data : #{data}
70
70
  end
71
71
  header.collect { |h| row[h] }
72
72
  end
73
- { :header => header, :data => only_data }
73
+ { table_name: table, header: header, data: only_data }
74
74
  end
75
75
 
76
76
  # mysql text and blobs fields are handled the same way internally
@@ -1,4 +1,4 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
  module Tapsoob
3
- VERSION = "0.2.11".freeze
3
+ VERSION = "0.3.2".freeze
4
4
  end
data/tapsoob.gemspec CHANGED
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
20
20
  s.require_paths = ["lib"]
21
21
 
22
22
  # Dependencies
23
- s.add_dependency "sequel", "~> 5.19.0"
23
+ s.add_dependency "sequel", "~> 5.21.0"
24
24
  s.add_dependency "thor", "~> 0.20.3"
25
25
 
26
26
  if (RUBY_PLATFORM =~ /java/).nil?
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tapsoob
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.11
4
+ version: 0.3.2
5
5
  platform: java
6
6
  authors:
7
7
  - Félix Bellanger
@@ -9,14 +9,14 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-04-11 00:00:00.000000000 Z
12
+ date: 2019-06-18 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: 5.19.0
19
+ version: 5.21.0
20
20
  name: sequel
21
21
  prerelease: false
22
22
  type: :runtime
@@ -24,7 +24,7 @@ dependencies:
24
24
  requirements:
25
25
  - - "~>"
26
26
  - !ruby/object:Gem::Version
27
- version: 5.19.0
27
+ version: 5.21.0
28
28
  - !ruby/object:Gem::Dependency
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
@@ -98,6 +98,9 @@ files:
98
98
  - lib/tapsoob.rb
99
99
  - lib/tapsoob/chunksize.rb
100
100
  - lib/tapsoob/cli.rb
101
+ - lib/tapsoob/cli/data_stream.rb
102
+ - lib/tapsoob/cli/root.rb
103
+ - lib/tapsoob/cli/schema.rb
101
104
  - lib/tapsoob/config.rb
102
105
  - lib/tapsoob/data_stream.rb
103
106
  - lib/tapsoob/errors.rb