tapsoob 0.8.6-java → 0.8.7-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/schema.rb +1 -1
- data/lib/tapsoob/version.rb +1 -1
- data/spec/spec_helper.rb +3 -2
- data/spec/support/operation_helpers.rb +43 -0
- data/spec/unit/tapsoob/base_spec.rb +222 -0
- data/spec/unit/tapsoob/cli_pipeline_spec.rb +380 -0
- data/spec/unit/tapsoob/config_spec.rb +54 -0
- data/spec/unit/tapsoob/data_stream_spec.rb +48 -0
- data/spec/unit/tapsoob/file_partition_spec.rb +117 -0
- data/spec/unit/tapsoob/keyed_spec.rb +121 -0
- data/spec/unit/tapsoob/progress_event_spec.rb +136 -0
- data/spec/unit/tapsoob/progress_spec.rb +335 -0
- data/spec/unit/tapsoob/pull_spec.rb +335 -0
- data/spec/unit/tapsoob/push_spec.rb +264 -0
- data/spec/unit/tapsoob/schema_spec.rb +154 -0
- data/spec/unit/tapsoob/utils_spec.rb +64 -0
- metadata +11 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 146ca85995071f912509d794d2436fd26f3c9adfcef5f72178af08025578fd94
|
|
4
|
+
data.tar.gz: 8d96d10769f796ba102dda47893625450dcd73ca73eb94bca0f302ff03fdcc12
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f7ab5c568c2c2c51356fab7aacce84f6fc41fcc898ed5f38d6bcef3f8e59b94452cac19a5fdcb9fa84e68d3084775deb4e50ef2a73834a9e066a40b15203c094
|
|
7
|
+
data.tar.gz: ea9d5d6185def0e29bc686ed088558df1098d8d99ac49312f1e33a8c3e484df8897c1b5bab76e49dc7aa9a35a43c4c63c7e2a7c5bd401862247a2beb9a31e426
|
data/lib/tapsoob/cli/schema.rb
CHANGED
|
@@ -29,7 +29,7 @@ module Tapsoob
|
|
|
29
29
|
|
|
30
30
|
desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
|
|
31
31
|
def dump_table(database_url, table)
|
|
32
|
-
puts Tapsoob::Schema.dump_table(database_url, table)
|
|
32
|
+
puts Tapsoob::Schema.dump_table(database_url, table, {})
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
desc "foreign_keys DATABASE_URL", "Dump foreign_keys from a database using a database URL"
|
data/lib/tapsoob/version.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
|
@@ -45,6 +45,7 @@ RSpec.configure do |config|
|
|
|
45
45
|
# Integration tests require a real DB — skip unless env vars are set.
|
|
46
46
|
config.filter_run_excluding :integration unless ENV['INTEGRATION_TESTS'] || ENV['SRC_DATABASE_URL']
|
|
47
47
|
|
|
48
|
-
config.include DbHelpers,
|
|
49
|
-
config.include RoundTripHelper,
|
|
48
|
+
config.include DbHelpers, :integration
|
|
49
|
+
config.include RoundTripHelper, :integration
|
|
50
|
+
config.include OperationHelpers
|
|
50
51
|
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
require 'tapsoob/operation/pull'
|
|
2
|
+
require 'tapsoob/operation/push'
|
|
3
|
+
|
|
4
|
+
# Shared helpers for unit specs that exercise Pull / Push / Base.
|
|
5
|
+
# Included automatically via spec_helper for all unit specs.
|
|
6
|
+
module OperationHelpers
|
|
7
|
+
# Default opts used across pull/push/base unit tests.
|
|
8
|
+
UNIT_OPTS = {
|
|
9
|
+
data: true,
|
|
10
|
+
schema: true,
|
|
11
|
+
indexes: false,
|
|
12
|
+
progress: false,
|
|
13
|
+
default_chunksize: 1000,
|
|
14
|
+
no_split: true,
|
|
15
|
+
}.freeze
|
|
16
|
+
|
|
17
|
+
# A pre-seeded in-memory SQLite DB with :users (5 rows) and :widgets (3 rows).
|
|
18
|
+
# Returns a new connection each call — callers own disconnection.
|
|
19
|
+
def seeded_sqlite_db
|
|
20
|
+
d = connect_sqlite
|
|
21
|
+
d.create_table(:users) { primary_key :id; String :name }
|
|
22
|
+
d.create_table(:widgets) { primary_key :id; Integer :qty }
|
|
23
|
+
5.times { |i| d[:users].insert(name: "user_#{i}") }
|
|
24
|
+
3.times { |i| d[:widgets].insert(qty: i * 10) }
|
|
25
|
+
d
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def build_pull(db, dump_dir, extra_opts = {})
|
|
29
|
+
op = Tapsoob::Operation::Pull.new(sqlite_memory_url, dump_dir, UNIT_OPTS.merge(extra_opts))
|
|
30
|
+
op.instance_variable_set(:@db, db)
|
|
31
|
+
op
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def build_push(db_url, dump_dir, extra_opts = {})
|
|
35
|
+
Tapsoob::Operation::Push.new(db_url, dump_dir, UNIT_OPTS.merge(extra_opts))
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def build_base(db, dump_dir, extra_opts = {})
|
|
39
|
+
op = Tapsoob::Operation::Pull.new(sqlite_memory_url, dump_dir, UNIT_OPTS.merge(extra_opts))
|
|
40
|
+
op.instance_variable_set(:@db, db)
|
|
41
|
+
op
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
require 'tapsoob/operation/base'
|
|
3
|
+
require 'tapsoob/operation/pull'
|
|
4
|
+
require 'tapsoob/operation/push'
|
|
5
|
+
|
|
6
|
+
RSpec.describe Tapsoob::Operation::Base do
|
|
7
|
+
let(:db) { seeded_sqlite_db }
|
|
8
|
+
let(:dump_dir) { Dir.mktmpdir("tapsoob_base_") }
|
|
9
|
+
|
|
10
|
+
after do
|
|
11
|
+
db.disconnect
|
|
12
|
+
FileUtils.rm_rf(dump_dir)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# ── format_number ────────────────────────────────────────────────────────────
|
|
16
|
+
|
|
17
|
+
describe '#format_number' do
|
|
18
|
+
it 'formats numbers with commas' do
|
|
19
|
+
op = build_base(db, dump_dir)
|
|
20
|
+
expect(op.format_number(1_000_000)).to eq("1,000,000")
|
|
21
|
+
expect(op.format_number(1234)).to eq("1,234")
|
|
22
|
+
expect(op.format_number(999)).to eq("999")
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# ── resuming? ────────────────────────────────────────────────────────────────
|
|
27
|
+
|
|
28
|
+
describe '#resuming?' do
|
|
29
|
+
it 'returns false by default' do
|
|
30
|
+
expect(build_base(db, dump_dir).resuming?).to be false
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it 'returns true when :resume is set' do
|
|
34
|
+
expect(build_base(db, dump_dir, resume: true).resuming?).to be true
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# ── parallel? / parallel_workers ─────────────────────────────────────────────
|
|
39
|
+
|
|
40
|
+
describe '#parallel?' do
|
|
41
|
+
it 'returns false when parallel is 1' do
|
|
42
|
+
expect(build_base(db, dump_dir, parallel: 1).parallel?).to be false
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
it 'returns true when parallel > 1' do
|
|
46
|
+
expect(build_base(db, dump_dir, parallel: 2).parallel?).to be true
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
describe '#parallel_workers' do
|
|
51
|
+
it 'defaults to 1' do
|
|
52
|
+
expect(build_base(db, dump_dir).parallel_workers).to eq(1)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
it 'returns the requested count' do
|
|
56
|
+
expect(build_base(db, dump_dir, parallel: 4).parallel_workers).to eq(4)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# ── table_parallel_workers ───────────────────────────────────────────────────
|
|
61
|
+
|
|
62
|
+
describe '#table_parallel_workers' do
|
|
63
|
+
it 'returns 1 when no_split is set' do
|
|
64
|
+
expect(build_base(db, dump_dir, no_split: true).table_parallel_workers(:users, 5_000_000)).to eq(1)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it 'returns 1 when dump_path is nil' do
|
|
68
|
+
op = Tapsoob::Operation::Pull.new(sqlite_memory_url, nil, { default_chunksize: 1000 })
|
|
69
|
+
expect(op.table_parallel_workers(:users, 5_000_000)).to eq(1)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
it 'returns 1 when row_count is below threshold' do
|
|
73
|
+
expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 50_000)).to eq(1)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
it 'returns >= 2 for a very large table' do
|
|
77
|
+
expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 5_000_000)).to be >= 2
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
it 'returns >= 2 for a 1M+ row table' do
|
|
81
|
+
expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 1_000_000)).to be >= 2
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
it 'returns >= 2 for a 500K+ row table' do
|
|
85
|
+
expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 500_000)).to be >= 2
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
it 'returns 2 for a table just over the 100K threshold' do
|
|
89
|
+
expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 150_000)).to eq(2)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# ── stream_state ─────────────────────────────────────────────────────────────
|
|
94
|
+
|
|
95
|
+
describe '#stream_state / #stream_state=' do
|
|
96
|
+
it 'defaults to empty hash' do
|
|
97
|
+
expect(build_base(db, dump_dir).stream_state).to eq({})
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
it 'stores and retrieves state' do
|
|
101
|
+
op = build_base(db, dump_dir)
|
|
102
|
+
op.stream_state = { table_name: :users }
|
|
103
|
+
expect(op.stream_state).to eq({ table_name: :users })
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# ── add_completed_table ──────────────────────────────────────────────────────
|
|
108
|
+
|
|
109
|
+
describe '#add_completed_table' do
|
|
110
|
+
it 'appends to completed_tables thread-safely' do
|
|
111
|
+
op = build_base(db, dump_dir)
|
|
112
|
+
op.add_completed_table(:users)
|
|
113
|
+
op.add_completed_table(:widgets)
|
|
114
|
+
expect(op.completed_tables).to include("users", "widgets")
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# ── max_intra_table_workers ──────────────────────────────────────────────────
|
|
119
|
+
|
|
120
|
+
describe '#max_intra_table_workers' do
|
|
121
|
+
it 'returns at least 2' do
|
|
122
|
+
expect(build_base(db, dump_dir).max_intra_table_workers).to be >= 2
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# ── catch_errors ─────────────────────────────────────────────────────────────
|
|
127
|
+
|
|
128
|
+
describe '#catch_errors' do
|
|
129
|
+
it 'yields and returns the block result' do
|
|
130
|
+
expect(build_base(db, dump_dir).send(:catch_errors) { 42 }).to eq(42)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
it 're-raises exceptions' do
|
|
134
|
+
op = build_base(db, dump_dir)
|
|
135
|
+
expect { op.send(:catch_errors) { raise ArgumentError, "boom" } }.to raise_error(ArgumentError, "boom")
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# ── apply_table_filter (array form) ──────────────────────────────────────────
|
|
140
|
+
|
|
141
|
+
describe '#apply_table_filter' do
|
|
142
|
+
it 'filters an array by table_filter' do
|
|
143
|
+
op = build_base(db, dump_dir, tables: ["users"])
|
|
144
|
+
expect(op.apply_table_filter(["users", "widgets"])).to eq(["users"])
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
it 'excludes tables from an array' do
|
|
148
|
+
op = build_base(db, dump_dir, exclude_tables: ["widgets"])
|
|
149
|
+
expect(op.apply_table_filter(["users", "widgets"])).to eq(["users"])
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# ── Base.factory ─────────────────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
describe '.factory' do
|
|
156
|
+
it 'returns a Pull instance for :pull type' do
|
|
157
|
+
expect(described_class.factory(:pull, sqlite_memory_url, dump_dir, { default_chunksize: 1000 })).to be_a(Tapsoob::Operation::Pull)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
it 'returns a Push instance for :push type' do
|
|
161
|
+
expect(described_class.factory(:push, sqlite_memory_url, dump_dir, { default_chunksize: 1000 })).to be_a(Tapsoob::Operation::Push)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
it 'raises for unknown type' do
|
|
165
|
+
expect { described_class.factory(:unknown, sqlite_memory_url, dump_dir, {}) }
|
|
166
|
+
.to raise_error(RuntimeError, /Unknown Operation Type/)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it 'returns a resume instance when opts[:resume] is true' do
|
|
170
|
+
op = build_pull(db, dump_dir)
|
|
171
|
+
op.initialize_dump_directory
|
|
172
|
+
op.pull_schema
|
|
173
|
+
|
|
174
|
+
# Pull#to_hash calls remote_tables_info which requires an active pull run;
|
|
175
|
+
# use the base to_hash binding to get just the serializable fields.
|
|
176
|
+
hash = Tapsoob::Operation::Base.instance_method(:to_hash).bind(op).call
|
|
177
|
+
resumed = described_class.factory(:pull, sqlite_memory_url, dump_dir,
|
|
178
|
+
hash.merge(resume: true, klass: "Tapsoob::Operation::Pull", default_chunksize: 1000))
|
|
179
|
+
expect(resumed).to be_a(Tapsoob::Operation::Pull)
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# ── exiting? / setup_signal_trap ─────────────────────────────────────────────
|
|
184
|
+
|
|
185
|
+
describe '#exiting?' do
|
|
186
|
+
it 'returns false initially' do
|
|
187
|
+
expect(build_base(db, dump_dir).exiting?).to be false
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
describe '#setup_signal_trap' do
|
|
192
|
+
it 'registers signal handlers without error' do
|
|
193
|
+
op = build_base(db, dump_dir)
|
|
194
|
+
expect { op.setup_signal_trap }.not_to raise_error
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# ── can_use_pk_partitioning? ─────────────────────────────────────────────────
|
|
199
|
+
|
|
200
|
+
describe '#can_use_pk_partitioning?' do
|
|
201
|
+
it 'returns true for a table with a single integer PK' do
|
|
202
|
+
op = build_base(db, dump_dir)
|
|
203
|
+
expect(op.can_use_pk_partitioning?(:users)).to be true
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# ── db / default_chunksize ───────────────────────────────────────────────────
|
|
208
|
+
|
|
209
|
+
describe '#default_chunksize' do
|
|
210
|
+
it 'returns the value from opts' do
|
|
211
|
+
expect(build_base(db, dump_dir, default_chunksize: 500).default_chunksize).to eq(500)
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
describe '#table_filter / #exclude_tables' do
|
|
216
|
+
it 'returns empty arrays by default' do
|
|
217
|
+
op = build_base(db, dump_dir)
|
|
218
|
+
expect(op.table_filter).to eq([])
|
|
219
|
+
expect(op.exclude_tables).to eq([])
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
require 'tapsoob/cli'
|
|
3
|
+
|
|
4
|
+
# CLI pipeline specs — invoke Thor commands the same way clients do in rake tasks:
|
|
5
|
+
#
|
|
6
|
+
# tapsoob schema dump <src_url>
|
|
7
|
+
# tapsoob schema load <dst_url>
|
|
8
|
+
# tapsoob schema indexes <src_url>
|
|
9
|
+
# tapsoob schema load_indexes <dst_url>
|
|
10
|
+
# tapsoob schema foreign_keys <src_url>
|
|
11
|
+
# tapsoob schema load_foreign_keys <dst_url>
|
|
12
|
+
# tapsoob data pull <src_url> [dump_path]
|
|
13
|
+
# tapsoob data push <dst_url> [dump_path]
|
|
14
|
+
# tapsoob schema reset_db_sequences <dst_url>
|
|
15
|
+
# tapsoob pull <dump_path> <src_url>
|
|
16
|
+
# tapsoob push <dump_path> <dst_url>
|
|
17
|
+
# tapsoob version
|
|
18
|
+
|
|
19
|
+
RSpec.describe "CLI pipelines" do
|
|
20
|
+
# ── helpers ──────────────────────────────────────────────────────────────────
|
|
21
|
+
|
|
22
|
+
def make_db(path)
|
|
23
|
+
url = DbHelpers.adapt_url("sqlite://#{path}")
|
|
24
|
+
db = Sequel.connect(url)
|
|
25
|
+
db.extension :schema_dumper
|
|
26
|
+
db
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def seed_db(db)
|
|
30
|
+
db.create_table(:users) { primary_key :id; String :name, null: false }
|
|
31
|
+
db.create_table(:widgets) { primary_key :id; Integer :qty, default: 0 }
|
|
32
|
+
3.times { |i| db[:users].insert(name: "user_#{i}") }
|
|
33
|
+
2.times { |i| db[:widgets].insert(qty: i * 5) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Invoke a Thor subclass with argv, swallowing stdout/stderr output.
|
|
37
|
+
def run_cli(klass, argv)
|
|
38
|
+
klass.start(argv, debug: false)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# ── shared setup ─────────────────────────────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
let(:tmp) { Dir.mktmpdir("tapsoob_cli_") }
|
|
44
|
+
let(:src_path) { File.join(tmp, "src.db") }
|
|
45
|
+
let(:dst_path) { File.join(tmp, "dst.db") }
|
|
46
|
+
let(:dump_dir) { File.join(tmp, "dump") }
|
|
47
|
+
let(:src_url) { DbHelpers.adapt_url("sqlite://#{src_path}") }
|
|
48
|
+
let(:dst_url) { DbHelpers.adapt_url("sqlite://#{dst_path}") }
|
|
49
|
+
|
|
50
|
+
let(:src_db) do
|
|
51
|
+
db = make_db(src_path)
|
|
52
|
+
seed_db(db)
|
|
53
|
+
db
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Ensure src_db is created before tests run and everything is cleaned up after.
|
|
57
|
+
before { src_db }
|
|
58
|
+
after do
|
|
59
|
+
src_db.disconnect rescue nil
|
|
60
|
+
FileUtils.rm_rf(tmp)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# ── tapsoob version ───────────────────────────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
describe "tapsoob version" do
|
|
66
|
+
it 'prints the version string' do
|
|
67
|
+
expect { run_cli(Tapsoob::CLI::Root, ["version"]) }.to output(/\d+\.\d+/).to_stdout
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# ── schema dump | schema load pipeline ───────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
describe "schema dump → load pipeline" do
|
|
74
|
+
it 'dumps schema to stdout and loads it into a fresh DB' do
|
|
75
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
76
|
+
expect(schema_text).to include("users", "widgets")
|
|
77
|
+
|
|
78
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
79
|
+
File.write(schema_file, schema_text)
|
|
80
|
+
|
|
81
|
+
dst_db = make_db(dst_path)
|
|
82
|
+
begin
|
|
83
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
84
|
+
expect(dst_db.table_exists?(:users)).to be true
|
|
85
|
+
expect(dst_db.table_exists?(:widgets)).to be true
|
|
86
|
+
ensure
|
|
87
|
+
dst_db.disconnect
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
it 'schema load is idempotent when destination is fresh' do
|
|
92
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
93
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
94
|
+
File.write(schema_file, schema_text)
|
|
95
|
+
|
|
96
|
+
dst_db = make_db(dst_path)
|
|
97
|
+
begin
|
|
98
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
99
|
+
expect(dst_db.table_exists?(:users)).to be true
|
|
100
|
+
expect(dst_db.table_exists?(:widgets)).to be true
|
|
101
|
+
ensure
|
|
102
|
+
dst_db.disconnect
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# ── indexes pipeline ──────────────────────────────────────────────────────────
|
|
108
|
+
|
|
109
|
+
describe "schema indexes → load_indexes pipeline" do
|
|
110
|
+
it 'dumps indexes and loads them without error' do
|
|
111
|
+
index_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["indexes", src_url]) }
|
|
112
|
+
index_file = File.join(tmp, "indexes.rb")
|
|
113
|
+
File.write(index_file, index_text)
|
|
114
|
+
|
|
115
|
+
# Load schema first so destination tables exist
|
|
116
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
117
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
118
|
+
File.write(schema_file, schema_text)
|
|
119
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
120
|
+
|
|
121
|
+
expect { run_cli(Tapsoob::CLI::Schema, ["load_indexes", dst_url, index_file]) }.not_to raise_error
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# ── foreign_keys pipeline ─────────────────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
describe "schema foreign_keys → load_foreign_keys pipeline" do
|
|
128
|
+
it 'dumps foreign keys and loads them without error' do
|
|
129
|
+
fk_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["foreign_keys", src_url]) }
|
|
130
|
+
fk_file = File.join(tmp, "fk.rb")
|
|
131
|
+
File.write(fk_file, fk_text)
|
|
132
|
+
|
|
133
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
134
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
135
|
+
File.write(schema_file, schema_text)
|
|
136
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
137
|
+
|
|
138
|
+
expect { run_cli(Tapsoob::CLI::Schema, ["load_foreign_keys", dst_url, fk_file]) }.not_to raise_error
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# ── reset_db_sequences ────────────────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
describe "schema reset_db_sequences" do
|
|
145
|
+
it 'resets sequences on the destination DB without error' do
|
|
146
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
147
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
148
|
+
File.write(schema_file, schema_text)
|
|
149
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
150
|
+
|
|
151
|
+
expect { run_cli(Tapsoob::CLI::Schema, ["reset_db_sequences", dst_url]) }.not_to raise_error
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# ── data pull → push pipeline (dump_path mode) ───────────────────────────────
|
|
156
|
+
|
|
157
|
+
describe "data pull → push pipeline" do
|
|
158
|
+
before do
|
|
159
|
+
%w[data schemas indexes].each { |d| FileUtils.mkdir_p(File.join(dump_dir, d)) }
|
|
160
|
+
ordered = src_db.send(:sort_dumped_tables, src_db.tables, {}).map(&:to_s)
|
|
161
|
+
File.write(File.join(dump_dir, "table_order.txt"), ordered.join("\n") + "\n")
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
it 'pulls data into dump_dir and pushes it to destination' do
|
|
165
|
+
# Load schema into dst first
|
|
166
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
167
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
168
|
+
File.write(schema_file, schema_text)
|
|
169
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
170
|
+
|
|
171
|
+
run_cli(Tapsoob::CLI::DataStream, ["pull", src_url, dump_dir, "--progress=false", "--chunksize=1000"])
|
|
172
|
+
run_cli(Tapsoob::CLI::DataStream, ["push", dst_url, dump_dir, "--progress=false", "--chunksize=1000"])
|
|
173
|
+
|
|
174
|
+
dst_db = make_db(dst_path)
|
|
175
|
+
begin
|
|
176
|
+
expect(dst_db[:users].count).to eq(3)
|
|
177
|
+
expect(dst_db[:widgets].count).to eq(2)
|
|
178
|
+
ensure
|
|
179
|
+
dst_db.disconnect
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# ── tapsoob pull → push (Root command, full round-trip) ──────────────────────
|
|
185
|
+
|
|
186
|
+
describe "tapsoob pull → push (Root commands)" do
|
|
187
|
+
it 'performs a full schema+data round-trip via pull/push commands' do
|
|
188
|
+
run_cli(Tapsoob::CLI::Root, ["pull", dump_dir, src_url,
|
|
189
|
+
"--progress=false", "--chunksize=1000", "--no-split"])
|
|
190
|
+
run_cli(Tapsoob::CLI::Root, ["push", dump_dir, dst_url,
|
|
191
|
+
"--progress=false", "--chunksize=1000"])
|
|
192
|
+
|
|
193
|
+
dst_db = make_db(dst_path)
|
|
194
|
+
begin
|
|
195
|
+
expect(dst_db.table_exists?(:users)).to be true
|
|
196
|
+
expect(dst_db[:users].count).to eq(3)
|
|
197
|
+
expect(dst_db[:widgets].count).to eq(2)
|
|
198
|
+
ensure
|
|
199
|
+
dst_db.disconnect
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# ── schema dump_table ─────────────────────────────────────────────────────────
|
|
205
|
+
|
|
206
|
+
describe "schema dump_table" do
|
|
207
|
+
it 'dumps a single table schema to stdout' do
|
|
208
|
+
output = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump_table", src_url, "users"]) }
|
|
209
|
+
expect(output).to include("users")
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# ── data push --purge flag ────────────────────────────────────────────────────
|
|
214
|
+
|
|
215
|
+
describe "data push --purge" do
|
|
216
|
+
before do
|
|
217
|
+
%w[data schemas indexes].each { |d| FileUtils.mkdir_p(File.join(dump_dir, d)) }
|
|
218
|
+
ordered = src_db.send(:sort_dumped_tables, src_db.tables, {}).map(&:to_s)
|
|
219
|
+
File.write(File.join(dump_dir, "table_order.txt"), ordered.join("\n") + "\n")
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
it 'truncates destination tables before inserting' do
|
|
223
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
224
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
225
|
+
File.write(schema_file, schema_text)
|
|
226
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
227
|
+
|
|
228
|
+
run_cli(Tapsoob::CLI::DataStream, ["pull", src_url, dump_dir, "--progress=false"])
|
|
229
|
+
run_cli(Tapsoob::CLI::DataStream, ["push", dst_url, dump_dir, "--progress=false", "--purge"])
|
|
230
|
+
|
|
231
|
+
dst_db = make_db(dst_path)
|
|
232
|
+
begin
|
|
233
|
+
expect(dst_db[:users].count).to eq(3)
|
|
234
|
+
ensure
|
|
235
|
+
dst_db.disconnect
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# ── schema indexes_individual ─────────────────────────────────────────────────
|
|
241
|
+
|
|
242
|
+
describe "schema indexes_individual" do
|
|
243
|
+
it 'dumps per-table index JSON without error' do
|
|
244
|
+
expect {
|
|
245
|
+
capture_stdout { run_cli(Tapsoob::CLI::Schema, ["indexes_individual", src_url]) }
|
|
246
|
+
}.not_to raise_error
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# ── schema load via STDIN ────────────────────────────────────────────────────
|
|
251
|
+
|
|
252
|
+
describe "schema load via STDIN" do
|
|
253
|
+
it 'reads schema from STDIN when no filename is given' do
|
|
254
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
255
|
+
|
|
256
|
+
stub_const("STDIN", StringIO.new(schema_text))
|
|
257
|
+
|
|
258
|
+
dst_db = make_db(dst_path)
|
|
259
|
+
begin
|
|
260
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url])
|
|
261
|
+
expect(dst_db.table_exists?(:users)).to be true
|
|
262
|
+
ensure
|
|
263
|
+
dst_db.disconnect
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# ── schema load_foreign_keys via STDIN ───────────────────────────────────────
|
|
269
|
+
|
|
270
|
+
describe "schema load_foreign_keys via STDIN" do
|
|
271
|
+
it 'reads foreign keys from STDIN when no filename is given' do
|
|
272
|
+
fk_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["foreign_keys", src_url]) }
|
|
273
|
+
|
|
274
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
275
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
276
|
+
File.write(schema_file, schema_text)
|
|
277
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
278
|
+
|
|
279
|
+
stub_const("STDIN", StringIO.new(fk_text))
|
|
280
|
+
expect { run_cli(Tapsoob::CLI::Schema, ["load_foreign_keys", dst_url]) }.not_to raise_error
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# ── schema load_indexes via STDIN ────────────────────────────────────────────
|
|
285
|
+
|
|
286
|
+
describe "schema load_indexes via STDIN" do
|
|
287
|
+
it 'reads indexes from STDIN when no filename is given' do
|
|
288
|
+
index_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["indexes", src_url]) }
|
|
289
|
+
|
|
290
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
291
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
292
|
+
File.write(schema_file, schema_text)
|
|
293
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
294
|
+
|
|
295
|
+
stub_const("STDIN", StringIO.new(index_text))
|
|
296
|
+
expect { run_cli(Tapsoob::CLI::Schema, ["load_indexes", dst_url]) }.not_to raise_error
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# ── root pull --resume with missing file (parse_opts error path) ─────────────
|
|
301
|
+
|
|
302
|
+
describe "root pull --resume with non-existent file" do
|
|
303
|
+
it 'raises when the resume file does not exist' do
|
|
304
|
+
expect {
|
|
305
|
+
run_cli(Tapsoob::CLI::Root, ["pull", dump_dir, src_url,
|
|
306
|
+
"--resume=/tmp/nonexistent_tapsoob_#{Process.pid}.dat",
|
|
307
|
+
"--progress=false"])
|
|
308
|
+
}.to raise_error(RuntimeError, /Unable to find resume file/)
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# ── root pull --config option ─────────────────────────────────────────────────
|
|
313
|
+
|
|
314
|
+
describe "root pull --config with a YAML config file" do
|
|
315
|
+
it 'loads options from a config YAML file' do
|
|
316
|
+
config_file = File.join(tmp, "tapsoob.yml")
|
|
317
|
+
File.write(config_file, { "progress" => false }.to_yaml)
|
|
318
|
+
|
|
319
|
+
expect {
|
|
320
|
+
run_cli(Tapsoob::CLI::Root, ["pull", dump_dir, src_url,
|
|
321
|
+
"--config=#{config_file}", "--progress=false", "--chunksize=1000", "--no-split"])
|
|
322
|
+
}.not_to raise_error
|
|
323
|
+
end
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# ── data push via STDIN ───────────────────────────────────────────────────────
|
|
327
|
+
|
|
328
|
+
describe "data push via STDIN" do
|
|
329
|
+
it 'imports rows from STDIN JSON when no dump_path is given' do
|
|
330
|
+
# Set up schema in destination first
|
|
331
|
+
schema_text = capture_stdout { run_cli(Tapsoob::CLI::Schema, ["dump", src_url]) }
|
|
332
|
+
schema_file = File.join(tmp, "schema.rb")
|
|
333
|
+
File.write(schema_file, schema_text)
|
|
334
|
+
run_cli(Tapsoob::CLI::Schema, ["load", dst_url, schema_file])
|
|
335
|
+
|
|
336
|
+
# Generate valid NDJSON for the users table
|
|
337
|
+
ndjson_line = JSON.generate({
|
|
338
|
+
table_name: "users",
|
|
339
|
+
header: ["id", "name"],
|
|
340
|
+
types: ["integer", "string"],
|
|
341
|
+
data: [[100, "stdin_user"]]
|
|
342
|
+
})
|
|
343
|
+
|
|
344
|
+
fake_stdin = StringIO.new(ndjson_line + "\n")
|
|
345
|
+
stub_const("STDIN", fake_stdin)
|
|
346
|
+
|
|
347
|
+
run_cli(Tapsoob::CLI::DataStream, ["push", dst_url, "--progress=false"])
|
|
348
|
+
|
|
349
|
+
dst_db = make_db(dst_path)
|
|
350
|
+
begin
|
|
351
|
+
expect(dst_db[:users].where(id: 100).first).not_to be_nil
|
|
352
|
+
ensure
|
|
353
|
+
dst_db.disconnect
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# ── data pull --parallel warning ──────────────────────────────────────────────
|
|
359
|
+
|
|
360
|
+
describe "data pull parallel-to-STDOUT warning" do
|
|
361
|
+
it 'falls back to serial (no error) when parallel > 1 and no dump_path' do
|
|
362
|
+
# The code emits a warning to STDERR and resets parallel to 1, then runs serial pull.
|
|
363
|
+
expect {
|
|
364
|
+
capture_stdout { run_cli(Tapsoob::CLI::DataStream, ["pull", src_url, "--parallel=2", "--progress=false"]) }
|
|
365
|
+
}.not_to raise_error
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# ── helper ───────────────────────────────────────────────────────────────────
|
|
370
|
+
|
|
371
|
+
def capture_stdout(&block)
|
|
372
|
+
old = $stdout
|
|
373
|
+
$stdout = StringIO.new
|
|
374
|
+
block.call
|
|
375
|
+
$stdout.string
|
|
376
|
+
ensure
|
|
377
|
+
$stdout = old
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
end
|