tapsoob 0.5.24-java → 0.5.25-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -0
- data/.rspec +0 -0
- data/Gemfile +1 -1
- data/README.md +0 -0
- data/Rakefile +0 -0
- data/lib/tapsoob/chunksize.rb +0 -0
- data/lib/tapsoob/cli/data_stream.rb +0 -0
- data/lib/tapsoob/cli/root.rb +0 -0
- data/lib/tapsoob/cli/schema.rb +0 -0
- data/lib/tapsoob/cli.rb +0 -0
- data/lib/tapsoob/config.rb +0 -0
- data/lib/tapsoob/data_stream.rb +26 -6
- data/lib/tapsoob/errors.rb +0 -0
- data/lib/tapsoob/log.rb +0 -0
- data/lib/tapsoob/operation.rb +3 -1
- data/lib/tapsoob/progress_bar.rb +0 -0
- data/lib/tapsoob/railtie.rb +0 -0
- data/lib/tapsoob/schema.rb +0 -0
- data/lib/tapsoob/utils.rb +9 -14
- data/lib/tapsoob/version.rb +1 -1
- data/lib/tapsoob.rb +0 -0
- data/lib/tasks/tapsoob.rake +0 -0
- data/spec/lib/tapsoob/chunksize_spec.rb +0 -0
- data/spec/lib/tapsoob/version_spec.rb +0 -0
- data/spec/spec_helper.rb +0 -0
- data/tapsoob.gemspec +8 -8
- metadata +18 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7598763d461706f95f3f4da66603dc989a3dccbf2bae2b43e32101a47d73fb7
|
4
|
+
data.tar.gz: fb5ab2c06783cf3ffc2ffd2ff2f68e904c2aaa1144760f17e5e17dbd6f110062
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5003fc09f13d21d339384da694297268107c67e59ba314ae1191b7997496daa73cda88355d5fc172bcb36f61f870cdd480d6cd12175c6d4ca5f33e303d6cbab0
|
7
|
+
data.tar.gz: 4623748891328b90001dee1dfd9f8c868bad4ad34dd528bb72c2fe191323fb7382a6b38b51abb085d02cd70c81b3f0d69e88ed3604ec71dfe4965838b5dd9b7a
|
data/.gitignore
CHANGED
File without changes
|
data/.rspec
CHANGED
File without changes
|
data/Gemfile
CHANGED
data/README.md
CHANGED
File without changes
|
data/Rakefile
CHANGED
File without changes
|
data/lib/tapsoob/chunksize.rb
CHANGED
File without changes
|
File without changes
|
data/lib/tapsoob/cli/root.rb
CHANGED
File without changes
|
data/lib/tapsoob/cli/schema.rb
CHANGED
File without changes
|
data/lib/tapsoob/cli.rb
CHANGED
File without changes
|
data/lib/tapsoob/config.rb
CHANGED
File without changes
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -87,14 +87,34 @@ module Tapsoob
|
|
87
87
|
|
88
88
|
def fetch_file(dump_path)
|
89
89
|
#state[:chunksize] = fetch_chunksize
|
90
|
-
|
91
|
-
|
90
|
+
# Read NDJSON format - each line is a separate JSON chunk
|
91
|
+
file_path = File.join(dump_path, "data", "#{table_name}.json")
|
92
|
+
|
93
|
+
# Parse all chunks and combine them
|
94
|
+
all_data = []
|
95
|
+
table_name_val = nil
|
96
|
+
header_val = nil
|
97
|
+
types_val = nil
|
98
|
+
|
99
|
+
File.readlines(file_path).each do |line|
|
100
|
+
chunk = JSON.parse(line.strip)
|
101
|
+
table_name_val ||= chunk["table_name"]
|
102
|
+
header_val ||= chunk["header"]
|
103
|
+
types_val ||= chunk["types"]
|
104
|
+
all_data.concat(chunk["data"]) if chunk["data"]
|
105
|
+
end
|
106
|
+
|
107
|
+
# Apply skip-duplicates if needed
|
108
|
+
all_data = all_data.uniq if @options[:"skip-duplicates"]
|
109
|
+
|
110
|
+
state[:size] = all_data.size
|
92
111
|
log.debug "DataStream#fetch_file"
|
112
|
+
|
93
113
|
rows = {
|
94
|
-
:table_name =>
|
95
|
-
:header =>
|
96
|
-
:data => (
|
97
|
-
:types =>
|
114
|
+
:table_name => table_name_val,
|
115
|
+
:header => header_val,
|
116
|
+
:data => (all_data[state[:offset], state[:chunksize]] || []),
|
117
|
+
:types => types_val
|
98
118
|
}
|
99
119
|
update_chunksize_stats
|
100
120
|
rows
|
data/lib/tapsoob/errors.rb
CHANGED
File without changes
|
data/lib/tapsoob/log.rb
CHANGED
File without changes
|
data/lib/tapsoob/operation.rb
CHANGED
@@ -375,7 +375,9 @@ module Tapsoob
|
|
375
375
|
idxs = {}
|
376
376
|
table_idxs = Dir.glob(File.join(dump_path, "indexes", "*.json")).map { |path| File.basename(path, '.json') }
|
377
377
|
table_idxs.each do |table_idx|
|
378
|
-
|
378
|
+
# Read NDJSON format - each line is a separate index
|
379
|
+
index_file = File.join(dump_path, "indexes", "#{table_idx}.json")
|
380
|
+
idxs[table_idx] = File.readlines(index_file).map { |line| JSON.parse(line.strip) }
|
379
381
|
end
|
380
382
|
|
381
383
|
return unless idxs.size > 0
|
data/lib/tapsoob/progress_bar.rb
CHANGED
File without changes
|
data/lib/tapsoob/railtie.rb
CHANGED
File without changes
|
data/lib/tapsoob/schema.rb
CHANGED
File without changes
|
data/lib/tapsoob/utils.rb
CHANGED
@@ -143,26 +143,21 @@ Data : #{data}
|
|
143
143
|
end
|
144
144
|
|
145
145
|
def export_indexes(dump_path, table, index_data)
|
146
|
-
|
147
|
-
|
148
|
-
previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
|
149
|
-
data = data + previous_data
|
150
|
-
end
|
146
|
+
# Use append-only writes to avoid O(n²) complexity
|
147
|
+
index_file = File.join(dump_path, "indexes", "#{table}.json")
|
151
148
|
|
152
|
-
File.open(
|
153
|
-
file.write(JSON.generate(
|
149
|
+
File.open(index_file, 'a') do |file|
|
150
|
+
file.write(JSON.generate(index_data) + "\n")
|
154
151
|
end
|
155
152
|
end
|
156
153
|
|
157
154
|
def export_rows(dump_path, table, row_data)
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
data[:data] = previous_data["data"] + row_data[:data] unless row_data[:data].nil?
|
162
|
-
end
|
155
|
+
# Use append-only writes to avoid O(n²) complexity
|
156
|
+
# Store metadata separately and append data chunks as NDJSON
|
157
|
+
data_file = File.join(dump_path, "data", "#{table}.json")
|
163
158
|
|
164
|
-
File.open(
|
165
|
-
file.write(JSON.generate(
|
159
|
+
File.open(data_file, 'a') do |file|
|
160
|
+
file.write(JSON.generate(row_data) + "\n")
|
166
161
|
end
|
167
162
|
end
|
168
163
|
|
data/lib/tapsoob/version.rb
CHANGED
data/lib/tapsoob.rb
CHANGED
File without changes
|
data/lib/tasks/tapsoob.rake
CHANGED
File without changes
|
File without changes
|
File without changes
|
data/spec/spec_helper.rb
CHANGED
File without changes
|
data/tapsoob.gemspec
CHANGED
@@ -20,18 +20,18 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.require_paths = ["lib"]
|
21
21
|
|
22
22
|
# Dependencies
|
23
|
-
s.add_dependency "sequel", "~> 5.
|
24
|
-
s.add_dependency "thor", "~> 1.
|
23
|
+
s.add_dependency "sequel", "~> 5.96.0"
|
24
|
+
s.add_dependency "thor", "~> 1.4.0"
|
25
25
|
|
26
26
|
if (RUBY_PLATFORM =~ /java/).nil?
|
27
|
-
s.add_development_dependency "mysql2", "~> 0.
|
28
|
-
s.add_development_dependency "pg", "~>
|
29
|
-
s.add_development_dependency "sqlite3", "~>
|
27
|
+
s.add_development_dependency "mysql2", "~> 0.5.7"
|
28
|
+
s.add_development_dependency "pg", "~> 1.6.2"
|
29
|
+
s.add_development_dependency "sqlite3", "~> 2.7.4"
|
30
30
|
else
|
31
31
|
s.platform = 'java'
|
32
32
|
|
33
|
-
s.add_dependency "jdbc-mysql", "~>
|
34
|
-
s.add_dependency "jdbc-postgres", "~> 42.
|
35
|
-
s.add_dependency "jdbc-sqlite3", "~> 3.
|
33
|
+
s.add_dependency "jdbc-mysql", "~> 9.1.0.1"
|
34
|
+
s.add_dependency "jdbc-postgres", "~> 42.6.0"
|
35
|
+
s.add_dependency "jdbc-sqlite3", "~> 3.46.1.1"
|
36
36
|
end
|
37
37
|
end
|
metadata
CHANGED
@@ -1,86 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tapsoob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.25
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Félix Bellanger
|
8
8
|
- Michael Chrisco
|
9
|
-
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: sequel
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 5.
|
20
|
-
name: sequel
|
19
|
+
version: 5.96.0
|
21
20
|
type: :runtime
|
22
21
|
prerelease: false
|
23
22
|
version_requirements: !ruby/object:Gem::Requirement
|
24
23
|
requirements:
|
25
24
|
- - "~>"
|
26
25
|
- !ruby/object:Gem::Version
|
27
|
-
version: 5.
|
26
|
+
version: 5.96.0
|
28
27
|
- !ruby/object:Gem::Dependency
|
28
|
+
name: thor
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 1.
|
34
|
-
name: thor
|
33
|
+
version: 1.4.0
|
35
34
|
type: :runtime
|
36
35
|
prerelease: false
|
37
36
|
version_requirements: !ruby/object:Gem::Requirement
|
38
37
|
requirements:
|
39
38
|
- - "~>"
|
40
39
|
- !ruby/object:Gem::Version
|
41
|
-
version: 1.
|
40
|
+
version: 1.4.0
|
42
41
|
- !ruby/object:Gem::Dependency
|
42
|
+
name: jdbc-mysql
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
48
|
-
name: jdbc-mysql
|
47
|
+
version: 9.1.0.1
|
49
48
|
type: :runtime
|
50
49
|
prerelease: false
|
51
50
|
version_requirements: !ruby/object:Gem::Requirement
|
52
51
|
requirements:
|
53
52
|
- - "~>"
|
54
53
|
- !ruby/object:Gem::Version
|
55
|
-
version:
|
54
|
+
version: 9.1.0.1
|
56
55
|
- !ruby/object:Gem::Dependency
|
56
|
+
name: jdbc-postgres
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 42.
|
62
|
-
name: jdbc-postgres
|
61
|
+
version: 42.6.0
|
63
62
|
type: :runtime
|
64
63
|
prerelease: false
|
65
64
|
version_requirements: !ruby/object:Gem::Requirement
|
66
65
|
requirements:
|
67
66
|
- - "~>"
|
68
67
|
- !ruby/object:Gem::Version
|
69
|
-
version: 42.
|
68
|
+
version: 42.6.0
|
70
69
|
- !ruby/object:Gem::Dependency
|
70
|
+
name: jdbc-sqlite3
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 3.
|
76
|
-
name: jdbc-sqlite3
|
75
|
+
version: 3.46.1.1
|
77
76
|
type: :runtime
|
78
77
|
prerelease: false
|
79
78
|
version_requirements: !ruby/object:Gem::Requirement
|
80
79
|
requirements:
|
81
80
|
- - "~>"
|
82
81
|
- !ruby/object:Gem::Version
|
83
|
-
version: 3.
|
82
|
+
version: 3.46.1.1
|
84
83
|
description: Simple tool to import/export databases inspired by taps but OOB, meaning
|
85
84
|
databases are imported/exported from the filesystem.
|
86
85
|
email: felix.bellanger@faveod.com
|
@@ -120,7 +119,6 @@ homepage: https://github.com/Keeguon/tapsoob
|
|
120
119
|
licenses:
|
121
120
|
- MIT
|
122
121
|
metadata: {}
|
123
|
-
post_install_message:
|
124
122
|
rdoc_options: []
|
125
123
|
require_paths:
|
126
124
|
- lib
|
@@ -135,8 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
135
133
|
- !ruby/object:Gem::Version
|
136
134
|
version: '0'
|
137
135
|
requirements: []
|
138
|
-
rubygems_version: 3.
|
139
|
-
signing_key:
|
136
|
+
rubygems_version: 3.6.9
|
140
137
|
specification_version: 4
|
141
138
|
summary: Simple tool to import/export databases.
|
142
139
|
test_files:
|