tapsoob 0.5.24-java → 0.5.26-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11fbd480c773fa8dc6e153610ce7ca4b942edcb78e1924f9d3f7f77fd80ccbb1
-  data.tar.gz: 176d7cfe422de41213a2b245db01598503855481b73d8c577aa29293f0a31c63
+  metadata.gz: fe2c90dbfc6fff88f1ee9b6cc612de126911267688b74573c08777da3b75d8e9
+  data.tar.gz: a0d8eb29b71d4e66ec95efd8d0dbbccbefafed14c0a34864b63a9332a1f5b54d
 SHA512:
-  metadata.gz: acdbb9432e0c046d114128bff2f1b5b828e3ef22208edca966c57cac532fbe6fc5007cfc2d50901802eee2c7bc5aed4e89a2920bd4b4bf6d2c4c45ecae43df25
-  data.tar.gz: 9be5ba4e60af48dc35bc6bdd392bcc875201ad11a20e85c63273b358389f9079d316cde0ecbb1d6ca7356e455299730d1e5acd67597a4b99cf1fc1062b12a84c
+  metadata.gz: 03ef33a4a575e953e2f5a91fa86154507a2cf3d43c63de48991361f8d10ea49e36e8ffc45e7679fa21b74b023d8032dd73ba7f2db99e633070a4dfa3795c6c6a
+  data.tar.gz: 3abad922203a7119f59e12cad794c9deefe34071ce0783b0f7743d2e3478c432b828d58bf1ce214af8e802f92c11350c7fa5e70786f60c0dc4217c9304d9ad94

data/Gemfile

@@ -35,7 +35,7 @@ end
 gemspec
 
 group :development do
-  gem 'warbler', '~> 2.0.4', platform: :jruby, require: false
+  gem 'warbler', '~> 2.0.5', platform: :jruby, github: "jruby/warbler", require: false
 end
 
 group :test do

data/lib/tapsoob/data_stream.rb

@@ -87,14 +87,34 @@ module Tapsoob
 
     def fetch_file(dump_path)
       #state[:chunksize] = fetch_chunksize
-      ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
-      state[:size] = ds["data"].size
+      # Read NDJSON format - each line is a separate JSON chunk
+      file_path = File.join(dump_path, "data", "#{table_name}.json")
+
+      # Parse all chunks and combine them
+      all_data = []
+      table_name_val = nil
+      header_val = nil
+      types_val = nil
+
+      File.readlines(file_path).each do |line|
+        chunk = JSON.parse(line.strip)
+        table_name_val ||= chunk["table_name"]
+        header_val ||= chunk["header"]
+        types_val ||= chunk["types"]
+        all_data.concat(chunk["data"]) if chunk["data"]
+      end
+
+      # Apply skip-duplicates if needed
+      all_data = all_data.uniq if @options[:"skip-duplicates"]
+
+      state[:size] = all_data.size
       log.debug "DataStream#fetch_file"
+
       rows = {
-        :table_name => ds["table_name"],
-        :header     => ds["header"],
-        :data       => ((@options[:"skip-duplicates"] ? ds["data"].uniq : ds["data"])[state[:offset], (state[:offset] + state[:chunksize])] || [ ]),
-        :types      => ds["types"]
+        :table_name => table_name_val,
+        :header     => header_val,
+        :data       => (all_data[state[:offset], state[:chunksize]] || []),
+        :types      => types_val
       }
       update_chunksize_stats
       rows

data/lib/tapsoob/operation.rb

@@ -249,6 +249,9 @@ module Tapsoob
             stream.fetch_data_from_database(data) do |rows|
               next if rows == {}
 
+              # Update progress bar immediately when data is ready, before I/O
+              progress.inc(row_size) if progress
+
               if dump_path.nil?
                 puts JSON.generate(rows)
               else
@@ -258,9 +261,9 @@ module Tapsoob
             log.debug "row size: #{row_size}"
             stream.error = false
             self.stream_state = stream.to_hash
-            
+
             c.idle_secs = (d1 + d2)
-            
+
             elapsed_time
           end
         rescue Tapsoob::CorruptedData => e
@@ -268,8 +271,6 @@ module Tapsoob
           stream.error = true
           next
         end
-
-        progress.inc(row_size) if progress
         
         break if stream.complete?
       end
@@ -375,7 +376,9 @@ module Tapsoob
       idxs = {}
       table_idxs = Dir.glob(File.join(dump_path, "indexes", "*.json")).map { |path| File.basename(path, '.json') }
       table_idxs.each do |table_idx|
-        idxs[table_idx] = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table_idx}.json")))
+        # Read NDJSON format - each line is a separate index
+        index_file = File.join(dump_path, "indexes", "#{table_idx}.json")
+        idxs[table_idx] = File.readlines(index_file).map { |line| JSON.parse(line.strip) }
       end
 
       return unless idxs.size > 0

data/lib/tapsoob/utils.rb

@@ -143,26 +143,21 @@ Data : #{data}
     end
 
     def export_indexes(dump_path, table, index_data)
-      data = [index_data]
-      if File.exist?(File.join(dump_path, "indexes", "#{table}.json"))
-        previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
-        data = data + previous_data
-      end
+      # Use append-only writes to avoid O(n²) complexity
+      index_file = File.join(dump_path, "indexes", "#{table}.json")
 
-      File.open(File.join(dump_path, "indexes", "#{table}.json"), 'w') do |file|
-        file.write(JSON.generate(data))
+      File.open(index_file, 'a') do |file|
+        file.write(JSON.generate(index_data) + "\n")
       end
     end
 
     def export_rows(dump_path, table, row_data)
-      data = row_data
-      if File.exist?(File.join(dump_path, "data", "#{table}.json"))
-        previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
-        data[:data] = previous_data["data"] + row_data[:data] unless row_data[:data].nil?
-      end
+      # Use append-only writes to avoid O(n²) complexity
+      # Store metadata separately and append data chunks as NDJSON
+      data_file = File.join(dump_path, "data", "#{table}.json")
 
-      File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
-        file.write(JSON.generate(data))
+      File.open(data_file, 'a') do |file|
+        file.write(JSON.generate(row_data) + "\n")
       end
     end
 

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.24".freeze
+  VERSION = "0.5.26".freeze
 end

data/tapsoob.gemspec

@@ -20,18 +20,18 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Dependencies
-  s.add_dependency "sequel", "~> 5.80.0"
-  s.add_dependency "thor", "~> 1.3.1"
+  s.add_dependency "sequel", "~> 5.96.0"
+  s.add_dependency "thor", "~> 1.4.0"
 
   if (RUBY_PLATFORM =~ /java/).nil?
-    s.add_development_dependency "mysql2",  "~> 0.4.10"
-    s.add_development_dependency "pg",      "~> 0.21.0"
-    s.add_development_dependency "sqlite3", "~> 1.3.11"
+    s.add_development_dependency "mysql2",  "~> 0.5.7"
+    s.add_development_dependency "pg",      "~> 1.6.2"
+    s.add_development_dependency "sqlite3", "~> 2.7.4"
   else
     s.platform = 'java'
 
-    s.add_dependency "jdbc-mysql",    "~> 5.1.44"
-    s.add_dependency "jdbc-postgres", "~> 42.2.25"
-    s.add_dependency "jdbc-sqlite3",  "~> 3.28.0"
+    s.add_dependency "jdbc-mysql",    "~> 9.1.0.1"
+    s.add_dependency "jdbc-postgres", "~> 42.6.0"
+    s.add_dependency "jdbc-sqlite3",  "~> 3.46.1.1"
   end
 end

metadata

@@ -1,86 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.24
+  version: 0.5.26
 platform: java
 authors:
 - Félix Bellanger
 - Michael Chrisco
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-04 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.80.0
-  name: sequel
+        version: 5.96.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.80.0
+        version: 5.96.0
 - !ruby/object:Gem::Dependency
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
-  name: thor
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
+  name: jdbc-mysql
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.1.44
-  name: jdbc-mysql
+        version: 9.1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.1.44
+        version: 9.1.0.1
 - !ruby/object:Gem::Dependency
+  name: jdbc-postgres
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 42.2.25
-  name: jdbc-postgres
+        version: 42.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 42.2.25
+        version: 42.6.0
 - !ruby/object:Gem::Dependency
+  name: jdbc-sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.28.0
-  name: jdbc-sqlite3
+        version: 3.46.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.28.0
+        version: 3.46.1.1
 description: Simple tool to import/export databases inspired by taps but OOB, meaning
   databases are imported/exported from the filesystem.
 email: felix.bellanger@faveod.com
@@ -120,7 +119,6 @@ homepage: https://github.com/Keeguon/tapsoob
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -135,8 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Simple tool to import/export databases.
 test_files: