tapsoob 0.5.24 → 0.5.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85d42c955109e84e5af46ab969b893d144d33ec4c022c94fb286ceed5aa50596
-  data.tar.gz: 67b2ed47364c35f7d8a5756374b0706a5fb67975c58c3b24a69e0581cbb7e734
+  metadata.gz: 9f19b27231e8231c5a11cb879256df846b2963712eb9a335549f7c3199144b74
+  data.tar.gz: 792abe05f841fe4aa5a55de950d371f0ba23bb0d1ebfcb4e7c6ab76b695cd47d
 SHA512:
-  metadata.gz: eeb40937fdb97fc9031537e555492686d5df9eb26edc8a2b3c3590ab4e3631bb7011008c16e8ce4c375f6fc857064d0ee4302cb61dc1bf01fa2b613eee1a8e9b
-  data.tar.gz: 937d994cbf16669db3d15e7e1520ddd26d15097909a7bb4168f68ca97df652e8415abe7750b9e31afa61008027d3538989b573695cf235aab696806a6ce6a236
+  metadata.gz: d0ca8097f4ba556ef856e33b8c9e6e03c68636af1dc2c6d1896d3c399d5835cedb2de3d889bef199e903efc09eb2a3da74a9f9e2806dcc7a178bd3cda37c175f
+  data.tar.gz: 6a9c81e0acc7eca5296e718bd3e20bc224b23cb788b16d8b9651e48eda87cde3bcece13159d0ffcae4ff4d2e8b71144d818b2eef276dd7e967e1fd6020a5292c

data/Gemfile

@@ -35,7 +35,7 @@ end
 gemspec
 
 group :development do
-  gem 'warbler', '~> 2.0.4', platform: :jruby, require: false
+  gem 'warbler', '~> 2.0.5', platform: :jruby, github: "jruby/warbler", require: false
 end
 
 group :test do

data/lib/tapsoob/data_stream.rb

@@ -87,14 +87,34 @@ module Tapsoob
 
     def fetch_file(dump_path)
       #state[:chunksize] = fetch_chunksize
-      ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
-      state[:size] = ds["data"].size
+      # Read NDJSON format - each line is a separate JSON chunk
+      file_path = File.join(dump_path, "data", "#{table_name}.json")
+
+      # Parse all chunks and combine them
+      all_data = []
+      table_name_val = nil
+      header_val = nil
+      types_val = nil
+
+      File.readlines(file_path).each do |line|
+        chunk = JSON.parse(line.strip)
+        table_name_val ||= chunk["table_name"]
+        header_val ||= chunk["header"]
+        types_val ||= chunk["types"]
+        all_data.concat(chunk["data"]) if chunk["data"]
+      end
+
+      # Apply skip-duplicates if needed
+      all_data = all_data.uniq if @options[:"skip-duplicates"]
+
+      state[:size] = all_data.size
       log.debug "DataStream#fetch_file"
+
       rows = {
-        :table_name => ds["table_name"],
-        :header     => ds["header"],
-        :data       => ((@options[:"skip-duplicates"] ? ds["data"].uniq : ds["data"])[state[:offset], (state[:offset] + state[:chunksize])] || [ ]),
-        :types      => ds["types"]
+        :table_name => table_name_val,
+        :header     => header_val,
+        :data       => (all_data[state[:offset], state[:chunksize]] || []),
+        :types      => types_val
       }
       update_chunksize_stats
       rows

data/lib/tapsoob/operation.rb

@@ -249,6 +249,9 @@ module Tapsoob
             stream.fetch_data_from_database(data) do |rows|
               next if rows == {}
 
+              # Update progress bar immediately when data is ready, before I/O
+              progress.inc(row_size) if progress
+
               if dump_path.nil?
                 puts JSON.generate(rows)
               else
@@ -258,9 +261,9 @@ module Tapsoob
             log.debug "row size: #{row_size}"
             stream.error = false
             self.stream_state = stream.to_hash
-            
+
             c.idle_secs = (d1 + d2)
-            
+
             elapsed_time
           end
         rescue Tapsoob::CorruptedData => e
@@ -268,8 +271,6 @@ module Tapsoob
           stream.error = true
           next
         end
-
-        progress.inc(row_size) if progress
         
         break if stream.complete?
       end
@@ -375,7 +376,9 @@ module Tapsoob
       idxs = {}
       table_idxs = Dir.glob(File.join(dump_path, "indexes", "*.json")).map { |path| File.basename(path, '.json') }
       table_idxs.each do |table_idx|
-        idxs[table_idx] = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table_idx}.json")))
+        # Read NDJSON format - each line is a separate index
+        index_file = File.join(dump_path, "indexes", "#{table_idx}.json")
+        idxs[table_idx] = File.readlines(index_file).map { |line| JSON.parse(line.strip) }
       end
 
       return unless idxs.size > 0

data/lib/tapsoob/utils.rb

@@ -143,26 +143,21 @@ Data : #{data}
     end
 
     def export_indexes(dump_path, table, index_data)
-      data = [index_data]
-      if File.exist?(File.join(dump_path, "indexes", "#{table}.json"))
-        previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
-        data = data + previous_data
-      end
+      # Use append-only writes to avoid O(n²) complexity
+      index_file = File.join(dump_path, "indexes", "#{table}.json")
 
-      File.open(File.join(dump_path, "indexes", "#{table}.json"), 'w') do |file|
-        file.write(JSON.generate(data))
+      File.open(index_file, 'a') do |file|
+        file.write(JSON.generate(index_data) + "\n")
       end
     end
 
     def export_rows(dump_path, table, row_data)
-      data = row_data
-      if File.exist?(File.join(dump_path, "data", "#{table}.json"))
-        previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
-        data[:data] = previous_data["data"] + row_data[:data] unless row_data[:data].nil?
-      end
+      # Use append-only writes to avoid O(n²) complexity
+      # Store metadata separately and append data chunks as NDJSON
+      data_file = File.join(dump_path, "data", "#{table}.json")
 
-      File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
-        file.write(JSON.generate(data))
+      File.open(data_file, 'a') do |file|
+        file.write(JSON.generate(row_data) + "\n")
       end
     end
 

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.24".freeze
+  VERSION = "0.5.26".freeze
 end

data/tapsoob.gemspec

@@ -20,18 +20,18 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Dependencies
-  s.add_dependency "sequel", "~> 5.80.0"
-  s.add_dependency "thor", "~> 1.3.1"
+  s.add_dependency "sequel", "~> 5.96.0"
+  s.add_dependency "thor", "~> 1.4.0"
 
   if (RUBY_PLATFORM =~ /java/).nil?
-    s.add_development_dependency "mysql2",  "~> 0.4.10"
-    s.add_development_dependency "pg",      "~> 0.21.0"
-    s.add_development_dependency "sqlite3", "~> 1.3.11"
+    s.add_development_dependency "mysql2",  "~> 0.5.7"
+    s.add_development_dependency "pg",      "~> 1.6.2"
+    s.add_development_dependency "sqlite3", "~> 2.7.4"
   else
     s.platform = 'java'
 
-    s.add_dependency "jdbc-mysql",    "~> 5.1.44"
-    s.add_dependency "jdbc-postgres", "~> 42.2.25"
-    s.add_dependency "jdbc-sqlite3",  "~> 3.28.0"
+    s.add_dependency "jdbc-mysql",    "~> 9.1.0.1"
+    s.add_dependency "jdbc-postgres", "~> 42.6.0"
+    s.add_dependency "jdbc-sqlite3",  "~> 3.46.1.1"
   end
 end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.24
+  version: 0.5.26
 platform: ruby
 authors:
 - Félix Bellanger
 - Michael Chrisco
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-04 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -17,70 +16,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.80.0
+        version: 5.96.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.80.0
+        version: 5.96.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.10
+        version: 0.5.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.10
+        version: 0.5.7
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.21.0
+        version: 1.6.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.21.0
+        version: 1.6.2
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.11
+        version: 2.7.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.11
+        version: 2.7.4
 description: Simple tool to import/export databases inspired by taps but OOB, meaning
   databases are imported/exported from the filesystem.
 email: felix.bellanger@faveod.com
@@ -120,7 +119,6 @@ homepage: https://github.com/Keeguon/tapsoob
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -135,8 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Simple tool to import/export databases.
 test_files: