tapsoob 0.5.23 → 0.5.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7a84b5a6f4398257b459a0a8b74ff67bfcac3f50620b7e7e72c9ebd513b5211
-  data.tar.gz: 037127603d8cae32df3a0b71d2ffd89f4631632ee1d5d726e6105d2e5e0fccee
+  metadata.gz: 87f86ef24606eac9b07a505455b518402d33d522b8e14420dfe36fbcc38beb78
+  data.tar.gz: 46722746c4d5c6ec5247f28cb5d05d33f55d78e3a8fd7ee560096c097351c9fd
 SHA512:
-  metadata.gz: 5158318d197305651b772044b98a614638d3c1e3e540ade7e1997b2799fef2607914442c1ce4050023516cef8fdec00c4e31379720288b6ea2714e59840243d5
-  data.tar.gz: 4bba24e4863a6034611da2d055766e87deb680e63291e32ef13993364937c109f1979799496700656edfb24dfa1173883d34303c188598842584f54340faca09
+  metadata.gz: 2de7d2c3841e96e5b322cec4079b6d70a6b8fe761fde5e4b3799676cb61af6f2f38efed8a95ff40470b783db0897e21e93e44da541507b351e0f7c814fb18242
+  data.tar.gz: 363345d8ad6ce7847e1480c70e26cfabc6f27a8d9581b1f33e7bd7c9df3d270b8d9f7cec4cb6e267e234fbc15f3f5b70a5e5d559f979fbf09e15621f8b235789

data/Gemfile

@@ -35,7 +35,7 @@ end
 gemspec
 
 group :development do
-  gem 'warbler', '~> 2.0.4', platform: :jruby, require: false
+  gem 'warbler', '~> 2.0.5', platform: :jruby, github: "jruby/warbler", require: false
 end
 
 group :test do

data/lib/tapsoob/cli/root.rb

@@ -59,11 +59,6 @@ module Tapsoob
         end
       end
 
-      def ripl
-        require 'ripl'
-        Ripl.start
-      end
-
       desc "version", "Show tapsoob version"
       def version
         puts Tapsoob::VERSION.dup

data/lib/tapsoob/cli/schema.rb

@@ -9,8 +9,9 @@ module Tapsoob
       desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
       def console(database_url)
         $db = Sequel.connect(database_url)
-        require 'ripl'
-        Ripl.start binding: binding
+        require 'irb'
+        ARGV.clear # otherwise all script parameters get passed to IRB
+        IRB.start
       end
 
       desc "dump DATABASE_URL", "Dump a database using a database URL"

data/lib/tapsoob/data_stream.rb

@@ -87,14 +87,34 @@ module Tapsoob
 
     def fetch_file(dump_path)
       #state[:chunksize] = fetch_chunksize
-      ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
-      state[:size] = ds["data"].size
+      # Read NDJSON format - each line is a separate JSON chunk
+      file_path = File.join(dump_path, "data", "#{table_name}.json")
+
+      # Parse all chunks and combine them
+      all_data = []
+      table_name_val = nil
+      header_val = nil
+      types_val = nil
+
+      File.readlines(file_path).each do |line|
+        chunk = JSON.parse(line.strip)
+        table_name_val ||= chunk["table_name"]
+        header_val ||= chunk["header"]
+        types_val ||= chunk["types"]
+        all_data.concat(chunk["data"]) if chunk["data"]
+      end
+
+      # Apply skip-duplicates if needed
+      all_data = all_data.uniq if @options[:"skip-duplicates"]
+
+      state[:size] = all_data.size
       log.debug "DataStream#fetch_file"
+
       rows = {
-        :table_name => ds["table_name"],
-        :header     => ds["header"],
-        :data       => ((@options[:"skip-duplicates"] ? ds["data"].uniq : ds["data"])[state[:offset], (state[:offset] + state[:chunksize])] || [ ]),
-        :types      => ds["types"]
+        :table_name => table_name_val,
+        :header     => header_val,
+        :data       => (all_data[state[:offset], state[:chunksize]] || []),
+        :types      => types_val
       }
       update_chunksize_stats
       rows

data/lib/tapsoob/operation.rb

@@ -375,7 +375,9 @@ module Tapsoob
       idxs = {}
       table_idxs = Dir.glob(File.join(dump_path, "indexes", "*.json")).map { |path| File.basename(path, '.json') }
       table_idxs.each do |table_idx|
-        idxs[table_idx] = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table_idx}.json")))
+        # Read NDJSON format - each line is a separate index
+        index_file = File.join(dump_path, "indexes", "#{table_idx}.json")
+        idxs[table_idx] = File.readlines(index_file).map { |line| JSON.parse(line.strip) }
       end
 
       return unless idxs.size > 0

data/lib/tapsoob/utils.rb

@@ -143,26 +143,21 @@ Data : #{data}
     end
 
     def export_indexes(dump_path, table, index_data)
-      data = [index_data]
-      if File.exist?(File.join(dump_path, "indexes", "#{table}.json"))
-        previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
-        data = data + previous_data
-      end
+      # Use append-only writes to avoid O(n²) complexity
+      index_file = File.join(dump_path, "indexes", "#{table}.json")
 
-      File.open(File.join(dump_path, "indexes", "#{table}.json"), 'w') do |file|
-        file.write(JSON.generate(data))
+      File.open(index_file, 'a') do |file|
+        file.write(JSON.generate(index_data) + "\n")
       end
     end
 
     def export_rows(dump_path, table, row_data)
-      data = row_data
-      if File.exist?(File.join(dump_path, "data", "#{table}.json"))
-        previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
-        data[:data] = previous_data["data"] + row_data[:data] unless row_data[:data].nil?
-      end
+      # Use append-only writes to avoid O(n²) complexity
+      # Store metadata separately and append data chunks as NDJSON
+      data_file = File.join(dump_path, "data", "#{table}.json")
 
-      File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
-        file.write(JSON.generate(data))
+      File.open(data_file, 'a') do |file|
+        file.write(JSON.generate(row_data) + "\n")
       end
     end
 

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.23".freeze
+  VERSION = "0.5.25".freeze
 end

data/tapsoob.gemspec

@@ -20,19 +20,18 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Dependencies
-  s.add_dependency "ripl", "~> 0.7.1"
-  s.add_dependency "sequel", "~> 5.70.0"
-  s.add_dependency "thor", "~> 1.2.1"
+  s.add_dependency "sequel", "~> 5.96.0"
+  s.add_dependency "thor", "~> 1.4.0"
 
   if (RUBY_PLATFORM =~ /java/).nil?
-    s.add_development_dependency "mysql2",  "~> 0.4.10"
-    s.add_development_dependency "pg",      "~> 0.21.0"
-    s.add_development_dependency "sqlite3", "~> 1.3.11"
+    s.add_development_dependency "mysql2",  "~> 0.5.7"
+    s.add_development_dependency "pg",      "~> 1.6.2"
+    s.add_development_dependency "sqlite3", "~> 2.7.4"
   else
     s.platform = 'java'
 
-    s.add_dependency "jdbc-mysql",    "~> 5.1.44"
-    s.add_dependency "jdbc-postgres", "~> 42.2.25"
-    s.add_dependency "jdbc-sqlite3",  "~> 3.28.0"
+    s.add_dependency "jdbc-mysql",    "~> 9.1.0.1"
+    s.add_dependency "jdbc-postgres", "~> 42.6.0"
+    s.add_dependency "jdbc-sqlite3",  "~> 3.46.1.1"
   end
 end

metadata

@@ -1,100 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.23
+  version: 0.5.25
 platform: ruby
 authors:
 - Félix Bellanger
 - Michael Chrisco
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: ripl
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.1
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.70.0
+        version: 5.96.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.70.0
+        version: 5.96.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.10
+        version: 0.5.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.10
+        version: 0.5.7
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.21.0
+        version: 1.6.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.21.0
+        version: 1.6.2
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.11
+        version: 2.7.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.11
+        version: 2.7.4
 description: Simple tool to import/export databases inspired by taps but OOB, meaning
   databases are imported/exported from the filesystem.
 email: felix.bellanger@faveod.com
@@ -134,7 +119,6 @@ homepage: https://github.com/Keeguon/tapsoob
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -149,8 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Simple tool to import/export databases.
 test_files: