tapsoob 0.5.27 → 0.5.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9e2d75dd19008094d30f87a528281d1a6df5c0b2bda8faa79a67dbeb2bb22ce
-  data.tar.gz: 940c827d7dd14127f99116e66ddaa9ceb0d7fcab0127876748e1b571cde7d140
+  metadata.gz: 748c9cf97135cae0c72f0ba298f7c3f197e65764ce4c44c0670d8ef2fe5181ed
+  data.tar.gz: 7bebc431915dcf8dd5bff0ee341cf22765646d1854485c23f80b5b9c642fc142
 SHA512:
-  metadata.gz: 537960e95fb5ae7b5e3d2e022c56fdf26ac989017274fea1f59fe8510e30248c8c8c3a060e70515d7b9db2882748962ae2423a3d9f6e1e63acf65c69b2741065
-  data.tar.gz: 027f3de686550ba21a5b44106d005526215e8f06667fcb80f8d6b5783423b453f1d1f4aef9be763bcf2e1ac7d4b25b8898a65d6f93798172203b91367d46e8d3
+  metadata.gz: 404478581e61ebdd2875c21d0af865acd5e9dba010292819046cd5e195c9c78cad675bdd50216ea26c5bba8923c0e532a8bd2593b4fe7eb0e71f85db865ef067
+  data.tar.gz: 3310489ebd6dcfd7efc76c4fd5d9b2ec6ffa9da12c6cb4161ffd02267f376927b957f5142dcf42ba81fd544c470c36e07f7e0263b66bcae6168b6c9dfe309b61

data/lib/tapsoob/operation.rb

@@ -202,7 +202,7 @@ module Tapsoob
           :chunksize  => default_chunksize,
           :table_name => table_name
         }, { :debug => opts[:debug] })
-        estimated_chunks = (count.to_f / default_chunksize).ceil
+        estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
         progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, estimated_chunks) : nil)
         pull_data_from_table(stream, progress)
       end
@@ -217,7 +217,7 @@ module Tapsoob
 
       stream = Tapsoob::DataStream.factory(db, stream_state)
       chunksize = stream_state[:chunksize] || default_chunksize
-      estimated_chunks = (record_count.to_f / chunksize).ceil
+      estimated_chunks = [(record_count.to_f / chunksize).ceil, 1].max
       progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, estimated_chunks) : nil)
       pull_data_from_table(stream, progress)
     end
@@ -423,8 +423,10 @@ module Tapsoob
       table_name = stream_state[:table_name]
       record_count = tables[table_name.to_s]
       log.info "Resuming #{table_name}, #{format_number(record_count)} records"
-      progress = ProgressBar.new(table_name.to_s, record_count)
       stream = Tapsoob::DataStream.factory(db, stream_state)
+      chunksize = stream_state[:chunksize] || default_chunksize
+      estimated_chunks = [(record_count.to_f / chunksize).ceil, 1].max
+      progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, estimated_chunks) : nil)
       push_data_from_file(stream, progress)
     end
 
@@ -434,7 +436,9 @@ module Tapsoob
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
       tables.each do |table_name, count|
-        next unless File.exist?(File.join(dump_path, "data", "#{table_name}.json")) || File.exist?(File.join(dump_path, "data", "#{table_name}.json")) && JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))["data"].size == 0
+        # Skip if data file doesn't exist or has no data
+        data_file = File.join(dump_path, "data", "#{table_name}.json")
+        next unless File.exist?(data_file) && count > 0
         db[table_name.to_sym].truncate if @opts[:purge]
         stream = Tapsoob::DataStream.factory(db, {
           :table_name => table_name,
@@ -445,7 +449,8 @@ module Tapsoob
           :purge => opts[:purge] || false,
           :debug => opts[:debug]
         })
-        progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
+        estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+        progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, estimated_chunks) : nil)
         push_data_from_file(stream, progress)
       end
     end
@@ -496,7 +501,8 @@ module Tapsoob
         end
         stream.state[:chunksize] = chunksize
 
-        progress.inc(row_size) if progress
+        # Update progress bar by 1 chunk
+        progress.inc(1) if progress
 
         break if stream.complete?
       end
@@ -528,8 +534,13 @@ module Tapsoob
       tbls = Dir.glob(File.join(dump_path, "schemas", "*")).map { |path| File.basename(path, ".rb") }
       tbls.each do |table|
         if File.exist?(File.join(dump_path, "data", "#{table}.json"))
-          data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
-          tables_with_counts[table] = data["data"].size
+          # Read NDJSON format - each line is a separate JSON chunk
+          total_rows = 0
+          File.readlines(File.join(dump_path, "data", "#{table}.json")).each do |line|
+            chunk = JSON.parse(line.strip)
+            total_rows += chunk["data"].size if chunk["data"]
+          end
+          tables_with_counts[table] = total_rows
         else
           tables_with_counts[table] = 0
         end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.27".freeze
+  VERSION = "0.5.29".freeze
 end

data/lib/tasks/tapsoob.rake

@@ -2,7 +2,7 @@ namespace :tapsoob do
   desc "Pulls a database to your filesystem"
   task :pull => :environment do
     # Default options
-    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false}
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true}
 
     # Get the dump_path
     dump_path = File.expand_path(Rails.root.join("db", Time.now.strftime("%Y%m%d%I%M%S%p"))).to_s
@@ -23,7 +23,7 @@ namespace :tapsoob do
   desc "Push a compatible dump on your filesystem to a database"
   task :push, [:timestamp] => :environment do |t, args|
     # Default options
-    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false}
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true}
 
     # Get the dumps
     dumps = Dir[Rails.root.join("db", "*/")].select { |e| e =~ /([0-9]{14})([A-Z]{2})/ }.sort

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.27
+  version: 0.5.29
 platform: ruby
 authors:
 - Félix Bellanger