tapsoob 0.4.23 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02ae7de93cd8aeddb465cd48bafe2e33eec932a72a946d6dbde9c05e5d8232d1
-  data.tar.gz: c7e6a34e96cfebb2a0cd16fdd3f6f30d84abd918b17f1a78392a08e53783762b
+  metadata.gz: af1c77e65193e1c885e35d4e0b7e2c28b933f915b9b2ad0d21398fe5fa7abd8c
+  data.tar.gz: 0a286d60df2babfe2282a708fb802d561afa9960ca7e8287030c1e9d9df3500e
 SHA512:
-  metadata.gz: af637e1c4486956987ac94dbac997b77d48d34789333161701618588849db8ef50396b607c29f74613bba88f410a608f7ae4f4856296e4a33e1672f2540146a4
-  data.tar.gz: 70b0f436ed34583a25a8715acd7c60dea54eac1728c7e615bc64ba4d9341ab066944806fd63bfcecf46c1b2120a10ad39e71109ea4b6225e676c9fe3fc2d03db
+  metadata.gz: 4fad86e7e1ec643b8d3f3a86ea06a69f2b5878cf20ae0c866f48bbbceae5af09f8d1da55533d875045b6e75470e3bf73be82512db90ef86be931616207c86a30
+  data.tar.gz: e4740da5ddbdbe77248707db361cb825afb9130a6db63a6dd212cad06fc0a266a11f6424503ecec6c676ba41d62f68c9c108a38c8b43109890da15f9512f83ed

data/lib/tapsoob/cli/data_stream.rb

@@ -12,7 +12,6 @@ module Tapsoob
     class DataStream < Thor
       desc "pull DATABASE_URL [DUMP_PATH]", "Pull data from a database."
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
-      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
@@ -24,11 +23,10 @@ module Tapsoob
 
       desc "push DATABASE_URL [DUMP_PATH]", "Push data to a database."
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
-      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
-      option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
+      option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
@@ -65,6 +63,7 @@ module Tapsoob
           # Default options
           opts = {
             progress: options[:progress],
+            tables: options[:tables],
             debug: options[:debug]
           }
 
@@ -78,15 +77,6 @@ module Tapsoob
             opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
           end
 
-          # Regex filter
-          opts[:table_filter] = options[:filter] if options[:filter]
-
-          # Table filter
-          if options[:tables]
-            r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
-            opts[:table_filter] = "#{r_tables}"
-          end
-
           # Exclude tables
           opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
 

data/lib/tapsoob/cli/root.rb

@@ -10,15 +10,16 @@ module Tapsoob
   module CLI
     class Root < Thor
       desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
-      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
+      option :data, desc: "Pull the data to the database", default: true, type: :boolean, aliases: '-d'
+      option :schema, desc: "Pull the schema to the database", default: true, type: :boolean, aliases: "-s"
       option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
       option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
       option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
-      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
-      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      option :progress, desc: "Show progress", default: true, type: :boolean
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def pull(dump_path, database_url)
         opts = parse_opts(options)
         Tapsoob.log.level = Logger::DEBUG if opts[:debug]
@@ -30,18 +31,19 @@ module Tapsoob
       end
 
       desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
-      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
+      option :data, desc: "Push the data to the database", default: true, type: :boolean, aliases: '-d'
+      option :schema, desc: "Push the schema to the database", default: true, type: :boolean, aliases: "-s"
       option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
       option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
       option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
-      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
-      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      option :progress, desc: "Show progress", default: true, type: :boolean
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def push(dump_path, database_url)
         opts = parse_opts(options)
         Tapsoob.log.level = Logger::DEBUG if opts[:debug]
@@ -67,9 +69,12 @@ module Tapsoob
         def parse_opts(options)
           # Default options
           opts = {
-            skip_schema: options[:"skip-schema"],
+            data: options[:data],
+            data: options[:schema],
             indexes_first: options[:"indexes_first"],
             disable_compression: options[:"disable-compression"],
+            tables: options[:tables],
+            progress: options[:progress],
             debug: options[:debug]
           }
 
@@ -92,15 +97,6 @@ module Tapsoob
             opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
           end
 
-          # Regex filter
-          opts[:table_filter] = options[:filter] if options[:filter]
-
-          # Table filter
-          if options[:tables]
-            r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
-            opts[:table_filter] = "#{r_tables}"
-          end
-
           # Exclude tables
           opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
 

data/lib/tapsoob/data_stream.rb

@@ -74,6 +74,7 @@ module Tapsoob
     def fetch_rows
       state[:chunksize] = fetch_chunksize
       ds = table.order(*order_by).limit(state[:chunksize], state[:offset])
+      state[:size] = table.count
       log.debug "DataStream#fetch_rows SQL -> #{ds.sql}"
       rows = Tapsoob::Utils.format_data(db, ds.all,
         :string_columns => string_columns,
@@ -87,6 +88,7 @@ module Tapsoob
     def fetch_file(dump_path)
       state[:chunksize] = fetch_chunksize
       ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
+      state[:size] = ds["data"].size
       log.debug "DataStream#fetch_file"
       rows = {
         :table_name => ds["table_name"],
@@ -132,71 +134,34 @@ module Tapsoob
       t2 = Time.now
       elapsed_time = t2 - t1
 
-      if opts[:type] == "file"
-        @complete = rows[:data] == [ ]
-      else
-        @complete = rows == { }
-      end
+      state[:offset] += (rows == {} ? 0 : rows[:data].size)
 
-      [encoded_data, (@complete ? 0 : rows[:data].size), elapsed_time]
+      [encoded_data, (rows == {} ? 0 : rows[:data].size), elapsed_time]
     end
 
     def complete?
-      @complete
+      state[:offset] >= state[:size]
     end
 
-    def fetch_database
-      params = fetch_from_database
+    def fetch_data_from_database(params)
       encoded_data = params[:encoded_data]
-      json = params[:json]
 
-      rows = parse_encoded_data(encoded_data, json[:checksum])
-
-      @complete = rows == { }
+      rows = parse_encoded_data(encoded_data, params[:checksum])
 
       # update local state
-      state.merge!(json[:state].merge(:chunksize => state[:chunksize]))
-
-      unless @complete
-        yield rows if block_given?
-        state[:offset] += rows[:data].size
-        rows[:data].size
-      else
-        0
-      end
-    end
+      state.merge!(params[:state].merge(:chunksize => state[:chunksize]))
 
-    def fetch_from_database
-      res = nil
-      log.debug "DataStream#fetch_from_database state -> #{state.inspect}"
-      state[:chunksize] = Tapsoob::Utils.calculate_chunksize(state[:chunksize]) do |c|
-        state[:chunksize] = c.to_i
-        encoded_data = fetch.first
-
-        checksum = Tapsoob::Utils.checksum(encoded_data).to_s
-
-        res = {
-          :json         => { :checksum => checksum, :state => to_hash },
-          :encoded_data => encoded_data
-        }
-      end
-
-      res
+      yield rows if block_given?
+      (rows == {} ? 0 : rows[:data].size)
     end
 
-    def fetch_data_in_database(params)
+    def fetch_data_to_database(params)
       encoded_data = params[:encoded_data]
 
       rows = parse_encoded_data(encoded_data, params[:checksum])
-
-      @complete = rows[:data] == [ ]
-
-      unless @complete
-        import_rows(rows)
-        rows[:data].size
-      else
-        0
-      end
+      
+      import_rows(rows)
+      (rows == {} ? 0 : rows[:data].size)
     end
 
     def self.parse_json(json)
@@ -266,7 +231,6 @@ module Tapsoob
       end
 
       table.import(columns, data, :commit_every => 100)
-      state[:offset] += rows[:data].size
     rescue Exception => ex
       case ex.message
       when /integer out of range/ then

data/lib/tapsoob/operation.rb

@@ -21,8 +21,12 @@ module Tapsoob
       "op"
     end
 
-    def skip_schema?
-      !!opts[:skip_schema]
+    def data?
+      opts[:data]
+    end
+
+    def schema?
+      opts[:schema]
     end
 
     def indexes_first?
@@ -30,7 +34,7 @@ module Tapsoob
     end
 
     def table_filter
-      opts[:table_filter]
+      opts[:tables] || []
     end
 
     def exclude_tables
@@ -38,19 +42,18 @@ module Tapsoob
     end
 
     def apply_table_filter(tables)
-      return tables unless table_filter || exclude_tables
+      return tables if table_filter.empty? && exclude_tables.empty?
 
-      re = table_filter ? Regexp.new(table_filter) : nil
       if tables.kind_of?(Hash)
         ntables = {}
         tables.each do |t, d|
-          if !exclude_tables.include?(t.to_s) && (!re || !re.match(t.to_s).nil?)
+          if !exclude_tables.include?(t.to_s) && (!table_filter.empty? && table_filter.include?(t.to_s))
             ntables[t] = d
           end
         end
         ntables
       else
-        tables.reject { |t| exclude_tables.include?(t.to_s) || (re && re.match(t.to_s).nil?) }
+        tables.reject { |t| exclude_tables.include?(t.to_s) }.select { |t| table_filter.include?(t.to_s) }
       end
     end
 
@@ -163,13 +166,13 @@ module Tapsoob
     def run
       catch_errors do
         unless resuming?
-          pull_schema if !skip_schema?
-          pull_indexes if indexes_first? && !skip_schema?
+          pull_schema if schema?
+          pull_indexes if indexes_first? && schema?
         end
         setup_signal_trap
-        pull_partial_data if resuming?
-        pull_data
-        pull_indexes if !indexes_first? && !skip_schema?
+        pull_partial_data if data? && resuming?
+        pull_data if data?
+        pull_indexes if !indexes_first? && schema?
         pull_reset_sequences
       end
     end
@@ -182,7 +185,7 @@ module Tapsoob
         schema_data = Tapsoob::Schema.dump_table(database_url, table_name)
         log.debug "Table: #{table_name}\n#{schema_data}\n"
         output = Tapsoob::Utils.export_schema(dump_path, table_name, schema_data)
-        puts output if output
+        puts output if dump_path.nil? && output
         progress.inc(1)
       end
       progress.finish
@@ -217,25 +220,56 @@ module Tapsoob
 
     def pull_data_from_table(stream, progress)
       loop do
-        begin
-          exit 0 if exiting?
+        if exiting?
+          store_session
+          exit 0
+        end
 
-          size = stream.fetch_database do |rows|
-            if dump_path.nil?
-              puts JSON.generate(rows)
-            else
-              Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
+        row_size = 0
+        chunksize = stream.state[:chunksize]
+
+        begin
+          chunksize = Tapsoob::Utils.calculate_chunksize(chunksize) do |c|
+            stream.state[:chunksize] = c.to_i
+            encoded_data, row_size, elapsed_time = nil
+            d1 = c.time_delta do
+              encoded_data, row_size, elapsed_time = stream.fetch
+            end
+        
+            data = nil
+            d2 = c.time_delta do
+              data = {
+                :state        => stream.to_hash,
+                :checksum     => Tapsoob::Utils.checksum(encoded_data).to_s,
+                :encoded_data => encoded_data
+              }
+            end
+        
+            stream.fetch_data_from_database(data) do |rows|
+              next if rows == {}
+
+              if dump_path.nil?
+                puts JSON.generate(rows)
+              else
+                Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
+              end
             end
+            log.debug "row size: #{row_size}"
+            stream.error = false
+            self.stream_state = stream.to_hash
+            
+            c.idle_secs = (d1 + d2)
+            
+            elapsed_time
           end
-          stream.error = false
-          self.stream_state = stream.to_hash
         rescue Tapsoob::CorruptedData => e
           log.info "Corrupted Data Received #{e.message}, retrying..."
           stream.error = true
           next
         end
 
-        progress.inc(size) if progress && !exiting?
+        progress.inc(row_size) if progress
+        
         break if stream.complete?
       end
 
@@ -290,7 +324,7 @@ module Tapsoob
     def pull_indexes
       log.info "Receiving indexes"
 
-      raw_idxs = Tapsoob::Utils.schema_bin(:indexes_individual, database_url)
+      raw_idxs = Tapsoob::Schema.indexes_individual(database_url)
       idxs     = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
 
       apply_table_filter(idxs).each do |table, indexes|
@@ -298,7 +332,7 @@ module Tapsoob
         progress = ProgressBar.new(table, indexes.size)
         indexes.each do |idx|
           output = Tapsoob::Utils.export_indexes(dump_path, table, idx)
-          puts output if output
+          puts output if dump_path.nil? && output
           progress.inc(1)
         end
         progress.finish
@@ -309,7 +343,7 @@ module Tapsoob
       log.info "Resetting sequences"
 
       output = Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
-      puts output if output
+      puts output if dump_path.nil? && output
     end
   end
 
@@ -325,13 +359,13 @@ module Tapsoob
     def run
       catch_errors do
         unless resuming?
-          push_schema if !skip_schema?
-          push_indexes if indexes_first? && !skip_schema?
+          push_schema if schema?
+          push_indexes if indexes_first? && schema?
         end
         setup_signal_trap
-        push_partial_data if resuming?
-        push_data
-        push_indexes if !indexes_first? && !skip_schema?
+        push_partial_data if data? && resuming?
+        push_data if data?
+        push_indexes if !indexes_first? && schema?
         push_reset_sequences
       end
     end
@@ -404,7 +438,7 @@ module Tapsoob
           :purge => opts[:purge] || false,
           :debug => opts[:debug]
         })
-        progress = ProgressBar.new(table_name.to_s, count)
+        progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
         push_data_from_file(stream, progress)
       end
     end
@@ -426,17 +460,17 @@ module Tapsoob
             d1 = c.time_delta do
               encoded_data, row_size, elapsed_time = stream.fetch({ :type => "file", :source => dump_path })
             end
-            break if stream.complete?
 
             data = nil
             d2 = c.time_delta do
               data = {
-                :state    => stream.to_hash,
-                :checksum => Tapsoob::Utils.checksum(encoded_data).to_s
+                :state        => stream.to_hash,
+                :checksum     => Tapsoob::Utils.checksum(encoded_data).to_s,
+                :encoded_data => encoded_data
               }
             end
 
-            row_size = stream.fetch_data_in_database({ :encoded_data => encoded_data, :checksum => data[:checksum] })
+            stream.fetch_data_to_database(data)
             log.debug "row size: #{row_size}"
             self.stream_state = stream.to_hash
 
@@ -455,13 +489,12 @@ module Tapsoob
         end
         stream.state[:chunksize] = chunksize
 
-        progress.inc(row_size)
+        progress.inc(row_size) if progress
 
-        stream.increment(row_size)
         break if stream.complete?
       end
 
-      progress.finish
+      progress.finish if progress
       completed_tables << stream.table_name.to_s
       self.stream_state = {}
     end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.4.23".freeze
+  VERSION = "0.5.3".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.4.23
+  version: 0.5.3
 platform: ruby
 authors:
 - Félix Bellanger
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2022-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ripl
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Simple tool to import/export databases.