tapsoob 0.3.20 → 0.3.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52d4bfd61d315b90beb05943e4ed8d450ea6fef3f3d8cf8e685a36aace4a67d8
-  data.tar.gz: 73a623f81cfd0b8b5238ecfa06e0933d0033128ea7bd573f29f801294049ed46
+  metadata.gz: cfc8db7ed412920a9e55e43d5ee66c57b0dd09a86e322c0b6f37bd90762ef261
+  data.tar.gz: 46dba518c294c033bf7c12c06f79278700be456d00554ad30d9149decaf8b62e
 SHA512:
-  metadata.gz: df06895063ad4caf4fd7270a27771dc82f52d2b2d7e035c2a7a5651c2e6dc336e11741285e8a86a12b4af205391e472029c3941cc96412c7f7dbacd3ba6fca07
-  data.tar.gz: 4ae7a6b34fce4d1f267f8fd95c2cf895608232d9dd687427c3e9ed33a7a35402f33c5db87a7bd9a6cb924cf8c961973fe73303a30d2d3b0f282c84127850c859
+  metadata.gz: c2ef008de5be1f5e169055026c65b04518163e7987e34f2e1be5c554d7114be96d99f4fe7ae3284798f4e11225c1554f461db8c9cc487d1befb09ef654e8d0ad
+  data.tar.gz: b532c43909abc6afda5314e55da082ddbceb3a3ed285b5ba133f024f969120ec0d17b3d6f75c0423588b3fb61c015763038219f7b6c025d224c340416f7abf6c

data/lib/tapsoob/cli/root.rb

@@ -39,6 +39,7 @@ module Tapsoob
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
+      option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(dump_path, database_url)
         opts = parse_opts(options)
@@ -71,8 +72,9 @@ module Tapsoob
             debug: options[:debug]
           }
 
-          # Purge (push only)
+          # Push only options
           opts[:purge] = options[:purge] if options.key?(:purge)
+          opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
 
           # Resume
           if options[:resume]

data/lib/tapsoob/data_stream.rb

@@ -6,9 +6,9 @@ module Tapsoob
   class DataStream
     DEFAULT_CHUNKSIZE = 1000
 
-    attr_reader :db, :state
+    attr_reader :db, :state, :options
 
-    def initialize(db, state)
+    def initialize(db, state, opts = {})
       @db = db
       @state = {
         :offset          => 0,
@@ -17,6 +17,7 @@ module Tapsoob
         :total_chunksize => 0
       }.merge(state)
       @state[:chunksize] ||= DEFAULT_CHUNKSIZE
+      @options = opts
       @complete = false
     end
 
@@ -218,6 +219,9 @@ module Tapsoob
     end
 
     def import_rows(rows)
+      columns = rows[:header]
+      data    = rows[:data]
+
       # Decode blobs
       if rows.has_key?(:types) && rows[:types].include?("blob")
         blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
@@ -227,8 +231,14 @@ module Tapsoob
           end
         end
       end
+
+      # Remove id column
+      if @options[:"discard-identity"]
+        columns = rows[:header] - ["id"]
+        data    = data.map { |d| d[1..-1] }
+      end
       
-      table.import(rows[:header], rows[:data], :commit_every => 100)
+      table.import(columns, data, :commit_every => 100)
       state[:offset] += rows[:data].size
     rescue Exception => ex
       case ex.message
@@ -246,19 +256,19 @@ module Tapsoob
       state[:offset] = table.count
     end
 
-    def self.factory(db, state)
+    def self.factory(db, state, opts)
       if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
         Sequel::MySQL.convert_invalid_date_time = :nil
       end
 
       if state.has_key?(:klass)
-        return eval(state[:klass]).new(db, state)
+        return eval(state[:klass]).new(db, state, opts)
       end
 
       if Tapsoob::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
-        DataStreamKeyed.new(db, state)
+        DataStreamKeyed.new(db, state, opts)
       else
-        DataStream.new(db, state)
+        DataStream.new(db, state, opts)
       end
     end
   end
@@ -266,8 +276,8 @@ module Tapsoob
   class DataStreamKeyed < DataStream
     attr_accessor :buffer
 
-    def initialize(db, state)
-      super(db, state)
+    def initialize(db, state, opts = {})
+      super(db, state, opts)
       @state = { :primary_key => order_by(state[:table_name]).first, :filter => 0 }.merge(@state)
       @state[:chunksize] ||= DEFAULT_CHUNKSIZE
       @buffer = []

data/lib/tapsoob/operation.rb

@@ -394,9 +394,9 @@ module Tapsoob
       tables.each do |table_name, count|
         next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
         db[table_name.to_sym].truncate if @opts[:purge]
-        stream = Tapsoob::DataStream.factory(db,
+        stream = Tapsoob::DataStream.factory(db, {
           :table_name => table_name,
-          :chunksize => default_chunksize)
+          :chunksize => default_chunksize }, { :"discard-identity" => @opts[:"discard-identity"] || false })
         progress = ProgressBar.new(table_name.to_s, count)
         push_data_from_file(stream, progress)
       end
@@ -477,7 +477,7 @@ module Tapsoob
 
     def fetch_local_tables_info
       tables_with_counts = {}
-      tbls = Dir.glob(File.join(dump_path, "schemas", "*")).map { |path| File.basename(path, ".rb") }
+      tbls = Dir.glob(File.join(dump_path, "data", "*")).map { |path| File.basename(path, ".json") }
       tbls.each do |table|
         if File.exists?(File.join(dump_path, "data", "#{table}.json"))
           data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.3.20".freeze
+  VERSION = "0.3.25".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.3.20
+  version: 0.3.25
 platform: ruby
 authors:
 - Félix Bellanger