tapsoob 0.2.11 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 200790feed652d24673973d69dfda95163f49c8a4d74eef1998a2e25e340195e
-  data.tar.gz: c765871d207f593f3cef4f6236b375e06fd988a2db5ace2949106fb96e4f6249
+  metadata.gz: 85443bd27ad39d0b3a1889034db99210842ff44293cfe664775b0b05cd999b7f
+  data.tar.gz: d40c5db1351ae7c6dea459a39631ada180705f736cd12049e2487dc7fec4c364
 SHA512:
-  metadata.gz: eca26ac7f192755d81ed2e4f76313d16e94c007611496e8142c9f0b9fdadd7db9928200c6c4123498e50c45bb270f621050e18dfe40259fead5f62530e872110
-  data.tar.gz: 179d0f90234522a9dd47a274d690600d5f906f2a7265983cfd0e2af86e95993bb6f40e38269505f08819df43ff8201a775996e24bce0b96fc27661b7d1b005f3
+  metadata.gz: fc8222892ca312f16989c8e062d47007d9625b3217f38b6b7ec41a9a1b73e26b737fd3195d5f6fcf64803b4df161324df67335a08ae0604f15aa2bb117e4524c
+  data.tar.gz: ed0af0aad824706bb2a0927b8816553cc968f34929d0a85c81e28239f1fc58154e65659efe82b5c76d9bcddde7a00b728eba68cf296ba733e6e4b7bb569e9611

data/lib/tapsoob/cli/data_stream.rb

@@ -0,0 +1,103 @@
+require 'json'
+require 'pathname'
+require 'thor'
+require 'sequel'
+
+require_relative '../data_stream.rb'
+require_relative '../log'
+require_relative '../operation'
+
+module Tapsoob
+  module CLI
+    class DataStream < Thor
+      desc "pull DATABASE_URL [DUMP_PATH]", "Pull data from a database."
+      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
+      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
+      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
+      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      def pull(database_url, dump_path = nil)
+        op = Tapsoob::Operation.factory(:pull, database_url, dump_path, parse_opts(options))
+        op.pull_data
+      end
+
+      desc "push DATABASE_URL [DUMP_PATH]", "Push data to a database."
+      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
+      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
+      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
+      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      def push(database_url, dump_path = nil)
+        # instantiate stuff
+        data = []
+        opts = parse_opts(options)
+
+        # read data from dump_path or from STDIN
+        if dump_path && Dir.exists?(dump_path)
+          files = Dir[Pathname.new(dump_path).join("*.json")]
+          files.each { |file| data << JSON.parse(File.read(file)) }
+        else
+          STDIN.each_line { |line| data << JSON.parse(line, symbolize_names: true) }
+        end
+
+        # import data
+        data.each do |table|
+          stream = Tapsoob::DataStream.factory(db(database_url, opts), {
+            table_name: table[:table_name],
+            chunksize: opts[:default_chunksize],
+            debug: opts[:debug]
+          })
+
+          begin
+            stream.import_rows(table)
+          rescue Exception => e
+            stream.log.debug e.message
+          end
+        end
+      end
+
+      private
+        def parse_opts(options)
+          # Default options
+          opts = {
+            progress: options[:progress],
+            debug: options[:debug]
+          }
+
+          # Default chunksize
+          if options[:chunksize]
+            opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
+          end
+
+          # Regex filter
+          opts[:table_filter] = options[:filter] if options[:filter]
+
+          # Table filter
+          if options[:tables]
+            r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
+            opts[:table_filter] = "#{r_tables}"
+          end
+
+          # Exclude tables
+          opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
+
+          opts
+        end
+
+        def db(database_url, opts = {})
+          @db ||= Sequel.connect(database_url)
+          @db.loggers << Tapsoob.log if opts[:debug]
+
+          # Set parameters
+          if @db.uri =~ /oracle/i
+            @db << "ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"
+            @db << "ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS:FF6'"
+          end
+
+          @db
+        end
+    end
+  end
+end

data/lib/tapsoob/cli/root.rb

@@ -0,0 +1,130 @@
+require 'thor'
+require 'fileutils'
+
+require_relative '../config'
+require_relative '../log'
+require_relative '../operation'
+require_relative '../version'
+
+module Tapsoob
+  module CLI
+    class Root < Thor
+      desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
+      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
+      option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
+      option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
+      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
+      option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
+      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
+      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
+      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      def pull(dump_path, database_url)
+        opts = parse_opts(options)
+        Tapsoob.log.level = Logger::DEBUG if opts[:debug]
+        if opts[:resume_filename]
+          clientresumexfer(:pull, dump_path, database_url, opts)
+        else
+          clientxfer(:pull, dump_path, database_url, opts)
+        end
+      end
+
+      desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
+      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
+      option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
+      option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
+      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
+      option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
+      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
+      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
+      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      def push(dump_path, database_url)
+        opts = parse_opts(options)
+        Tapsoob.log.level = Logger::DEBUG if opts[:debug]
+        if opts[:resume_filename]
+          clientresumexfer(:push, dump_path, database_url, opts)
+        else
+          clientxfer(:push, dump_path, database_url, opts)
+        end
+      end
+
+      desc "version", "Show tapsoob version"
+      def version
+        puts Tapsoob::VERSION.dup
+      end
+
+      desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
+      subcommand "schema", Schema
+
+      desc "data SUBCOMMAND ...ARGS", "Pull/Push data with internal Tapsoob classes"
+      subcommand "data", DataStream
+
+      private
+        def parse_opts(options)
+          # Default options
+          opts = {
+            skip_schema: options[:"skip-schema"],
+            indexes_first: options[:"indexes_first"],
+            disable_compression: options[:"disable-compression"],
+            debug: options[:debug]
+          }
+
+          # Resume
+          if options[:resume]
+            if File.exists?(options[:resume])
+              opts[:resume_file] = options[:resume]
+            else
+              raise "Unable to find resume file."
+            end
+          end
+
+          # Default chunksize
+          if options[:chunksize]
+            opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
+          end
+
+          # Regex filter
+          opts[:table_filter] = options[:filter] if options[:filter]
+
+          # Table filter
+          if options[:tables]
+            r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
+            opts[:table_filter] = "#{r_tables}"
+          end
+
+          # Exclude tables
+          opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
+
+          opts
+        end
+
+        def clientxfer(method, dump_path, database_url, opts)
+          Tapsoob::Config.verify_database_url(database_url)
+
+          FileUtils.mkpath "#{dump_path}/schemas"
+          FileUtils.mkpath "#{dump_path}/data"
+          FileUtils.mkpath "#{dump_path}/indexes"
+
+          Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
+        end
+
+        def clientresumexfer(method, dump_path, database_url, opts)
+          session = JSON.parse(File.read(opts.delete(:resume_filename)))
+          session.symbolize_recursively!
+
+          dump_path = dump_path || session.delete(:dump_path)
+
+          require 'taps/operation'
+
+          newsession = session.merge({
+            :default_chunksize => opts[:default_chunksize],
+            :disable_compression => opts[:disable_compression],
+            :resume => true
+          })
+
+          Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
+        end
+    end
+  end
+end

data/lib/tapsoob/cli/schema.rb

@@ -0,0 +1,73 @@
+require 'thor'
+require 'sequel'
+
+require_relative '../schema'
+
+module Tapsoob
+  module CLI
+    class Schema < Thor
+      desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
+      def console(database_url)
+        $db = Sequel.connect(database_url)
+        require 'irb'
+        require 'irb/completion'
+        IRB.start
+      end
+
+      desc "dump DATABASE_URL", "Dump a database using a database URL"
+      def dump(database_url)
+        puts Tapsoob::Schema.dump(database_url)
+      end
+
+      desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
+      def dump_table(database_url, table)
+        puts Tapsoob::Schema.dump_table(database_url, table)
+      end
+
+      desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
+      def indexes(database_url)
+        puts Tapsoob::Schema.indexes(database_url)
+      end
+
+      desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
+      def indexes_individual(database_url)
+        puts Tapsoob::Schema.indexes_individual(database_url)
+      end
+
+      desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
+      def reset_db_sequences(database_url)
+        Tapsoob::Schema.reset_db_sequences(database_url)
+      end
+
+      desc "load DATABASE_URL [FILENAME]", "Load a database schema from a file to a database using a database URL"
+      def load(database_url, filename = nil)
+        schema = if filename && File.exists?(filename)
+          File.read(filename)
+        else
+          STDIN.read
+        end
+
+        begin
+          Tapsoob::Schema.load(database_url, schema)
+        rescue Exception => e
+          throw e
+        end
+      end
+
+      desc "load_indexes DATABASE_URL [FILENAME]", "Load indexes from a file to a database using a database URL"
+      def load_indexes(database_url, filename = nil)
+        indexes = if filename && File.exists?(filename)
+          File.read(filename)
+        else
+          STDIN.read
+        end
+
+        begin
+          Tapsoob::Schema.load_indexes(database_url, indexes)
+        rescue Exception => e
+          throw e
+        end
+      end
+    end
+  end
+end

data/lib/tapsoob/cli.rb

@@ -1,17 +1,8 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'fileutils'
-require 'sequel'
-require 'tempfile'
 require 'thor'
+require 'tempfile'
 
 # tapsoob deps
-require 'tapsoob/config'
-require 'tapsoob/log'
-require 'tapsoob/operation'
-require 'tapsoob/schema'
-require 'tapsoob/version'
+require_relative 'config'
 
 Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
   # this is dirty but it solves a weird problem where the tempfile disappears mid-process
@@ -21,172 +12,7 @@ Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
   "sqlite://#{$__taps_database.path}"
 end
 
-module Tapsoob
-  module CLI
-    class Schema < Thor
-      desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
-      def console(database_url)
-        $db = Sequel.connect(database_url)
-        require 'irb'
-        require 'irb/completion'
-        IRB.start
-      end
-
-      desc "dump DATABASE_URL", "Dump a database using a database URL"
-      def dump(database_url)
-        puts Tapsoob::Schema.dump(database_url)
-      end
-
-      desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
-      def dump_table(database_url, table)
-        puts Tapsoob::Schema.dump_table(database_url, table)
-      end
-
-      desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
-      def indexes(database_url)
-        puts Tapsoob::Schema.indexes(database_url)
-      end
-
-      desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
-      def indexes_individual(database_url)
-        puts Tapsoob::Schema.indexes_individual(database_url)
-      end
-
-      desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
-      def reset_db_sequences(database_url)
-        Tapsoob::Schema.reset_db_sequences(database_url)
-      end
-
-      desc "load DATABASE_URL FILENAME", "Load a database schema from a file to a database using a database URL"
-      def load(database_url, filename)
-        schema = File.read(filename) rescue help
-        Tapsoob::Schema.load(database_url, schema)
-      end
-
-      desc "load_indexes DATABASE_URL FILENAME", "Load indexes from a file to a database using a database URL"
-      def load_indexes(database_url, filename)
-        indexes = File.read(filename) rescue help
-        Tapsoob::Schema.load_indexes(database_url, indexes)
-      end
-    end
-
-    class Root < Thor
-      desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
-      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
-      option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
-      option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
-      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
-      option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
-      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
-      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
-      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
-      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
-      def pull(dump_path, database_url)
-        opts = parse_opts(options)
-        Tapsoob.log.level = Logger::DEBUG if opts[:debug]
-        if opts[:resume_filename]
-          clientresumexfer(:pull, dump_path, database_url, opts)
-        else
-          clientxfer(:pull, dump_path, database_url, opts)
-        end
-      end
-
-      desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
-      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
-      option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
-      option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
-      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
-      option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
-      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
-      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
-      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
-      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
-      def push(dump_path, database_url)
-        opts = parse_opts(options)
-        Tapsoob.log.level = Logger::DEBUG if opts[:debug]
-        if opts[:resume_filename]
-          clientresumexfer(:push, dump_path, database_url, opts)
-        else
-          clientxfer(:push, dump_path, database_url, opts)
-        end
-      end
-
-      desc "version", "Show tapsoob version"
-      def version
-        puts Tapsoob::VERSION.dup
-      end
-
-      desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
-      subcommand "schema", Schema
-
-      private
-        def parse_opts(options)
-          # Default options
-          opts = {
-            skip_schema: options[:"skip-schema"],
-            indexes_first: options[:"indexes_first"],
-            disable_compression: options[:"disable-compression"],
-            debug: options[:debug]
-          }
-
-          # Resume
-          if options[:resume]
-            if File.exists?(options[:resume])
-              opts[:resume_file] = options[:resume]
-            else
-              raise "Unable to find resume file."
-            end
-          end
-
-          # Default chunksize
-          if options[:chunksize]
-            opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
-          end
-
-          # Regex filter
-          opts[:table_filter] = options[:filter] if options[:filter]
-
-          # Table filter
-          if options[:tables]
-            r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
-            opts[:table_filter] = "#{r_tables}"
-          end
-
-          # Exclude tables
-          opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
-
-          opts
-        end
-
-        def clientxfer(method, dump_path, database_url, opts)
-          Tapsoob::Config.verify_database_url(database_url)
-
-          FileUtils.mkpath "#{dump_path}/schemas"
-          FileUtils.mkpath "#{dump_path}/data"
-          FileUtils.mkpath "#{dump_path}/indexes"
-
-          require 'tapsoob/operation'
-
-          Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
-        end
-
-        def clientresumexfer(method, dump_path, database_url, opts)
-          session = JSON.parse(File.read(opts.delete(:resume_filename)))
-          session.symbolize_recursively!
-
-          dump_path = dump_path || session.delete(:dump_path)
-
-          require 'taps/operation'
-
-          newsession = session.merge({
-            :default_chunksize => opts[:default_chunksize],
-            :disable_compression => opts[:disable_compression],
-            :resume => true
-          })
-
-          Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
-        end
-    end
-  end
-end
-
+# CLI modules
+require_relative 'cli/schema'
+require_relative 'cli/data_stream'
+require_relative 'cli/root'

data/lib/tapsoob/data_stream.rb

@@ -21,6 +21,7 @@ module Tapsoob
     end
 
     def log
+      Tapsoob.log.level = Logger::DEBUG if state[:debug]
       Tapsoob.log
     end
 
@@ -87,8 +88,9 @@ module Tapsoob
       ds = JSON.parse(File.read(File.join(dump_path, "data", "#{table_name}.json")))
       log.debug "DataStream#fetch_file"
       rows = {
-        :header => ds["header"],
-        :data   => ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]
+        :table_name => ds["table_name"],
+        :header     => ds["header"],
+        :data       => ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]
       }
       update_chunksize_stats
       rows
@@ -141,7 +143,7 @@ module Tapsoob
       @complete
     end
 
-    def fetch_database(dump_path)
+    def fetch_database
       params = fetch_from_database
       encoded_data = params[:encoded_data]
       json = params[:json]
@@ -154,7 +156,7 @@ module Tapsoob
       state.merge!(json[:state].merge(:chunksize => state[:chunksize]))
 
       unless @complete
-        Tapsoob::Utils.export_rows(dump_path, table_name, rows)
+        yield rows if block_given?
         state[:offset] += rows[:data].size
         rows[:data].size
       else

data/lib/tapsoob/log.rb

@@ -7,8 +7,8 @@ module Tapsoob
   def self.log
     @@log ||= begin
       require 'logger'
-      log = Logger.new($stderr)
-      log.level = Logger::ERROR
+      log = Logger.new(STDERR)
+      log.level = Logger::INFO
       log.datetime_format = "%Y-%m-%d %H:%M:%S"
       log
     end

data/lib/tapsoob/operation.rb

@@ -10,7 +10,7 @@ module Tapsoob
   class Operation
     attr_reader :database_url, :dump_path, :opts
 
-    def initialize(database_url, dump_path, opts={})
+    def initialize(database_url, dump_path = nil, opts={})
       @database_url = database_url
       @dump_path    = dump_path
       @opts         = opts
@@ -55,12 +55,13 @@ module Tapsoob
     end
 
     def log
+      Tapsoob.log.level = Logger::DEBUG if opts[:debug]
       Tapsoob.log
     end
 
     def store_session
       file = "#{file_prefix}_#{Time.now.strftime("%Y%m%d%H%M")}.dat"
-      puts "\nSaving session to #{file}..."
+      log.info "\nSaving session to #{file}..."
       File.open(file, 'w') do |f|
         f.write(JSON.generate(to_hash))
       end
@@ -174,7 +175,7 @@ module Tapsoob
     end
 
     def pull_schema
-      puts "Receiving schema"
+      log.info "Receiving schema"
 
       progress = ProgressBar.new('Schema', tables.size)
       tables.each do |table_name, count|
@@ -188,12 +189,12 @@ module Tapsoob
     end
 
     def pull_data
-      puts "Receiving data"
+      log.info "Receiving data"
 
-      puts "#{tables.size} tables, #{format_number(record_count)} records"
+      log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
       tables.each do |table_name, count|
-        progress = ProgressBar.new(table_name.to_s, count)
+        progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, count) : nil)
         stream   = Tapsoob::DataStream.factory(db, {
           :chunksize  => default_chunksize,
           :table_name => table_name
@@ -207,9 +208,9 @@ module Tapsoob
 
       table_name = stream_state[:table_name]
       record_count = tables[table_name.to_s]
-      puts "Resuming #{table_name}, #{format_number(record_count)} records"
+      log.info "Resuming #{table_name}, #{format_number(record_count)} records"
 
-      progress = ProgressBar.new(table_name.to_s, record_count)
+      progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, record_count) : nil)
       stream = Tapsoob::DataStream.factory(db, stream_state)
       pull_data_from_table(stream, progress)
     end
@@ -219,19 +220,25 @@ module Tapsoob
         begin
           exit 0 if exiting?
 
-          size = stream.fetch_database(dump_path)
+          size = stream.fetch_database do |rows|
+            if dump_path.nil?
+              puts JSON.generate(rows)
+            else
+              Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
+            end
+          end
           break if stream.complete?
-          progress.inc(size) unless exiting?
+          progress.inc(size) if progress && !exiting?
           stream.error = false
           self.stream_state = stream.to_hash
         rescue Tapsoob::CorruptedData => e
-          puts "Corrupted Data Received #{e.message}, retrying..."
+          log.info "Corrupted Data Received #{e.message}, retrying..."
           stream.error = true
           next
         end
       end
 
-      progress.finish
+      progress.finish if progress
       completed_tables << stream.table_name.to_s
       self.stream_state = {}
     end
@@ -280,7 +287,7 @@ module Tapsoob
     end
 
     def pull_indexes
-      puts "Receiving indexes"
+      log.info "Receiving indexes"
 
       raw_idxs = Tapsoob::Utils.schema_bin(:indexes_individual, database_url)
       idxs     = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
@@ -298,7 +305,7 @@ module Tapsoob
     end
 
     def pull_reset_sequences
-      puts "Resetting sequences"
+      log.info "Resetting sequences"
 
       output = Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
       puts output if output
@@ -337,7 +344,7 @@ module Tapsoob
 
       return unless idxs.size > 0
 
-      puts "Sending indexes"
+      log.info "Sending indexes"
 
       apply_table_filter(idxs).each do |table, indexes|
         next unless indexes.size > 0
@@ -351,7 +358,7 @@ module Tapsoob
     end
 
     def push_schema
-      puts "Sending schema"
+      log.info "Sending schema"
 
       progress = ProgressBar.new('Schema', tables.size)
       tables.each do |table, count|
@@ -363,7 +370,7 @@ module Tapsoob
     end
 
     def push_reset_sequences
-      puts "Resetting sequences"
+      log.info "Resetting sequences"
 
       Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
     end
@@ -373,16 +380,16 @@ module Tapsoob
 
       table_name = stream_state[:table_name]
       record_count = tables[table_name.to_s]
-      puts "Resuming #{table_name}, #{format_number(record_count)} records"
+      log.info "Resuming #{table_name}, #{format_number(record_count)} records"
       progress = ProgressBar.new(table_name.to_s, record_count)
       stream = Tapsoob::DataStream.factory(db, stream_state)
       push_data_from_file(stream, progress)
     end
 
     def push_data
-      puts "Sending data"
+      log.info "Sending data"
 
-      puts "#{tables.size} tables, #{format_number(record_count)} records"
+      log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
       tables.each do |table_name, count|
         next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))

data/lib/tapsoob/schema.rb

@@ -67,7 +67,7 @@ END_MIG
       Sequel.connect(database_url) do |db|
         db.extension :schema_dumper
         klass = eval(schema)
-        klass.apply(db, :down)
+        #klass.apply(db, :down)
         klass.apply(db, :up)
       end
     end

data/lib/tapsoob/utils.rb

@@ -70,7 +70,7 @@ Data : #{data}
         end
         header.collect { |h| row[h] }
       end
-      { :header => header, :data => only_data }
+      { table_name: table, header: header, data: only_data }
     end
 
     # mysql text and blobs fields are handled the same way internally

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.2.11".freeze
+  VERSION = "0.3.2".freeze
 end

data/tapsoob.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Dependencies
-  s.add_dependency "sequel", "~> 5.19.0"
+  s.add_dependency "sequel", "~> 5.21.0"
   s.add_dependency "thor", "~> 0.20.3"
 
   if (RUBY_PLATFORM =~ /java/).nil?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.3.2
 platform: ruby
 authors:
 - Félix Bellanger
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-11 00:00:00.000000000 Z
+date: 2019-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +98,9 @@ files:
 - lib/tapsoob.rb
 - lib/tapsoob/chunksize.rb
 - lib/tapsoob/cli.rb
+- lib/tapsoob/cli/data_stream.rb
+- lib/tapsoob/cli/root.rb
+- lib/tapsoob/cli/schema.rb
 - lib/tapsoob/config.rb
 - lib/tapsoob/data_stream.rb
 - lib/tapsoob/errors.rb