tapsoob 0.2.7-java

data/lib/tapsoob/operation.rb

@@ -0,0 +1,482 @@
+# -*- encoding : utf-8 -*-
+require 'sequel'
+
+require 'tapsoob/progress_bar'
+require 'tapsoob/schema'
+require 'tapsoob/data_stream'
+
+module Tapsoob
+  class Operation
+    attr_reader :database_url, :dump_path, :opts
+
+    def initialize(database_url, dump_path, opts={})
+      @database_url = database_url
+      @dump_path    = dump_path
+      @opts         = opts
+      @exiting      = false
+    end
+
+    def file_prefix
+      "op"
+    end
+
+    def skip_schema?
+      !!opts[:skip_schema]
+    end
+
+    def indexes_first?
+      !!opts[:indexes_first]
+    end
+
+    def table_filter
+      opts[:table_filter]
+    end
+
+    def exclude_tables
+      opts[:exclude_tables] || []
+    end
+
+    def apply_table_filter(tables)
+      return tables unless table_filter || exclude_tables
+
+      re = table_filter ? Regexp.new(table_filter) : nil
+      if tables.kind_of?(Hash)
+        ntables = {}
+        tables.each do |t, d|
+          if !exclude_tables.include?(t.to_s) && (!re || !re.match(t.to_s).nil?)
+            ntables[t] = d
+          end
+        end
+        ntables
+      else
+        tables.reject { |t| exclude_tables.include?(t.to_s) || (re && re.match(t.to_s).nil?) }
+      end
+    end
+
+    def log
+      Tapsoob.log
+    end
+
+    def store_session
+      file = "#{file_prefix}_#{Time.now.strftime("%Y%m%d%H%M")}.dat"
+      puts "\nSaving session to #{file}..."
+      File.open(file, 'w') do |f|
+        f.write(JSON.generate(to_hash))
+      end
+    end
+
+    def to_hash
+      {
+        :klass            => self.class.to_s,
+        :database_url     => database_url,
+        :stream_state     => stream_state,
+        :completed_tables => completed_tables,
+        :table_filter     => table_filter,
+      }
+    end
+
+    def exiting?
+      !!@exiting
+    end
+
+    def setup_signal_trap
+      trap("INT") {
+        puts "\nCompleting current action..."
+        @exiting = true
+      }
+
+      trap("TERM") {
+        puts "\nCompleting current action..."
+        @exiting = true
+      }
+    end
+
+    def resuming?
+      opts[:resume] == true
+    end
+
+    def default_chunksize
+      opts[:default_chunksize]
+    end
+
+    def completed_tables
+      opts[:completed_tables] ||= []
+    end
+
+    def stream_state
+      opts[:stream_state] ||= {}
+    end
+
+    def stream_state=(val)
+      opts[:stream_state] = val
+    end
+
+    def db
+      @db ||= Sequel.connect(database_url)
+
+      # Set parameters
+      if @db.uri =~ /oracle/i
+        @db << "ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH24:MI:SS'"
+        @db << "ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MM-DD HH24:MI:SS:FF6'"
+      end
+
+      @db
+    end
+
+    def format_number(num)
+      num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
+    end
+
+    def catch_errors(&blk)
+      begin
+        blk.call
+      rescue Exception => e
+        raise e
+      end
+    end
+
+    def self.factory(type, database_url, dump_path, opts)
+      type = :resume if opts[:resume]
+      klass = case type
+        when :pull   then Tapsoob::Pull
+        when :push   then Tapsoob::Push
+        when :resume then eval(opts[:klass])
+        else raise "Unknown Operation Type -> #{type}"
+      end
+
+      klass.new(database_url, dump_path, opts)
+    end
+  end
+
+  class Pull < Operation
+    def file_prefix
+      "pull"
+    end
+
+    def to_hash
+      super.merge(:remote_tables_info => remote_tables_info)
+    end
+
+    def run
+      catch_errors do
+        unless resuming?
+          pull_schema if !skip_schema?
+          pull_indexes if indexes_first? && !skip_schema?
+        end
+        setup_signal_trap
+        pull_partial_data if resuming?
+        pull_data
+        pull_indexes if !indexes_first? && !skip_schema?
+        pull_reset_sequences
+      end
+    end
+
+    def pull_schema
+      puts "Receiving schema"
+
+      progress = ProgressBar.new('Schema', tables.size)
+      tables.each do |table_name, count|
+        schema_data = Tapsoob::Schema.dump_table(database_url, table_name)
+        log.debug "Table: #{table_name}\n#{schema_data}\n"
+        output = Tapsoob::Utils.export_schema(dump_path, table_name, schema_data)
+        puts output if output
+        progress.inc(1)
+      end
+      progress.finish
+    end
+
+    def pull_data
+      puts "Receiving data"
+
+      puts "#{tables.size} tables, #{format_number(record_count)} records"
+
+      tables.each do |table_name, count|
+        progress = ProgressBar.new(table_name.to_s, count)
+        stream   = Tapsoob::DataStream.factory(db, {
+          :chunksize  => default_chunksize,
+          :table_name => table_name
+        })
+        pull_data_from_table(stream, progress)
+      end
+    end
+
+    def pull_partial_data
+      return if stream_state == {}
+
+      table_name = stream_state[:table_name]
+      record_count = tables[table_name.to_s]
+      puts "Resuming #{table_name}, #{format_number(record_count)} records"
+
+      progress = ProgressBar.new(table_name.to_s, record_count)
+      stream = Tapsoob::DataStream.factory(db, stream_state)
+      pull_data_from_table(stream, progress)
+    end
+
+    def pull_data_from_table(stream, progress)
+      loop do
+        begin
+          exit 0 if exiting?
+
+          size = stream.fetch_database(dump_path)
+          break if stream.complete?
+          progress.inc(size) unless exiting?
+          stream.error = false
+          self.stream_state = stream.to_hash
+        rescue Tapsoob::CorruptedData => e
+          puts "Corrupted Data Received #{e.message}, retrying..."
+          stream.error = true
+          next
+        end
+      end
+
+      progress.finish
+      completed_tables << stream.table_name.to_s
+      self.stream_state = {}
+    end
+
+    def tables
+      h = {}
+      tables_info.each do |table_name, count|
+        next if completed_tables.include?(table_name.to_s)
+        h[table_name.to_s] = count
+      end
+      h
+    end
+
+    def record_count
+      tables_info.values.inject(:+)
+    end
+
+    def tables_info
+      opts[:tables_info] ||= fetch_tables_info
+    end
+
+    def fetch_tables_info
+      tables = db.tables
+
+      data = {}
+      apply_table_filter(tables).each do |table_name|
+        data[table_name] = db[table_name].count
+      end
+      data
+    end
+
+    def self.factory(db, state)
+      if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
+        Sequel::MySQL.convert_invalid_date_time = :nil
+      end
+
+      if state.has_key?(:klass)
+        return eval(state[:klass]).new(db, state)
+      end
+
+      if Tapsoob::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
+        DataStreamKeyed.new(db, state)
+      else
+        DataStream.new(db, state)
+      end
+    end
+
+    def pull_indexes
+      puts "Receiving indexes"
+
+      raw_idxs = Tapsoob::Utils.schema_bin(:indexes_individual, database_url)
+      idxs     = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
+
+      apply_table_filter(idxs).each do |table, indexes|
+        next unless indexes.size > 0
+        progress = ProgressBar.new(table, indexes.size)
+        indexes.each do |idx|
+          output = Tapsoob::Utils.export_indexes(dump_path, table, idx)
+          puts output if output
+          progress.inc(1)
+        end
+        progress.finish
+      end
+    end
+
+    def pull_reset_sequences
+      puts "Resetting sequences"
+
+      output = Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
+      puts output if output
+    end
+  end
+
+  class Push < Operation
+    def file_prefix
+      "push"
+    end
+
+    def to_hash
+      super.merge(:local_tables_info => local_tables_info)
+    end
+
+    def run
+      catch_errors do
+        unless resuming?
+          push_schema if !skip_schema?
+          push_indexes if indexes_first? && !skip_schema?
+        end
+        setup_signal_trap
+        push_partial_data if resuming?
+        push_data
+        push_indexes if !indexes_first? && !skip_schema?
+        push_reset_sequences
+      end
+    end
+
+    def push_indexes
+      idxs = {}
+      table_idxs = Dir.glob(File.join(dump_path, "indexes", "*.json")).map { |path| File.basename(path, '.json') }
+      table_idxs.each do |table_idx|
+        idxs[table_idx] = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table_idx}.json")))
+      end
+
+      return unless idxs.size > 0
+
+      puts "Sending indexes"
+
+      apply_table_filter(idxs).each do |table, indexes|
+        next unless indexes.size > 0
+        progress = ProgressBar.new(table, indexes.size)
+        indexes.each do |idx|
+          Tapsoob::Utils.load_indexes(database_url, idx)
+          progress.inc(1)
+        end
+        progress.finish
+      end
+    end
+
+    def push_schema
+      puts "Sending schema"
+
+      progress = ProgressBar.new('Schema', tables.size)
+      tables.each do |table, count|
+        log.debug "Loading '#{table}' schema\n"
+        Tapsoob::Utils.load_schema(dump_path, database_url, table)
+        progress.inc(1)
+      end
+      progress.finish
+    end
+
+    def push_reset_sequences
+      puts "Resetting sequences"
+
+      Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
+    end
+
+    def push_partial_data
+      return if stream_state == {}
+
+      table_name = stream_state[:table_name]
+      record_count = tables[table_name.to_s]
+      puts "Resuming #{table_name}, #{format_number(record_count)} records"
+      progress = ProgressBar.new(table_name.to_s, record_count)
+      stream = Tapsoob::DataStream.factory(db, stream_state)
+      push_data_from_file(stream, progress)
+    end
+
+    def push_data
+      puts "Sending data"
+
+      puts "#{tables.size} tables, #{format_number(record_count)} records"
+
+      tables.each do |table_name, count|
+        next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
+        stream = Tapsoob::DataStream.factory(db,
+          :table_name => table_name,
+          :chunksize => default_chunksize)
+        progress = ProgressBar.new(table_name.to_s, count)
+        push_data_from_file(stream, progress)
+      end
+    end
+
+    def push_data_from_file(stream, progress)
+      loop do
+        if exiting?
+          store_session
+          exit 0
+        end
+
+        row_size = 0
+        chunksize = stream.state[:chunksize]
+
+        begin
+          chunksize = Tapsoob::Utils.calculate_chunksize(chunksize) do |c|
+            stream.state[:chunksize] = c.to_i
+            encoded_data, row_size, elapsed_time = nil
+            d1 = c.time_delta do
+              encoded_data, row_size, elapsed_time = stream.fetch({ :type => "file", :source => dump_path })
+            end
+            break if stream.complete?
+
+            data = nil
+            d2 = c.time_delta do
+              data = {
+                :state    => stream.to_hash,
+                :checksum => Tapsoob::Utils.checksum(encoded_data).to_s
+              }
+            end
+
+            size = stream.fetch_data_in_database({ :encoded_data => encoded_data, :checksum => data[:checksum] })
+            self.stream_state = stream.to_hash
+
+            c.idle_secs = (d1 + d2)
+
+            elapsed_time
+          end
+        rescue Tapsoob::CorruptedData => e
+          # retry the same data, it got corrupted somehow.
+          next
+        rescue Tapsoob::DuplicatePrimaryKeyError => e
+          # verify the stream and retry it
+          stream.verify_stream
+          stream = JSON.generate({ :state => stream.to_hash })
+          next
+        end
+        stream.state[:chunksize] = chunksize
+
+        progress.inc(row_size)
+
+        stream.increment(row_size)
+        break if stream.complete?
+      end
+
+      progress.finish
+      completed_tables << stream.table_name.to_s
+      self.stream_state = {}
+    end
+
+    def local_tables_info
+      opts[:local_tables_info] ||= fetch_local_tables_info
+    end
+
+    def tables
+      h = {}
+      local_tables_info.each do |table_name, count|
+        next if completed_tables.include?(table_name.to_s)
+        h[table_name.to_s] = count
+      end
+      h
+    end
+
+    def record_count
+      @record_count ||= local_tables_info.values.inject(0) { |a,c| a += c }
+    end
+
+    def fetch_local_tables_info
+      tables_with_counts = {}
+      tbls = Dir.glob(File.join(dump_path, "schemas", "*")).map { |path| File.basename(path, ".rb") }
+      tbls.each do |table|
+        if File.exists?(File.join(dump_path, "data", "#{table}.json"))
+          data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
+          tables_with_counts[table] = data.size
+        else
+          tables_with_counts[table] = 0
+        end
+      end
+      apply_table_filter(tables_with_counts)
+    end
+  end
+end

data/lib/tapsoob/progress_bar.rb

@@ -0,0 +1,236 @@
+# -*- encoding : utf-8 -*-
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDOUT)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|",
+            @bar_mark * bar_width,
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols > 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method|
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage ||
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end

data/lib/tapsoob/railtie.rb

@@ -0,0 +1,11 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob'
+require 'rails'
+
+module Tapsoob
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/tapsoob.rake"
+    end
+  end
+end