tapsoob 0.1.10

data/lib/tapsoob/progress_bar.rb

@@ -0,0 +1,236 @@
+# -*- encoding : utf-8 -*-
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|",
+            @bar_mark * bar_width,
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols > 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end

data/lib/tapsoob/railtie.rb

@@ -0,0 +1,11 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob'
+require 'rails'
+
+module Tapsoob
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/tapsoob.rake"
+    end
+  end
+end

data/lib/tapsoob/schema.rb

@@ -0,0 +1,83 @@
+# -*- encoding : utf-8 -*-
+require 'sequel'
+require 'sequel/extensions/schema_dumper'
+require 'sequel/extensions/migration'
+require 'json'
+
+module Tapsoob
+  module Schema
+    extend self
+
+    def dump(database_url)
+      db = Sequel.connect(database_url)
+      db.dump_schema_migration(:indexes => false)
+    end
+
+    def dump_table(database_url, table)
+      table = table.to_sym
+      Sequel.connect(database_url) do |db|
+        <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{db.dump_table_schema(table.identifier, :indexes => false)}
+  end
+
+  def down
+    drop_table("#{table}") if @db.table_exists?("#{table}")
+  end
+end
+END_MIG
+      end
+    end
+
+    def indexes(database_url)
+      db = Sequel.connect(database_url)
+      db.dump_indexes_migration
+    end
+
+    def indexes_individual(database_url)
+      idxs = {}
+      Sequel.connect(database_url) do |db|
+        tables = db.tables
+        tables.each do |table|
+          idxs[table] = db.send(:dump_table_indexes, table, :add_index, {}).split("\n")
+        end
+      end
+
+      idxs.each do |table, indexes|
+        idxs[table] = indexes.map do |idx|
+          <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{idx}
+  end
+end
+END_MIG
+        end
+      end
+      JSON.generate(idxs)
+    end
+
+    def load(database_url, schema)
+      Sequel.connect(database_url) do |db|
+        klass = eval(schema)
+        klass.apply(db, :down)
+        klass.apply(db, :up)
+      end
+    end
+
+    def load_indexes(database_url, indexes)
+      Sequel.connect(database_url) do |db|
+        eval(indexes).apply(db, :up)
+      end
+    end
+
+    def reset_db_sequences(database_url)
+      db = Sequel.connect(database_url)
+      return unless db.respond_to?(:reset_primary_key_sequence)
+      db.tables.each do |table|
+        db.reset_primary_key_sequence(table)
+      end
+    end
+  end
+end

data/lib/tapsoob/utils.rb

@@ -0,0 +1,179 @@
+# -*- encoding : utf-8 -*-
+require 'zlib'
+
+require 'tapsoob/errors'
+require 'tapsoob/chunksize'
+require 'tapsoob/schema'
+
+module Tapsoob
+  module Utils
+    extend self
+
+    def windows?
+      return @windows if defined?(@windows)
+      require 'rbconfig'
+      @windows = !!(::RbConfig::CONFIG['host_os'] =~ /mswin|mingw/)
+    end
+
+    def bin(cmd)
+      cmd = "#{cmd}.cmd" if windows?
+      cmd
+    end
+
+    def checksum(data)
+      Zlib.crc32(data)
+    end
+
+    def valid_data?(data, crc32)
+      Zlib.crc32(data) == crc32.to_i
+    end
+
+    def base64encode(data)
+      [data].pack("m")
+    end
+
+    def base64decode(data)
+      data.unpack("m").first
+    end
+
+    def format_data(data, opts = {})
+      return {} if data.size == 0
+      string_columns = opts[:string_columns] || []
+      schema = opts[:schema] || []
+      table = opts[:table]
+
+      max_lengths = schema.inject({}) do |hash, (column, meta)|
+        if meta[:db_type] =~ /^varchar\((\d+)\)/
+          hash.update(column => $1.to_i)
+        end
+        hash
+      end
+
+      header = data[0].keys
+      only_data = data.collect do |row|
+        row = blobs_to_string(row, string_columns)
+        row.each do |column, data|
+          if data.to_s.length > (max_lengths[column] || data.to_s.length)
+            raise Tapsoob::InvalidData.new(<<-ERROR)
+Detected data that exceeds the length limitation of its column. This is
+generally due to the fact that SQLite does not enforce length restrictions.
+
+Table : #{table}
+Column : #{column}
+Type : #{schema.detect{|s| s.first == column}.last[:db_type]}
+Data : #{data}
+            ERROR
+          end
+        end
+        header.collect { |h| row[h] }
+      end
+      { :header => header, :data => only_data }
+    end
+
+    # mysql text and blobs fields are handled the same way internally
+    # this is not true for other databases so we must check if the field is
+    # actually text and manually convert it back to a string
+    def incorrect_blobs(db, table)
+      return [] if (db.url =~ /mysql:\/\//).nil?
+
+      columns = []
+      db.schema(table).each do |data|
+        column, cdata = data
+        columns << column if cdata[:db_type] =~ /text/
+      end
+      columns
+    end
+
+    def blobs_to_string(row, columns)
+      return row if columns.size == 0
+      columns.each do |c|
+        row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
+      end
+      row
+    end
+
+    def calculate_chunksize(old_chunksize)
+      c = Tapsoob::Chunksize.new(old_chunksize)
+
+      begin
+        c.start_time = Time.now
+        c.time_in_db = yield c
+      rescue Errno::EPIPE
+        c.retries += 1
+        raise if c.retries > 2
+
+        # we got disconnected, the chunksize could be too large
+        # reset the chunksize based on the number of retries
+        c.reset_chunksize
+        retry
+      end
+
+      c.end_time = Time.now
+      c.calc_new_chunksize
+    end
+
+    def export_schema(dump_path, table, schema_data)
+      File.open(File.join(dump_path, "schemas", "#{table}.rb"), 'w') do |file|
+        file.write(schema_data)
+      end
+    end
+
+    def export_indexes(dump_path, table, index_data)
+      data = [index_data]
+      if File.exists?(File.join(dump_path, "indexes", "#{table}.json"))
+        previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
+        data = data + previous_data
+      end
+
+      File.open(File.join(dump_path, "indexes", "#{table}.json"), 'w') do |file|
+        file.write(JSON.generate(data))
+      end
+    end
+
+    def export_rows(dump_path, table, row_data)
+      data = row_data
+      if File.exists?(File.join(dump_path, "data", "#{table}.json"))
+        previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
+        data[:data] = previous_data["data"] + row_data[:data]
+      end
+
+      File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
+        file.write(JSON.generate(data))
+      end
+    end
+
+    def load_schema(dump_path, database_url, table)
+      schema = File.join(dump_path, "schemas", "#{table}.rb")
+      schema_bin(:load, database_url, schema.to_s)
+    end
+
+    def load_indexes(database_url, index)
+      Tapsoob::Schema.load_indexes(database_url, index)
+    end
+
+    def schema_bin(*args)
+      bin_path = File.expand_path("#{File.dirname(__FILE__)}/../../bin/#{bin('schema')}")
+      `"#{bin_path}" #{args.map { |a| "'#{a}'" }.join(' ')}`
+    end
+
+    def primary_key(db, table)
+      db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c[0] }
+    end
+
+    def single_integer_primary_key(db, table)
+      table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
+      keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
+      not keys.nil? and keys.size == 1
+    end
+
+    def order_by(db, table)
+      pkey = primary_key(db, table)
+      if pkey
+        pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
+      else
+        table = table.to_sym.identifier unless table.kind_of?(Sequel::SQL::Identifier)
+        db[table].columns
+      end
+    end
+  end
+end