taps-taps 0.3.24

data/lib/taps/progress_bar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols > 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/taps/schema.rb

@@ -0,0 +1,82 @@
+require 'sequel'
+require 'sequel/extensions/schema_dumper'
+require 'sequel/extensions/migration'
+require 'vendor/okjson'
+
+module Taps
+module Schema
+  extend self
+
+  def dump(database_url)
+    db = Sequel.connect(database_url)
+    db.dump_schema_migration(:indexes => false)
+  end
+
+  def dump_table(database_url, table)
+    table = table.to_sym
+    Sequel.connect(database_url) do |db|
+      <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{db.dump_table_schema(table.identifier, :indexes => false)}
+  end
+
+  def down
+    drop_table("#{table}") if @db.table_exists?("#{table}")
+  end
+end
+END_MIG
+    end
+  end
+
+  def indexes(database_url)
+    db = Sequel.connect(database_url)
+    db.dump_indexes_migration
+  end
+
+  def indexes_individual(database_url)
+    idxs = {}
+    Sequel.connect(database_url) do |db|
+      tables = db.tables
+      tables.each do |table|
+        idxs[table] = db.send(:dump_table_indexes, table, :add_index, {}).split("\n")
+      end
+    end
+
+    idxs.each do |table, indexes|
+      idxs[table] = indexes.map do |idx|
+        <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{idx}
+  end
+end
+END_MIG
+      end
+    end
+    OkJson.encode(idxs)
+  end
+
+  def load(database_url, schema)
+    Sequel.connect(database_url) do |db|
+      klass = eval(schema)
+      klass.apply(db, :down)
+      klass.apply(db, :up)
+    end
+  end
+
+  def load_indexes(database_url, indexes)
+    Sequel.connect(database_url) do |db|
+      eval(indexes).apply(db, :up)
+    end
+  end
+
+  def reset_db_sequences(database_url)
+    db = Sequel.connect(database_url)
+    return unless db.respond_to?(:reset_primary_key_sequence)
+    db.tables.each do |table|
+      db.reset_primary_key_sequence(table)
+    end
+  end
+end
+end

data/lib/taps/server.rb

@@ -0,0 +1,210 @@
+require 'sinatra/base'
+require 'taps/config'
+require 'taps/utils'
+require 'taps/db_session'
+require 'taps/data_stream'
+
+module Taps
+class Server < Sinatra::Base
+  use Rack::Auth::Basic do |login, password|
+    login == Taps::Config.login && password == Taps::Config.password
+  end
+
+  use Rack::Deflater unless ENV['NO_DEFLATE']
+
+  set :raise_errors => false
+  set :show_exceptions => false
+
+  error do
+    e = request.env['sinatra.error']
+    puts "ERROR: #{e.class}: #{e.message}"
+    begin
+      require 'hoptoad_notifier'
+      HoptoadNotifier.configure do |config|
+        config.api_key = ENV["HOPTOAD_API_KEY"]
+      end
+      HoptoadNotifier.notify(e)
+      puts "  notified Hoptoad"
+    rescue LoadError
+      puts "An error occurred but Hoptoad was not notified. To use Hoptoad, please"
+      puts "install the 'hoptoad_notifier' gem and set ENV[\"HOPTOAD_API_KEY\"]"
+    end
+    if e.kind_of?(Taps::BaseError)
+      content_type "application/json"
+      halt 412, ::OkJson.encode({ 'error_class' => e.class.to_s, 'error_message' => e.message, 'error_backtrace' => e.backtrace.join("\n") })
+    else
+      "Taps Server Error: #{e}\n#{e.backtrace}"
+    end
+  end
+
+  before do
+    unless request.path_info == '/health'
+      major, minor, patch = request.env['HTTP_TAPS_VERSION'].split('.') rescue []
+      unless "#{major}.#{minor}" == Taps.compatible_version && patch.to_i >= 23
+        halt 417, "Taps >= v#{Taps.compatible_version}.22 is required for this server"
+      end
+    end
+  end
+
+  get '/health' do
+    content_type 'application/json'
+    ::OkJson.encode({ :ok => true })
+  end
+
+  get '/' do
+    "hello"
+  end
+
+  post '/sessions' do
+    key = rand(9999999999).to_s
+
+    if ENV['NO_DEFAULT_DATABASE_URL']
+      database_url = request.body.string
+    else
+      database_url = Taps::Config.database_url || request.body.string
+    end
+
+    DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
+
+    "/sessions/#{key}"
+  end
+
+  post '/sessions/:key/push/verify_stream' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    state = DataStream.parse_json(params[:state])
+    stream = nil
+
+    size = 0
+    session.conn do |db|
+      Taps::Utils.server_error_handling do
+        stream = DataStream.factory(db, state)
+        stream.verify_stream
+      end
+    end
+
+    content_type 'application/json'
+    ::OkJson.encode({ :state => stream.to_hash })
+  end
+
+  post '/sessions/:key/push/table' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    json = DataStream.parse_json(params[:json])
+
+    size = 0
+    session.conn do |db|
+      Taps::Utils.server_error_handling do
+        stream = DataStream.factory(db, json[:state])
+        size = stream.fetch_remote_in_server(params)
+      end
+    end
+
+    # TODO: return the stream's state with the size
+    size.to_s
+  end
+
+  post '/sessions/:key/push/reset_sequences' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    Taps::Utils.schema_bin(:reset_db_sequences, session.database_url)
+  end
+
+  post '/sessions/:key/push/schema' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    schema_data = request.body.read
+    Taps::Utils.load_schema(session.database_url, schema_data)
+  end
+
+  post '/sessions/:key/push/indexes' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    index_data = request.body.read
+    Taps::Utils.load_indexes(session.database_url, index_data)
+  end
+
+  post '/sessions/:key/pull/schema' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    Taps::Utils.schema_bin(:dump_table, session.database_url, params[:table_name])
+  end
+
+  get '/sessions/:key/pull/indexes' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    content_type 'application/json'
+    Taps::Utils.schema_bin(:indexes_individual, session.database_url)
+  end
+
+  get '/sessions/:key/pull/table_names' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    tables = []
+    session.conn do |db|
+      tables = db.tables
+    end
+
+    content_type 'application/json'
+    ::OkJson.encode(tables)
+  end
+
+  post '/sessions/:key/pull/table_count' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    count = 0
+    session.conn do |db|
+      count = db[ params[:table].to_sym.identifier ].count
+    end
+    count.to_s
+  end
+
+  post '/sessions/:key/pull/table' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    encoded_data = nil
+    stream = nil
+
+    session.conn do |db|
+      state = ::OkJson.decode(params[:state]).symbolize_keys
+      stream = Taps::DataStream.factory(db, state)
+      encoded_data = stream.fetch.first
+    end
+
+    checksum = Taps::Utils.checksum(encoded_data).to_s
+    json = ::OkJson.encode({ :checksum => checksum, :state => stream.to_hash })
+
+    content, content_type_value = Taps::Multipart.create do |r|
+      r.attach :name => :encoded_data,
+        :payload => encoded_data,
+        :content_type => 'application/octet-stream'
+      r.attach :name => :json,
+        :payload => json,
+        :content_type => 'application/json'
+    end
+
+    content_type content_type_value
+    content
+  end
+
+  delete '/sessions/:key' do
+    session = DbSession.filter(:key => params[:key]).first
+    halt 404 unless session
+
+    session.destroy
+
+    "ok"
+  end
+
+end
+end