taps2 0.5.1

data/lib/taps/progress_bar.rb

@@ -0,0 +1,235 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|",
+            @bar_mark * bar_width,
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        _, cols = data.unpack("SSSS")
+        if cols > 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method|
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage ||
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end

data/lib/taps/schema.rb

@@ -0,0 +1,82 @@
+require 'sequel'
+require 'sequel/extensions/schema_dumper'
+require 'sequel/extensions/migration'
+require 'vendor/okjson'
+
+module Taps
+  module Schema
+    extend self
+
+    def dump(database_url)
+      db = Sequel.connect(database_url)
+      db.dump_schema_migration(:indexes => false)
+    end
+
+    def dump_table(database_url, table)
+      table = table.to_sym
+      Sequel.connect(database_url) do |db|
+        <<END_MIG
+  Class.new(Sequel::Migration) do
+    def up
+      #{db.extension(:schema_dumper).dump_table_schema(Sequel.identifier(table), :indexes => false)}
+    end
+
+    def down
+      drop_table("#{table}") if @db.table_exists?("#{table}")
+    end
+  end
+  END_MIG
+      end
+    end
+
+    def indexes(database_url)
+      db = Sequel.connect(database_url)
+      db.dump_indexes_migration
+    end
+
+    def indexes_individual(database_url)
+      idxs = {}
+      Sequel.connect(database_url) do |db|
+        tables = db.tables
+        tables.each do |table|
+          idxs[table] = db.extension(:schema_dumper).send(:dump_table_indexes, table, :add_index, {}).split("\n")
+        end
+      end
+
+      idxs.each do |table, indexes|
+        idxs[table] = indexes.map do |idx|
+          <<END_MIG
+  Class.new(Sequel::Migration) do
+    def up
+      #{idx}
+    end
+  end
+  END_MIG
+        end
+      end
+      ::OkJson.encode(idxs)
+    end
+
+    def load(database_url, schema)
+      Sequel.connect(database_url) do |db|
+        klass = eval(schema)
+        klass.apply(db, :down)
+        klass.apply(db, :up)
+      end
+    end
+
+    def load_indexes(database_url, indexes)
+      Sequel.connect(database_url) do |db|
+        eval(indexes).apply(db, :up)
+      end
+    end
+
+    def reset_db_sequences(database_url)
+      db = Sequel.connect(database_url)
+      return unless db.respond_to?(:reset_primary_key_sequence)
+      db.tables.each do |table|
+        db.reset_primary_key_sequence(table)
+      end
+    end
+  end
+end

data/lib/taps/server.rb

@@ -0,0 +1,200 @@
+require 'sinatra/base'
+require 'taps/config'
+require 'taps/utils'
+require 'taps/db_session'
+require 'taps/data_stream'
+
+module Taps
+  class Server < Sinatra::Base
+    use Rack::Auth::Basic do |login, password|
+      login == Taps::Config.login && password == Taps::Config.password
+    end
+
+    use Rack::Deflater unless ENV['NO_DEFLATE']
+
+    set :raise_errors => false
+    set :show_exceptions => false
+
+    error do
+      e = request.env['sinatra.error']
+
+      puts "ERROR: #{e.class}: #{e.message}"
+
+      if e.kind_of?(Taps::BaseError)
+        content_type "application/json"
+        halt 412, ::OkJson.encode({ 'error_class' => e.class.to_s, 'error_message' => e.message, 'error_backtrace' => e.backtrace.join("\n") })
+      else
+        "Taps Server Error: #{e}\n#{e.backtrace}"
+      end
+    end
+
+    before do
+      unless request.path_info == '/health'
+        major, minor = request.env['HTTP_TAPS_VERSION'].split('.') rescue []
+        unless "#{major}.#{minor}" == Taps.compatible_version
+          halt 417, "Taps >= v#{Taps.compatible_version}.x is required for this server"
+        end
+      end
+    end
+
+    get '/health' do
+      content_type 'application/json'
+      ::OkJson.encode({ :ok => true })
+    end
+
+    get '/' do
+      "hello"
+    end
+
+    post '/sessions' do
+      key = rand(9999999999).to_s
+
+      if ENV['NO_DEFAULT_DATABASE_URL']
+        database_url = request.body.string
+      else
+        database_url = Taps::Config.database_url || request.body.string
+      end
+
+      DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
+
+      "/sessions/#{key}"
+    end
+
+    post '/sessions/:key/push/verify_stream' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      state = DataStream.parse_json(params[:state])
+      stream = nil
+
+      session.conn do |db|
+        Taps::Utils.server_error_handling do
+          stream = DataStream.factory(db, state)
+          stream.verify_stream
+        end
+      end
+
+      content_type 'application/json'
+      ::OkJson.encode({ :state => stream.to_hash })
+    end
+
+    post '/sessions/:key/push/table' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      json = DataStream.parse_json(params[:json])
+
+      size = 0
+      session.conn do |db|
+        Taps::Utils.server_error_handling do
+          stream = DataStream.factory(db, json[:state])
+          size = stream.fetch_remote_in_server(params)
+        end
+      end
+
+      # TODO: return the stream's state with the size
+      size.to_s
+    end
+
+    post '/sessions/:key/push/reset_sequences' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      Taps::Utils.schema_bin(:reset_db_sequences, session.database_url)
+    end
+
+    post '/sessions/:key/push/schema' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      schema_data = request.body.read
+      Taps::Utils.load_schema(session.database_url, schema_data)
+    end
+
+    post '/sessions/:key/push/indexes' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      index_data = request.body.read
+      Taps::Utils.load_indexes(session.database_url, index_data)
+    end
+
+    post '/sessions/:key/pull/schema' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      Taps::Utils.schema_bin(:dump_table, session.database_url, params[:table_name])
+    end
+
+    get '/sessions/:key/pull/indexes' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      content_type 'application/json'
+      Taps::Utils.schema_bin(:indexes_individual, session.database_url)
+    end
+
+    get '/sessions/:key/pull/table_names' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      tables = []
+      session.conn do |db|
+        tables = db.tables
+      end
+
+      content_type 'application/json'
+      ::OkJson.encode(tables)
+    end
+
+    post '/sessions/:key/pull/table_count' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      count = 0
+      session.conn do |db|
+        count = db[ params[:table].to_sym.identifier ].count
+      end
+      count.to_s
+    end
+
+    post '/sessions/:key/pull/table' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      encoded_data = nil
+      stream = nil
+
+      session.conn do |db|
+        state = ::OkJson.decode(params[:state]).symbolize_keys
+        stream = Taps::DataStream.factory(db, state)
+        encoded_data = stream.fetch.first
+      end
+
+      checksum = Taps::Utils.checksum(encoded_data).to_s
+      json = ::OkJson.encode({ :checksum => checksum, :state => stream.to_hash })
+
+      content, content_type_value = Taps::Multipart.create do |r|
+        r.attach :name => :encoded_data,
+          :payload => encoded_data,
+          :content_type => 'application/octet-stream'
+        r.attach :name => :json,
+          :payload => json,
+          :content_type => 'application/json'
+      end
+
+      content_type content_type_value
+      content
+    end
+
+    delete '/sessions/:key' do
+      session = DbSession.filter(:key => params[:key]).first
+      halt 404 unless session
+
+      session.destroy
+
+      "ok"
+    end
+
+  end
+end