matthewtodd-taps 0.2.19

data/lib/taps/config.rb

@@ -0,0 +1,33 @@
+require 'sequel'
+require 'sqlite3'
+
+module Taps
+	def self.version_yml
+		@@version_yml ||= YAML.load(File.read(File.dirname(__FILE__) + '/../../VERSION.yml'))
+	end
+
+	def self.version
+		"#{version_yml[:major]}.#{version_yml[:minor]}.#{version_yml[:patch]}"
+	end
+
+	def self.compatible_version
+		"#{version_yml[:major]}.#{version_yml[:minor]}"
+	end
+
+	class Config
+		class << self
+			attr_accessor :taps_database_url
+			attr_accessor :login, :password, :database_url, :remote_url
+			attr_accessor :chunksize
+
+			def verify_database_url
+				db = Sequel.connect(self.database_url)
+				db.tables
+				db.disconnect
+			rescue Object => e
+				puts "Failed to connect to database:\n  #{e.class} -> #{e}"
+				exit 1
+			end
+		end
+	end
+end

data/lib/taps/db_session.rb

@@ -0,0 +1,62 @@
+require 'thread'
+
+Sequel::Model.db = Sequel.connect(Taps::Config.taps_database_url)
+
+class DbSession < Sequel::Model
+	plugin :schema
+	set_schema do
+		primary_key :id
+		text :key
+		text :database_url
+		timestamp :started_at
+		timestamp :last_access
+	end
+
+	@@connections = {}
+	@@mutex = Mutex.new
+
+	def connection
+		@@mutex.synchronize {
+			conn =
+				if @@connections.key?(key)
+					@@connections[key].first
+				else
+					Sequel.connect(database_url)
+				end
+			@@connections[key] = [conn, Time.now]
+			return conn
+		}
+	end
+
+	def disconnect
+		@@mutex.synchronize {
+			if @@connections.key?(key)
+				conn, time = @@connections.delete(key)
+				conn.disconnect
+			end
+		}
+	end
+
+	# Removes connections that have not been accessed within the
+	# past thirty seconds.
+	def self.cleanup
+		@@mutex.synchronize {
+			now = Time.now
+			@@connections.each do |key, (conn, time)|
+				if now - time > 30
+					@@connections.delete(key)
+					conn.disconnect
+				end
+			end
+		}
+	end
+
+	Thread.new {
+		while true
+			sleep 30
+			cleanup
+		end
+	}.run
+end
+
+DbSession.create_table! unless DbSession.table_exists?

data/lib/taps/progress_bar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols > 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/taps/schema.rb

@@ -0,0 +1,94 @@
+require 'active_record'
+require 'active_support'
+require 'stringio'
+require 'uri'
+
+require 'taps/adapter_hacks'
+
+module Taps
+module Schema
+	extend self
+
+	def create_config(url)
+		uri = URI.parse(url)
+		adapter = uri.scheme
+		adapter = 'postgresql' if adapter == 'postgres'
+		adapter = 'sqlite3' if adapter == 'sqlite'
+		config = {
+			'adapter' => adapter,
+			'database' => uri.path.blank? ? uri.host : uri.path.split('/')[1],
+			'username' => uri.user,
+			'password' => uri.password,
+			'host' => uri.host,
+		}
+		config = sqlite_config(url) if config['adapter'] == 'sqlite3'
+		config
+	end
+
+	def sqlite_config(url)
+		m = %r{(sqlite3?)://(.+)}.match(url)
+		database = m[2]
+		database, q = database.split('?')
+		{ 'adapter' => 'sqlite3', 'database' => database }
+	end
+
+	def connection(database_url)
+		config = create_config(database_url)
+		c = ActiveRecord::Base.establish_connection(config)
+		Taps::AdapterHacks.load(config['adapter'])
+		c
+	end
+
+	def dump(database_url)
+		connection(database_url)
+
+		stream = StringIO.new
+		ActiveRecord::SchemaDumper.ignore_tables = []
+		ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
+		stream.string
+	end
+
+	def dump_without_indexes(database_url)
+		schema = dump(database_url)
+		schema.split("\n").collect do |line|
+			if line =~ /^\s+add_index/
+				line = "##{line}"
+			end
+			line
+		end.join("\n")
+	end
+
+	def indexes(database_url)
+		schema = dump(database_url)
+		schema.split("\n").collect do |line|
+			line if line =~ /^\s+add_index/
+		end.uniq.join("\n")
+	end
+
+	def load(database_url, schema)
+		connection(database_url)
+		eval(schema)
+	end
+
+	def load_indexes(database_url, indexes)
+		connection(database_url)
+
+		schema =<<EORUBY
+ActiveRecord::Schema.define do
+	#{indexes}
+end
+EORUBY
+		eval(schema)
+	end
+
+	def reset_db_sequences(database_url)
+		connection(database_url)
+
+		if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
+			ActiveRecord::Base.connection.tables.each do |table|
+				ActiveRecord::Base.connection.reset_pk_sequence!(table)
+			end
+		end
+	end
+end
+end

data/lib/taps/server.rb

@@ -0,0 +1,138 @@
+require 'sinatra/base'
+require 'taps/config'
+require 'taps/utils'
+require 'taps/db_session'
+require 'taps/schema'
+
+module Taps
+class Server < Sinatra::Default
+	use Rack::Auth::Basic do |login, password|
+		login == Taps::Config.login && password == Taps::Config.password
+	end
+
+	error do
+		e = request.env['sinatra.error']
+		"Taps Server Error: #{e}"
+	end
+
+	before do
+		major, minor, patch = request.env['HTTP_TAPS_VERSION'].split('.') rescue []
+		unless "#{major}.#{minor}" == Taps.compatible_version
+			halt 417, "Taps v#{Taps.compatible_version}.x is required for this server"
+		end
+	end
+
+	get '/' do
+		"hello"
+	end
+
+	post '/sessions' do
+		key = rand(9999999999).to_s
+		database_url = Taps::Config.database_url || request.body.string
+
+		DbSession.create(:key => key, :database_url => database_url, :started_at => Time.now, :last_access => Time.now)
+
+		"/sessions/#{key}"
+	end
+
+	post '/sessions/:key/tables/:table' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		gzip_data = request.body.read
+		halt 412 unless Taps::Utils.valid_data?(gzip_data, request.env['HTTP_TAPS_CHECKSUM'])
+
+		rows = Marshal.load(Taps::Utils.gunzip(gzip_data))
+
+		db = session.connection
+		table = db[params[:table].to_sym]
+		table.import(rows[:header], rows[:data])
+
+		"#{rows[:data].size}"
+	end
+
+	post '/sessions/:key/reset_sequences' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		Taps::Schema.reset_db_sequences(session.database_url)
+	end
+
+	post '/sessions/:key/schema' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		schema_data = request.body.read
+		Taps::Schema.load(session.database_url, schema_data)
+	end
+
+	post '/sessions/:key/indexes' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		index_data = request.body.read
+		Taps::Schema.load_indexes(session.database_url, index_data)
+	end
+
+	get '/sessions/:key/schema' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		Taps::Schema.dump_without_indexes(session.database_url)
+	end
+
+	get '/sessions/:key/indexes' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		Taps::Schema.indexes(session.database_url)
+	end
+
+	get '/sessions/:key/tables' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		db = session.connection
+		tables = db.tables
+
+		tables_with_counts = tables.inject({}) do |accum, table|
+			accum[table] = db[table].count
+			accum
+		end
+
+		Marshal.dump(tables_with_counts)
+	end
+
+	get '/sessions/:key/tables/:table/:chunk' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		chunk = params[:chunk].to_i
+		chunk = 500 if chunk < 1
+
+		offset = params[:offset].to_i
+		offset = 0 if offset < 0
+
+		db = session.connection
+		table = db[params[:table].to_sym]
+		order = Taps::Utils.order_by(db, params[:table].to_sym)
+		string_columns = Taps::Utils.incorrect_blobs(db, params[:table].to_sym)
+		raw_data = Marshal.dump(Taps::Utils.format_data(table.order(*order).limit(chunk, offset).all, string_columns))
+		gzip_data = Taps::Utils.gzip(raw_data)
+		response['Taps-Checksum'] = Taps::Utils.checksum(gzip_data).to_s
+		response['Content-Type'] = "application/octet-stream"
+		gzip_data
+	end
+
+	delete '/sessions/:key' do
+		session = DbSession.filter(:key => params[:key]).first
+		halt 404 unless session
+
+		session.disconnect
+		session.destroy
+
+		"ok"
+	end
+
+end
+end