dbfire 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: df593da02e1d142e3ac1f674f02bcf9d411c4f4c
+  data.tar.gz: 07ae4472da933e06aa32cc696d4fb1d193a047dd
+SHA512:
+  metadata.gz: 88ded2d6f4fc0d89da6a51937932658dbe1d08de2fcc204709e134a9c9f70fc05a3e5edc590a8081e46ba54b5ddc5bad8e7f29a1b2a984e696a6bb687388f3ea
+  data.tar.gz: 90acc5f7c14db69f6bfc2d2220f66df71bdd55b5655ed3701f5fda27701880cb4bf7b6f1c22b0d1f48cb857c437c3d710dc2a893e41389606896eee98de7d05a

data/bin/fire

@@ -0,0 +1,260 @@
+#!/usr/bin/ruby
+
+load File.dirname(__FILE__) + '/../lib/dbfire/filesize.rb'
+
+Thread.abort_on_exception = true
+
+$blocksize = 65536
+
+TYPE_PREFIXES = %w{Ki Mi Gi Ti Pi Ei Zi Yi}
+
+class Fire
+  def self.usage
+    puts "Database load testing toolkit by m4rkw\n\n"
+    puts "Usage: fire <file1> [file2] [..]\n\n"
+    puts "To split your general.log:\n\n"
+    puts "fire -s <general log> <threads> [all]\n\n"
+    exit
+  end
+
+  def initialize(args)
+    if args[0] == '-s'
+      if args.length <3
+        self.class.usage
+      end
+      split(args)
+      return
+    end
+
+    @cli_string = ENV['CLI']
+
+    if !@cli_string
+      raise "The environment variable CLI is not set. Please set this to your cli command to invoke the database client."
+    end
+
+    @queue = Queue.new
+
+    @start_time = Time.now.to_i
+
+    fire(ARGV)
+    display
+    monitor
+  end
+
+  def split(args)
+    thread_id = 0
+
+    File.open(args[1],"r:ascii-8bit").each do |line|
+      line.chomp!
+
+      m = line.match(/[\s\t]+[0-9]+ Query[\t\s]+(\/\*.*?\*\/)?(.*)/)
+
+      if m
+          word = m[2].split(" ").first
+
+          if ["SELECT","INSERT","UPDATE","DELETE"].include? word
+              if args[3] == 'all'
+                  target = "ALL.#{thread_id}"
+              else
+                  target = "#{word}.#{thread_id}"
+              end
+
+              File.open(target,'a+') do |out|
+                  out.write("#{m[2]};\n")
+              end
+
+              thread_id += 1
+
+              if thread_id >= args[2].to_i
+                  thread_id = 0
+              end
+          end
+      end
+    end
+  end
+
+  def fire(files)
+    @threads = []
+    @progress = []
+
+    for thread_id in 0...files.length
+      @progress[thread_id] = {
+        :total_bytes => File.size(files[thread_id]),
+        :sent => 0,
+        :sent_at_last_interval => 0,
+        :sent_speed => 0,
+        :recv => 0,
+        :recv_at_last_interval => 0,
+        :recv_speed => 0,
+        :timestamp => Time.now.to_i
+      }
+
+      thread = Thread.new(thread_id, files[thread_id]) do |tid, thread_file|
+        file_thread(tid, thread_file)
+      end
+
+      @threads.push thread
+    end
+  end
+
+  def file_thread(tid, thread_file)
+    bytes_sent = 0
+    bytes_read = 0
+
+    File.open(thread_file, "r:utf-8") do |file|
+      IO.popen(@cli_string, "a+") do |db|
+        data = nil
+
+        while not file.eof?
+          if !data
+            data = file.readline
+          end
+
+          begin
+            db.write_nonblock(data)
+            bytes_sent += data.length
+            data = nil
+            @queue << {:thread_id => tid, :sent => bytes_sent, :recv => bytes_read}
+          rescue IO::WaitWritable
+            begin
+              recv = db.read_nonblock($blocksize)
+              bytes_read += recv.length
+              @queue << {:thread_id => tid, :sent => bytes_sent, :recv => bytes_read}
+            rescue IO::WaitReadable
+            end
+          end
+        end
+
+        db.close_write
+
+        while not db.eof?
+          begin
+            recv = db.read_nonblock($blocksize)
+            bytes_read += recv.length
+            @queue << {:thread_id => tid, :sent => bytes_sent, :recv => bytes_read}
+          rescue
+            IO::WaitReadable
+          end
+        end
+      end
+    end
+  end
+
+  def monitor
+    Thread.new do
+      while 1
+        update
+        sleep 0.5
+      end
+    end
+
+    Thread.new do
+      last_update = nil
+
+      while 1
+        begin
+          value = @queue.pop
+          process_update(value)
+        rescue
+        end
+      end
+    end
+
+    @threads.each do |thread|
+      thread.join
+    end
+
+    execution_time = Time.at(Time.now.to_i - @start_time).utc.strftime("%H:%M:%S")
+
+    puts "\nFinished in: #{execution_time}\n\n"
+  end
+
+  def process_update(value)
+    @progress[value[:thread_id]][:sent] = value[:sent]
+    @progress[value[:thread_id]][:recv] = value[:recv]
+
+    time_now = Time.now.to_i
+    seconds_elapsed = time_now - @progress[value[:thread_id]][:timestamp]
+
+    if seconds_elapsed > 0
+      sent_diff = value[:sent] - @progress[value[:thread_id]][:sent_at_last_interval]
+      recv_diff = value[:recv] - @progress[value[:thread_id]][:recv_at_last_interval]
+
+      @progress[value[:thread_id]][:sent_speed] = sent_diff / seconds_elapsed
+      @progress[value[:thread_id]][:recv_speed] = recv_diff / seconds_elapsed
+      @progress[value[:thread_id]][:timestamp] = time_now
+      @progress[value[:thread_id]][:sent_at_last_interval] = value[:sent]
+      @progress[value[:thread_id]][:recv_at_last_interval] = value[:recv]
+    end
+  end
+
+  def display
+    output = []
+
+    for i in 0...@progress.length
+      pro = @progress[i][:sent]
+      total = @progress[i][:total_bytes]
+      pc = sprintf("%.2f", pro / (total.to_f / 100))
+      data_out = Filesize.from("#{@progress[i][:sent]} B").pretty
+      data_in = Filesize.from("#{@progress[i][:recv]} B").pretty
+
+      out_speed = Filesize.from("#{@progress[i][:sent_speed]} B").pretty + "/s"
+      in_speed = Filesize.from("#{@progress[i][:recv_speed]} B").pretty + "/s"
+
+      time_now = Time.now.to_i
+
+      if pro > 0 and (time_now - @start_time) >0
+        overall_speed = pro / (time_now - @start_time)
+
+        if overall_speed > 0
+          eta = (total - pro) / overall_speed
+          @progress[i][:eta] = Time.at(eta).utc.strftime("%H:%M:%S")
+        else
+          @progress[i][:eta] = "--"
+        end
+      else
+        @progress[i][:eta] = "--"
+      end
+
+      output.push([data_out, pc, out_speed, data_in, in_speed, @progress[i][:eta]])
+    end
+
+    format_output output, "%d: out: [%s] %s%% (%s)  in: [%s] (%s)  eta: %s"
+  end
+
+  def format_output(data, format)
+    width = []
+
+    for i in 0...data.length
+      for j in 0...data[i].length
+        if !width[j]
+          width[j] = data[i][j].length
+        elsif data[i][j].length > width[j]
+          width[j] = data[i][j].length
+        end
+      end
+    end
+
+    for i in 0...data.length
+      for j in 0...data[i].length
+        data[i][j] = data[i][j].rjust(width[j], ' ')
+      end
+      puts sprintf(format, i, *data[i])
+    end
+  end
+
+  def update
+    for i in 0...@progress.length
+      print "\e[A"
+      print "\e[K"
+    end
+
+    display
+  end
+end
+
+if ARGV.length == 0
+  Fire.usage
+end
+
+Fire.new(ARGV)

data/lib/dbfire/filesize.rb

@@ -0,0 +1,179 @@
+class Filesize
+  include Comparable
+
+  TYPE_PREFIXES = {
+    # Unit prefixes used for SI file sizes.
+    :SI => %w{k M G T P E Z Y},
+    # Unit prefixes used for binary file sizes.
+    :BINARY => %w{Ki Mi Gi Ti Pi Ei Zi Yi}
+  }
+
+  # @deprecated Please use TYPE_PREFIXES[:SI] instead
+  PREFIXES = TYPE_PREFIXES[:SI]
+
+  # Set of rules describing file sizes according to SI units.
+  SI = {
+    :regexp => /^([\d,.]+)?\s?([kmgtpezy]?)b$/i,
+    :multiplier => 1000,
+    :prefixes => TYPE_PREFIXES[:SI],
+    :presuffix => '' # deprecated
+  }
+  # Set of rules describing file sizes according to binary units.
+  BINARY = {
+    :regexp => /^([\d,.]+)?\s?(?:([kmgtpezy])i)?b$/i,
+    :multiplier => 1024,
+    :prefixes => TYPE_PREFIXES[:BINARY],
+    :presuffix => 'i' # deprecated
+  }
+
+  # @param [Number] size A file size, in bytes.
+  # @param [SI, BINARY] type Which type to use for conversions.
+  def initialize(size, type = BINARY)
+    @bytes = size.to_i
+    @type  = type
+  end
+
+  # @return [Number] Returns the size in bytes.
+  def to_i
+    @bytes
+  end
+  alias_method :to_int, :to_i
+
+  # @param [String] unit Which unit to convert to.
+  # @return [Float] Returns the size in a given unit.
+  def to(unit = 'B')
+    to_parts = self.class.parse(unit)
+    prefix   = to_parts[:prefix]
+
+    if prefix == 'B' or prefix.empty?
+      return to_i.to_f
+    end
+
+    to_type = to_parts[:type]
+    size    = @bytes
+
+    pos = (@type[:prefixes].map { |s| s[0].chr.downcase }.index(prefix.downcase) || -1) + 1
+
+    size = size/(to_type[:multiplier].to_f**(pos)) unless pos < 1
+  end
+  alias_method :to_f, :to
+
+  # @param (see #to_f)
+  # @return [String] Same as {#to_f}, but as a string, with the unit appended.
+  # @see #to_f
+  def to_s(unit = 'B')
+    "%.2f %s" % [to(unit).to_f.to_s, unit]
+  end
+
+  # Same as {#to_s} but with an automatic determination of the most
+  # sensible unit.
+  #
+  # @return [String]
+  # @see #to_s
+  def pretty
+    size = @bytes
+    if size < @type[:multiplier]
+      unit = "B"
+    else
+      pos = (Math.log(size) / Math.log(@type[:multiplier])).floor
+      pos = @type[:prefixes].size-1 if pos > @type[:prefixes].size - 1
+
+      unit = @type[:prefixes][pos-1] + "B"
+    end
+
+    to_s(unit)
+  end
+
+  # @return [Filesize]
+  def +(other)
+    self.class.new(@bytes + other.to_i, @type)
+  end
+
+  # @return [Filesize]
+  def -(other)
+    self.class.new(@bytes - other.to_i, @type)
+  end
+
+  # @return [Filesize]
+  def *(other)
+    self.class.new(@bytes * other.to_i, @type)
+  end
+
+  # @return [Filesize]
+  def /(other)
+    result = @bytes / other.to_f
+    if other.is_a? Filesize
+      result
+    else
+      self.class.new(result, @type)
+    end
+  end
+
+  def <=>(other)
+    self.to_i <=> other.to_i
+  end
+
+  # @return [Array<self, other>]
+  # @api private
+  def coerce(other)
+    return self, other
+  end
+
+  class << self
+    # Parses a string, which describes a file size, and returns a
+    # Filesize object.
+    #
+    # @param [String] arg A file size to parse.
+    # @raise [ArgumentError] Raised if the file size cannot be parsed properly.
+    # @return [Filesize]
+    def from(arg)
+      parts  = parse(arg)
+      prefix = parts[:prefix]
+      size   = parts[:size]
+      type   = parts[:type]
+
+      raise ArgumentError, "Unparseable filesize" unless type
+
+      offset = (type[:prefixes].map { |s| s[0].chr.downcase }.index(prefix.downcase) || -1) + 1
+
+      new(size * (type[:multiplier] ** (offset)), type)
+    end
+
+    # @return [Hash<:prefix, :size, :type>]
+    # @api private
+    def parse(string)
+      type = nil
+      # in this order, so we prefer binary :)
+      [BINARY, SI].each { |_type|
+        if string =~ _type[:regexp]
+          type    =  _type
+          break
+        end
+      }
+
+      prefix = $2 || ''
+      size   = ($1 || 0).to_f
+
+      return { :prefix => prefix, :size => size, :type => type}
+    end
+  end
+
+  # The size of a floppy disk
+  Floppy = Filesize.from("1474 KiB")
+  # The size of a CD
+  CD     = Filesize.from("700 MB")
+  # The size of a common DVD
+  DVD_5  = Filesize.from("4.38 GiB")
+  # The same as a DVD 5
+  DVD    = DVD_5
+  # The size of a single-sided dual-layer DVD
+  DVD_9  = Filesize.from("7.92 GiB")
+  # The size of a double-sided single-layer DVD
+  DVD_10 = DVD_5 * 2
+  # The size of a double-sided DVD, combining a DVD-9 and a DVD-5
+  DVD_14 = DVD_9 + DVD_5
+  # The size of a double-sided dual-layer DVD
+  DVD_18 = DVD_14 * 2
+  # The size of a Zip disk
+  ZIP    = Filesize.from("100 MB")
+end

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: dbfire
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- m4rkw
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-29 00:00:00.000000000 Z
+dependencies: []
+description: Simple, fast, standalone database load testing toolkid
+email: m@rkw.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/fire
+- lib/dbfire/filesize.rb
+homepage: https://github.com/m4rkw/dbfire
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: DBFire
+test_files: []