shiba 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e14e046d4bcec0e2025d671461517dabd547bc0a
+  data.tar.gz: 1e5b9e2043c8ecb2f2a46d2dc581be196c2daf75
+SHA512:
+  metadata.gz: 51f1c7929ab2306a7889d37f11d9fcfc73091c7b9325a70cf17e9f57293e4463a9780ce04702ad1ef8392b3c325c0f9a82a7d0e999779d9a989eaeacbc4c6e3d
+  data.tar.gz: 4e7ddb0dd6e3f5e555995e2ab0a9505428770dc335489c7929bdab4b7138ac28b94c9e5b91aad2b7a75d3568c3fe9b80ef4b655274c42b6c1ecdcb52fece5efa

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.*.sw*
+node_modules

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Contributor Covenant Code of Conduct
+
+We, the burrito brothers, find this humorous and odd.

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql2"
+gem "byebug"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    shiba (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (10.0.2)
+    mysql2 (0.5.2)
+    rake (10.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  byebug
+  mysql2
+  rake (~> 10.0)
+  shiba!
+
+BUNDLED WITH
+   2.0.1

data/README.md

@@ -0,0 +1,53 @@
+# Shiba
+
+Shiba is a tool that helps you to understand and write better SQL.  Integrate
+the gem into your test suite, give Shiba a bit of data about your indexes, and Shiba
+will let you know the impact of your queries on production, with the goal of catching
+poorly performing queries before they hit production.
+
+## Installation
+
+You can run shiba either as a gem in your test suite, or as a standalone utility.
+
+### Gem
+
+Add this line to your application's Gemfile:
+
+```ruby
+group :test do
+  gem 'shiba'
+end
+
+bundle
+
+# Run some some code using shiba to generate a SQL report
+bundle exec shiba analyze rake test:functional
+```
+
+### Standalone:
+
+```
+# 1.  Get shiba.
+local:$ git clone git@github.com:burrito-brothers/shiba.git
+
+# 2.  Get production data.
+# Shiba *can* work without any further data, but it's really best if you can
+# dump index statistics from a production database, or a staging database with
+# that resembles production.
+
+local:$ ssh production_host
+production_host:$ mysql -ABe "select * from information_schema.statistics where table_schema = 'DATABASE'" > shiba_schema_stats.tsv
+local:$ scp production_host:shiba_schema_stats.tsv shiba/
+
+# 3.  Analyze your queries.
+# set shiba loose on your queries!
+# If you can't do step #2, just leave off the '-s' option
+
+local:$ cd shiba
+local:$ bin/analyze.rb -h 127.0.0.1 -d TESTDB -u MYSQLUSER -p MYSQLPASS -s shiba_schema_stats.tsv -f ~/src/MYPROJECT/log/test.log > results.json
+
+# analyze the results with `jq`, whynot
+
+local:$ jq -C -s 'sort_by(.cost) | reverse' results.json | less -R
+
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/TODO

@@ -0,0 +1,12 @@
+===
+this query is throwing the optimizer for a serious loop;
+it says it can use an index on `ipb_address` but when we force key it
+still table scans.  Not clear on whether the OR in there is fucking us over or
+if it's a test-data issue.
+
+
+# SELECT  ipb_id,ipb_address,ipb_timestamp,ipb_auto,ipb_anon_only,ipb_create_account,ipb_enable_autoblock,ipb_expiry,ipb_deleted,ipb_block_email,ipb_allow_usertalk,ipb_parent_block_id,ipb_sitewide,comment_ipb_reason.comment_text AS `ipb_reason_text`,comment_ipb_reason.comment_data AS `ipb_reason_data`,comment_ipb_reason.comment_id AS `ipb_reason_cid`,ipb_by,ipb_by_text,NULL AS `ipb_by_actor`  FROM `ipblocks` FORCE KEY(`ipb_address`) JOIN `comment` `comment_ipb_reason` ON ((comment_ipb_reason.comment_id = ipb_reason_id))   WHERE ipb_address = '127.0.0.1' OR ((ipb_range_start  LIKE '7F00%' ESCAPE '`' ) AND (ipb_range_start <= '7F000001') AND (ipb_range_end >= '7F000001'));
+
+===
+need to use format=json and see how much of an index we're using.
+

data/bin/analyze

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'optionparser'
+require 'shiba/configure'
+require 'shiba/output'
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "analyze <command>. Creates report of SQL from the running process."
+
+  opts.on("-h","--host HOST", "sql host") do |h|
+    options["host"] = h
+  end
+
+  opts.on("-d","--database DATABASE", "sql database") do |d|
+    options["database"] = d
+  end
+
+  opts.on("-u","--user USER", "sql user") do |u|
+    options["user"] = u
+  end
+
+  opts.on("-p","--password PASSWORD", "sql password") do |p|
+    options["password"] = p
+  end
+
+  opts.on("-i","--index INDEX", "index of query to inspect") do |i|
+    options["index"] = i.to_i
+  end
+
+  opts.on("-l", "--limit NUM", "stop after processing NUM queries") do |l|
+    options["limit"] = l.to_i
+  end
+
+  opts.on("-s","--stats FILES", "location of index statistics tsv file") do |f|
+    options["stats"] = f
+  end
+
+  opts.on("-f", "--file FILE", "location of file containing queries") do |f|
+    options["file"] = f
+  end
+
+  opts.on("-o", "--output FILE", "write to file instead of stdout") do |f|
+    options["output"] = f
+  end
+
+  opts.on("-t", "--test", "analyze queries at --file instead of analyzing a process") do |f|
+    options["test"] = true
+  end
+
+end
+
+parser.parse!
+
+if options["test"] && !options["file"]
+  $stderr.puts "--file <query log> is required for test mode"
+  $stderr.puts parser.banner
+  exit 1
+end
+
+# Automagic configuration goes here
+if !options["database"]
+    config = Shiba::Configure.activerecord_configuration
+
+    if tc = config && config['test']
+      $stderr.puts "Reading configuration from '#{`pwd`.chomp}/config/database.yml'[:test]."
+      options['host']     ||= tc['host']
+      options['database'] ||= tc['database']
+      options['user'] ||= tc['username']
+      options['password'] ||= tc['password']
+    end
+  end
+
+  if !options["file"]
+    options["file"] = `mktemp /tmp/shiba-query.log-#{Time.now.to_i}`.chomp
+  end
+
+  if !options["output"]
+    options["output"] = `mktemp /tmp/shiba-explain.log-#{Time.now.to_i}`.chomp
+  end
+
+# Log process queries
+if !options.delete("test")
+  if ARGV.empty?
+    $stderr.puts "The name of a command must be passed in to generate SQL logs."
+    $stderr.puts "Example: shiba analyze rake spec"
+    $stderr.puts ""
+    $stderr.puts "For static analysis, try the --test option."
+    exit 1
+  end
+
+  path = "#{File.dirname(__FILE__)}/watch.rb"
+  watch_args = ARGV + [ "--file", options["file"] ]
+  pid = fork do
+    Signal.trap("INT") { exit 1 }
+    exec(path, *watch_args)
+  end
+
+  Signal.trap("INT", "IGNORE")
+  Process.wait(pid)
+  Signal.trap("INT", "DEFAULT")
+end
+
+# Explain
+$stderr.puts "Analyzing SQL to '#{options["output"]}'..."
+path = "#{File.dirname(__FILE__)}/explain"
+args = options.select { |_,v| !v.nil? }.map { |k,v| [ "--#{k}", v ] }.flatten
+
+$stderr.puts ([path] + args).join(" ")
+if !system(path, *args)
+  exit 1
+end
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "shiba"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/explain

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'shiba'
+require 'shiba/analyzer'
+require 'shiba/index'
+require 'shiba/configure'
+require 'shiba/output'
+
+require 'optionparser'
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: explain -h HOST -d DB -u USER -p PASS [-f QUERY_FILE] [-s STATS_FILE] "
+
+  opts.on("-h","--host HOST", "sql host") do |h|
+    options["host"] = h
+  end
+
+  opts.on("-d","--database DATABASE", "sql database") do |d|
+    options["database"] = d
+  end
+
+  opts.on("-u","--user USER", "sql user") do |u|
+    options["username"] = u
+  end
+
+  opts.on("-p","--password PASSWORD", "sql password") do |p|
+    options["password"] = p
+  end
+
+  opts.on("-i","--index INDEX", "index of query to inspect") do |i|
+    options["index"] = i.to_i
+  end
+
+  opts.on("-l", "--limit NUM", "stop after processing NUM queries") do |l|
+    options["limit"] = l.to_i
+  end
+
+  opts.on("-s","--stats FILES", "location of index statistics tsv file") do |f|
+    options["stats_file"] = f
+  end
+
+  opts.on("-f", "--file FILE", "location of file containing queries") do |f|
+    options["file"] = f
+  end
+
+  opts.on("-o", "--output FILE", "write to file instead of stdout") do |f|
+    options["output"] = f
+  end
+
+  opts.on("--debug") do
+    options["debug"] = true
+  end
+end
+
+parser.parse!
+
+["database", "username"].each do |opt|
+  if !options[opt]
+    $stderr.puts "Required: #{opt}"
+    $stderr.puts parser.banner
+    exit 1
+  end
+end
+
+file = options.delete("file")
+file = File.open(file, "r") if file
+
+output = options.delete("output")
+output = File.open(output, 'w') if output
+
+Shiba.configure(options)
+
+schema_stats_fname = options["stats_file"]
+
+if schema_stats_fname && !File.exist?(schema_stats_fname)
+  $stderr.puts "No such file: #{schema_stats_fname}"
+  exit 1
+end
+
+if schema_stats_fname
+  schema_stats = Shiba::Index.parse(schema_stats_fname)
+
+  local_db_stats = Shiba::Index.query(Shiba.connection)
+  Shiba::Index.fuzz!(local_db_stats)
+  local_db_stats.each do |table, values|
+    schema_stats[table] = values unless schema_stats[table]
+  end
+else
+  schema_stats = Shiba::Index.query(Shiba.connection)
+
+  if Shiba::Index.insufficient_stats?(schema_stats)
+    $stderr.puts "WARN: insufficient stats available in the #{options["database"]} database, guessing at numbers."
+    $stderr.puts "To get better analysis please specify an index statistics file."
+    sleep 0.5
+    Shiba::Index.fuzz!(schema_stats)
+  end
+end
+
+file = $stdin if file.nil?
+output = $stdout if output.nil?
+
+queries = Shiba::Analyzer.analyze(file, output, schema_stats, options)
+Shiba::Output.new(queries).make_web!

data/bin/fingerprint

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# run from script path
+cd "${0%/*}"
+
+case "$OSTYPE" in
+  linux*)   ./../cmd/builds/fingerprint.linux-amd64 ;;
+  darwin*)  ./../cmd/builds/fingerprint.darwin-amd64 ;; 
+  *)        echo "unsupported system $OSTYPE" & exit 1;;
+esac

data/bin/redmine/sample_redmine.rb

@@ -0,0 +1,165 @@
+require 'bundler/setup'
+
+require 'activeresource'
+require 'logger'
+require 'pp'
+require 'mysql2'
+
+ActiveResource::Base.logger = Logger.new($stdout)
+
+class Redmine < ActiveResource::Base
+  self.site = 'http://www.redmine.org/'
+  self.user = 'osheroff'
+  self.password = `cat ~/.redmine_pass`.chomp
+  def enhance(hash, owner)
+  end
+end
+
+class Attachment < Redmine
+  def enhance(hash, owner)
+    hash['disk_filename'] = self.attributes['content_url']
+  end
+end
+
+class Comment < Redmine; end
+
+class Changeset < Redmine
+  @@changeset_id = 1
+
+  def enhance(hash, owner)
+    hash['id'] = @@changeset_id
+    @@changeset_id += 1
+    hash['repository_id'] = 1
+    hash['scmid'] = 1
+  end
+end
+
+class Issue < Redmine; end
+
+class Journal < Redmine
+  def enhance(hash, owner)
+    hash['journalized_type'] = 'Issue'
+    hash['journalized_id'] = owner.id
+  end
+end
+
+class Watcher < Redmine; end
+class Relation < Redmine; end
+class User < Redmine; end
+
+class Sampler
+  def initialize(interested_in)
+    @results = {}
+    @interested_in = interested_in
+  end
+
+  def extract_hash(instance, owner = nil)
+    h = {}
+    table_name = instance.class.name.tableize
+    attrs = @interested_in[table_name]
+
+    instance.attributes.each do |k, v|
+      if k == "id" || attrs.include?(k)
+        h[k] = v
+      elsif attrs.include?("#{k}_id")
+        h[k + "_id"] = v.id
+      elsif v.is_a?(Array) && @interested_in[k]
+        v.each do |obj|
+          extract_hash(obj, instance)
+        end
+      end
+    end
+
+    instance.enhance(h, owner)
+
+    @results[table_name] ||= []
+    @results[table_name] << h
+  end
+
+  def get_instance(model, id, params = {})
+    model.find(id, params: params) rescue nil
+  end
+
+  def find_max(model, start, offset, params)
+    last_instance = nil
+    while instance = get_instance(model, start + offset, params)
+      last_instance = instance
+      extract_hash(instance)
+      offset *= 2
+      sleep(0.5)
+    end
+
+    1.upto(6) do |i|
+      # check for gaps
+      instance = get_instance(model, start + offset + (i**2), params)
+      if instance
+        last_instance = instance
+        extract_hash(instance)
+        return find_max(model, start + offset + (i**2), offset, params) || last_instance
+      end
+    end
+
+    last_instance
+  end
+
+  def sample_model(model, params = {})
+    # go exponentially up from 1.
+    last = find_max(model, 0, 1, params)
+    # go up from the last hit value.
+    if last
+      find_max(model, last.id, 1, params)
+    end
+  end
+
+  def output
+    @results.each do |table, res|
+      File.open("/tmp/redmine/#{table}.json", "w+") do |f|
+        res.each do |row|
+          f.puts(row.to_json)
+        end
+      end
+    end
+  end
+end
+
+client = Mysql2::Client.new(username: 'root', password: '123456', database: 'redmine_test', host: '127.0.0.1')
+
+
+tables = {
+  'attachments' => {
+    model: Attachment
+  },
+  'issues' => {
+    model: Issue,
+    params: { include: ['journals', 'comments', 'changesets', 'watchers'] },
+  },
+  'users' => {
+    model: User
+  }
+}
+
+
+all_tables = tables.map do |k, v|
+  a = [k]
+  a += v[:params][:include] if v[:params] && v[:params][:include]
+  a
+end.flatten
+
+table_indexes = {}
+all_tables.each do |table|
+  indexes = client.query("show indexes from #{table}")
+  interested_columns = indexes.to_a.map { |i| i["Column_name"] }.flatten.uniq
+  table_indexes[table] = interested_columns
+end
+
+sampler = Sampler.new(table_indexes)
+
+tables.each do |table, hash|
+  sampler.sample_model(hash[:model], hash[:params])
+end
+
+sampler.output
+
+#sample_model(Issue, %w(project_id status_id category_id priority_id author_id))
+
+