shiba 0.1.0

data/bin/release

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+for cmd in ["fingerprint"]; do
+    GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o cmd/builds/fingerprint.linux-amd64 cmd/fingerprint.go
+    GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w"  -o cmd/builds/fingerprint.darwin-amd64 cmd/fingerprint.go
+done

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/shiba

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'optionparser'
+
+APP = File.basename(__FILE__)
+
+commands = {
+    "analyze" => { :bin => "analyze", :description => "Analyze logged SQL queries" },
+    "watch"   => { :bin => "watch.rb", :description => "Log queries from Rails command" }
+}
+
+global = OptionParser.new do |opts|
+  opts.banner = "usage: #{APP} [--help] <command> [<args>]"
+  opts.separator ""
+  opts.separator "These are the commands available:"
+  commands.each do |name,info|
+    opts.separator "  #{name}     #{info[:description]}"
+  end
+  opts.separator ""
+  opts.separator "See #{APP} --help <command> to read about a specific command."
+  opts.separator ""
+end
+
+global.order!
+
+command = ARGV.shift
+
+if command.nil?
+  puts global.to_s
+  exit
+end
+
+if !commands.key?(command)
+    puts "#{APP}: '#{command}' is not a '#{APP}' command. See '#{APP} --help'."
+    exit 1
+end
+
+path = "#{File.dirname(__FILE__)}/#{commands[command][:bin]}"
+
+Kernel.exec(path, *ARGV)

data/bin/watch.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'optionparser'
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "watch <command>. Create SQL logs for a running process"
+
+  opts.on("-f", "--file FILE", "write to file") do |f|
+    options["file"] = f
+  end
+
+end
+
+parser.parse!
+
+$stderr.puts "Recording SQL queries to '#{options["file"]}'..."
+ENV['SHIBA_OUT'] = options["file"]
+Kernel.exec(ARGV.join(" "))

data/cmd/check.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"bufio"
+	"encoding/csv"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/xwb1989/sqlparser"
+)
+
+var indexFile = flag.String("i", "", "index files to check queries against")
+
+func init() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Returns unindexed queries. Reads select sql statements from stdin and checks the provided index file which is the result of analyze.\n")
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+
+		flag.PrintDefaults()
+	}
+}
+
+type tableStats map[string][]string
+
+func main() {
+	flag.Parse()
+
+	if *indexFile == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	stats := readTableIndexes(*indexFile)
+	r := bufio.NewReader(os.Stdin)
+
+	analyzeQueries(r, stats)
+
+}
+
+func readTableIndexes(path string) tableStats {
+	f, err := os.Open(path)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "index list not found at", path)
+		os.Exit(1)
+	}
+
+	br := bufio.NewReader(f)
+
+	r := csv.NewReader(br)
+
+	indexes := map[string][]string{}
+
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table := record[0]
+
+		indexes[table] = append(indexes[table], record[1])
+	}
+
+	return indexes
+}
+
+func analyzeQueries(r io.Reader, stats tableStats) {
+	scanner := bufio.NewScanner(r)
+
+	for scanner.Scan() {
+		sql := scanner.Text()
+		stmt, err := sqlparser.Parse(sql)
+
+		if err != nil {
+			fmt.Println("Unable to parse line", sql)
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		switch q := stmt.(type) {
+		case *sqlparser.Select:
+			if !hasIndex(q, stats) {
+				fmt.Println(sql)
+			}
+		default:
+			fmt.Fprintln(os.Stderr, "Only Select queries are supported", sqlparser.String(stmt))
+			os.Exit(1)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "reading standard input:", err)
+	}
+}
+
+func hasIndex(q *sqlparser.Select, stats tableStats) bool {
+	// fixme: assume simple table from
+	table := sqlparser.String(q.From[0])
+	// expr :=
+	if _, ok := stats[table]; !ok {
+		fmt.Fprintln(os.Stderr, "Table does not appear to have an index", sqlparser.String(q))
+		return false
+	}
+
+	indexed := stats[table]
+
+	var found bool
+	sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) {
+		switch node.(type) {
+		case *sqlparser.ColName:
+			// we're getting fully qualified column back
+			c := sqlparser.String(node)
+			for _, v := range indexed {
+				// quick hack because I'm parsing columns wrong somehow
+				if v == c || (table+"."+v) == c {
+					found = true
+					return false, nil
+				}
+			}
+		}
+
+		return true, nil
+	}, q.Where)
+
+	return found
+}
+
+func checkErr(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

data/cmd/fingerprint.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/percona/go-mysql/query"
+)
+
+func main() {
+	r := bufio.NewReader(os.Stdin)
+
+	for {
+		sql, err := r.ReadString('\n')
+
+		switch err {
+		case nil:
+			fmt.Println(query.Fingerprint(sql))
+		case io.EOF:
+			os.Exit(0)
+		default:
+			fmt.Fprintln(os.Stderr, "reading standard input:", err)
+		}
+	}
+
+}

data/cmd/inspect.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/csv"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+var user = flag.String("u", "", "user name for database connection")
+var password = flag.String("p", "", "password for database connection")
+var database = flag.String("db", "", "database name e.g. app_test")
+
+func init() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Reads indexes from the given database and outputs the results in CSV.\n")
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+
+		flag.PrintDefaults()
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	if *user == "" {
+		fmt.Println("missing -u database user")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	if *database == "" {
+		fmt.Println("missing -db database name")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	db, err := sql.Open("mysql", *user+":"+*password+"@/information_schema?charset=utf8")
+	defer db.Close()
+
+	if err != nil {
+		if *password == "" {
+			fmt.Println("missing -p database password")
+			flag.Usage()
+			os.Exit(1)
+		}
+	}
+
+	checkErr(err)
+
+	// query
+	// select column_name, column_key from columns where table_schema = "zammad_test";
+	rows, err := db.Query("SELECT table_name, column_name FROM columns where TABLE_SCHEMA = ? AND COLUMN_KEY is not null", *database)
+	checkErr(err)
+
+	w := csv.NewWriter(os.Stdout)
+
+	count := 0
+	for rows.Next() {
+		count++
+		result := make([]string, 2)
+
+		err = rows.Scan(&result[0], &result[1])
+		checkErr(err)
+
+		if err := w.Write(result); err != nil {
+			log.Fatalln("error writing record to csv:", err)
+		}
+	}
+
+	// Write any buffered data to the underlying writer (standard output).
+	w.Flush()
+
+	if err := w.Error(); err != nil {
+		log.Fatal(err)
+	}
+
+	if count == 0 {
+		log.Fatal("No rows found for database. It's possible the user does not have permissions to view it.")
+	}
+
+}
+
+func checkErr(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

data/cmd/parse.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/xwb1989/sqlparser"
+)
+
+type query struct {
+	Table   string   `json:"table"`
+	Columns []string `json:"columns"`
+}
+
+func init() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Takes queries from stdin and converst them to json.\n")
+	}
+}
+
+func main() {
+	parseQueries(os.Stdin)
+}
+
+func parseQueries(r io.Reader) {
+	scanner := bufio.NewScanner(r)
+
+	enc := json.NewEncoder(os.Stdout)
+
+	for scanner.Scan() {
+		sql := scanner.Text()
+		stmt, err := sqlparser.Parse(sql)
+
+		if err != nil {
+			fmt.Println("Unable to parse line", sql)
+			fmt.Fprintln(os.Stderr, err)
+		}
+
+		switch q := stmt.(type) {
+		case *sqlparser.Select:
+			parsed := parse(q)
+			enc.Encode(parsed)
+		case *sqlparser.Insert:
+			enc.Encode(parseTable(q))
+		default:
+			fmt.Fprintln(os.Stderr, "Query not supported: ", sqlparser.String(stmt))
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "reading standard input:", err)
+	}
+}
+
+func parseTable(q *sqlparser.Insert) string {
+	return sqlparser.String(q.Table)
+}
+
+func parse(q *sqlparser.Select) query {
+	// fixme: assume simple table from
+	//tables := sqlparser.String(q.From[0])
+	table := sqlparser.String(q.From)
+	columns := []string{}
+	sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) {
+		switch node.(type) {
+		case *sqlparser.ColName:
+			c := sqlparser.String(node)
+			columns = append(columns, c)
+		}
+
+		return true, nil
+	}, q.Where)
+
+	return query{Table: table, Columns: columns}
+}

data/lib/shiba.rb

@@ -0,0 +1,21 @@
+require "shiba/version"
+require "mysql2"
+
+module Shiba
+  class Error < StandardError; end
+
+  def self.configure(connection_hash)
+    @connection_hash = connection_hash
+  end
+
+  def self.connection
+    @connection ||= Mysql2::Client.new(@connection_hash)
+  end
+
+  def self.root
+    File.dirname(__dir__)
+  end
+end
+
+# This goes at the end so that Shiba.root is defined.
+require "shiba/railtie" if defined?(Rails)