shiba 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/bin/release ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env bash
2
+
3
+ for cmd in ["fingerprint"]; do
4
+ GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o cmd/builds/fingerprint.linux-amd64 cmd/fingerprint.go
5
+ GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o cmd/builds/fingerprint.darwin-amd64 cmd/fingerprint.go
6
+ done
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/bin/shiba ADDED
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'optionparser'
4
+
5
+ APP = File.basename(__FILE__)
6
+
7
+ commands = {
8
+ "analyze" => { :bin => "analyze", :description => "Analyze logged SQL queries" },
9
+ "watch" => { :bin => "watch.rb", :description => "Log queries from Rails command" }
10
+ }
11
+
12
+ global = OptionParser.new do |opts|
13
+ opts.banner = "usage: #{APP} [--help] <command> [<args>]"
14
+ opts.separator ""
15
+ opts.separator "These are the commands available:"
16
+ commands.each do |name,info|
17
+ opts.separator " #{name} #{info[:description]}"
18
+ end
19
+ opts.separator ""
20
+ opts.separator "See #{APP} --help <command> to read about a specific command."
21
+ opts.separator ""
22
+ end
23
+
24
+ global.order!
25
+
26
+ command = ARGV.shift
27
+
28
+ if command.nil?
29
+ puts global.to_s
30
+ exit
31
+ end
32
+
33
+ if !commands.key?(command)
34
+ puts "#{APP}: '#{command}' is not a '#{APP}' command. See '#{APP} --help'."
35
+ exit 1
36
+ end
37
+
38
+ path = "#{File.dirname(__FILE__)}/#{commands[command][:bin]}"
39
+
40
+ Kernel.exec(path, *ARGV)
data/bin/watch.rb ADDED
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'optionparser'
4
+
5
+ options = {}
6
+ parser = OptionParser.new do |opts|
7
+ opts.banner = "watch <command>. Create SQL logs for a running process"
8
+
9
+ opts.on("-f", "--file FILE", "write to file") do |f|
10
+ options["file"] = f
11
+ end
12
+
13
+ end
14
+
15
+ parser.parse!
16
+
17
+ $stderr.puts "Recording SQL queries to '#{options["file"]}'..."
18
+ ENV['SHIBA_OUT'] = options["file"]
19
+ Kernel.exec(ARGV.join(" "))
Binary file
Binary file
data/cmd/check.go ADDED
@@ -0,0 +1,138 @@
1
+ package main
2
+
3
+ import (
4
+ "bufio"
5
+ "encoding/csv"
6
+ "flag"
7
+ "fmt"
8
+ "io"
9
+ "log"
10
+ "os"
11
+
12
+ "github.com/xwb1989/sqlparser"
13
+ )
14
+
15
+ var indexFile = flag.String("i", "", "index files to check queries against")
16
+
17
+ func init() {
18
+ flag.Usage = func() {
19
+ fmt.Fprintf(os.Stderr, "Returns unindexed queries. Reads select sql statements from stdin and checks the provided index file which is the result of analyze.\n")
20
+ fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
21
+
22
+ flag.PrintDefaults()
23
+ }
24
+ }
25
+
26
+ type tableStats map[string][]string
27
+
28
+ func main() {
29
+ flag.Parse()
30
+
31
+ if *indexFile == "" {
32
+ flag.Usage()
33
+ os.Exit(1)
34
+ }
35
+
36
+ stats := readTableIndexes(*indexFile)
37
+ r := bufio.NewReader(os.Stdin)
38
+
39
+ analyzeQueries(r, stats)
40
+
41
+ }
42
+
43
+ func readTableIndexes(path string) tableStats {
44
+ f, err := os.Open(path)
45
+ if err != nil {
46
+ fmt.Fprintln(os.Stderr, "index list not found at", path)
47
+ os.Exit(1)
48
+ }
49
+
50
+ br := bufio.NewReader(f)
51
+
52
+ r := csv.NewReader(br)
53
+
54
+ indexes := map[string][]string{}
55
+
56
+ for {
57
+ record, err := r.Read()
58
+ if err == io.EOF {
59
+ break
60
+ }
61
+ if err != nil {
62
+ log.Fatal(err)
63
+ }
64
+
65
+ table := record[0]
66
+
67
+ indexes[table] = append(indexes[table], record[1])
68
+ }
69
+
70
+ return indexes
71
+ }
72
+
73
+ func analyzeQueries(r io.Reader, stats tableStats) {
74
+ scanner := bufio.NewScanner(r)
75
+
76
+ for scanner.Scan() {
77
+ sql := scanner.Text()
78
+ stmt, err := sqlparser.Parse(sql)
79
+
80
+ if err != nil {
81
+ fmt.Println("Unable to parse line", sql)
82
+ fmt.Fprintln(os.Stderr, err)
83
+ os.Exit(1)
84
+ }
85
+
86
+ switch q := stmt.(type) {
87
+ case *sqlparser.Select:
88
+ if !hasIndex(q, stats) {
89
+ fmt.Println(sql)
90
+ }
91
+ default:
92
+ fmt.Fprintln(os.Stderr, "Only Select queries are supported", sqlparser.String(stmt))
93
+ os.Exit(1)
94
+ }
95
+ }
96
+
97
+ if err := scanner.Err(); err != nil {
98
+ fmt.Fprintln(os.Stderr, "reading standard input:", err)
99
+ }
100
+ }
101
+
102
+ func hasIndex(q *sqlparser.Select, stats tableStats) bool {
103
+ // fixme: assume simple table from
104
+ table := sqlparser.String(q.From[0])
105
+ // expr :=
106
+ if _, ok := stats[table]; !ok {
107
+ fmt.Fprintln(os.Stderr, "Table does not appear to have an index", sqlparser.String(q))
108
+ return false
109
+ }
110
+
111
+ indexed := stats[table]
112
+
113
+ var found bool
114
+ sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) {
115
+ switch node.(type) {
116
+ case *sqlparser.ColName:
117
+ // we're getting fully qualified column back
118
+ c := sqlparser.String(node)
119
+ for _, v := range indexed {
120
+ // quick hack because I'm parsing columns wrong somehow
121
+ if v == c || (table+"."+v) == c {
122
+ found = true
123
+ return false, nil
124
+ }
125
+ }
126
+ }
127
+
128
+ return true, nil
129
+ }, q.Where)
130
+
131
+ return found
132
+ }
133
+
134
+ func checkErr(err error) {
135
+ if err != nil {
136
+ panic(err)
137
+ }
138
+ }
@@ -0,0 +1,28 @@
1
+ package main
2
+
3
+ import (
4
+ "bufio"
5
+ "fmt"
6
+ "io"
7
+ "os"
8
+
9
+ "github.com/percona/go-mysql/query"
10
+ )
11
+
12
+ func main() {
13
+ r := bufio.NewReader(os.Stdin)
14
+
15
+ for {
16
+ sql, err := r.ReadString('\n')
17
+
18
+ switch err {
19
+ case nil:
20
+ fmt.Println(query.Fingerprint(sql))
21
+ case io.EOF:
22
+ os.Exit(0)
23
+ default:
24
+ fmt.Fprintln(os.Stderr, "reading standard input:", err)
25
+ }
26
+ }
27
+
28
+ }
data/cmd/inspect.go ADDED
@@ -0,0 +1,92 @@
1
+ package main
2
+
3
+ import (
4
+ "database/sql"
5
+ "encoding/csv"
6
+ "flag"
7
+ "fmt"
8
+ "log"
9
+ "os"
10
+
11
+ _ "github.com/go-sql-driver/mysql"
12
+ )
13
+
14
+ var user = flag.String("u", "", "user name for database connection")
15
+ var password = flag.String("p", "", "password for database connection")
16
+ var database = flag.String("db", "", "database name e.g. app_test")
17
+
18
+ func init() {
19
+ flag.Usage = func() {
20
+ fmt.Fprintf(os.Stderr, "Reads indexes from the given database and outputs the results in CSV.\n")
21
+ fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
22
+
23
+ flag.PrintDefaults()
24
+ }
25
+ }
26
+
27
+ func main() {
28
+ flag.Parse()
29
+
30
+ if *user == "" {
31
+ fmt.Println("missing -u database user")
32
+ flag.Usage()
33
+ os.Exit(1)
34
+ }
35
+
36
+ if *database == "" {
37
+ fmt.Println("missing -db database name")
38
+ flag.Usage()
39
+ os.Exit(1)
40
+ }
41
+
42
+ db, err := sql.Open("mysql", *user+":"+*password+"@/information_schema?charset=utf8")
43
+ defer db.Close()
44
+
45
+ if err != nil {
46
+ if *password == "" {
47
+ fmt.Println("missing -p database password")
48
+ flag.Usage()
49
+ os.Exit(1)
50
+ }
51
+ }
52
+
53
+ checkErr(err)
54
+
55
+ // query
56
+ // select column_name, column_key from columns where table_schema = "zammad_test";
57
+ rows, err := db.Query("SELECT table_name, column_name FROM columns where TABLE_SCHEMA = ? AND COLUMN_KEY is not null", *database)
58
+ checkErr(err)
59
+
60
+ w := csv.NewWriter(os.Stdout)
61
+
62
+ count := 0
63
+ for rows.Next() {
64
+ count++
65
+ result := make([]string, 2)
66
+
67
+ err = rows.Scan(&result[0], &result[1])
68
+ checkErr(err)
69
+
70
+ if err := w.Write(result); err != nil {
71
+ log.Fatalln("error writing record to csv:", err)
72
+ }
73
+ }
74
+
75
+ // Write any buffered data to the underlying writer (standard output).
76
+ w.Flush()
77
+
78
+ if err := w.Error(); err != nil {
79
+ log.Fatal(err)
80
+ }
81
+
82
+ if count == 0 {
83
+ log.Fatal("No rows found for database. It's possible the user does not have permissions to view it.")
84
+ }
85
+
86
+ }
87
+
88
+ func checkErr(err error) {
89
+ if err != nil {
90
+ panic(err)
91
+ }
92
+ }
data/cmd/parse.go ADDED
@@ -0,0 +1,79 @@
1
+ package main
2
+
3
+ import (
4
+ "bufio"
5
+ "encoding/json"
6
+ "flag"
7
+ "fmt"
8
+ "io"
9
+ "os"
10
+
11
+ "github.com/xwb1989/sqlparser"
12
+ )
13
+
14
+ type query struct {
15
+ Table string `json:"table"`
16
+ Columns []string `json:"columns"`
17
+ }
18
+
19
+ func init() {
20
+ flag.Usage = func() {
21
+ fmt.Fprintf(os.Stderr, "Takes queries from stdin and converst them to json.\n")
22
+ }
23
+ }
24
+
25
+ func main() {
26
+ parseQueries(os.Stdin)
27
+ }
28
+
29
+ func parseQueries(r io.Reader) {
30
+ scanner := bufio.NewScanner(r)
31
+
32
+ enc := json.NewEncoder(os.Stdout)
33
+
34
+ for scanner.Scan() {
35
+ sql := scanner.Text()
36
+ stmt, err := sqlparser.Parse(sql)
37
+
38
+ if err != nil {
39
+ fmt.Println("Unable to parse line", sql)
40
+ fmt.Fprintln(os.Stderr, err)
41
+ }
42
+
43
+ switch q := stmt.(type) {
44
+ case *sqlparser.Select:
45
+ parsed := parse(q)
46
+ enc.Encode(parsed)
47
+ case *sqlparser.Insert:
48
+ enc.Encode(parseTable(q))
49
+ default:
50
+ fmt.Fprintln(os.Stderr, "Query not supported: ", sqlparser.String(stmt))
51
+ }
52
+ }
53
+
54
+ if err := scanner.Err(); err != nil {
55
+ fmt.Fprintln(os.Stderr, "reading standard input:", err)
56
+ }
57
+ }
58
+
59
+ func parseTable(q *sqlparser.Insert) string {
60
+ return sqlparser.String(q.Table)
61
+ }
62
+
63
+ func parse(q *sqlparser.Select) query {
64
+ // fixme: assume simple table from
65
+ //tables := sqlparser.String(q.From[0])
66
+ table := sqlparser.String(q.From)
67
+ columns := []string{}
68
+ sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) {
69
+ switch node.(type) {
70
+ case *sqlparser.ColName:
71
+ c := sqlparser.String(node)
72
+ columns = append(columns, c)
73
+ }
74
+
75
+ return true, nil
76
+ }, q.Where)
77
+
78
+ return query{Table: table, Columns: columns}
79
+ }
data/lib/shiba.rb ADDED
@@ -0,0 +1,21 @@
1
+ require "shiba/version"
2
+ require "mysql2"
3
+
4
+ module Shiba
5
+ class Error < StandardError; end
6
+
7
+ def self.configure(connection_hash)
8
+ @connection_hash = connection_hash
9
+ end
10
+
11
+ def self.connection
12
+ @connection ||= Mysql2::Client.new(@connection_hash)
13
+ end
14
+
15
+ def self.root
16
+ File.dirname(__dir__)
17
+ end
18
+ end
19
+
20
+ # This goes at the end so that Shiba.root is defined.
21
+ require "shiba/railtie" if defined?(Rails)