shiba 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/CODE_OF_CONDUCT.md +3 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +24 -0
- data/README.md +53 -0
- data/Rakefile +2 -0
- data/TODO +12 -0
- data/bin/analyze +116 -0
- data/bin/check +0 -0
- data/bin/console +14 -0
- data/bin/explain +105 -0
- data/bin/fingerprint +10 -0
- data/bin/inspect +0 -0
- data/bin/parse +0 -0
- data/bin/redmine/sample_redmine.rb +165 -0
- data/bin/release +6 -0
- data/bin/setup +8 -0
- data/bin/shiba +40 -0
- data/bin/watch.rb +19 -0
- data/cmd/builds/fingerprint.darwin-amd64 +0 -0
- data/cmd/builds/fingerprint.linux-amd64 +0 -0
- data/cmd/check.go +138 -0
- data/cmd/fingerprint.go +28 -0
- data/cmd/inspect.go +92 -0
- data/cmd/parse.go +79 -0
- data/lib/shiba.rb +21 -0
- data/lib/shiba/analyzer.rb +100 -0
- data/lib/shiba/configure.rb +31 -0
- data/lib/shiba/explain.rb +234 -0
- data/lib/shiba/index.rb +159 -0
- data/lib/shiba/output.rb +67 -0
- data/lib/shiba/output/tags.yaml +44 -0
- data/lib/shiba/query.rb +34 -0
- data/lib/shiba/query_watcher.rb +79 -0
- data/lib/shiba/railtie.rb +20 -0
- data/lib/shiba/version.rb +3 -0
- data/shiba.gemspec +38 -0
- data/web/bootstrap.min.css +7 -0
- data/web/dist/bundle.js +167 -0
- data/web/main.css +18 -0
- data/web/main.js +5 -0
- data/web/package-lock.json +4100 -0
- data/web/package.json +19 -0
- data/web/results.html.erb +199 -0
- data/web/vue.js +11055 -0
- data/web/webpack.config.js +14 -0
- metadata +121 -0
data/bin/release
ADDED
@@ -0,0 +1,6 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
for cmd in ["fingerprint"]; do
|
4
|
+
GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o cmd/builds/fingerprint.linux-amd64 cmd/fingerprint.go
|
5
|
+
GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o cmd/builds/fingerprint.darwin-amd64 cmd/fingerprint.go
|
6
|
+
done
|
data/bin/setup
ADDED
data/bin/shiba
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optionparser'
|
4
|
+
|
5
|
+
APP = File.basename(__FILE__)
|
6
|
+
|
7
|
+
commands = {
|
8
|
+
"analyze" => { :bin => "analyze", :description => "Analyze logged SQL queries" },
|
9
|
+
"watch" => { :bin => "watch.rb", :description => "Log queries from Rails command" }
|
10
|
+
}
|
11
|
+
|
12
|
+
global = OptionParser.new do |opts|
|
13
|
+
opts.banner = "usage: #{APP} [--help] <command> [<args>]"
|
14
|
+
opts.separator ""
|
15
|
+
opts.separator "These are the commands available:"
|
16
|
+
commands.each do |name,info|
|
17
|
+
opts.separator " #{name} #{info[:description]}"
|
18
|
+
end
|
19
|
+
opts.separator ""
|
20
|
+
opts.separator "See #{APP} --help <command> to read about a specific command."
|
21
|
+
opts.separator ""
|
22
|
+
end
|
23
|
+
|
24
|
+
global.order!
|
25
|
+
|
26
|
+
command = ARGV.shift
|
27
|
+
|
28
|
+
if command.nil?
|
29
|
+
puts global.to_s
|
30
|
+
exit
|
31
|
+
end
|
32
|
+
|
33
|
+
if !commands.key?(command)
|
34
|
+
puts "#{APP}: '#{command}' is not a '#{APP}' command. See '#{APP} --help'."
|
35
|
+
exit 1
|
36
|
+
end
|
37
|
+
|
38
|
+
path = "#{File.dirname(__FILE__)}/#{commands[command][:bin]}"
|
39
|
+
|
40
|
+
Kernel.exec(path, *ARGV)
|
data/bin/watch.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optionparser'
|
4
|
+
|
5
|
+
options = {}
|
6
|
+
parser = OptionParser.new do |opts|
|
7
|
+
opts.banner = "watch <command>. Create SQL logs for a running process"
|
8
|
+
|
9
|
+
opts.on("-f", "--file FILE", "write to file") do |f|
|
10
|
+
options["file"] = f
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
parser.parse!
|
16
|
+
|
17
|
+
$stderr.puts "Recording SQL queries to '#{options["file"]}'..."
|
18
|
+
ENV['SHIBA_OUT'] = options["file"]
|
19
|
+
Kernel.exec(ARGV.join(" "))
|
Binary file
|
Binary file
|
data/cmd/check.go
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
package main
|
2
|
+
|
3
|
+
import (
|
4
|
+
"bufio"
|
5
|
+
"encoding/csv"
|
6
|
+
"flag"
|
7
|
+
"fmt"
|
8
|
+
"io"
|
9
|
+
"log"
|
10
|
+
"os"
|
11
|
+
|
12
|
+
"github.com/xwb1989/sqlparser"
|
13
|
+
)
|
14
|
+
|
15
|
+
var indexFile = flag.String("i", "", "index files to check queries against")
|
16
|
+
|
17
|
+
func init() {
|
18
|
+
flag.Usage = func() {
|
19
|
+
fmt.Fprintf(os.Stderr, "Returns unindexed queries. Reads select sql statements from stdin and checks the provided index file which is the result of analyze.\n")
|
20
|
+
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
|
21
|
+
|
22
|
+
flag.PrintDefaults()
|
23
|
+
}
|
24
|
+
}
|
25
|
+
|
26
|
+
type tableStats map[string][]string
|
27
|
+
|
28
|
+
func main() {
|
29
|
+
flag.Parse()
|
30
|
+
|
31
|
+
if *indexFile == "" {
|
32
|
+
flag.Usage()
|
33
|
+
os.Exit(1)
|
34
|
+
}
|
35
|
+
|
36
|
+
stats := readTableIndexes(*indexFile)
|
37
|
+
r := bufio.NewReader(os.Stdin)
|
38
|
+
|
39
|
+
analyzeQueries(r, stats)
|
40
|
+
|
41
|
+
}
|
42
|
+
|
43
|
+
func readTableIndexes(path string) tableStats {
|
44
|
+
f, err := os.Open(path)
|
45
|
+
if err != nil {
|
46
|
+
fmt.Fprintln(os.Stderr, "index list not found at", path)
|
47
|
+
os.Exit(1)
|
48
|
+
}
|
49
|
+
|
50
|
+
br := bufio.NewReader(f)
|
51
|
+
|
52
|
+
r := csv.NewReader(br)
|
53
|
+
|
54
|
+
indexes := map[string][]string{}
|
55
|
+
|
56
|
+
for {
|
57
|
+
record, err := r.Read()
|
58
|
+
if err == io.EOF {
|
59
|
+
break
|
60
|
+
}
|
61
|
+
if err != nil {
|
62
|
+
log.Fatal(err)
|
63
|
+
}
|
64
|
+
|
65
|
+
table := record[0]
|
66
|
+
|
67
|
+
indexes[table] = append(indexes[table], record[1])
|
68
|
+
}
|
69
|
+
|
70
|
+
return indexes
|
71
|
+
}
|
72
|
+
|
73
|
+
func analyzeQueries(r io.Reader, stats tableStats) {
|
74
|
+
scanner := bufio.NewScanner(r)
|
75
|
+
|
76
|
+
for scanner.Scan() {
|
77
|
+
sql := scanner.Text()
|
78
|
+
stmt, err := sqlparser.Parse(sql)
|
79
|
+
|
80
|
+
if err != nil {
|
81
|
+
fmt.Println("Unable to parse line", sql)
|
82
|
+
fmt.Fprintln(os.Stderr, err)
|
83
|
+
os.Exit(1)
|
84
|
+
}
|
85
|
+
|
86
|
+
switch q := stmt.(type) {
|
87
|
+
case *sqlparser.Select:
|
88
|
+
if !hasIndex(q, stats) {
|
89
|
+
fmt.Println(sql)
|
90
|
+
}
|
91
|
+
default:
|
92
|
+
fmt.Fprintln(os.Stderr, "Only Select queries are supported", sqlparser.String(stmt))
|
93
|
+
os.Exit(1)
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
if err := scanner.Err(); err != nil {
|
98
|
+
fmt.Fprintln(os.Stderr, "reading standard input:", err)
|
99
|
+
}
|
100
|
+
}
|
101
|
+
|
102
|
+
func hasIndex(q *sqlparser.Select, stats tableStats) bool {
|
103
|
+
// fixme: assume simple table from
|
104
|
+
table := sqlparser.String(q.From[0])
|
105
|
+
// expr :=
|
106
|
+
if _, ok := stats[table]; !ok {
|
107
|
+
fmt.Fprintln(os.Stderr, "Table does not appear to have an index", sqlparser.String(q))
|
108
|
+
return false
|
109
|
+
}
|
110
|
+
|
111
|
+
indexed := stats[table]
|
112
|
+
|
113
|
+
var found bool
|
114
|
+
sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) {
|
115
|
+
switch node.(type) {
|
116
|
+
case *sqlparser.ColName:
|
117
|
+
// we're getting fully qualified column back
|
118
|
+
c := sqlparser.String(node)
|
119
|
+
for _, v := range indexed {
|
120
|
+
// quick hack because I'm parsing columns wrong somehow
|
121
|
+
if v == c || (table+"."+v) == c {
|
122
|
+
found = true
|
123
|
+
return false, nil
|
124
|
+
}
|
125
|
+
}
|
126
|
+
}
|
127
|
+
|
128
|
+
return true, nil
|
129
|
+
}, q.Where)
|
130
|
+
|
131
|
+
return found
|
132
|
+
}
|
133
|
+
|
134
|
+
func checkErr(err error) {
|
135
|
+
if err != nil {
|
136
|
+
panic(err)
|
137
|
+
}
|
138
|
+
}
|
data/cmd/fingerprint.go
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
package main
|
2
|
+
|
3
|
+
import (
|
4
|
+
"bufio"
|
5
|
+
"fmt"
|
6
|
+
"io"
|
7
|
+
"os"
|
8
|
+
|
9
|
+
"github.com/percona/go-mysql/query"
|
10
|
+
)
|
11
|
+
|
12
|
+
func main() {
|
13
|
+
r := bufio.NewReader(os.Stdin)
|
14
|
+
|
15
|
+
for {
|
16
|
+
sql, err := r.ReadString('\n')
|
17
|
+
|
18
|
+
switch err {
|
19
|
+
case nil:
|
20
|
+
fmt.Println(query.Fingerprint(sql))
|
21
|
+
case io.EOF:
|
22
|
+
os.Exit(0)
|
23
|
+
default:
|
24
|
+
fmt.Fprintln(os.Stderr, "reading standard input:", err)
|
25
|
+
}
|
26
|
+
}
|
27
|
+
|
28
|
+
}
|
data/cmd/inspect.go
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
package main
|
2
|
+
|
3
|
+
import (
|
4
|
+
"database/sql"
|
5
|
+
"encoding/csv"
|
6
|
+
"flag"
|
7
|
+
"fmt"
|
8
|
+
"log"
|
9
|
+
"os"
|
10
|
+
|
11
|
+
_ "github.com/go-sql-driver/mysql"
|
12
|
+
)
|
13
|
+
|
14
|
+
var user = flag.String("u", "", "user name for database connection")
|
15
|
+
var password = flag.String("p", "", "password for database connection")
|
16
|
+
var database = flag.String("db", "", "database name e.g. app_test")
|
17
|
+
|
18
|
+
func init() {
|
19
|
+
flag.Usage = func() {
|
20
|
+
fmt.Fprintf(os.Stderr, "Reads indexes from the given database and outputs the results in CSV.\n")
|
21
|
+
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
|
22
|
+
|
23
|
+
flag.PrintDefaults()
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
func main() {
|
28
|
+
flag.Parse()
|
29
|
+
|
30
|
+
if *user == "" {
|
31
|
+
fmt.Println("missing -u database user")
|
32
|
+
flag.Usage()
|
33
|
+
os.Exit(1)
|
34
|
+
}
|
35
|
+
|
36
|
+
if *database == "" {
|
37
|
+
fmt.Println("missing -db database name")
|
38
|
+
flag.Usage()
|
39
|
+
os.Exit(1)
|
40
|
+
}
|
41
|
+
|
42
|
+
db, err := sql.Open("mysql", *user+":"+*password+"@/information_schema?charset=utf8")
|
43
|
+
defer db.Close()
|
44
|
+
|
45
|
+
if err != nil {
|
46
|
+
if *password == "" {
|
47
|
+
fmt.Println("missing -p database password")
|
48
|
+
flag.Usage()
|
49
|
+
os.Exit(1)
|
50
|
+
}
|
51
|
+
}
|
52
|
+
|
53
|
+
checkErr(err)
|
54
|
+
|
55
|
+
// query
|
56
|
+
// select column_name, column_key from columns where table_schema = "zammad_test";
|
57
|
+
rows, err := db.Query("SELECT table_name, column_name FROM columns where TABLE_SCHEMA = ? AND COLUMN_KEY is not null", *database)
|
58
|
+
checkErr(err)
|
59
|
+
|
60
|
+
w := csv.NewWriter(os.Stdout)
|
61
|
+
|
62
|
+
count := 0
|
63
|
+
for rows.Next() {
|
64
|
+
count++
|
65
|
+
result := make([]string, 2)
|
66
|
+
|
67
|
+
err = rows.Scan(&result[0], &result[1])
|
68
|
+
checkErr(err)
|
69
|
+
|
70
|
+
if err := w.Write(result); err != nil {
|
71
|
+
log.Fatalln("error writing record to csv:", err)
|
72
|
+
}
|
73
|
+
}
|
74
|
+
|
75
|
+
// Write any buffered data to the underlying writer (standard output).
|
76
|
+
w.Flush()
|
77
|
+
|
78
|
+
if err := w.Error(); err != nil {
|
79
|
+
log.Fatal(err)
|
80
|
+
}
|
81
|
+
|
82
|
+
if count == 0 {
|
83
|
+
log.Fatal("No rows found for database. It's possible the user does not have permissions to view it.")
|
84
|
+
}
|
85
|
+
|
86
|
+
}
|
87
|
+
|
88
|
+
func checkErr(err error) {
|
89
|
+
if err != nil {
|
90
|
+
panic(err)
|
91
|
+
}
|
92
|
+
}
|
data/cmd/parse.go
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
package main
|
2
|
+
|
3
|
+
import (
|
4
|
+
"bufio"
|
5
|
+
"encoding/json"
|
6
|
+
"flag"
|
7
|
+
"fmt"
|
8
|
+
"io"
|
9
|
+
"os"
|
10
|
+
|
11
|
+
"github.com/xwb1989/sqlparser"
|
12
|
+
)
|
13
|
+
|
14
|
+
type query struct {
|
15
|
+
Table string `json:"table"`
|
16
|
+
Columns []string `json:"columns"`
|
17
|
+
}
|
18
|
+
|
19
|
+
func init() {
|
20
|
+
flag.Usage = func() {
|
21
|
+
fmt.Fprintf(os.Stderr, "Takes queries from stdin and converst them to json.\n")
|
22
|
+
}
|
23
|
+
}
|
24
|
+
|
25
|
+
func main() {
|
26
|
+
parseQueries(os.Stdin)
|
27
|
+
}
|
28
|
+
|
29
|
+
func parseQueries(r io.Reader) {
|
30
|
+
scanner := bufio.NewScanner(r)
|
31
|
+
|
32
|
+
enc := json.NewEncoder(os.Stdout)
|
33
|
+
|
34
|
+
for scanner.Scan() {
|
35
|
+
sql := scanner.Text()
|
36
|
+
stmt, err := sqlparser.Parse(sql)
|
37
|
+
|
38
|
+
if err != nil {
|
39
|
+
fmt.Println("Unable to parse line", sql)
|
40
|
+
fmt.Fprintln(os.Stderr, err)
|
41
|
+
}
|
42
|
+
|
43
|
+
switch q := stmt.(type) {
|
44
|
+
case *sqlparser.Select:
|
45
|
+
parsed := parse(q)
|
46
|
+
enc.Encode(parsed)
|
47
|
+
case *sqlparser.Insert:
|
48
|
+
enc.Encode(parseTable(q))
|
49
|
+
default:
|
50
|
+
fmt.Fprintln(os.Stderr, "Query not supported: ", sqlparser.String(stmt))
|
51
|
+
}
|
52
|
+
}
|
53
|
+
|
54
|
+
if err := scanner.Err(); err != nil {
|
55
|
+
fmt.Fprintln(os.Stderr, "reading standard input:", err)
|
56
|
+
}
|
57
|
+
}
|
58
|
+
|
59
|
+
func parseTable(q *sqlparser.Insert) string {
|
60
|
+
return sqlparser.String(q.Table)
|
61
|
+
}
|
62
|
+
|
63
|
+
func parse(q *sqlparser.Select) query {
|
64
|
+
// fixme: assume simple table from
|
65
|
+
//tables := sqlparser.String(q.From[0])
|
66
|
+
table := sqlparser.String(q.From)
|
67
|
+
columns := []string{}
|
68
|
+
sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) {
|
69
|
+
switch node.(type) {
|
70
|
+
case *sqlparser.ColName:
|
71
|
+
c := sqlparser.String(node)
|
72
|
+
columns = append(columns, c)
|
73
|
+
}
|
74
|
+
|
75
|
+
return true, nil
|
76
|
+
}, q.Where)
|
77
|
+
|
78
|
+
return query{Table: table, Columns: columns}
|
79
|
+
}
|
data/lib/shiba.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require "shiba/version"
|
2
|
+
require "mysql2"
|
3
|
+
|
4
|
+
module Shiba
|
5
|
+
class Error < StandardError; end
|
6
|
+
|
7
|
+
def self.configure(connection_hash)
|
8
|
+
@connection_hash = connection_hash
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.connection
|
12
|
+
@connection ||= Mysql2::Client.new(@connection_hash)
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.root
|
16
|
+
File.dirname(__dir__)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# This goes at the end so that Shiba.root is defined.
|
21
|
+
require "shiba/railtie" if defined?(Rails)
|