SQLDependencyGrapher 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Manifest.txt +7 -0
- data/Rakefile +66 -0
- data/bin/sql_dep_graph +6 -0
- data/lib/sql_dep_grapher.rb +116 -0
- data/lib/sql_dep_grapher/graph.rb +82 -0
- data/test/test.log +22 -0
- data/test/test_sql_dep_grapher.rb +102 -0
- metadata +44 -0
data/Manifest.txt
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/testtask'
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
require 'rake/gempackagetask'
|
6
|
+
require 'rake/contrib/sshpublisher'
|
7
|
+
|
8
|
+
$:.unshift 'lib'
|
9
|
+
require 'sql_dep_grapher'
|
10
|
+
|
11
|
+
$VERBOSE = nil
|
12
|
+
|
13
|
+
spec = Gem::Specification.new do |s|
|
14
|
+
s.name = "SQLDependencyGrapher"
|
15
|
+
s.version = SQLDependencyGrapher::VERSION
|
16
|
+
s.summary = "Graphs table dependencies based on usage from SQL logs"
|
17
|
+
s.author = "Eric Hodel"
|
18
|
+
s.email = "eric@robotcoop.com"
|
19
|
+
|
20
|
+
s.has_rdoc = true
|
21
|
+
s.files = File.read("Manifest.txt").split($/)
|
22
|
+
s.require_path = 'lib'
|
23
|
+
s.executables = ["sql_dep_graph"]
|
24
|
+
s.default_executable = "sql_dep_graph"
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "Run tests"
|
28
|
+
task :default => [ :test ]
|
29
|
+
|
30
|
+
Rake::TestTask.new("test") do |t|
|
31
|
+
t.libs << "test"
|
32
|
+
t.libs << "lib"
|
33
|
+
t.pattern = "test/test_*.rb"
|
34
|
+
t.verbose = true
|
35
|
+
end
|
36
|
+
|
37
|
+
desc "Generate RDoc"
|
38
|
+
Rake::RDocTask.new :rdoc do |rd|
|
39
|
+
rd.rdoc_dir = "doc"
|
40
|
+
rd.rdoc_files.add "lib"
|
41
|
+
rd.main = "SQLDependencyGrapher"
|
42
|
+
rd.options << "-d" if `which dot` =~ /\/dot/
|
43
|
+
end
|
44
|
+
|
45
|
+
desc "Build Gem"
|
46
|
+
Rake::GemPackageTask.new spec do |pkg|
|
47
|
+
pkg.need_zip = true
|
48
|
+
pkg.need_tar = true
|
49
|
+
end
|
50
|
+
|
51
|
+
desc "Sends RDoc to RubyForge"
|
52
|
+
task :send_rdoc => [ :rerdoc ] do
|
53
|
+
publisher = Rake::SshDirPublisher.new('drbrain@rubyforge.org',
|
54
|
+
'/var/www/gforge-projects/rails-analyzer/sql_dep_graph',
|
55
|
+
'doc')
|
56
|
+
publisher.upload
|
57
|
+
end
|
58
|
+
|
59
|
+
desc "Clean up"
|
60
|
+
task :clean => [ :clobber_rdoc, :clobber_package ]
|
61
|
+
|
62
|
+
desc "Clean up"
|
63
|
+
task :clobber => [ :clean ]
|
64
|
+
|
65
|
+
# vim: ts=4 sts=4 sw=4 syntax=Ruby
|
66
|
+
|
data/bin/sql_dep_graph
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
$TESTING = false unless defined? $TESTING
|
2
|
+
|
3
|
+
require 'sql_dep_grapher/graph'
|
4
|
+
|
5
|
+
##
|
6
|
+
# SQLDependencyGrapher allows you to visualize the query dependencies between
|
7
|
+
# your database tables to better understand how they actually get used. It
|
8
|
+
# generates a graph of the connections between tables based on joins found in
|
9
|
+
# a SQL query log.
|
10
|
+
#
|
11
|
+
# To generate a graph, you run the sql_dep_graph command whiche creates a dot
|
12
|
+
# file that you can render with Graphviz or OmniGraffle.
|
13
|
+
#
|
14
|
+
# Usage:
|
15
|
+
#
|
16
|
+
# sql_dep_graph log/production.log > sql_deps.dot
|
17
|
+
#
|
18
|
+
# dot -Tpng sql_deps.dot > sql_deps.png
|
19
|
+
#
|
20
|
+
# Then open sql_deps.png in your favorite image viewer.
|
21
|
+
#
|
22
|
+
# You can find Graphviz here:
|
23
|
+
#
|
24
|
+
# http://www.graphviz.org/
|
25
|
+
#
|
26
|
+
# And can download it for various platforms from here:
|
27
|
+
#
|
28
|
+
# http://www.graphviz.org/Download.php
|
29
|
+
|
30
|
+
module SQLDependencyGrapher
|
31
|
+
|
32
|
+
##
|
33
|
+
# The Version of SQLDependencyGrapher
|
34
|
+
|
35
|
+
VERSION = '1.0.0'
|
36
|
+
|
37
|
+
##
|
38
|
+
# Builds a Graph from a SQL query log given to +stream+.
|
39
|
+
|
40
|
+
def self.build_graph(stream = ARGF)
|
41
|
+
data = collect stream
|
42
|
+
counts = count data
|
43
|
+
return graph(data, counts)
|
44
|
+
end
|
45
|
+
|
46
|
+
private unless $TESTING
|
47
|
+
|
48
|
+
##
|
49
|
+
# Returns an Array of SQL joins from +stream+.
|
50
|
+
|
51
|
+
def self.collect(stream)
|
52
|
+
data = []
|
53
|
+
|
54
|
+
stream.each_line do |line|
|
55
|
+
line.grep(/FROM\s+(.*?)\s+WHERE/) do
|
56
|
+
tables = $1.split(',').reject { |t| t =~ /\(/ }
|
57
|
+
tables = tables.map { |t| t.split(' ').first }
|
58
|
+
data << tables if tables.size > 1
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
return data
|
63
|
+
end
|
64
|
+
|
65
|
+
##
|
66
|
+
# Counts the number of times a join between two tables occurs. Returns a
|
67
|
+
# Hash of pair => count.
|
68
|
+
|
69
|
+
def self.count(data)
|
70
|
+
counts = Hash.new 0
|
71
|
+
|
72
|
+
data.each do |tables|
|
73
|
+
tables = tables.sort
|
74
|
+
curr = tables.shift
|
75
|
+
tables.each do |table|
|
76
|
+
counts[[curr, table]] += 1
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
return counts
|
81
|
+
end
|
82
|
+
|
83
|
+
##
|
84
|
+
# Creates a Graph of +data+ using +counts+ for edge weights.
|
85
|
+
|
86
|
+
def self.graph(data, counts)
|
87
|
+
graph = Graph.new
|
88
|
+
|
89
|
+
max_count = counts.values.max
|
90
|
+
solid = Math.log10(max_count).floor
|
91
|
+
dotted = solid / 3
|
92
|
+
dashed = solid / 3 * 2
|
93
|
+
|
94
|
+
counts.each do |(first, second), count|
|
95
|
+
graph[first] << second
|
96
|
+
edge = ["weight=#{count}", "label=\"#{count}\"", "dir=none"]
|
97
|
+
|
98
|
+
case Math.log10 count
|
99
|
+
when 0.0..dotted then
|
100
|
+
edge << "style = dotted"
|
101
|
+
when dotted..dashed then
|
102
|
+
edge << "style = dashed"
|
103
|
+
when dashed..solid then
|
104
|
+
edge << "style = solid"
|
105
|
+
else
|
106
|
+
edge << "style = bold"
|
107
|
+
end
|
108
|
+
|
109
|
+
graph.edge[first][second].push(*edge)
|
110
|
+
end
|
111
|
+
|
112
|
+
return graph
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
@@ -0,0 +1,82 @@
|
|
1
|
+
#!/usr/local/bin/ruby -w
|
2
|
+
|
3
|
+
require 'pp'
|
4
|
+
|
5
|
+
class Graph < Hash
|
6
|
+
|
7
|
+
attr_reader :attribs
|
8
|
+
attr_reader :prefix
|
9
|
+
attr_reader :order
|
10
|
+
attr_reader :edge
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
super { |h,k| h[k] = [] }
|
14
|
+
@prefix = []
|
15
|
+
@attribs = Hash.new { |h,k| h[k] = [] }
|
16
|
+
@edge = Hash.new { |h,k| h[k] = Hash.new { |h2,k2| h2[k2] = [] } }
|
17
|
+
@order = []
|
18
|
+
end
|
19
|
+
|
20
|
+
def []=(key, val)
|
21
|
+
@order << key unless self.has_key? key
|
22
|
+
super(key, val)
|
23
|
+
end
|
24
|
+
|
25
|
+
def each_pair
|
26
|
+
@order.each do |from|
|
27
|
+
self[from].each do |to|
|
28
|
+
yield(from, to)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def invert
|
34
|
+
result = self.class.new
|
35
|
+
each_pair do |from, to|
|
36
|
+
result[to] << from
|
37
|
+
end
|
38
|
+
result
|
39
|
+
end
|
40
|
+
|
41
|
+
def counts
|
42
|
+
result = Hash.new(0)
|
43
|
+
each_pair do |from, to|
|
44
|
+
result[from] += 1
|
45
|
+
end
|
46
|
+
result
|
47
|
+
end
|
48
|
+
|
49
|
+
def keys_by_count
|
50
|
+
counts.sort_by { |x,y| y }.map {|x| x.first }
|
51
|
+
end
|
52
|
+
|
53
|
+
def to_s
|
54
|
+
result = []
|
55
|
+
result << "digraph absent"
|
56
|
+
result << " {"
|
57
|
+
@prefix.each do |line|
|
58
|
+
result << line
|
59
|
+
end
|
60
|
+
@attribs.sort.each do |node, attribs|
|
61
|
+
result << " #{node.inspect} [ #{attribs.join(',')} ]"
|
62
|
+
end
|
63
|
+
each_pair do |from, to|
|
64
|
+
edge = @edge[from][to].join(", ")
|
65
|
+
edge = " [ #{edge} ]" unless edge.empty?
|
66
|
+
result << " #{from.inspect} -> #{to.inspect}#{edge};"
|
67
|
+
end
|
68
|
+
result << " }"
|
69
|
+
result.join("\n")
|
70
|
+
end
|
71
|
+
|
72
|
+
def save(path, type="png")
|
73
|
+
File.open(path + ".dot", "w") do |f|
|
74
|
+
f.puts self.to_s
|
75
|
+
f.flush
|
76
|
+
cmd = "/usr/local/bin/dot -T#{type} #{path}.dot > #{path}.#{type}"
|
77
|
+
system cmd
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
data/test/test.log
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
FROM goals g, related_goal_summaries rgs WHERE
|
2
|
+
FROM tags_teams j, tags t WHERE
|
3
|
+
FROM tags_teams, tags, teams WHERE
|
4
|
+
FROM entries e, teams t WHERE
|
5
|
+
FROM entries e, teams t WHERE
|
6
|
+
FROM goals g, related_goal_summaries rgs WHERE
|
7
|
+
FROM teams, team_members WHERE
|
8
|
+
FROM tags_teams, tags, teams WHERE
|
9
|
+
FROM tags_teams, tags, teams, goals g WHERE
|
10
|
+
FROM tags t, tag_similarities ts, tags_teams tt, teams tm WHERE
|
11
|
+
FROM tags_teams j, tags t WHERE
|
12
|
+
FROM tags_teams, tags, teams WHERE
|
13
|
+
FROM tags_teams j, tags t WHERE
|
14
|
+
FROM tags_teams, tags, teams WHERE
|
15
|
+
FROM goals g, goal_similarities gs WHERE
|
16
|
+
FROM goals g, related_goal_summaries rgs WHERE
|
17
|
+
FROM goals g, related_goal_summaries rgs WHERE
|
18
|
+
FROM tags_teams j, tags t WHERE
|
19
|
+
FROM tags_teams, tags, teams, goals g WHERE
|
20
|
+
FROM tags_teams, tags, teams WHERE
|
21
|
+
FROM entries e, teams t WHERE
|
22
|
+
FROM entries e, teams t WHERE
|
@@ -0,0 +1,102 @@
|
|
1
|
+
$TESTING = true
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
require 'sql_dep_grapher'
|
5
|
+
|
6
|
+
class TestSQLDependencyGrapher < Test::Unit::TestCase
|
7
|
+
|
8
|
+
DATA = [
|
9
|
+
%w[goals related_goal_summaries],
|
10
|
+
%w[tags_teams tags],
|
11
|
+
%w[tags_teams tags teams],
|
12
|
+
%w[entries teams],
|
13
|
+
%w[entries teams],
|
14
|
+
%w[goals related_goal_summaries],
|
15
|
+
%w[teams team_members],
|
16
|
+
%w[tags_teams tags teams],
|
17
|
+
%w[tags_teams tags teams goals],
|
18
|
+
%w[tags tag_similarities tags_teams teams],
|
19
|
+
%w[tags_teams tags],
|
20
|
+
%w[tags_teams tags teams],
|
21
|
+
%w[tags_teams tags],
|
22
|
+
%w[tags_teams tags teams],
|
23
|
+
%w[goals goal_similarities],
|
24
|
+
%w[goals related_goal_summaries],
|
25
|
+
%w[goals related_goal_summaries],
|
26
|
+
%w[tags_teams tags],
|
27
|
+
%w[tags_teams tags teams goals],
|
28
|
+
%w[tags_teams tags teams],
|
29
|
+
%w[entries teams],
|
30
|
+
%w[entries teams],
|
31
|
+
]
|
32
|
+
|
33
|
+
COUNTS = {
|
34
|
+
%w[entries teams] => 4,
|
35
|
+
%w[goal_similarities goals] => 1,
|
36
|
+
%w[goals related_goal_summaries] => 4,
|
37
|
+
%w[goals tags] => 2,
|
38
|
+
%w[goals tags_teams] => 2,
|
39
|
+
%w[goals teams] => 2,
|
40
|
+
%w[tag_similarities tags] => 1,
|
41
|
+
%w[tag_similarities tags_teams] => 1,
|
42
|
+
%w[tag_similarities teams] => 1,
|
43
|
+
%w[tags tags_teams] => 9,
|
44
|
+
%w[tags teams] => 5,
|
45
|
+
%w[team_members teams] => 1,
|
46
|
+
}
|
47
|
+
|
48
|
+
GRAPH = "digraph absent
|
49
|
+
{
|
50
|
+
\"tag_similarities\" -> \"tags_teams\" [ weight=1, label=\"1\", dir=none, style = dotted ];
|
51
|
+
\"tag_similarities\" -> \"teams\" [ weight=1, label=\"1\", dir=none, style = dotted ];
|
52
|
+
\"tag_similarities\" -> \"tags\" [ weight=1, label=\"1\", dir=none, style = dotted ];
|
53
|
+
\"goals\" -> \"tags_teams\" [ weight=2, label=\"2\", dir=none, style = bold ];
|
54
|
+
\"goals\" -> \"related_goal_summaries\" [ weight=4, label=\"4\", dir=none, style = bold ];
|
55
|
+
\"goals\" -> \"teams\" [ weight=2, label=\"2\", dir=none, style = bold ];
|
56
|
+
\"goals\" -> \"tags\" [ weight=2, label=\"2\", dir=none, style = bold ];
|
57
|
+
\"goal_similarities\" -> \"goals\" [ weight=1, label=\"1\", dir=none, style = dotted ];
|
58
|
+
\"tags\" -> \"teams\" [ weight=5, label=\"5\", dir=none, style = bold ];
|
59
|
+
\"tags\" -> \"tags_teams\" [ weight=9, label=\"9\", dir=none, style = bold ];
|
60
|
+
\"entries\" -> \"teams\" [ weight=4, label=\"4\", dir=none, style = bold ];
|
61
|
+
\"team_members\" -> \"teams\" [ weight=1, label=\"1\", dir=none, style = dotted ];
|
62
|
+
}"
|
63
|
+
|
64
|
+
def setup
|
65
|
+
@sdg = SQLDependencyGrapher
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_build_graph
|
69
|
+
graph = :junk
|
70
|
+
File.open 'test/test.log' do |fp|
|
71
|
+
graph = @sdg.build_graph fp
|
72
|
+
end
|
73
|
+
|
74
|
+
# HACK until I can get Graph#== working
|
75
|
+
flunk "Equality is hard"
|
76
|
+
#assert_equal GRAPH, graph.to_s
|
77
|
+
end
|
78
|
+
|
79
|
+
def test_collect
|
80
|
+
data = :junk
|
81
|
+
|
82
|
+
File.open 'test/test.log' do |fp|
|
83
|
+
data = @sdg.collect fp
|
84
|
+
end
|
85
|
+
|
86
|
+
assert_equal DATA, data
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_count
|
90
|
+
counts = @sdg.count DATA
|
91
|
+
|
92
|
+
assert_equal COUNTS, counts
|
93
|
+
end
|
94
|
+
|
95
|
+
def test_graph
|
96
|
+
graph = @sdg.graph DATA, COUNTS
|
97
|
+
|
98
|
+
assert_equal GRAPH, graph.to_s # Graph#== won't do what I want yet, I think
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
|
metadata
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.8.10
|
3
|
+
specification_version: 1
|
4
|
+
name: SQLDependencyGrapher
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 1.0.0
|
7
|
+
date: 2005-06-03
|
8
|
+
summary: Graphs table dependencies based on usage from SQL logs
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: eric@robotcoop.com
|
12
|
+
homepage:
|
13
|
+
rubyforge_project:
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable: sql_dep_graph
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: true
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
-
|
22
|
+
- ">"
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.0.0
|
25
|
+
version:
|
26
|
+
platform: ruby
|
27
|
+
authors:
|
28
|
+
- Eric Hodel
|
29
|
+
files:
|
30
|
+
- Manifest.txt
|
31
|
+
- Rakefile
|
32
|
+
- bin/sql_dep_graph
|
33
|
+
- lib/sql_dep_grapher.rb
|
34
|
+
- lib/sql_dep_grapher/graph.rb
|
35
|
+
- test/test.log
|
36
|
+
- test/test_sql_dep_grapher.rb
|
37
|
+
test_files: []
|
38
|
+
rdoc_options: []
|
39
|
+
extra_rdoc_files: []
|
40
|
+
executables:
|
41
|
+
- sql_dep_graph
|
42
|
+
extensions: []
|
43
|
+
requirements: []
|
44
|
+
dependencies: []
|