sql_dep_graph 1.0.0

data/Manifest.txt

@@ -0,0 +1,7 @@
+Manifest.txt
+Rakefile
+bin/sql_dep_graph
+lib/sql_dep_grapher.rb
+lib/sql_dep_grapher/graph.rb
+test/test.log
+test/test_sql_dep_grapher.rb

data/Rakefile

@@ -0,0 +1,66 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/contrib/sshpublisher'
+
+$:.unshift 'lib'
+require 'sql_dep_grapher'
+
+$VERBOSE = nil
+
+spec = Gem::Specification.new do |s|
+    s.name = "sql_dep_graph"
+    s.version = SQLDependencyGrapher::VERSION
+    s.summary = "Graphs table dependencies based on usage from SQL logs"
+    s.author = "Eric Hodel"
+    s.email = "eric@robotcoop.com"
+
+    s.has_rdoc = true
+    s.files = File.read("Manifest.txt").split($/)
+    s.require_path = 'lib'
+    s.executables = ["sql_dep_graph"]
+    s.default_executable = "sql_dep_graph"
+end
+
+desc "Run tests"
+task :default => [ :test ]
+
+Rake::TestTask.new("test") do |t|
+    t.libs << "test"
+    t.libs << "lib"
+    t.pattern = "test/test_*.rb"
+    t.verbose = true
+end
+
+desc "Generate RDoc"
+Rake::RDocTask.new :rdoc do |rd|
+    rd.rdoc_dir = "doc"
+    rd.rdoc_files.add "lib"
+    rd.main = "SQLDependencyGrapher"
+    rd.options << "-d" if `which dot` =~ /\/dot/
+end
+
+desc "Build Gem"
+Rake::GemPackageTask.new spec do |pkg|
+    pkg.need_zip = true
+    pkg.need_tar = true
+end
+
+desc "Sends RDoc to RubyForge"
+task :send_rdoc => [ :rerdoc ] do
+    publisher = Rake::SshDirPublisher.new('drbrain@rubyforge.org',
+                    '/var/www/gforge-projects/rails-analyzer/sql_dep_graph',
+                    'doc')
+    publisher.upload
+end
+
+desc "Clean up"
+task :clean => [ :clobber_rdoc, :clobber_package ]
+
+desc "Clean up"
+task :clobber => [ :clean ]
+
+# vim: ts=4 sts=4 sw=4 syntax=Ruby
+

data/bin/sql_dep_graph

@@ -0,0 +1,6 @@
+#!/usr/local/bin/ruby -w
+
+require 'sql_dep_grapher'
+
+puts SQLDependencyGrapher.build_graph
+

data/lib/sql_dep_grapher.rb

@@ -0,0 +1,116 @@
+$TESTING = false unless defined? $TESTING
+
+require 'sql_dep_grapher/graph'
+
+##
+# SQLDependencyGrapher allows you to visualize the query dependencies between
+# your database tables to better understand how they actually get used.  It
+# generates a graph of the connections between tables based on joins found in
+# a SQL query log.
+#
+# To generate a graph, you run the sql_dep_graph command whiche creates a dot
+# file that you can render with Graphviz or OmniGraffle.
+#
+# Usage:
+#
+#   sql_dep_graph log/production.log > sql_deps.dot
+#
+#   dot -Tpng sql_deps.dot > sql_deps.png
+#
+# Then open sql_deps.png in your favorite image viewer.
+#
+# You can find Graphviz here:
+#
+# http://www.graphviz.org/
+#
+# And can download it for various platforms from here:
+#
+# http://www.graphviz.org/Download.php
+
+module SQLDependencyGrapher
+
+  ##
+  # The Version of SQLDependencyGrapher
+
+  VERSION = '1.0.0'
+
+  ##
+  # Builds a Graph from a SQL query log given to +stream+.
+
+  def self.build_graph(stream = ARGF)
+    data = collect stream
+    counts = count data
+    return graph(data, counts)
+  end
+
+  private unless $TESTING
+
+  ##
+  # Returns an Array of SQL joins from +stream+.
+
+  def self.collect(stream)
+    data = []
+
+    stream.each_line do |line|
+      line.grep(/FROM\s+(.*?)\s+WHERE/) do
+        tables = $1.split(',').reject { |t| t =~ /\(/ }
+        tables = tables.map { |t| t.split(' ').first }
+        data << tables if tables.size > 1
+      end
+    end
+
+    return data
+  end
+
+  ##
+  # Counts the number of times a join between two tables occurs.  Returns a
+  # Hash of pair => count.
+
+  def self.count(data)
+    counts = Hash.new 0
+
+    data.each do |tables|
+      tables = tables.sort
+      curr = tables.shift
+      tables.each do |table|
+        counts[[curr, table]] += 1
+      end
+    end
+
+    return counts
+  end
+
+  ##
+  # Creates a Graph of +data+ using +counts+ for edge weights.
+
+  def self.graph(data, counts)
+    graph = Graph.new
+
+    max_count = counts.values.max
+    solid = Math.log10(max_count).floor
+    dotted = solid / 3
+    dashed = solid / 3 * 2
+
+    counts.each do |(first, second), count|
+      graph[first] << second
+      edge = ["weight=#{count}", "label=\"#{count}\"", "dir=none"]
+
+      case Math.log10 count
+      when 0.0..dotted then
+        edge << "style = dotted"
+      when dotted..dashed then
+        edge << "style = dashed"
+      when dashed..solid then
+        edge << "style = solid"
+      else
+        edge << "style = bold"
+      end
+
+      graph.edge[first][second].push(*edge)
+    end
+
+    return graph
+  end
+
+end
+

data/lib/sql_dep_grapher/graph.rb

@@ -0,0 +1,82 @@
+#!/usr/local/bin/ruby -w
+
+require 'pp'
+
+class Graph < Hash
+
+  attr_reader :attribs
+  attr_reader :prefix
+  attr_reader :order
+  attr_reader :edge
+
+  def initialize
+    super { |h,k| h[k] = [] }
+    @prefix = []
+    @attribs = Hash.new { |h,k| h[k] = [] }
+    @edge = Hash.new { |h,k| h[k] = Hash.new { |h2,k2| h2[k2] = [] } }
+    @order = []
+  end
+
+  def []=(key, val)
+    @order << key unless self.has_key? key
+    super(key, val)
+  end
+
+  def each_pair
+    @order.each do |from|
+      self[from].each do |to|
+        yield(from, to)
+      end
+    end
+  end
+  
+  def invert
+    result = self.class.new
+    each_pair do |from, to|
+      result[to] << from
+    end
+    result
+  end
+
+  def counts
+    result = Hash.new(0)
+    each_pair do |from, to|
+      result[from] += 1
+    end
+    result
+  end
+  
+  def keys_by_count
+    counts.sort_by { |x,y| y }.map {|x| x.first }
+  end
+
+  def to_s
+    result = []
+    result << "digraph absent"
+    result << "  {"
+    @prefix.each do |line|
+      result << line
+    end
+    @attribs.sort.each do |node, attribs|
+      result << "    #{node.inspect} [ #{attribs.join(',')} ]"
+    end
+    each_pair do |from, to|
+      edge = @edge[from][to].join(", ")
+      edge = " [ #{edge} ]" unless edge.empty?
+      result << "    #{from.inspect} -> #{to.inspect}#{edge};"
+    end
+    result << "  }"
+    result.join("\n")
+  end
+
+  def save(path, type="png")
+    File.open(path + ".dot", "w") do |f|
+      f.puts self.to_s
+      f.flush
+      cmd = "/usr/local/bin/dot -T#{type} #{path}.dot > #{path}.#{type}"
+      system cmd
+    end
+  end
+
+end
+

data/test/test.log

@@ -0,0 +1,22 @@
+FROM goals g, related_goal_summaries rgs WHERE
+FROM tags_teams j, tags t WHERE
+FROM tags_teams, tags, teams WHERE 
+FROM entries e, teams t WHERE
+FROM entries e, teams t WHERE
+FROM goals g, related_goal_summaries rgs WHERE
+FROM teams, team_members WHERE
+FROM tags_teams, tags, teams WHERE
+FROM tags_teams, tags, teams, goals g WHERE
+FROM tags t, tag_similarities ts, tags_teams tt, teams tm WHERE
+FROM tags_teams j, tags t WHERE
+FROM tags_teams, tags, teams WHERE
+FROM tags_teams j, tags t WHERE
+FROM tags_teams, tags, teams WHERE
+FROM goals g, goal_similarities gs WHERE
+FROM goals g, related_goal_summaries rgs WHERE
+FROM goals g, related_goal_summaries rgs WHERE
+FROM tags_teams j, tags t WHERE
+FROM tags_teams, tags, teams, goals g WHERE
+FROM tags_teams, tags, teams WHERE
+FROM entries e, teams t WHERE
+FROM entries e, teams t WHERE

data/test/test_sql_dep_grapher.rb

@@ -0,0 +1,102 @@
+$TESTING = true
+
+require 'test/unit'
+require 'sql_dep_grapher'
+
+class TestSQLDependencyGrapher < Test::Unit::TestCase
+
+  DATA = [
+    %w[goals related_goal_summaries],
+    %w[tags_teams tags],
+    %w[tags_teams tags teams],
+    %w[entries teams],
+    %w[entries teams],
+    %w[goals related_goal_summaries],
+    %w[teams team_members],
+    %w[tags_teams tags teams],
+    %w[tags_teams tags teams goals],
+    %w[tags tag_similarities tags_teams teams],
+    %w[tags_teams tags],
+    %w[tags_teams tags teams],
+    %w[tags_teams tags],
+    %w[tags_teams tags teams],
+    %w[goals goal_similarities],
+    %w[goals related_goal_summaries],
+    %w[goals related_goal_summaries],
+    %w[tags_teams tags],
+    %w[tags_teams tags teams goals],
+    %w[tags_teams tags teams],
+    %w[entries teams],
+    %w[entries teams],
+  ]
+
+  COUNTS = {
+    %w[entries teams] => 4,
+    %w[goal_similarities goals] => 1,
+    %w[goals related_goal_summaries] => 4,
+    %w[goals tags] => 2,
+    %w[goals tags_teams] => 2,
+    %w[goals teams] => 2,
+    %w[tag_similarities tags] => 1,
+    %w[tag_similarities tags_teams] => 1,
+    %w[tag_similarities teams] => 1,
+    %w[tags tags_teams] => 9,
+    %w[tags teams] => 5,
+    %w[team_members teams] => 1,
+  }
+
+  GRAPH = "digraph absent
+  {
+    \"tag_similarities\" -> \"tags_teams\" [ weight=1, label=\"1\", dir=none, style = dotted ];
+    \"tag_similarities\" -> \"teams\" [ weight=1, label=\"1\", dir=none, style = dotted ];
+    \"tag_similarities\" -> \"tags\" [ weight=1, label=\"1\", dir=none, style = dotted ];
+    \"goals\" -> \"tags_teams\" [ weight=2, label=\"2\", dir=none, style = bold ];
+    \"goals\" -> \"related_goal_summaries\" [ weight=4, label=\"4\", dir=none, style = bold ];
+    \"goals\" -> \"teams\" [ weight=2, label=\"2\", dir=none, style = bold ];
+    \"goals\" -> \"tags\" [ weight=2, label=\"2\", dir=none, style = bold ];
+    \"goal_similarities\" -> \"goals\" [ weight=1, label=\"1\", dir=none, style = dotted ];
+    \"tags\" -> \"teams\" [ weight=5, label=\"5\", dir=none, style = bold ];
+    \"tags\" -> \"tags_teams\" [ weight=9, label=\"9\", dir=none, style = bold ];
+    \"entries\" -> \"teams\" [ weight=4, label=\"4\", dir=none, style = bold ];
+    \"team_members\" -> \"teams\" [ weight=1, label=\"1\", dir=none, style = dotted ];
+  }"
+
+  def setup
+    @sdg = SQLDependencyGrapher
+  end
+
+  def test_build_graph
+    graph = :junk
+    File.open 'test/test.log' do |fp|
+      graph = @sdg.build_graph fp
+    end
+
+    # HACK until I can get Graph#== working
+    flunk "Equality is hard"
+    #assert_equal GRAPH, graph.to_s
+  end
+
+  def test_collect
+    data = :junk
+
+    File.open 'test/test.log' do |fp|
+      data = @sdg.collect fp
+    end
+
+    assert_equal DATA, data
+  end
+
+  def test_count
+    counts = @sdg.count DATA
+
+    assert_equal COUNTS, counts
+  end
+
+  def test_graph
+    graph = @sdg.graph DATA, COUNTS
+
+    assert_equal GRAPH, graph.to_s # Graph#== won't do what I want yet, I think
+  end
+
+end
+

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.10
+specification_version: 1
+name: sql_dep_graph
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2005-06-04
+summary: Graphs table dependencies based on usage from SQL logs
+require_paths: 
+  - lib
+email: eric@robotcoop.com
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: 
+default_executable: sql_dep_graph
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - Eric Hodel
+files: 
+  - Manifest.txt
+  - Rakefile
+  - bin/sql_dep_graph
+  - lib/sql_dep_grapher.rb
+  - lib/sql_dep_grapher/graph.rb
+  - test/test.log
+  - test/test_sql_dep_grapher.rb
+test_files: []
+rdoc_options: []
+extra_rdoc_files: []
+executables: 
+  - sql_dep_graph
+extensions: []
+requirements: []
+dependencies: []