piggly 1.2.1 → 2.0.0

data/lib/piggly/command/trace.rb

@@ -0,0 +1,90 @@
+module Piggly
+  module Command
+
+    #
+    # This command connects to the database, dumps all stored procedures, compiles them
+    # with instrumentation code, and finally installs the instrumented code.
+    #
+    class Trace < Base
+    end
+
+    class << Trace
+      def main(argv)
+        config     = configure(argv)
+        connection = connect(config)
+        index      = Dumper::Index.new(config)
+
+        dump(connection, index)
+
+        procedures = filter(config, index)
+
+        if procedures.empty?
+          if config.filters.empty?
+            abort "no stored procedures in the cache"
+          else
+            abort "no stored procedures in the cache matched your criteria"
+          end
+        elsif config.dry_run?
+          puts procedures.map{|p| p.signature }
+          exit 0
+        end
+
+        trace(config, procedures)
+        install(Installer.new(config, connection), procedures, Profile.new)
+      end
+
+      # Writes all stored procedures in the database to disk
+      #   @return [void]
+      def dump(connection, index)
+        index.update(Dumper::ReifiedProcedure.all(connection))
+      end
+
+      # Compiles all the stored procedures on disk and installs them
+      #   @return [void]
+      def trace(config, procedures)
+        puts "compiling #{procedures.size} procedures"
+
+        compiler = Compiler::TraceCompiler.new(config)
+        queue    = Util::ProcessQueue.new
+        procedures.each{|p| queue.add { compiler.compile(p) }}
+
+        # Force parser to load before we start forking
+        Parser.parser
+        queue.execute
+      end
+
+      def install(installer, procedures, profile)
+        puts "tracing #{procedures.size} procedures"
+        installer.install(procedures, profile)
+      end
+
+      # Parses command-line options
+      #   @return [Config]
+      def configure(argv, config = Config.new)
+        p = OptionParser.new do |o|
+          o.on("-t", "--dry-run",           "only print the names of matching procedures", &o_dry_run(config))
+          o.on("-s", "--select PATTERN",    "select procedures matching PATTERN", &o_select(config))
+          o.on("-r", "--reject PATTERN",    "ignore procedures matching PATTERN", &o_reject(config))
+          o.on("-c", "--cache-root PATH",   "local cache directory", &o_cache_root(config))
+          o.on("-o", "--report-root PATH",  "report output directory", &o_report_root(config))
+          o.on("-d", "--database PATH",     "read database adapter settings from YAML/JSON file", &o_database_yml(config))
+          o.on("-k", "--connection NAME",   "use connection adapter NAME", &o_connection_name(config))
+          o.on("-V", "--version",           "show version", &o_version(config))
+          o.on("-h", "--help",              "show this message") { abort o.to_s }
+        end
+
+        begin
+          p.parse! argv
+          config
+        rescue OptionParser::InvalidOption,
+               OptionParser::InvalidArgument,
+               OptionParser::MissingArgument
+          puts p
+          puts
+          puts $!
+          exit! 1
+        end
+      end
+    end
+  end
+end

data/lib/piggly/command/untrace.rb

@@ -0,0 +1,78 @@
+module Piggly
+  module Command
+
+    class Untrace < Base
+    end
+
+    class << Untrace
+      def main(argv)
+        config      = configure(argv)
+        index       = Dumper::Index.new(config)
+        connection  = connect(config)
+        procedures  = filter(config, index)
+
+        if procedures.empty?
+          if config.filters.empty?
+            abort "no stored procedures in the cache"
+          else
+            abort "no stored procedures in the cache matched your criteria"
+          end
+        elsif config.dry_run?
+          puts procedures.map{|x| x.signature }
+          exit 0
+        end
+
+        untrace(Installer.new(config, connection), procedures)
+      end
+
+      #
+      # Restores database procedures from file cache
+      #
+      def untrace(installer, procedures)
+        puts "restoring #{procedures.size} procedures"
+        installer.uninstall(procedures)
+      end
+
+      #
+      # Returns a list of Procedure values that satisfy at least one of the given filters
+      #
+      def find_procedures(filters, index)
+        if filters.empty?
+          index.procedures
+        else
+          filters.inject(Set.new){|set, filter| set | index.procedures.select(&filter) }
+        end
+      end
+
+      #
+      # Parses command-line options
+      #
+      def configure(argv, config = Config.new)
+        p = OptionParser.new do |o|
+          o.on("-t", "--dry-run",         "only print the names of matching procedures", &o_dry_run(config))
+          o.on("-s", "--select PATTERN",  "select procedures matching PATTERN", &o_select(config))
+          o.on("-r", "--reject PATTERN",  "ignore procedures matching PATTERN", &o_reject(config))
+          o.on("-c", "--cache-root PATH", "local cache directory", &o_cache_root(config))
+          o.on("-d", "--database PATH",   "read 'piggly' database adapter settings from YAML file", &o_database_yml(config))
+          o.on("-k", "--connection NAME", "use connection adapter NAME", &o_connection_name(config))
+          o.on("-V", "--version",         "show version", &o_version(config))
+          o.on("-h", "--help",            "show this message") { abort o.to_s }
+        end
+
+        begin
+          p.parse! argv
+          config
+        rescue OptionParser::InvalidOption,
+               OptionParser::InvalidArgument,
+               OptionParser::MissingArgument
+          puts p
+          puts
+          puts $!
+          exit! 1
+        end
+      end
+
+    end
+
+  end
+end

data/lib/piggly/compiler.rb

@@ -1,5 +1,7 @@
-require File.join(File.dirname(__FILE__), *%w[compiler cache])
-require File.join(File.dirname(__FILE__), *%w[compiler tags])
-require File.join(File.dirname(__FILE__), *%w[compiler trace])
-require File.join(File.dirname(__FILE__), *%w[compiler pretty])
-require File.join(File.dirname(__FILE__), *%w[compiler queue])
+module Piggly
+  module Compiler
+    autoload :CacheDir,       "piggly/compiler/cache_dir"
+    autoload :TraceCompiler,  "piggly/compiler/trace_compiler"
+    autoload :CoverageReport, "piggly/compiler/coverage_report"
+  end
+end

data/lib/piggly/compiler/cache_dir.rb

@@ -0,0 +1,119 @@
+# encoding: ascii-8bit
+
+module Piggly
+  module Compiler
+
+    #
+    # Each cache unit (any group of data that should be expired and created
+    # together) can be broken apart, to prevent unmarshaling a huge block of
+    # data all at once.
+    #
+    # The interface works like a Hash, so the compile method should return a
+    # hash of objects. Each object is writen to a different file (named by the
+    # hash key) within the same directory. String objects are (usually) read
+    # and written directly to disk, while all other objects are (un-)Marshal'd
+    #
+    # Cache invalidation is done by comparing mtime timestamps on the cached
+    # object's file to all the "source" files (ruby libs, input files, etc)
+    # required to regenerate the data.
+    #
+    class CacheDir
+      # Non-printable ASCII char indicates data should be Marshal'd
+      HINT = /[\000-\010\016-\037\177-\300]/
+
+      def initialize(dir)
+        @dir  = dir
+        @data = Hash.new do |h, k|
+          path = File.join(@dir, k.to_s)
+          if File.exists?(path)
+            h[k.to_s] = File.open(path, "rb") do |io|
+              # Detect Marshal'd data
+              if io.read(2) !~ HINT
+                io.rewind
+                io.read
+              else
+                io.rewind
+                Marshal.load(io)
+              end
+            end
+          end
+        end
+      end
+
+      # Load given key from file system into memory if needed
+      #   @return [Object]
+      def [](key)
+        @data[key.to_s]
+      end
+
+      # Writes through to file system
+      #   @return [void]
+      def []=(key, value)
+        @data[key.to_s] = value
+        write(key.to_s => value)
+      end
+
+      # Writes through to file system and returns self
+      #   @return [CacheDir] self
+      def update(hash)
+        hash.each{|k,v| self[k] = v }
+        self
+      end
+
+      # @return [void]
+      def delete(key)
+        path = File.join(@dir, key.to_s)
+        File.unlink(path) if File.exists?(path)
+        @data.delete(key)
+      end
+
+      # @return [Array<String>]
+      def keys
+        Dir[@dir + "/*"].map{|f| File.basename(f) }
+      end
+
+      # Creates cachedir, destroys its contents, and returns self
+      #   @return [CacheDir] self
+      def clear
+        @data.clear
+
+        if File.exists?(@dir)
+          FileUtils.rm(Dir["#{@dir}/*"])
+          FileUtils.touch(@dir)
+        else
+          FileUtils.mkdir(@dir) 
+        end
+
+        self
+      end
+
+      # Clears entire cache, replaces contents, and returns self
+      #   @return [CacheDir] self
+      def replace(hash)
+        clear
+        update(hash)
+      end
+
+    private
+
+      # Serializes each entry to disk
+      #   @return [void]
+      def write(hash)
+        FileUtils.mkdir(@dir) unless File.exists?(@dir)
+        FileUtils.touch(@dir) # Update mtime
+
+        hash.each do |key, value|
+          File.open(File.join(@dir, key.to_s), "wb") do |io|
+            # Marshal if the first two bytes contain non-ASCII
+            if value.is_a?(String) and value[0,2] !~ HINT
+              io.write(value)
+            else
+              Marshal.dump(value, io)
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/piggly/compiler/coverage_report.rb

@@ -0,0 +1,63 @@
+module Piggly
+  module Compiler
+
+    #
+    # Produces HTML output to report coverage of tagged nodes in the tree
+    #
+    class CoverageReport
+      include Reporter::HtmlDsl
+
+      def initialize(config)
+        @config = config
+      end
+
+      def compile(procedure, profile)
+        trace = Compiler::TraceCompiler.new(@config)
+
+        if trace.stale?(procedure)
+          raise StaleCacheError,
+            "stale cached syntax tree for #{procedure.name}"
+        end
+
+        # Get (copies of) the tagged nodes from the compiled tree
+        data = trace.compile(procedure)
+
+        return :html  => traverse(data[:tree], profile),
+               :lines => 1 .. procedure.source(@config).count("\n") + 1
+      end
+
+    protected
+
+      # @return [String]
+      def traverse(node, profile, string = "")
+        if node.tagged?
+          tag = profile[node.tag_id]
+
+          if tag.complete?
+            string << %[<span class="#{tag.style}" id="T#{tag.id}">]
+          else
+            string << %[<span class="#{tag.style}" id="T#{tag.id}" title="#{tag.description}">]
+          end
+        end
+
+        if node.terminal?
+          if style = node.style
+            string << %[<span class="#{style}">#{e(node.text_value)}</span>]
+          else
+            string << e(node.text_value)
+          end
+        else
+          # Non-terminals never write their text_value
+          node.elements.each{|child| traverse(child, profile, string) }
+        end
+
+        if node.tagged?
+          string << %[</span>]
+        end
+
+        string
+      end
+
+    end
+  end
+end

data/lib/piggly/compiler/trace_compiler.rb

@@ -0,0 +1,105 @@
+module Piggly
+  module Compiler
+
+    #
+    # Walks the parse tree, attaching Tag values and rewriting source code to ping them.
+    #
+    class TraceCompiler
+      include Util::Cacheable
+
+      def initialize(config)
+        @config = config
+      end
+
+      # Is the cache_path is older than its source path or the other files?
+      def stale?(procedure)
+        Util::File.stale?(cache_path(procedure.source_path(@config)),
+                          procedure.source_path(@config),
+                          *self.class.cache_sources)
+      end
+
+      def compile(procedure)
+        cache = CacheDir.new(cache_path(procedure.source_path(@config)))
+
+        if stale?(procedure)
+          $stdout.puts "Compiling #{procedure.name}"
+          tree = Parser.parse(IO.read(procedure.source_path(@config)))
+          tree = tree.force! if tree.respond_to?(:thunk?)
+
+          tags = []
+          code = traverse(tree, procedure.oid, tags)
+
+          cache.replace(:tree => tree, :code => code, :tags => tags)
+        end
+
+        cache
+      end
+
+    protected
+
+      # Rewrites the parse tree to call instrumentation helpers, and destructively
+      # updates `tags` by appending the tags of instrumented nodes
+      #   @return [String]
+      def traverse(node, oid, tags)
+        if node.terminal? or node.expression?
+          node.source_text
+        else
+          if node.respond_to?(:condStub) and node.respond_to?(:cond)
+            # Preserve opening parenthesis and whitespace before injecting code. This way 
+            # IF(test) becomes IF(piggly_cond(TAG, test)) instead of IFpiggly_cond(TAG, (test))
+            pre, cond = node.cond.expr.text_value.match(/\A(\(?[\t\n\r ]*)(.+)\z/m).captures
+            node.cond.source_text = ""
+            
+            tags << node.cond.tag(oid)
+
+            node.condStub.source_text  = "#{pre}piggly_cond($PIGGLY$#{node.cond.tag_id}$PIGGLY$, (#{cond}))"
+            node.condStub.source_text << traverse(node.cond.tail, oid, tags) # preserve trailing whitespace
+          end
+
+          if node.respond_to?(:bodyStub)
+            if node.respond_to?(:exitStub) and node.respond_to?(:cond)
+              tags << node.body.tag(oid)
+              tags << node.cond.tag(oid)
+
+              # Hack to simulate a loop conditional statement in stmtForLoop and stmtLoop.
+              # signal condition is true when body is executed, and false when exit stub is reached
+              node.bodyStub.source_text  = "perform piggly_cond($PIGGLY$#{node.cond.tag_id}$PIGGLY$, true);#{node.indent(:bodySpace)}"
+              node.bodyStub.source_text << "perform piggly_branch($PIGGLY$#{node.body.tag_id}$PIGGLY$);#{node.indent(:bodySpace)}"
+
+              if node.respond_to?(:doneStub)
+                # Signal the end of an iteration was reached
+                node.doneStub.source_text  = "#{node.indent(:bodySpace)}perform piggly_signal($PIGGLY$#{node.cond.tag_id}$PIGGLY$, $PIGGLY$@$PIGGLY$);"
+                node.doneStub.source_text << node.body.indent
+              end
+
+              # Signal the loop terminated
+              node.exitStub.source_text  = "\n#{node.indent}perform piggly_cond($PIGGLY$#{node.cond.tag_id}$PIGGLY$, false);"
+            elsif node.respond_to?(:body)
+              # Unconditional branches (or blocks)
+              #   BEGIN ... END;
+              #   ... ELSE ... END;
+              #   CONTINUE label;
+              #   EXIT label;
+              tags << node.body.tag(oid)
+              node.bodyStub.source_text = "perform piggly_branch($PIGGLY$#{node.body.tag_id}$PIGGLY$);#{node.indent(:bodySpace)}"
+            end
+          end
+
+          # Traverse children (in which we just injected code)
+          node.elements.map{|e| traverse(e, oid, tags) }.join
+        end
+      end
+    end
+
+    class << TraceCompiler
+
+      # Each of these files' mtimes are used to determine when another file is stale
+      def cache_sources
+        [Parser.grammar_path,
+         Parser.parser_path,
+         Parser.nodes_path]
+      end
+    end
+
+  end
+end