skylight 0.2.0.beta.3 → 0.2.0.beta.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b6ffd4680bf0bfa2f2e725a0ed6601c70d8da33
-  data.tar.gz: 7467f5495f63fe61cf9ebf342c1b9970e69f4ac5
+  metadata.gz: 6c34ce7c2bd1892ae5f05d4e9cc0ee5054cbb9f1
+  data.tar.gz: ce556c192f1f5b1d72be7680d358d8373a80cfe7
 SHA512:
-  metadata.gz: 63aac94d7f9c0ed6f337901c37b8bbe3e4a519d800092a6a1c023e7881d095740130d299b23e9def21e7c2e9cc7c89664c26c425e6b4fdbcebd76f101a099de5
-  data.tar.gz: 93b7b2607ddc3a677b04b648c6246b5c8201b46355c01d34892e9291d0cf1555c97308e21019cdf5ec2609b85c10821c644e86dc0f9d6cbe5096fcf2136e476c
+  metadata.gz: a9b4be609c1d76a2075cbb9ad116d724fc35be01dfe133e00978c3b09a8e32765a78d2db0ccb2f3c43a60182891a57e42ee8bcecd53ec8d50750a6fc2f38f138
+  data.tar.gz: 082e9dd9bfc81ea18b53b830c52bf26796b90943c6af65b78723b21a8d1207478f87ac835a8386c4f0741a4fb97d9d89200ed2746b8fdadb8d9945b869c1a975

data/bin/skylight

@@ -1,3 +1,16 @@
 #!/usr/bin/env ruby
-require "skylight/cli"
-Skylight::CLI.start(ARGV)
+#
+# This file was generated by Bundler.
+#
+# The application 'skylight' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('skylight', 'skylight')

data/lib/skylight.rb

@@ -59,9 +59,10 @@ module Skylight
     noise
     other)
 
-  TIER_REGEX = /^(?:#{TIERS.join('|')})(?:\.|$)/
-  CATEGORY_REGEX = /^[a-z0-9_-]+(?:\.[a-z0-9_-]+)*$/i
+  TIER_REGEX = /^(?:#{TIERS.join('|')})(?:\.|$)/u
+  CATEGORY_REGEX = /^[a-z0-9_-]+(?:\.[a-z0-9_-]+)*$/iu
   DEFAULT_CATEGORY = "app.block".freeze
+  DEFAULT_OPTIONS = { category: DEFAULT_CATEGORY }
 
   def self.start!(*args)
     Instrumenter.start!(*args)
@@ -71,23 +72,27 @@ module Skylight
     Instrumenter.stop!(*args)
   end
 
-  def self.trace(*args, &blk)
+  def self.trace(title=nil, desc=nil, annot=nil)
     unless inst = Instrumenter.instance
       return yield if block_given?
       return
     end
 
-    inst.trace(*args, &blk)
+    if block_given?
+      inst.trace(title, desc, annot) { yield }
+    else
+      inst.trace(title, desc, annot)
+    end
   end
 
-  def self.instrument(opts = {}, &blk)
+  def self.instrument(opts = DEFAULT_OPTIONS)
     unless inst = Instrumenter.instance
       return yield if block_given?
       return
     end
 
     if Hash === opts
-      category = opts.delete(:category) || DEFAULT_CATEGORY
+      category = opts.delete(:category)
       title    = opts.delete(:title)
       desc     = opts.delete(:description)
     else
@@ -96,16 +101,20 @@ module Skylight
       desc     = nil
     end
 
-    inst.instrument(category, title, desc, &blk)
+    if block_given?
+      inst.instrument(category, title, desc) { yield }
+    else
+      inst.instrument(category, title, desc)
+    end
   end
 
-  def self.disable(&block)
+  def self.disable
     unless inst = Instrumenter.instance
       return yield if block_given?
       return
     end
 
-    inst.disable(&block)
+    inst.disable { yield }
   end
 
   RUBYBIN = File.join(

data/lib/skylight/instrumenter.rb

@@ -1,5 +1,6 @@
 require 'thread'
 require 'set'
+require 'base64'
 
 module Skylight
   class Instrumenter
@@ -55,7 +56,7 @@ module Skylight
       @subscriber = Subscriber.new(config, self)
 
       @trace_info = @config[:trace_info] || TraceInfo.new
-      @descriptions = Hash.new { |h,k| h[k] = Set.new }
+      @descriptions = Hash.new { |h,k| h[k] = {} }
     end
 
     def current_trace
@@ -87,7 +88,7 @@ module Skylight
       @worker.shutdown
     end
 
-    def trace(endpoint, cat, *args)
+    def trace(endpoint, cat, title=nil, desc=nil, annot=nil)
       # If a trace is already in progress, continue with that one
       if trace = @trace_info.current
         t { "already tracing" }
@@ -96,7 +97,7 @@ module Skylight
       end
 
       begin
-        trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.micros, cat, *args)
+        trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.micros, cat, title, desc, annot)
       rescue Exception => e
         error e.message
         t { e.backtrace.join("\n") }
@@ -126,7 +127,17 @@ module Skylight
       @disabled
     end
 
-    def instrument(cat, *args)
+    @scanner = StringScanner.new('')
+    def self.match?(string, regex)
+      @scanner.string = string
+      @scanner.match?(regex)
+    end
+
+    def match?(string, regex)
+      self.class.match?(string, regex)
+    end
+
+    def instrument(cat, title=nil, desc=nil, annot=nil)
       unless trace = @trace_info.current
         return yield if block_given?
         return
@@ -134,15 +145,15 @@ module Skylight
 
       cat = cat.to_s
 
-      unless cat =~ CATEGORY_REGEX
+      unless match?(cat, CATEGORY_REGEX)
         warn "invalid skylight instrumentation category; value=%s", cat
         return yield if block_given?
         return
       end
 
-      cat = "other.#{cat}" unless cat =~ TIER_REGEX
+      cat = "other.#{cat}" unless match?(cat, TIER_REGEX)
 
-      unless sp = trace.instrument(cat, *args)
+      unless sp = trace.instrument(cat, title, desc, annot)
         return yield if block_given?
         return
       end
@@ -157,16 +168,17 @@ module Skylight
     end
 
     def limited_description(description)
+      endpoint = nil
       endpoint = @trace_info.current.endpoint
 
       DESC_LOCK.synchronize do
         set = @descriptions[endpoint]
 
         if set.size >= 100
-          return TOO_MANY_UNIQUES if set.size >= 100
+          return TOO_MANY_UNIQUES
         end
 
-        set << description
+        set[description] = true
         description
       end
     end
@@ -174,6 +186,10 @@ module Skylight
     def error(reason, body)
       t { fmt "processing error; reason=%s; body=%s", reason, body }
 
+      if body.encoding == Encoding::BINARY || !body.valid_encoding?
+        body = Base64.encode64(body)
+      end
+
       message = Skylight::Messages::Error.new(reason: reason, body: body)
 
       unless @worker.submit(message)

data/lib/skylight/messages/trace.rb

@@ -13,14 +13,17 @@ module Skylight
 
         include Util::Logging
 
-        attr_accessor :endpoint
-        attr_reader   :spans, :notifications
+        attr_reader   :endpoint, :spans, :notifications
 
-        def initialize(instrumenter, endpoint, start, cat, *args)
+        def endpoint=(value)
+          @endpoint = value.freeze
+        end
+
+        def initialize(instrumenter, endpoint, start, cat, title=nil, desc=nil, annot=nil)
           raise ArgumentError, 'instrumenter is required' unless instrumenter
 
           @instrumenter = instrumenter
-          @endpoint     = endpoint
+          @endpoint     = endpoint.freeze
           @start        = start
           @spans        = []
           @stack        = []
@@ -32,9 +35,13 @@ module Skylight
           # Track time
           @last_seen_time = start
 
-          annot = args.pop if Hash === args
-          title = args.shift
-          desc  = args.shift
+          if Hash === title
+            annot = title
+            title = desc = nil
+          elsif Hash === desc
+            annot = desc
+            desc = nil
+          end
 
           # Create the root node
           @root = start(@start, cat, title, desc, annot)
@@ -60,13 +67,27 @@ module Skylight
           nil
         end
 
-        def instrument(cat, *args)
-          annot = args.pop if Hash === args.last
-          title = args.shift
-          desc  = args.shift
-          now   = adjust_for_skew(Util::Clock.micros)
+        def instrument(cat, title=nil, desc=nil, annot=nil)
+          if Hash === title
+            annot = title
+            title = desc = nil
+          elsif Hash === desc
+            annot = desc
+            desc = nil
+          end
 
-          desc = @instrumenter.limited_description(desc)
+          title.freeze
+          desc.freeze
+
+          original_desc = desc
+          now           = adjust_for_skew(Util::Clock.micros)
+          desc          = @instrumenter.limited_description(desc)
+
+          if desc == Instrumenter::TOO_MANY_UNIQUES
+            debug "[SKYLIGHT] A payload description produced <too many uniques>"
+            debug "original desc=%s", original_desc
+            debug "cat=%s, title=%s, desc=%s, annot=%s", cat, title, desc, annot.inspect
+          end
 
           start(now - gc_time, cat, title, desc, annot)
         end

data/lib/skylight/normalizers.rb

@@ -1,4 +1,5 @@
 require 'skylight/normalizers/default'
+require 'skylight/util/allocation_free'
 
 module Skylight
   # Convert AS::N events to Skylight events
@@ -40,6 +41,8 @@ module Skylight
     end
 
     class RenderNormalizer < Normalizer
+      include AllocationFree
+
       def setup
         @paths = config['normalizers.render.view_paths'] || []
       end
@@ -54,19 +57,45 @@ module Skylight
       end
 
       def relative_path(path, annotations)
-        return path if Pathname.new(path).relative?
+        return path if relative_path?(path)
 
-        root = @paths.find { |p| path.start_with?(p) }
+        root = array_find(@paths) { |p| path.start_with?(p) }
 
         if root
-          relative = path[root.size..-1]
-          relative = relative[1..-1] if relative.start_with?("/")
-          relative
+          start = root.size
+          start += 1 if path.getbyte(start) == SEPARATOR_BYTE
+          path[start, path.size]
         else
           annotations[:skylight_error] = ["absolute_path", path]
           "Absolute Path"
         end
       end
+
+    private
+      def relative_path?(path)
+        !absolute_path?(path)
+      end
+
+      SEPARATOR_BYTE = File::SEPARATOR.ord
+
+      if File::NULL == "NUL"
+        ALT_SEPARATOR_BYTE = File::ALT_SEPARATOR && File::ALT_SEPARATOR.ord
+        COLON_BYTE = ":".ord
+        def absolute_path?(path)
+          if alpha?(path.getbyte(0)) && path.getbyte(1) == COLON_BYTE
+            byte2 = path.getbyte(2)
+            byte2 == SEPARATOR_BYTE || byte2 == ALT_SEPARATOR_BYTE
+          end
+        end
+
+        def alpha?(byte)
+          byte >= 65 and byte <= 90 || byte >= 97 and byte <= 122
+        end
+      else
+        def absolute_path?(path)
+          path.getbyte(0) == SEPARATOR_BYTE
+        end
+      end
     end
 
     class Container

data/lib/skylight/normalizers/process_action.rb

@@ -3,9 +3,11 @@ module Skylight
     class ProcessAction < Normalizer
       register "process_action.action_controller"
 
+      CAT = "app.controller.request".freeze
+
       def normalize(trace, name, payload)
         trace.endpoint = controller_action(payload)
-        [ "app.controller.request", trace.endpoint, nil, normalize_payload(payload) ]
+        [ CAT, trace.endpoint, nil, normalize_payload(payload) ]
       end
 
     private
@@ -17,7 +19,9 @@ module Skylight
       def normalize_payload(payload)
         normalized = {}
 
-        payload.each do |key, value|
+        payload.each_key do |key|
+          value = payload[key]
+
           value = value.inspect unless value.is_a?(String) || value.is_a?(Numeric)
           normalized[key] = value
         end

data/lib/skylight/normalizers/render_collection.rb

@@ -3,9 +3,11 @@ module Skylight
     class RenderCollection < RenderNormalizer
       register "render_collection.action_view"
 
+      CAT = "view.render.collection".freeze
+
       def normalize(trace, name, payload)
         normalize_render(
-          "view.render.collection",
+          CAT,
           payload,
           count: payload[:count])
       end

data/lib/skylight/normalizers/render_partial.rb

@@ -3,9 +3,11 @@ module Skylight
     class RenderPartial < RenderNormalizer
       register "render_partial.action_view"
 
+      CAT = "view.render.template".freeze
+
       def normalize(trace, name, payload)
         normalize_render(
-          "view.render.template",
+          CAT,
           payload,
           partial: 1)
       end

data/lib/skylight/normalizers/render_template.rb

@@ -3,9 +3,11 @@ module Skylight
     class RenderTemplate < RenderNormalizer
       register "render_template.action_view"
 
+      CAT = "view.render.template".freeze
+
       def normalize(trace, name, payload)
         normalize_render(
-          "view.render.template",
+          CAT,
           payload,
           partial: 0)
       end

data/lib/skylight/normalizers/send_file.rb

@@ -10,6 +10,9 @@ module Skylight
       class SendFile < Normalizer
         register "send_file.action_controller"
 
+        CAT = "app.controller.send_file".freeze
+        TITLE = "send file".freeze
+
         def normalize(trace, name, payload)
           path = payload[:path]
 
@@ -20,31 +23,47 @@ module Skylight
             disposition: normalize_disposition(payload),
             status:      normalize_status(payload) }
 
-          title = "send file"
+          title = TITLE
 
           # depending on normalization, we probably want this to eventually
           # include the full path, but we need to make sure we have a good
           # deduping strategy first.
           desc = nil
 
-          [ "app.controller.send_file", title, desc, annotations ]
+          [ CAT, title, desc, annotations ]
         end
 
       private
 
+        OCTET_STREAM = "application/octet-stream".freeze
+        ATTACHMENT = "attachment".freeze
+
+        def initialize(*)
+          super
+
+          @mimes = Mime::SET.reduce({}) do |hash, mime|
+            hash[mime.symbol] = mime.to_s.dup.freeze
+            hash
+          end
+        end
+
         def normalize_type(payload)
-          type = payload[:type] || "application/octet-stream"
-          type = Mime[type].to_s if type.is_a?(Symbol)
+          type = payload[:type] || OCTET_STREAM
+          type = @mimes[type] if type.is_a?(Symbol)
           type
         end
 
+        def mime_for(type)
+          @mimes[type] ||= Mime[type].to_s.freeze
+        end
+
         def normalize_status(payload)
           status = payload[:status] || 200
           Rack::Utils.status_code(status)
         end
 
         def normalize_disposition(payload)
-          payload[:disposition] || "attachment"
+          payload[:disposition] || ATTACHMENT
         end
       end
 

data/lib/skylight/normalizers/sql.rb

@@ -6,25 +6,24 @@ module Skylight
     class SQL < Normalizer
       register "sql.active_record"
 
+      CAT = "db.sql.query".freeze
+
       def normalize(trace, name, payload)
         case payload[:name]
         when "SCHEMA", "CACHE"
           return :skip
         else
-          name  = "db.sql.query"
+          name  = CAT
           title = payload[:name] || "SQL"
         end
 
-        if payload[:binds].empty?
-          extracted_title, payload[:sql], binds, error = extract_binds(payload)
-        else
-          extracted_title, _, _, error = extract_binds(payload)
-          binds = payload[:binds].map { |col, val| val.inspect }
+        unless payload[:binds].empty?
+          payload[:binds] = payload[:binds].map { |col, val| val.inspect }
         end
 
+        extracted_title, payload[:sql], binds, error = extract_binds(payload, payload[:binds])
         title = extracted_title if extracted_title
 
-
         if payload[:sql]
           annotations = {
             sql:   payload[:sql],
@@ -40,8 +39,8 @@ module Skylight
       end
 
     private
-      def extract_binds(payload)
-        title, sql, binds = SqlLexer::Lexer.bindify(payload[:sql])
+      def extract_binds(payload, precalculated)
+        title, sql, binds = SqlLexer::Lexer.bindify(payload[:sql], precalculated)
         [ title, sql, binds, nil ]
       rescue
         [ nil, nil, nil, ["sql_parse", payload[:sql]] ]

data/lib/skylight/util/allocation_free.rb

@@ -0,0 +1,15 @@
+module Skylight
+  module AllocationFree
+    def array_find(array)
+      i = 0
+
+      while i < array.size
+        item = array[i]
+        return item if yield item
+        i += 1
+      end
+
+      nil
+    end
+  end
+end

data/lib/skylight/vendor/beefcake.rb

@@ -54,13 +54,20 @@ module Skylight
         end
 
         def field(rule, name, type, fn, opts)
-          fields[fn] = Field.new(rule, name, type, fn, opts)
+          field = Field.new(rule, name, type, fn, opts)
+          indexed_fields[fn] = field
+          fields << field
+          fields.sort!
           __write_initializer
           attr_accessor name
         end
 
         def fields
-          @fields ||= {}
+          @fields ||= []
+        end
+
+        def indexed_fields
+          @indexed_fields ||= {}
         end
 
         def __write_initializer
@@ -68,7 +75,7 @@ module Skylight
 
           lines << "def initialize(attrs=nil)"
           lines << "return unless attrs"
-          fields.values.each do |fld|
+          fields.each do |fld|
             lines << "@#{fld.name} = attrs[:#{fld.name}]"
           end
 
@@ -93,7 +100,7 @@ module Skylight
 
           # TODO: Error if any required fields at nil
 
-          fields.values.sort.each do |fld|
+          fields.each do |fld|
             if fld.opts[:packed]
               bytes = encode!(Buffer.new, fld, 0)
               buf.append_info(fld.fn, Buffer.wire_for(fld.type))
@@ -109,27 +116,36 @@ module Skylight
 
         def encode!(buf, fld, fn)
           v = self[fld.name]
-          v = v.is_a?(Array) ? v : [v]
-
-          v.compact.each do |val|
-            case fld.type
-            when Class # encodable
-              # TODO: raise error if type != val.class
-              buf.append(:string, val.encode, fn)
-            when Module # enum
-              if ! valid_enum?(fld.type, val)
-                raise InvalidValueError.new(fld.name, val)
-              end
 
-              buf.append(:int32, val, fn)
-            else
-              buf.append(fld.type, val, fn)
+          if v.is_a?(Array)
+            v.each do |val|
+              encode_field!(buf, fld.type, val, fn)
             end
+          else
+            encode_field!(buf, fld.type, v, fn)
           end
 
           buf
         end
 
+        def encode_field!(buf, type, val, fn)
+          return if val.nil?
+
+          case type
+          when Class # encodable
+            # TODO: raise error if type != val.class
+            buf.append(:string, val.encode, fn)
+          when Module # enum
+            if ! valid_enum?(type, val)
+              raise InvalidValueError.new(fld.name, val)
+            end
+
+            buf.append(:int32, val, fn)
+          else
+            buf.append(type, val, fn)
+          end
+        end
+
         def valid_enum?(mod, val)
           !!name_for(mod, val)
         end
@@ -144,13 +160,12 @@ module Skylight
         end
 
         def validate!
-          fields.values.each do |fld|
+          fields.each do |fld|
             if fld.rule == :required && self[fld.name].nil?
               raise RequiredFieldNotSetError, fld.name
             end
           end
         end
-
       end
 
 
@@ -165,7 +180,7 @@ module Skylight
           while buf.length > 0
             fn, wire = buf.read_info
 
-            fld = fields[fn]
+            fld = indexed_fields[fn]
 
             # We don't have a field for with index fn.
             # Ignore this data and move on.
@@ -196,7 +211,7 @@ module Skylight
           end
 
           # Set defaults
-          fields.values.each do |f|
+          fields.each do |f|
             next if o[f.name] == false
             o[f.name] ||= f.opts[:default]
           end
@@ -218,6 +233,10 @@ module Skylight
         self.class.fields
       end
 
+      def indexed_fields
+        self.class.indexed_fields
+      end
+
       def [](k)
         __send__(k)
       end
@@ -229,7 +248,7 @@ module Skylight
       def ==(o)
         return false if (o == nil) || (o == false)
         return false unless o.respond_to?(:[])
-        fields.values.all? do |fld|
+        fields.all? do |fld|
           if fld.rule == :repeated
             Array(self[fld.name]) == Array(o[fld.name])
           else
@@ -239,7 +258,7 @@ module Skylight
       end
 
       def inspect
-        set = fields.values.select {|fld| self[fld.name] != nil }
+        set = fields.select {|fld| self[fld.name] != nil }
 
         flds = set.map do |fld|
           val = self[fld.name]
@@ -259,7 +278,7 @@ module Skylight
       end
 
       def to_hash
-        fields.values.inject({}) do |h, fld|
+        fields.inject({}) do |h, fld|
           if v = self[fld.name]
             h[fld.name] = v
           end

data/lib/skylight/vendor/beefcake/buffer.rb

@@ -11,6 +11,8 @@ module Skylight
       MinInt64  = -(1<<63)
       MaxInt64  =  (1<<63)-1
 
+      MaxFixnum =  (1 << (1.size * 8 - 2) - 1)
+
       def self.wire_for(type)
         case type
         when Class
@@ -70,7 +72,7 @@ module Skylight
 
       if ''.respond_to?(:force_encoding)
         def buf=(buf)
-          @buf = buf.force_encoding('BINARY')
+          @buf = buf.force_encoding(BINARY)
         end
       end
 
@@ -78,7 +80,7 @@ module Skylight
         @buf.respond_to?(:bytesize) ? @buf.bytesize : @buf.length
       end
 
-      BINARY = 'BINARY'
+      BINARY = 'BINARY'.freeze
 
       # Detect a ruby encodings bug, as far as I know, this exists in
       # most versions fo JRuby as well as 1.9.2

data/lib/skylight/vendor/beefcake/encode.rb

@@ -10,7 +10,7 @@ module Skylight
           append_info(fn, wire)
         end
 
-        __send__("append_#{type}", val)
+        __send__(HANDLERS[type], val)
       end
 
       def append_info(fn, wire)
@@ -26,7 +26,7 @@ module Skylight
       end
 
       def append_fixed64(n)
-        if n < MinUint64 || n > MaxUint64
+        if uint64?(n)
           raise OutOfRangeError, n
         end
 
@@ -77,8 +77,18 @@ module Skylight
         append_fixed64((n << 1) ^ (n >> 63))
       end
 
+      def uint64?(n)
+        if n < MinUint64
+          false
+        elsif n < MaxFixnum
+          true
+        else
+          n <= MaxUint64
+        end
+      end
+
       def append_uint64(n)
-        if n < MinUint64 || n > MaxUint64
+        unless uint64?(n)
           raise OutOfRangeError, n
         end
 
@@ -110,6 +120,13 @@ module Skylight
       end
       alias :append_bytes :append_string
 
+      HANDLERS = instance_methods.reduce({}) do |hash, meth|
+        if meth.to_s =~ /^append_(.*)$/
+          hash[$1.to_sym] = meth
+        end
+
+        hash
+      end
     end
   end
 end

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '0.2.0.beta.3'
+  VERSION = '0.2.0.beta.4'
 end
 

data/lib/skylight/worker/collector.rb

@@ -67,6 +67,9 @@ module Skylight
       def send_error(msg)
         res = @http_auth.post("/agent/error?hostname=#{escape(config[:'hostname'])}", reason: msg.reason, body: msg.body)
 
+        # error already handled in Util::HTTP
+        return unless res
+
         unless res.success?
           if (400..499).include? res.status
             warn "error wasn't sent successfully; status=%s", res.status
@@ -108,6 +111,9 @@ module Skylight
       def refresh_report_token(now)
         res = @http_auth.get("/agent/authenticate?hostname=#{escape(config[:'hostname'])}")
 
+        # error already handled in Util::HTTP
+        return unless res
+
         unless res.success?
           if (400..499).include? res.status
             warn "token request rejected; status=%s", res.status

data/lib/sql_lexer/lexer.rb

@@ -59,21 +59,22 @@ module SqlLexer
 
     Literals      = %Q<(?:NULL|TRUE|FALSE)(?=(?:[#{WS}]|#{OpPart}|#{End}))>
 
-    TkWS          = %r<[#{WS}]+>
-    TkOptWS       = %r<[#{WS}]*>
-    TkOp          = %r<[#{OpPart}]>
-    TkPlaceholder = %r<#{Placeholder}>
-    TkNonBind     = %r<#{NonBind}>
-    TkQuotedTable = %r<#{QuotedTable}>i
-    TkUpdateTable = %r<UPDATE#{TableNext}>i
-    TkInsertTable = %r<INSERT[#{WS}]+INTO#{TableNext}>i
-    TkDeleteTable = %r<DELETE[#{WS}]+FROM#{TableNext}>i
-    TkFromTable   = %r<FROM#{TableNext}>i
-    TkID          = %r<#{ID}>
-    TkEnd         = %r<;?[#{WS}]*>
-    TkBind        = %r<#{String}|#{Number}|#{Literals}>
-    TkIn          = %r<#{InOp}>i
-    TkSpecialOp   = %r<#{SpecialOps}>i
+    TkWS          = %r<[#{WS}]+>u
+    TkOptWS       = %r<[#{WS}]*>u
+    TkOp          = %r<[#{OpPart}]>u
+    TkPlaceholder = %r<#{Placeholder}>u
+    TkNonBind     = %r<#{NonBind}>u
+    TkQuotedTable = %r<#{QuotedTable}>iu
+    TkUpdateTable = %r<UPDATE#{TableNext}>iu
+    TkInsertTable = %r<INSERT[#{WS}]+INTO#{TableNext}>iu
+    TkDeleteTable = %r<DELETE[#{WS}]+FROM#{TableNext}>iu
+    TkFromTable   = %r<FROM#{TableNext}>iu
+    TkID          = %r<#{ID}>u
+    TkEnd         = %r<;?[#{WS}]*>u
+    TkBind        = %r<#{String}|#{Number}|#{Literals}>u
+    TkIn          = %r<#{InOp}>iu
+    TkSpecialOp   = %r<#{SpecialOps}>iu
+    TkStartSelect = %r<SELECT(?=(?:[#{WS}]|#{OpPart}))>iu
 
     STATE_HANDLERS = {
       begin:       :process_begin,
@@ -81,91 +82,159 @@ module SqlLexer
       tokens:      :process_tokens,
       bind:        :process_bind,
       non_bind:    :process_non_bind,
+      placeholder: :process_placeholder,
       table_name:  :process_table_name,
       end:         :process_end,
       special:     :process_special,
       in:          :process_in
     }
 
-    def self.bindify(string)
-      new(string).tap do |scanner|
-        scanner.process
-        return scanner.title, scanner.output, scanner.binds
-      end
+    def self.bindify(string, binds=nil)
+      scanner = instance(string)
+      scanner.process(binds)
+      [scanner.title, scanner.output, scanner.binds]
     end
 
     attr_reader :output, :binds, :title
 
-    def initialize(string)
-      @scanner = StringScanner.new(string)
+    def self.pooled_value(name, default)
+      key = :"__skylight_sql_#{name}"
+
+      singleton_class.class_eval do
+        define_method(name) do
+          value = Thread.current[key] ||= default.dup
+          value.clear
+          value
+        end
+      end
+
+      __send__(name)
+    end
+
+    SCANNER_KEY = :__skylight_sql_scanner
+    LEXER_KEY   = :__skylight_sql_lexer
+
+    def self.scanner(string='')
+      scanner = Thread.current[SCANNER_KEY] ||= StringScanner.new('')
+      scanner.string = string
+      scanner
+    end
+
+    def self.instance(string)
+      lexer = Thread.current[LEXER_KEY] ||= new
+      lexer.init(string)
+      lexer
+    end
+
+    pooled_value :binds, []
+    pooled_value :table, "*" * 20
+
+    SPACE = " ".freeze
+
+    DEBUG = ENV["DEBUG"]
+
+    def init(string)
       @state   = :begin
-      @input   = string
-      @output  = string.dup
-      @binds   = []
+      @debug   = DEBUG
+      @binds   = self.class.binds
+      @table   = self.class.table
+      @title   = nil
+      @bind    = 0
+
+      self.string = string
+    end
+
+    def string=(value)
+      @input   = value
+
+      @scanner = self.class.scanner(value)
+
+      # intentionally allocates; we need to return a new
+      # string as part of this API
+      @output = value.dup
     end
 
-    def process
-      @operation = @table = @title = nil
+    PLACEHOLDER = "?".freeze
+    UNKNOWN = "<unknown>".freeze
+
+    def process(binds)
+      @operation = nil
+      @provided_binds = binds
 
       while @state
-        if ENV["DEBUG"]
+        if @debug
           p @state
           p @scanner
         end
 
-        send STATE_HANDLERS[@state]
+        __send__ STATE_HANDLERS[@state]
       end
 
       pos = 0
       removed = 0
+
+      # intentionally allocates; the returned binds must
+      # be in a newly produced array
       extracted_binds = Array.new(@binds.size / 2)
 
-      if @operation && @table
-        table = @input[@table[0], @table[1] - @table[0]]
-        @title = "#{@operation} #{table}"
+      if @operation && !@table.empty?
+        @title = "" << @operation << SPACE << @table
       end
 
       while pos < @binds.size
-        slice = @output.slice!(@binds[pos] - removed, @binds[pos+1])
-        @output.insert(@binds[pos] - removed, '?')
-        extracted_binds[pos/2] = slice
-        removed += slice.size - 1
+        if @binds[pos] == nil
+          extracted_binds[pos/2] = @binds[pos+1]
+        else
+          slice = @output[@binds[pos] - removed, @binds[pos+1]]
+          @output[@binds[pos] - removed, @binds[pos+1]] = PLACEHOLDER
+
+          extracted_binds[pos/2] = slice
+          removed += (@binds[pos+1] - 1)
+        end
+
         pos += 2
       end
 
       @binds = extracted_binds
+      nil
     end
 
+    EMPTY = "".freeze
+
     def process_begin
-      @scanner.scan(TkOptWS)
+      @scanner.skip(TkOptWS)
       @state = :first_token
     end
 
+    OP_SELECT_FROM = "SELECT FROM".freeze
+    OP_UPDATE      = "UPDATE".freeze
+    OP_INSERT_INTO = "INSERT INTO".freeze
+    OP_DELETE_FROM = "DELETE FROM".freeze
+
     def process_first_token
-      if @scanner.skip(/SELECT\s+/i)
-        @operation = :"SELECT FROM"
+      if @scanner.skip(TkStartSelect)
+        @operation = OP_SELECT_FROM
         @state = :tokens
-        return
-      end
+      else
+        if @scanner.skip(TkUpdateTable)
+          @operation = OP_UPDATE
+        elsif @scanner.skip(TkInsertTable)
+          @operation = OP_INSERT_INTO
+        elsif @scanner.skip(TkDeleteTable)
+          @operation = OP_DELETE_FROM
+        end
 
-      if @scanner.skip(TkUpdateTable)
-        @operation = :UPDATE
-      elsif @scanner.skip(TkInsertTable)
-        @operation = :"INSERT INTO"
-      elsif @scanner.skip(TkDeleteTable)
-        @operation = :"DELETE FROM"
+        @state = :table_name
       end
-
-      @state = :table_name
     end
 
     def process_table_name
       pos = @scanner.pos
 
       if @scanner.skip(TkQuotedTable)
-        @table = [pos + 1, @scanner.pos - 1]
+        copy_substr(@input, @table, pos + 1, @scanner.pos - 1)
       elsif @scanner.skip(TkID)
-        @table = [pos, @scanner.pos]
+        copy_substr(@input, @table, pos, @scanner.pos)
       end
 
       @state = :tokens
@@ -174,12 +243,14 @@ module SqlLexer
     def process_tokens
       @scanner.skip(TkOptWS)
 
-      if @operation == :"SELECT FROM" && !@table && @scanner.skip(TkFromTable)
+      if @operation == OP_SELECT_FROM && @table.empty? && @scanner.skip(TkFromTable)
         @state = :table_name
       elsif @scanner.match?(TkSpecialOp)
         @state = :special
       elsif @scanner.match?(TkBind)
         @state = :bind
+      elsif @scanner.match?(TkPlaceholder)
+        @state = :placeholder
       elsif @scanner.match?(TkNonBind)
         @state = :non_bind
       else
@@ -187,10 +258,28 @@ module SqlLexer
       end
     end
 
+    def process_placeholder
+      @scanner.skip(TkPlaceholder)
+
+      binds << nil
+
+      if !@provided_binds
+        @binds << UNKNOWN
+      elsif !@provided_binds[@bind]
+        @binds << UNKNOWN
+      else
+        @binds << @provided_binds[@bind]
+      end
+
+      @bind += 1
+
+      @state = :tokens
+    end
+
     def process_special
       if @scanner.skip(TkIn)
         @scanner.skip(TkOptWS)
-        @scanner.skip(/\(/)
+        @scanner.skip(/\(/u)
         @state = :in
       end
     end
@@ -209,16 +298,16 @@ module SqlLexer
           raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in IN"
         end
 
-        if ENV["DEBUG"]
+        if @debug
           p @state
           p @scanner
           p nest
         end
 
-        if @scanner.skip(/\(/)
+        if @scanner.skip(/\(/u)
           nest += 1
           process_tokens
-        elsif @scanner.skip(/\)/)
+        elsif @scanner.skip(/\)/u)
           nest -= 1
           break if nest.zero?
           process_tokens
@@ -263,5 +352,19 @@ module SqlLexer
 
       @state = nil
     end
+
+  private
+    def copy_substr(source, target, start_pos, end_pos)
+      pos = start_pos
+
+      while pos < end_pos
+        target.concat source.getbyte(pos)
+        pos += 1
+      end
+    end
+
+    scanner
+    instance('')
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skylight
 version: !ruby/object:Gem::Version
-  version: 0.2.0.beta.3
+  version: 0.2.0.beta.4
 platform: ruby
 authors:
 - Tilde, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-03 00:00:00.000000000 Z
+date: 2013-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -62,6 +62,7 @@ files:
 - lib/skylight/normalizers/sql.rb
 - lib/skylight/railtie.rb
 - lib/skylight/subscriber.rb
+- lib/skylight/util/allocation_free.rb
 - lib/skylight/util/clock.rb
 - lib/skylight/util/gzip.rb
 - lib/skylight/util/http.rb
@@ -153,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.1.9
 signing_key: 
 specification_version: 4
 summary: Skylight is a ruby application monitoring tool.