skylight 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54d9a76b353b8d0a2a6fecce094a8a32802a1b9d
-  data.tar.gz: c773bed69979daa2fe511a1fef1698699aa6ba42
+  metadata.gz: 38880fd1b62fad365c461e3fb6517e979f14baa6
+  data.tar.gz: 9c62ce619478170a9e1b38e788bc35e61cd41b94
 SHA512:
-  metadata.gz: ad2f6c9f9edec7ec1417a89faa026cafbb5b36b9adc6e47c35063793178b8ff4bd75d683c404463fb37e0581be526eeeb7443e9df2314be060fd78d9419a51b6
-  data.tar.gz: f4b98d9d9b50b111b7c5f33ae57343c7985be52bcaa0cced2781c322bc1231b2186dd8f1ffda1a3b9eb098afec2c98e3572ed3492e21c39b4c8693acd3399dc8
+  metadata.gz: f0aaab6c04ff20b4fcb06da895808f9c84ae9fda59fd33a62639355b9ae5933ad417a80224c2409de46121a5dcd6fe8d45d09434fbd02d04cf7007659e6a0254
+  data.tar.gz: 2c7a866a02d2746130638408246c9d751a46dd7395f6ae98533081e984a7dffd3334de731a8dfa5b88a52b019af25e69c0a32f463d202f0dfa6b68ac7ffb68d8

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.3 (December 20, 2013)
+
+* Fix SQL lexing for comments, arrays, double-colon casting, and multiple queries
+* Handle template paths from gems
+* Status and exception reports for agent debugging
+
 ## 0.2.2 (December 10, 2013)
 
 * Added support for Mongoid/Moped

data/lib/skylight/config.rb

@@ -30,6 +30,7 @@ module Skylight
       'AGENT_SAMPLE_SIZE'   => :'agent.sample',
       'AGENT_SOCKFILE_PATH' => :'agent.sockfile_path',
       'AGENT_STRATEGY'      => :'agent.strategy',
+      'AGENT_MAX_MEMORY'    => :'agent.max_memory',
       'REPORT_HOST'         => :'report.host',
       'REPORT_PORT'         => :'report.port',
       'REPORT_SSL'          => :'report.ssl',
@@ -49,6 +50,7 @@ module Skylight
       :'agent.keepalive'     => 60,
       :'agent.interval'      => 5,
       :'agent.sample'        => 200,
+      :'agent.max_memory'    => 256, # MB
       :'report.host'         => 'agent.skylight.io'.freeze,
       :'report.port'         => 443,
       :'report.ssl'          => true,

data/lib/skylight/instrumenter.rb

@@ -79,7 +79,7 @@ module Skylight
       self
 
     rescue Exception => e
-      error "failed to start instrumenter; msg=%s", e.message
+      log_error "failed to start instrumenter; msg=%s", e.message
       nil
     end
 
@@ -99,7 +99,7 @@ module Skylight
       begin
         trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.micros, cat, title, desc, annot)
       rescue Exception => e
-        error e.message
+        log_error e.message
         t { e.backtrace.join("\n") }
         return
       end
@@ -183,14 +183,10 @@ module Skylight
       end
     end
 
-    def error(reason, body)
-      t { fmt "processing error; reason=%s; body=%s", reason, body }
+    def error(type, description, details=nil)
+      t { fmt "processing error; type=%s; description=%s", type, description }
 
-      if body.encoding == Encoding::BINARY || !body.valid_encoding?
-        body = Base64.encode64(body)
-      end
-
-      message = Skylight::Messages::Error.new(reason: reason, body: body)
+      message = Skylight::Messages::Error.new(type: type, description: description, details: details && details.to_json)
 
       unless @worker.submit(message)
         warn "failed to submit error to worker"

data/lib/skylight/messages/error.rb

@@ -3,8 +3,9 @@ require 'skylight/messages/base'
 module Skylight
   module Messages
     class Error < Base
-      required :reason, :string, 1
-      required :body,   :string, 2
+      required :type,        :string, 1
+      required :description, :string, 2
+      optional :details,     :string, 3
     end
   end
 end

data/lib/skylight/normalizers.rb

@@ -60,11 +60,21 @@ module Skylight
         return path if relative_path?(path)
 
         root = array_find(@paths) { |p| path.start_with?(p) }
+        type = :project
+
+        unless root
+          root = array_find(Gem.path) { |p| path.start_with?(p) }
+          type = :gem
+        end
 
         if root
           start = root.size
           start += 1 if path.getbyte(start) == SEPARATOR_BYTE
-          path[start, path.size]
+          if type == :gem
+            "$GEM_PATH/#{path[start, path.size]}"
+          else
+            path[start, path.size]
+          end
         else
           annotations[:skylight_error] = ["absolute_path", path]
           "Absolute Path"

data/lib/skylight/normalizers/sql.rb

@@ -45,9 +45,8 @@ module Skylight
         title, sql, binds = SqlLexer::Lexer.bindify(payload[:sql], precalculated)
         [ title, sql, binds, nil ]
       rescue
-        # Encode this since we may have improperly incoded strings that can't be to_json'ed
-        encoded = encode(payload: payload, precalculated: precalculated)
-        [ nil, nil, nil, ["sql_parse", encoded.to_json] ]
+        error = ["sql_parse", encode(payload[:sql]), encode(payload: payload, precalculated: precalculated)]
+        [ nil, nil, nil, error ]
       end
 
       def encode(body)

data/lib/skylight/util/http.rb

@@ -87,8 +87,8 @@ module Skylight
       end
 
       def execute(req, body=nil)
-        t { fmt "executing HTTP request; host=%s; port=%s; body=%s",
-              @host, @port, body && body.bytesize }
+        t { fmt "executing HTTP request; host=%s; port=%s; path=%s, body=%s",
+              @host, @port, req.path, body && body.bytesize }
 
         if body
           body = Gzip.compress(body) if @deflate

data/lib/skylight/util/logging.rb

@@ -35,7 +35,13 @@ module Skylight
         log :error, msg, *args
       end
 
-      alias fmt sprintf
+      alias log_trace trace
+      alias log_debug debug
+      alias log_info  info
+      alias log_warn  warn
+      alias log_error error
+
+      alias fmt       sprintf
 
       def log(level, msg, *args)
         return unless respond_to?(:config)

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
 end
 

data/lib/skylight/worker/collector.rb

@@ -62,27 +62,37 @@ module Skylight
         true
       end
 
+      def send_status(status)
+        post_data(:status, status)
+      end
+
+      def send_exception(exception)
+        post_data(:exception, exception)
+      end
+
     private
 
-      def send_error(msg)
-        res = @http_auth.post("/agent/error?hostname=#{escape(config[:'hostname'])}", reason: msg.reason, body: msg.body)
+      def post_data(type, data)
+        t { "posting data (#{type}): #{data.inspect}" }
+
+        res = @http_auth.post("/agent/#{type}?hostname=#{escape(config[:'hostname'])}", data)
 
         # error already handled in Util::HTTP
         return unless res
 
         unless res.success?
-          if (400..499).include? res.status
-            warn "error wasn't sent successfully; status=%s", res.status
-          end
-
-          warn "could not fetch report session token; status=%s", res.status
-          return
+          warn "#{type} wasn't sent successfully; status=%s", res.status
         end
       rescue Exception => e
         error "exception; msg=%s; class=%s", e.message, e.class
         t { e.backtrace.join("\n") }
       end
 
+      def send_error(msg)
+        details = msg.details ? JSON.parse(msg.details) : nil
+        post_data(:error, type: msg.type, description: msg.description, details: details)
+      end
+
       def finish
         t { fmt "collector finishing up" }
 

data/lib/skylight/worker/server.rb

@@ -2,8 +2,6 @@ require 'socket'
 
 module Skylight
   module Worker
-    # TODO:
-    #   - Shutdown if no connections for over a minute
     class Server
       LOCKFILE_PATH      = 'SKYLIGHT_LOCKFILE_PATH'.freeze
       LOCKFILE_ENV_KEY   = 'SKYLIGHT_LOCKFILE_FD'.freeze
@@ -18,7 +16,10 @@ module Skylight
         :config,
         :keepalive,
         :lockfile_path,
-        :sockfile_path
+        :sockfile_path,
+        :status_interval,
+        :last_status_update,
+        :max_memory
 
       def initialize(config, lockfile, srv, lockfile_path)
         unless lockfile && srv
@@ -37,6 +38,8 @@ module Skylight
         @connections   = {}
         @lockfile_path = lockfile_path
         @sockfile_path = @config[:'agent.sockfile_path']
+        @status_interval = 60
+        @max_memory      = @config[:'agent.max_memory']
       end
 
       # Called from skylight.rb on require
@@ -132,6 +135,7 @@ module Skylight
         now = Time.now.to_i
         next_sanity_check_at = now + tick
         had_client_at = now
+        last_status_update = now
 
         trace "starting IO loop"
         begin
@@ -179,9 +183,16 @@ module Skylight
           if keepalive < now - had_client_at
             info "no clients for #{keepalive} sec - shutting down"
             @run = false
-          elsif next_sanity_check_at <= now
-            next_sanity_check_at = now + tick
-            sanity_check
+          else
+            if next_sanity_check_at <= now
+              next_sanity_check_at = now + tick
+              sanity_check
+            end
+
+            if status_interval < now - last_status_update
+              last_status_update = now
+              status_check
+            end
           end
 
         rescue SignalException => e
@@ -193,9 +204,11 @@ module Skylight
         rescue Exception => e
           error "Loop exception: %s (%s)", e.message, e.class
           puts e.backtrace
+          @collector.send_exception(class_name: e.class.name, message: e.message, backtrace: e.backtrace)
           return false
         rescue Object => o
           error "Unknown object thrown: `%s`", o.to_s
+          @collector.send_exception(class_name: o.class.name)
           return false
         end while @run
 
@@ -301,6 +314,22 @@ module Skylight
           raise WorkerStateError, "sockfile gone"
         end
       end
+
+      def status_check
+        memory_usage = get_memory_usage
+
+        @collector.send_status(memory: memory_usage, max_memory: max_memory)
+
+        if memory_usage > max_memory
+          raise WorkerStateError, "Memory limit exceeded: #{memory_usage} (max: #{max_memory})"
+        end
+      end
+
+      def get_memory_usage
+        `ps -o rss= -p #{Process.pid}`.to_i / 1024
+      rescue Errno::ENOENT
+        0
+      end
     end
   end
 end

data/lib/sql_lexer/lexer.rb

@@ -19,7 +19,10 @@ module SqlLexer
     End           = %Q<;|$>
 
     InOp          = %q<IN>
-    SpecialOps    = %Q<#{InOp}(?=[#{WS}])>
+    ArrayOp       = %q<ARRAY>
+    ColonColonOp  = %Q<::(?=[#{StartID}])>
+    ArrayIndexOp  = %q<\\[(?:\-?\d+(?::\-?\d+)?|NULL)\\]>
+    SpecialOps    = %Q<#{InOp}(?=[#{WS}])|#{ColonColonOp}|#{ArrayOp}|#{ArrayIndexOp}>
 
     StartQuotedID = %Q<">
     StartTickedID = %Q<`>
@@ -35,13 +38,14 @@ module SqlLexer
     StartAnyId    = %Q<"#{StartID}>
     Placeholder   = %q<\$\p{Digit}+>
 
-    AfterID       = %Q<[#{WS};#{StartNonBind}]|(?:#{OpPart})|$>
+    AfterID       = %Q<[#{WS};#{StartNonBind}]|(?:#{OpPart})|(?:#{ColonColonOp})|(?:#{ArrayIndexOp})|$>
     ID            = %Q<[#{StartID}][#{PartID}]*(?=#{AfterID})>
     AfterOp       = %Q<[#{WS}]|[#{StartAnyId}]|[#{StartBind}]|(#{StartNonBind})|$>
     Op            = %Q<(?:#{OpPart})+(?=#{AfterOp})>
     QuotedID      = %Q<#{StartQuotedID}(?:[^"]|"")*">
     TickedID      = %Q<#{StartTickedID}(?:[^`]|``)*`>
     NonBind       = %Q<#{ID}|#{Op}|#{QuotedID}|#{TickedID}|#{Placeholder}>
+    Type          = %Q<[#{StartID}][#{PartID}]*(?:\\(\d+\\)|\\[\\])?(?=#{AfterID})>
     QuotedTable   = %Q<#{TickedID}|#{QuotedID}>
 
     StringBody    = %q<(?:''|\x5C'|[^'])*>
@@ -62,8 +66,12 @@ module SqlLexer
     TkWS          = %r<[#{WS}]+>u
     TkOptWS       = %r<[#{WS}]*>u
     TkOp          = %r<[#{OpPart}]>u
+    TkComment     = %r<^#{OptWS}--.*$>u
+    TkBlockCommentStart = %r</\*>u
+    TkBlockCommentEnd   = %r<\*/>u
     TkPlaceholder = %r<#{Placeholder}>u
     TkNonBind     = %r<#{NonBind}>u
+    TkType        = %r<#{Type}>u
     TkQuotedTable = %r<#{QuotedTable}>iu
     TkUpdateTable = %r<UPDATE#{TableNext}>iu
     TkInsertTable = %r<INSERT[#{WS}]+INTO#{TableNext}>iu
@@ -73,6 +81,9 @@ module SqlLexer
     TkEnd         = %r<;?[#{WS}]*>u
     TkBind        = %r<#{String}|#{Number}|#{Literals}>u
     TkIn          = %r<#{InOp}>iu
+    TkColonColon  = %r<#{ColonColonOp}>u
+    TkArray       = %r<#{ArrayOp}>iu
+    TkArrayIndex  = %r<#{ArrayIndexOp}>iu
     TkSpecialOp   = %r<#{SpecialOps}>iu
     TkStartSelect = %r<SELECT(?=(?:[#{WS}]|#{OpPart}))>iu
 
@@ -86,7 +97,8 @@ module SqlLexer
       table_name:  :process_table_name,
       end:         :process_end,
       special:     :process_special,
-      in:          :process_in
+      in:          :process_in,
+      array:       :process_array
     }
 
     def self.bindify(string, binds=nil)
@@ -158,6 +170,8 @@ module SqlLexer
     UNKNOWN = "<unknown>".freeze
 
     def process(binds)
+      process_comments
+
       @operation = nil
       @provided_binds = binds
 
@@ -201,6 +215,64 @@ module SqlLexer
 
     EMPTY = "".freeze
 
+    def process_comments
+      # SQL treats comments as similar to whitespace
+      # Here we replace all comments with spaces of the same length so as to not affect binds
+
+      # Remove block comments
+      # SQL allows for nested comments so this takes a bit more work
+      while @scanner.skip_until(TkBlockCommentStart)
+        count = 1
+        pos = @scanner.pos - 2
+
+        while true
+          # Determine whether we close the comment or start nesting
+          next_open  = @scanner.skip_until(TkBlockCommentStart)
+          @scanner.unscan if next_open
+          next_close = @scanner.skip_until(TkBlockCommentEnd)
+          @scanner.unscan if next_close
+
+          if next_open && next_open < next_close
+            # We're nesting
+            count += 1
+            @scanner.skip_until(TkBlockCommentStart)
+          else
+            # We're closing
+            count -= 1
+            @scanner.skip_until(TkBlockCommentEnd)
+          end
+
+          if count > 10_000
+            raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in block comments"
+          end
+
+          if count == 0
+            # We've closed all comments
+            length = @scanner.pos - pos
+            # Dup the string if necessary so we aren't destructive to the original value
+            @scanner.string = @input.dup if @scanner.string == @input
+            # Replace the comment with spaces
+            @scanner.string[pos, length] = SPACE*length
+            break
+          end
+        end
+      end
+
+      @scanner.reset
+
+      # Remove single line comments
+      while @scanner.skip_until(TkComment)
+        pos = @scanner.pos
+        len = @scanner.matched_size
+        # Dup the string if necessary so we aren't destructive to the original value
+        @scanner.string = @input.dup if @scanner.string == @input
+        # Replace the comment with spaces
+        @scanner.string[pos-len, len] = SPACE*len
+      end
+
+      @scanner.reset
+    end
+
     def process_begin
       @scanner.skip(TkOptWS)
       @state = :first_token
@@ -281,6 +353,17 @@ module SqlLexer
         @scanner.skip(TkOptWS)
         @scanner.skip(/\(/u)
         @state = :in
+      elsif @scanner.skip(TkArray)
+        @scanner.skip(/\[/u)
+        @state = :array
+      elsif @scanner.skip(TkColonColon)
+        if @scanner.skip(TkType)
+          @state = :tokens
+        else
+          @state = :end
+        end
+      elsif @scanner.skip(TkArrayIndex)
+        @state = :tokens
       end
     end
 
@@ -315,7 +398,57 @@ module SqlLexer
           process_tokens
         end
 
-        send STATE_HANDLERS[@state]
+        __send__ STATE_HANDLERS[@state]
+      end
+
+      @binds << pos
+      @binds << @scanner.pos - pos
+
+      @skip_binds = false
+
+      @state = :tokens
+    end
+
+    def process_array
+      nest = 1
+      iterations = 0
+
+      @skip_binds = true
+      pos = @scanner.pos - 6
+
+      while nest > 0
+        iterations += 1
+
+        if iterations > 10_000
+          raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in ARRAY"
+        end
+
+        if @debug
+          p "array loop"
+          p @state
+          p @scanner
+        end
+
+        if @scanner.skip(/\[/u)
+          nest += 1
+        elsif @scanner.skip(/\]/u)
+          nest -= 1
+
+          break if nest.zero?
+
+          # End of final nested array
+          next if @scanner.skip(/#{TkOptWS}(?=\])/u)
+        end
+
+        # A NULL array
+        next if @scanner.skip(/NULL/iu)
+
+        # Another nested array
+        next if @scanner.skip(/#{TkOptWS},#{TkOptWS}(?=\[)/u)
+
+        process_tokens
+
+        __send__ STATE_HANDLERS[@state]
       end
 
       @binds << pos
@@ -344,13 +477,18 @@ module SqlLexer
     end
 
     def process_end
-      @scanner.skip(TkEnd)
+      if @scanner.skip(TkEnd)
+        if @scanner.eos?
+          @state = nil
+        else
+          process_tokens
+        end
+      end
 
-      unless @scanner.eos?
+      # We didn't hit EOS and we couldn't process any tokens
+      if @state == :end
         raise "The SQL '#{@scanner.string}' could not be parsed"
       end
-
-      @state = nil
     end
 
   private

data/lib/sql_lexer/version.rb

@@ -1,3 +1,3 @@
 module SqlLexer
-  VERSION = "0.0.1"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skylight
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Tilde, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-10 00:00:00.000000000 Z
+date: 2013-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport