skylight 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3bbad2e6ff11878c301f0f6c35a19a700eb873e
-  data.tar.gz: 6c5088d5d960e65149f89ced6ee48a5c7ff3c4dc
+  metadata.gz: 5ba19fdc679a43bb87c5cf95b843b05ce58878cb
+  data.tar.gz: 7cdf7d06b0eba50a07cbea357929bcd8b8231f73
 SHA512:
-  metadata.gz: fd2dd401324a03a77b3ddc63f38d93cb4cfa7740f6603932e8c0b65f119c71d963637ffeb6f299d7eb427515232e230400014d5dec7b198340cf5f05835f4a01
-  data.tar.gz: 746417c9a7a2600875374f2a783add8e2d08f6e5344b7f70f4fc6ba6292a3807535e5d4c0f80b27c1239c2e5e1b099ff250fef544ed7aaeb52a678465359313c
+  metadata.gz: c59d80d86beb6ff52709bb48a3182373ae2557f901419e4603dcaac64072f6144b3c6735fc2aef5fae3fd487647f230dd6c582f67b54052961e080aba307f120
+  data.tar.gz: 260f99fcf7b8a7b7074538e2083df7b334bbd4e1afd4b1e6874dbbaafc2616567127507e00c6bd7724c8a55c32b9cf479759d6eac0961fda6a8bbe10b02d5914

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
 end
 

data/lib/skylight/worker/server.rb

@@ -327,7 +327,7 @@ module Skylight
 
       def get_memory_usage
         `ps -o rss= -p #{Process.pid}`.to_i / 1024
-      rescue Errno::ENOENT
+      rescue Errno::ENOENT, Errno::EINTR
         0
       end
     end

data/lib/sql_lexer.rb

@@ -2,4 +2,5 @@ require "sql_lexer/version"
 require "sql_lexer/lexer"
 
 module SqlLexer
+  # Your code goes here...
 end

data/lib/sql_lexer/lexer.rb

@@ -29,8 +29,6 @@ module SqlLexer
     StartString   = %Q<'>
     StartDigit    = %q<[\p{Digit}\.]>
 
-    StartSelect   = %Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
-
     # Binds that are also IDs do not need to be included here, since AfterOp (which uses StartBind)
     # also checks for StartAnyId
     StartBind     = %Q<#{StartString}|#{StartDigit}|#{SpecialOps}>
@@ -87,9 +85,7 @@ module SqlLexer
     TkArray       = %r<#{ArrayOp}>iu
     TkArrayIndex  = %r<#{ArrayIndexOp}>iu
     TkSpecialOp   = %r<#{SpecialOps}>iu
-    TkStartSelect = %r<#{StartSelect}>iu
-    TkStartSubquery = %r<\(#{OptWS}#{StartSelect}>iu
-    TkCloseParen  = %r<#{OptWS}\)>u
+    TkStartSelect = %r<SELECT(?=(?:[#{WS}]|#{OpPart}))>iu
 
     STATE_HANDLERS = {
       begin:       :process_begin,
@@ -101,7 +97,6 @@ module SqlLexer
       table_name:  :process_table_name,
       end:         :process_end,
       special:     :process_special,
-      subquery:    :process_subquery,
       in:          :process_in,
       array:       :process_array
     }
@@ -356,12 +351,8 @@ module SqlLexer
     def process_special
       if @scanner.skip(TkIn)
         @scanner.skip(TkOptWS)
-        if @scanner.skip(TkStartSubquery)
-          @state = :subquery
-        else
-          @scanner.skip(/\(/u)
-          @state = :in
-        end
+        @scanner.skip(/\(/u)
+        @state = :in
       elsif @scanner.skip(TkArray)
         @scanner.skip(/\[/u)
         @state = :array
@@ -371,48 +362,11 @@ module SqlLexer
         else
           @state = :end
         end
-      elsif @scanner.skip(TkStartSubquery)
-        @state = :subquery
       elsif @scanner.skip(TkArrayIndex)
         @state = :tokens
       end
     end
 
-    def process_subquery
-      nest = 1
-      iterations = 0
-
-      while nest > 0
-        iterations += 1
-
-        if iterations > 10_000
-          raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in subquery"
-        end
-
-        if @debug
-          p @state
-          p @scanner
-          p nest
-          p @scanner.peek(1)
-        end
-
-        if @scanner.skip(TkStartSubquery)
-          nest += 1
-          @state = :tokens
-        elsif @scanner.skip(TkCloseParen)
-          nest -= 1
-          break if nest.zero?
-          @state = :tokens
-        elsif @state == :subquery
-          @state = :tokens
-        end
-
-        __send__ STATE_HANDLERS[@state]
-      end
-
-      @state = :tokens
-    end
-
     def process_in
       nest = 1
       iterations = 0
@@ -436,7 +390,7 @@ module SqlLexer
         if @scanner.skip(/\(/u)
           nest += 1
           process_tokens
-        elsif @scanner.skip(TkCloseParen)
+        elsif @scanner.skip(/\)/u)
           nest -= 1
           break if nest.zero?
           process_tokens

data/lib/sql_lexer/version.rb

@@ -1,3 +1,3 @@
 module SqlLexer
-  VERSION = "0.0.4"
+  VERSION = "0.0.3"
 end

data/too-many-sockets.md

@@ -0,0 +1,62 @@
+We received a notification (Zendesk #280) from a customer that they were unable to SSH
+into their box, and that their service provider discovered that Skylight
+was triggering socket buffer limits.
+
+The customer immediately disabled Skylight, which fixed the bug for him.
+
+Customer Information:
+
+* Using Skylight 0.2.3
+* Ruby 2.0.0p353
+* Also had New Relic installed (Bug #46)
+* Recently upgraded to Rails 4
+* Passenger Enterprise with a max-pool size of 12 and zero-downtime deploys
+* He had a maxsockbuf of 32MB
+
+The customer also reported 134 open sockets in the server process.
+
+Separately, we discovered that New Relic was making our payloads
+extremely large.
+
+Our working hypothesis was that the large New Relic payloads were
+triggering the max buffer condition, which was then cascading into more
+failures.
+
+Upon further investigation, we discovered a few things:
+
+* If the client hits a kernel buffer limit (via EWOULDBLOCK that lasts
+  more than 5s), it closes down its socket
+* The server expects to recover from this situation by getting the
+  client socket in its read list from IO.select. It would then try to
+  read from it, get an EOF, and close the socket.
+* If the server does not get the client socket in its read list for some
+  (unknown) reason, this would result in an ever-growing list of sockets
+  on the server side.
+
+We are not sure why the sockets would not appear in the read list, but
+we hypothesize that when the OpenVZ limit is reached, the kernel no
+longer includes the socket in the read list, causing the server to never
+close the socket.
+
+Mitigation strategies:
+
+* The New Relic fix (#46) should reduce the likelihood of encountering
+  this in the first place
+* We want to add a Hello heartbeat from the client. If the server
+  doesn't receive a message every 1m (or 2m, TBD), it will close the
+  socket even if it's not in the read list.
+
+We are worried that if the server process ever gets stuck, this
+condition can occur. We are also considering a server heartbeat back to
+the clients, so they can take corrective action if the agent gets stuck.
+
+Some mitigations for this situation:
+
+* Kill the server process when a respawn is necessary. This will ensure
+  that a stuck agent process doesn't duplicate into N servers.
+* Before connecting to the server socket, pre-check the server process 
+  with kill -0.
+
+For next time: 
+* Request skylight.log for easier debugging
+* Form a best practices for customer bug reporting on doc site

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skylight
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Tilde, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-21 00:00:00.000000000 Z
+date: 2014-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -142,6 +142,7 @@ files:
 - lib/sql_lexer.rb
 - lib/sql_lexer/lexer.rb
 - lib/sql_lexer/version.rb
+- too-many-sockets.md
 homepage: http://www.skylight.io
 licenses: []
 metadata: {}