skylight 0.3.0.rc.5 → 0.3.0.rc.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b72ea644f301ea7e25dde40d6f14ccc986e05211
-  data.tar.gz: 89a4c7bd22804de66170f69ec5690c7ab1224ba5
+  metadata.gz: 0f6a3addbdf4a8213bc76ea6430030ab9ff2925f
+  data.tar.gz: 6e7ed7c562fce41983579e0fbb3d619ef742dac7
 SHA512:
-  metadata.gz: 812242c812c7d018a903ca44e05969ddf4997caff99cc138fac6954a5ae461f254bfd66012d4735ea69cee84076aebd876ca3eec431dee8985f1064d4e33ace5
-  data.tar.gz: 36363fc2011fc6f82b28a4eff8f8f45d95b5829bfd741ffcd9b86b009d4767b72e12391cc56653d7cde1c13054793a02bc4a32b520f592a145f75663e2441dbe
+  metadata.gz: c596d66b61b7c9a36dfd60d3e95fb549daaa8a7d5db6307e79892dd0b62d3b8692f0bb511872265afd7ad8e65f172d3f1140561fa98fae0dc07478e4f2e9c502
+  data.tar.gz: 150660085cfd209fc249ebadde9b46cc9707aed19651ae72e0c712b51fc70552f15871dd880a7eafb4ea4c1c5ce2444acbc0c6f062e6aad56f59c0ca1fe0ce87

data/ext/checksums.yml

@@ -1,3 +1,3 @@
 ---
-dc29745.x86_64-linux: "4f0b9ef0d700d2da590ad0994c2067304a50d7b290f70353c394e4a040cdf0a5"
-dc29745.i686-linux  : "9b7beebe61ef3201f4d7ad2cf5e92dc69cedefda033b9d081384c71700927a37"
+x86_64-linux: "4f0b9ef0d700d2da590ad0994c2067304a50d7b290f70353c394e4a040cdf0a5"
+i686-linux  : "9b7beebe61ef3201f4d7ad2cf5e92dc69cedefda033b9d081384c71700927a37"

data/ext/extconf.rb

@@ -47,7 +47,7 @@ unless File.exist?("libskylight.a")
   end
 
   unless missing_a
-    expected_checksum = checksums["#{rust_version}.#{arch}"]
+    expected_checksum = checksums[arch]
     actual_checksum = Digest::SHA2.hexdigest(archive)
 
     if expected_checksum == actual_checksum
@@ -77,6 +77,7 @@ else
 
   find_library("skylight", "factory", ".")
 
+  $CFLAGS << " -Werror"
   if RbConfig::CONFIG["arch"] =~ /darwin(\d+)?/
     $LDFLAGS << " -lpthread"
   else

data/lib/skylight/normalizers/process_action.rb

@@ -21,7 +21,9 @@ module Skylight
         normalized = {}
 
         PAYLOAD_KEYS.each do |key|
-          normalized[key] = payload[key]
+          val = payload[key]
+          val = val.inspect unless val.is_a?(String) || val.is_a?(Numeric)
+          normalized[key] = val
         end
 
         normalized

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '0.3.0.rc.5'
+  VERSION = '0.3.0.rc.6'
 end
 

data/lib/skylight/worker/server.rb

@@ -328,7 +328,7 @@ module Skylight
 
       def get_memory_usage
         `ps -o rss= -p #{Process.pid}`.to_i / 1024
-      rescue Errno::ENOENT
+      rescue Errno::ENOENT, Errno::EINTR
         0
       end
     end

data/lib/sql_lexer/lexer.rb

@@ -26,7 +26,7 @@ module SqlLexer
 
     StartQuotedID = %Q<">
     StartTickedID = %Q<`>
-    StartString   = %Q<'>
+    StartString   = %Q<[a-zA-Z]?'>
     StartDigit    = %q<[\p{Digit}\.]>
 
     StartSelect   = %Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
@@ -50,7 +50,7 @@ module SqlLexer
     Type          = %Q<[#{StartID}][#{PartID}]*(?:\\(\d+\\)|\\[\\])?(?=#{AfterID})>
     QuotedTable   = %Q<#{TickedID}|#{QuotedID}>
 
-    StringBody    = %q<(?:''|\x5C'|[^'])*>
+    StringBody    = %q{(?:''|(?<!\x5C)(?:\x5C\x5C)*\x5C'|[^'])*}
     String        = %Q<#{StartString}#{StringBody}'>
 
     Digits        = %q<\p{Digit}+>

data/lib/sql_lexer/version.rb

@@ -1,3 +1,3 @@
 module SqlLexer
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

data/too-many-sockets.md

@@ -0,0 +1,62 @@
+We received a notification (Zendesk #280) from a customer that they were unable to SSH
+into their box, and that their service provider discovered that Skylight
+was triggering socket buffer limits.
+
+The customer immediately disabled Skylight, which fixed the bug for him.
+
+Customer Information:
+
+* Using Skylight 0.2.3
+* Ruby 2.0.0p353
+* Also had New Relic installed (Bug #46)
+* Recently upgraded to Rails 4
+* Passenger Enterprise with a max-pool size of 12 and zero-downtime deploys
+* He had a maxsockbuf of 32MB
+
+The customer also reported 134 open sockets in the server process.
+
+Separately, we discovered that New Relic was making our payloads
+extremely large.
+
+Our working hypothesis was that the large New Relic payloads were
+triggering the max buffer condition, which was then cascading into more
+failures.
+
+Upon further investigation, we discovered a few things:
+
+* If the client hits a kernel buffer limit (via EWOULDBLOCK that lasts
+  more than 5s), it closes down its socket
+* The server expects to recover from this situation by getting the
+  client socket in its read list from IO.select. It would then try to
+  read from it, get an EOF, and close the socket.
+* If the server does not get the client socket in its read list for some
+  (unknown) reason, this would result in an ever-growing list of sockets
+  on the server side.
+
+We are not sure why the sockets would not appear in the read list, but
+we hypothesize that when the OpenVZ limit is reached, the kernel no
+longer includes the socket in the read list, causing the server to never
+close the socket.
+
+Mitigation strategies:
+
+* The New Relic fix (#46) should reduce the likelihood of encountering
+  this in the first place
+* We want to add a Hello heartbeat from the client. If the server
+  doesn't receive a message every 1m (or 2m, TBD), it will close the
+  socket even if it's not in the read list.
+
+We are worried that if the server process ever gets stuck, this
+condition can occur. We are also considering a server heartbeat back to
+the clients, so they can take corrective action if the agent gets stuck.
+
+Some mitigations for this situation:
+
+* Kill the server process when a respawn is necessary. This will ensure
+  that a stuck agent process doesn't duplicate into N servers.
+* Before connecting to the server socket, pre-check the server process 
+  with kill -0.
+
+For next time: 
+* Request skylight.log for easier debugging
+* Form a best practices for customer bug reporting on doc site

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skylight
 version: !ruby/object:Gem::Version
-  version: 0.3.0.rc.5
+  version: 0.3.0.rc.6
 platform: ruby
 authors:
 - Tilde, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-18 00:00:00.000000000 Z
+date: 2014-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -139,6 +139,7 @@ files:
 - lib/sql_lexer.rb
 - lib/sql_lexer/lexer.rb
 - lib/sql_lexer/version.rb
+- too-many-sockets.md
 homepage: http://www.skylight.io
 licenses: []
 metadata: {}