framed_rails 0.1.3 → 0.1.4

data/README.md

@@ -1,13 +1,12 @@
 `framed_rails`
 ------------
 
-`framed_rails` is a gem to add Framed instrumentation to your rails4
-app.  For each pageview, it sends an event to Framed.
+`framed_rails` is a gem to add Framed instrumentation to your Rails 4
+app. For each request, it sends an event to Framed.
 
-To use this with rails:
+To use this in your Rails project:
 
-
- * Add `framed_rails` to your Gemfile.
+ * Add `gem 'framed_rails', '~> 0.1.4'` to your Gemfile.
  * Add the following to `config/initializers/framed_rails.rb`:
 
 ```ruby
@@ -18,18 +17,7 @@ Framed.configure do |config|
 end
 ```
 
-Note that the threaded emitter works on a background thread.  It is
-possible that events will be unreported when your containing process
-ends, unless you drain at process shutdown, i.e.
-
-```ruby
-Framed.drain
-```
-
-If reporting fails, the exception will be logged to STDERR by default,
-unless Rails is detected.  In that case, the `Rails.logger` will be
-used.
-
+If reporting fails, the exception will be logged to `Rails.logger` by default.
 
 Configuration
 -------------
@@ -45,63 +33,70 @@ Configuration
     <td>:consumer</td>
     <td>The emitter to be used for reporting.  See the Emitters
     section below.</td>
-    <td>Framed::Emitters::Blocking</td>
-  </tr>
-
-	<tr>
-    <td>:endpoint</td>
-    <td>The URL to POST to, using basic auth with your write key</td>
-    <td>Framed::SEGMENT_API</td>
+    <td>`Framed::Emitters::Blocking`</td>
   </tr>
 
   <tr>
     <td>:user_id_controller_method</td>
     <td>The name of a controller method which returns the user ID, if
     any</td>
-    <td>'framed_devise_user_id'</td>
+    <td>`framed_current_user_id`</td>
   </tr>
 
 	<tr>
     <td>:logger</td>
     <td>A Logger for reporting errors.</td>
-    <td>STDERR, or Rails.logger if detected.</td>
+    <td>`Rails.logger`</td>
   </tr>
 
 	<tr>
     <td>:anonymous_cookie</td>
     <td>The name of the in signed cookie for anonymous user IDs.
     Long-lived anonymous user IDs are issued anonymous users.</td>
-    <td>Framed::COOKIE_NAME</td>
+    <td>`Framed::COOKIE_NAME`</td>
   </tr>
 
 	<tr>
     <td>:user_id_controller_method</td>
     <td>The name of a controller method which can provide the current
-    User ID.  Devise just works.</td>
-    <td>'framed_devise_user_id'</td>
+    User ID. (Also works with Devise).</td>
+    <td>'framed_current_user_id'</td>
   </tr>
+
+  <tr>
+    <td>:include_xhr</td>
+    <td>Whether to include requests sent via AJAX. (Turbolinks are always included.)</td>
+    <td>false</td>
+  </tr>
+
 </table>
 
 Emitters
 --------
 
-By default, the report is sent with a blocking Emitter.  If you would
-prefer a non-blocking emitter, you can include the following line in
+By default, events are sent with a blocking emitter, which sends each request to Framed
+as it happens. If you would prefer a non-blocking emitter, you can include the following line in
 your configure block:
 
 
 ```ruby
-  config[:consumer] => Framed::Emitters::Threaded
+config[:consumer] = Framed::Emitters::Buffered
 ```
 
-If you'd like to implement your own, see `lib/framed/emitters.rb`,
-particularly Base and Blocking as examples.
-
 Emitters included in this gem:
 
+ * `Framed::Emitters::Blocking` - Logs each request to Framed using a single blocking request (default)
+ * `Framed::Emitters::Buffered` - Logs to Framed 1) if no request is in progress, immediately 2) otherwise in batches of up to 100 as soon as the previous request completes.  All requests are sent on a background thread.
  * `Framed::Emitters::InMemory` - stores reported events in memory,
-   rather than transmitting them.   Events are later available as
-   Framed.consumer.reported.
- * `Framed::Emitters::Logger` - Logs an info message to config[:logger].
- * `Framed::Emitters::Blocking` - Logs to Framed, using a blocking request.
- * `Framed::Emitters::Threaded` - Logs to Framed, using a background-threaded request.
+   rather than transmitting them.  Events are later available as `Framed.consumer.reported`.
+ * `Framed::Emitters::Logger` - Logs an info message to `config[:logger]`.
+
+Both `InMemory` and `Logger` should be considered for debugging/diagnostic purposes only.
+
+Note that the `Buffered` emitter works on a background thread. It is
+possible that events will be unreported when your containing process
+ends, unless you explicitly call `#drain` at process shutdown, i.e.
+
+```ruby
+Framed.drain
+```

data/lib/framed/client.rb

@@ -16,6 +16,8 @@ module Framed
     end
 
     def track(data)
+      Framed.logger.info("Client#track #{data.length}")
+
       creds = Base64.strict_encode64(@config[:api_key] + ':')
       payload = JSON.generate(data)
       response = Excon.post(@config[:endpoint],

data/lib/framed/emitters.rb

@@ -21,10 +21,12 @@ module Framed
 
       private
 
-      def transmit(event)
+      def transmit(events)
+        return unless events && events.length > 0
+
         begin
-          @client.track(event)
-        rescue StandardError => exc
+          @client.track(events)
+        rescue Exception => exc
           Framed.logger.error("framed_rails: transmit failed: #{exc}")
         end
       end
@@ -49,64 +51,136 @@ module Framed
       end
     end
 
-    class Threaded < Base
+    class Blocking < Base
+      def start
+      end
+      def stop(drain = false)
+      end
+      def enqueue(event)
+        transmit([event])
+      end
+    end
+
+    class Buffered < Base
+      MAX_REQUEST_BATCH_SIZE = 100
+      MAX_QUEUE_SIZE = 10_000
+
       def initialize(client)
         super
-        @queue = Queue.new
-      end
+        @event_queue = Queue.new
+        @batch_lock = Mutex.new
 
-      def enqueue(event)
-        @queue << event
-        start
+        @request_queue = Queue.new
+        @request_pending = Mutex.new
+
+        @request_thread = nil
       end
 
       def start
-        return if @thread
+        if @request_thread and !@request_thread.alive?
+          Framed.logger.error("Starting request thread due to dead thread")
+        end
 
-        @thread = Thread.new do
+        @request_thread = Thread.new do
           while true
-            begin
-              process_events
-            rescue StandardError => exc
-              Framed.logger.error("framed_rails: run_thread failed: #{exc}")
-              stop
+            pending = @request_queue.pop
+
+            @request_pending.synchronize do
+              transmit(pending)
             end
-            sleep(0.5)
+
+            start_request
           end
         end
       end
 
       def stop(drain = false)
-        if @thread
-          @thread.kill
+        if drain
+          # start batch requests if needed
+          while @event_queue.length > 0
+            start_request
+          end
+
+          # wait for pending requests if needed:
+          while @request_queue.length > 0
+            sleep(0.1)
+          end
+
+          # and wait for the final request, if needed
+          @request_pending.synchronize do
+            stop_request_thread
+          end
+        else
+          stop_request_thread
         end
-        @thread = nil
+      end
+
+      def warn_full(event)
+        Framed.logger.error("Queued #{event} to framed, but queue is full.  Dropping event.")
+      end
+
+      def enqueue(event)
+        queue_full = false
+        @batch_lock.synchronize do
+          # To avoid logging inside the lock (since loggers can block)
+          #  we remember if the queue is full and log outside the lock.
+          queue_full = @event_queue.length >= MAX_QUEUE_SIZE
+
+          if !queue_full
+            @event_queue << event
+          end
 
-        if drain && @queue.length
-          process_events
+          # don't start a new request if one is already in progress.
+          if @request_pending.locked?
+            return
+          end
         end
+
+        warn_full if queue_full
+
+        start_request
       end
 
       private
 
-      def dequeue
-        @queue.pop
-      end
+      def ensure_request_thread
+        return if @request_thread && @request_thread.alive?
 
-      def process_events
-        while @queue.length > 0
-          transmit(dequeue)
+        @request_pending.synchronize do
+          start
         end
       end
-    end
 
-    class Blocking < Base
-      def start
+      def stop_request_thread
+        if @request_thread
+          @request_thread.kill
+          @request_thread = nil
+        end
       end
-      def stop
+
+      def start_request
+        ensure_request_thread
+
+        @batch_lock.synchronize do
+          return if @event_queue.empty?
+
+          pending = []
+          while pending.length < MAX_REQUEST_BATCH_SIZE && @event_queue.length > 0
+             pending << @event_queue.pop
+          end
+
+          @request_queue << pending
+        end
       end
-      def enqueue(event)
-        transmit(event)
+
+      def transmit(events)
+        return unless events && events.length > 0
+
+        begin
+          @client.track(events)
+        rescue Exception => exc
+          Framed.logger.error("framed_rails: transmit failed: #{exc}")
+        end
       end
     end
   end

data/lib/framed/exceptions.rb

@@ -10,4 +10,4 @@ module Framed
 
   class RequestError < Error
   end
-end
+end

data/lib/framed/rails.rb

@@ -1,27 +1,37 @@
 ActionController::Base.class_eval do
 
-  after_filter :framed_report_page_view
+  after_filter :framed_report_request
 
-  def pv_event_name
+  def framed_event_name
     "#{request.method}_#{params[:controller]}\##{params[:action]}"
   end
 
-  def framed_report_page_view
+  def framed_included?(request)
+    return true if Framed.configuration[:include_xhr]
+    # include Turbolinks requests (which are a special kind of XHR)
+    return true if request.headers.include?('X-XHR-Referer')
+
+    !request.xhr?
+  end
+
+  def framed_report_request
     begin
       anonymous_id = cookies.signed[Framed.anonymous_cookie]
       user_id = send(Framed.user_id_controller_method)
 
       if user_id.nil? && anonymous_id.nil?
         anonymous_id = Framed.new_anonymous_id
-        cookies.signed.permanent[Framed.anonymous_cookie] = { :value => anonymous_id, :httponly => true}
+        cookies.signed.permanent[Framed.anonymous_cookie] = {:value => anonymous_id, :httponly => true}
       end
 
+      return unless framed_included?(request)
+
       cleaned_params = params.except(:controller, :action).to_h
-      Framed.report({
+      event = {
         :type => :track,
         :anonymous_id => anonymous_id,
         :user_id => user_id,
-        :event   => pv_event_name,
+        :event   => framed_event_name,
         :context => {
           :path => request.path,
           :request_method => request.method,
@@ -31,13 +41,19 @@ ActionController::Base.class_eval do
         :properties => Framed::Utils.flattened_hash({
           :params => cleaned_params
         })
-      })
-    rescue StandardError => exc
-      Framed.logger.error("Failed to report page_view #{exc}")
+      }
+
+      Framed.report(event)
+    rescue Exception => exc
+      Framed.logger.error("Failed to report request #{exc}")
     end
   end
 
-  def framed_devise_user_id
-    current_user.try(:id)
+  def framed_current_user_id
+    begin
+      current_user.try(:id)
+    rescue
+      nil
+    end
   end
-end
+end

data/lib/framed/utils.rb

@@ -8,47 +8,47 @@ end
 
 module Framed
   module Utils
-    class << self
-      def uuid
-        begin
-          UUID.new.generate
-        rescue NameError
-          SecureRandom.uuid
-        end
-      end
+    extend self
 
-      # Adapted from Rails in case it isn't available.
-      def try(o, *a, &b)
-        try!(o, *a, &b) if a.empty? || o.respond_to?(a.first)
+    def uuid
+      begin
+        UUID.new.generate
+      rescue NameError
+        SecureRandom.uuid
       end
+    end
 
-      def try!(o, *a, &b)
-        if a.empty? && block_given?
-          if b.arity.zero?
-            o.instance_eval(&b)
-          else
-            yield o
-          end
+    # Adapted from Rails in case it isn't available.
+    def try(o, *a, &b)
+      try!(o, *a, &b) if a.empty? || o.respond_to?(a.first)
+    end
+
+    def try!(o, *a, &b)
+      if a.empty? && block_given?
+        if b.arity.zero?
+          o.instance_eval(&b)
         else
-          o.public_send(*a, &b)
+          yield o
         end
+      else
+        o.public_send(*a, &b)
       end
+    end
 
-      def serialize_date(dt)
-        dt.utc.iso8601
-      end
+    def serialize_date(dt)
+      dt.utc.iso8601
+    end
 
-      def flattened_hash(h, namespace = '', memo = {})
-        h.reduce(memo) { |memo, (key, value)|
-          value = value.to_h if value.respond_to?(:to_h)
-          if value.instance_of?(Hash)
-            memo.merge!(flattened_hash(value, "#{namespace}#{key}_", memo))
-          else
-            memo["#{namespace}#{key}"] = value
-          end
-          memo
-        }
-      end
+    def flattened_hash(h, namespace = '', memo = {})
+      h.reduce(memo) { |memo, (key, value)|
+        value = value.to_h if value.respond_to?(:to_h)
+        if value.instance_of?(Hash)
+          memo.merge!(flattened_hash(value, "#{namespace}#{key}_", memo))
+        else
+          memo["#{namespace}#{key}"] = value
+        end
+        memo
+      }
     end
   end
-end
+end

data/lib/framed/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Framed
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/framed_rails.rb

@@ -19,10 +19,11 @@ module Framed
     def configuration
       @configuration ||= {
         :consumer => Framed::Emitters::Blocking,
-        :user_id_controller_method => 'framed_devise_user_id',
+        :user_id_controller_method => 'framed_current_user_id',
         :endpoint => Framed::FRAMED_API_ENDPOINT,
         :logger => Logger.new(STDERR),
-        :anonymous_cookie => Framed::COOKIE_NAME
+        :anonymous_cookie => Framed::COOKIE_NAME,
+        :include_xhr => false
       }
     end
 
@@ -30,21 +31,23 @@ module Framed
       yield configuration
       self.client = Client.new(configuration)
 
-      @consumer.stop if @consumer
+      @consumer.stop(true) if @consumer
       @consumer = configuration[:consumer].new(self.client)
     end
 
     def report(event)
       event[:lib] =  "framed_ruby"
       event[:lib_version] = Framed::VERSION
+      event[:type] ||= :track
       event[:context] ||= {}
       event[:context].merge!({
         :channel => 'server',
       })
 
+      event[:properties] ||= {}
+
       # fill in if needed, in case it sits in queue for a while.
       event[:timestamp] ||= Framed::Utils.serialize_date(Time.now)
-
       @consumer.enqueue(event)
     end
 
@@ -53,7 +56,7 @@ module Framed
     end
 
     def drain
-      @consumer.stop(true)
+      @consumer.stop(true) if @consumer
     end
 
     def user_id_controller_method

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: framed_rails
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-13 00:00:00.000000000 Z
+date: 2015-08-26 00:00:00.000000000 Z
 dependencies: []
 description: ! 'TK
 
@@ -31,7 +31,6 @@ files:
 - framed_rails.gemspec
 - lib/framed/client.rb
 - lib/framed/emitters.rb
-- lib/framed/example.rb
 - lib/framed/exceptions.rb
 - lib/framed/okjson.rb
 - lib/framed/rails.rb

data/lib/framed/example.rb

@@ -1,17 +0,0 @@
-require 'framed_rails'
-
-Framed.configure do |config|
-  config[:api_key] = 'XWkzILLq5gLUKXQUhAl4DJej1wkxqiBy'
-#  config[:consumer] = Framed::Blocking
-end
-
-data = { 
-  anonymousId: "anon1",
-  userId: "user1",
-  event: "signup",
-  properties: {
-    name: "value"
-  }
-}
-
-Framed.report(data)