clickstream 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a0a3540d2bb32230a701e96604bc9373f324a1c0
+  data.tar.gz: cb89f0d5ebdc17d813de72850d600e3d2afdd558
+SHA512:
+  metadata.gz: e755219b1e4145876c81cbca82b7f6704e7a958c6cac1b667effd4a916a01dfbba7b723f1737f8f78082a744163f6bae36dcaa18e18c82606dbea483fd928bbd
+  data.tar.gz: e26ed131e5ef6db5b7dec9cb0472dbe494d24fed6476f2e8d85d838feb059ca274fc803da50a13f42532165920dc544f4ed4568afe03b18f773ca962fe4c1c98

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2012, Jerome Touffe-Blin
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+* Neither the name of Jerome Touffe-Blin nor the names of its contributors
+  may be used to endorse or promote products derived from this software 
+  without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,67 @@
+# clickstream-rb
+
+The *clickstream* gem includes a middleware
+that captures users browsing sessions for Rack applications.
+
+## Disclaimer
+
+This is an alpha release, it is tested with Sinatra and Rails 3 only.
+
+## Using with Rack application
+
+*Clickstream* can be used with any Rack application,
+for example with a **Sinatra** application.
+If your application includes a rackup file
+or uses *Rack::Builder* to construct the application pipeline, 
+simply require and use as follows:
+
+    require 'clickstream'
+    use Clickstream::Capture, {
+      capture: true,
+      bench: true,
+      api_key: 'your-private-api-key',
+      logger: 'log/clickstream.log'
+    }
+    run app
+
+## Using with Rails 3
+
+In order to use, include the following in a Rails application
+**Gemfile** file:
+
+    gem 'clickstream'
+
+**config/application.rb** file:
+
+    require 'clickstream'
+    config.middleware.insert 0, Clickstream::Capture, {
+      capture: Rails.env.production?,
+      api_key: 'your-private-api-key',
+      logger: 'log/clickstream.log',
+      filter_uri: ['admin']
+    }
+
+Check the Rack configuration:
+
+    rake middleware
+
+## Options
+
+- `api_key`: the api key for authentication (mandatory),
+- `capture`: set to true to collect data, default `false`
+- `bench`: set to true to benchmark middleware overhead, default `false`
+- `logger`: file to write logs to, default `env['rack.errors']`
+- `capture_crawlers`: set to true to capture hits from crawlers, default `false`
+- `api_uri`: overwrite api endpoint uri
+- `crawlers`: overwrite crawlers user agent regex
+- `filter_params`: array of parameters to filter, for `Rails` default to `Rails.configuration.filter_parameters`
+- `filter_uri`: array of uri for which **not** to capture data
+
+## Author
+
+Jerome Touffe-Blin, [@jtblin](https://twitter.com/jtlbin), [http://www.linkedin.com/in/jtblin](http://www.linkedin.com/in/jtblin)
+
+## License
+
+clickstream-rb is copyright 2013 Jerome Touffe-Blin and contributors. It is licensed under the BSD license. See the include LICENSE file for details.
+

data/lib/clickstream.rb

@@ -0,0 +1,16 @@
+require 'clickstream/version'
+require 'clickstream/api_client'
+require 'clickstream/capture'
+require 'clickstream/inspector'
+require 'clickstream/log_writer'
+require 'clickstream/compressor'
+
+module Clickstream
+  def self.logger
+    @logger
+  end
+
+  def self.logger=(log = STDOUT)
+    @logger = Clickstream::LogWriter.new log
+  end
+end

data/lib/clickstream/api_client.rb

@@ -0,0 +1,49 @@
+require 'net/http'
+require 'json'
+
+module Clickstream
+  class APIClient
+
+    API_URI = "http://localhost:15080".freeze
+
+    def initialize(api_key, api_uri)
+      @api_key = api_key
+      @api_uri = api_uri || API_URI
+    end
+
+    def handshake
+      Thread.new do
+        headers = { "Content-Type" => "application/json; charset=utf-8" }
+
+        uri = URI(@api_uri + '/' + @api_key + '/handshake')
+        Net::HTTP.new(uri.host, uri.port).start do |http|
+          response = http.get(uri.request_uri, headers)
+          json = JSON.parse(response.body)
+          json.each { |k, v| yield k, v }
+        end
+      end
+    end
+
+    def post_data(hash)
+      headers = {
+          "Accept-Encoding" => "gzip, deflate",
+          "Content-Encoding" => "deflate",
+          "Content-Type" => "application/json; charset=utf-8"
+      }
+
+      zlib = Clickstream::Compressor
+      json = hash.merge({api_key: @api_key}).to_json
+      payload = zlib.deflate(json)
+      uri = URI(@api_uri + '/' + @api_key + '/capture')
+
+      start = Time.now
+      Net::HTTP.new(uri.host, uri.port).start do |http|
+        response = http.post(uri.request_uri, payload, headers)
+        stop = Time.now
+        duration = ((stop-start) * 1000).round(3)
+        zlib.unzip(response.body, response['Content-Encoding']) + ' - Time: ' + duration.to_s + 'ms'
+      end
+    end
+
+  end
+end

data/lib/clickstream/capture.rb

@@ -0,0 +1,148 @@
+require 'clickstream'
+require 'rack/utils'
+require 'rack/logger'
+require 'securerandom'
+
+module Clickstream
+  class Capture
+    include Rack::Utils
+
+    attr_reader :client, :filter_params, :filter_uri
+
+    FORMAT = %{[Clickstream #{Clickstream::VERSION}] [%s] %s - %s "%s%s %s\n}
+
+    def initialize(app, opts={})
+      @app = app
+      # Options
+      @capture          = !!opts[:capture]
+      @bench            = opts[:capture] && opts[:bench]
+      capture_crawlers  = opts[:capture_crawlers]
+      crawlers          = opts[:crawlers] || "(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg|bot|crawler|spider|robot|crawling|facebook|w3c|coccoc|Daumoa|panopta)"
+      api_key           = opts[:api_key]
+      api_uri           = opts[:api_uri]
+      @filter_params    = opts[:filter_params] || []
+      @filter_uri       = opts[:filter_uri] || []
+
+      @cookie_name = 'clickstream.io'
+      filter_params.concat(Rails.configuration.filter_parameters || []) if defined?(Rails)
+
+      Clickstream.logger = opts[:logger] if opts[:logger]
+
+      raise ArgumentError, 'API key missing.' if api_key.nil?
+
+      @inspector = Clickstream::Inspector.new api_key, api_uri, crawlers, capture_crawlers, filter_params
+      @cookie_regex = Regexp.new "#{@cookie_name}="
+
+      @client = {}
+      Clickstream::APIClient.new(api_key, api_uri).handshake { |k, v| @client[k] = v}
+    end
+
+    def call(env)
+      dup._call(env)
+    end
+
+    def _call(env)
+      start_processing = Time.now
+      status, headers, response = @app.call(env)
+      stop_processing = Time.now
+
+      start = Time.now if @bench
+
+      headers = HeaderHash.new(headers)
+
+      if @capture && !STATUS_WITH_NO_ENTITY_BODY.include?(status) && !headers['transfer-encoding'] && headers['content-type'] && (
+        headers['content-type'].include?('text/html') || headers['content-type'].include?('application/json') ||
+            headers['content-type'].include?('application/xml') || headers['content-type'].include?('text/javascript') ||
+            headers['content-type'].include?('text/plain')
+      ) && !filtered_uri?(env['REQUEST_URI'])
+
+        cookie = session_cookie(env, headers)
+        pid = SecureRandom.uuid
+        body = response.clone
+
+        Thread.abort_on_exception = false
+        Thread.new do
+          begin
+            result = @inspector.investigate env, status, headers, body, start_processing, stop_processing, cookie, pid
+            log env, result
+          rescue Exception => e
+            log_error env, e
+          end
+        end
+
+        response = insert_js(response, headers, cookie, pid) if headers['content-type'].include?('text/html') #&& headers['content-length'].to_i > 0
+      end
+
+      if @bench
+        stop = Time.now
+        duration = ((stop-start) * 1000).round(3)
+        headers['Clickstream'] = "version #{Clickstream::VERSION}, time #{duration}ms"
+        Thread.new { log(env, "Time: #{duration}ms") }
+      end
+
+      [status, headers, response]
+    end
+
+    private
+
+    def filtered_uri?(uri)
+      filter_uri.select {|filter| uri.match filter}.size > 0
+    end
+
+    def session_cookie(env, headers)
+      cookie = extract_cookie(env['HTTP_COOKIE'])
+      set_cookie(headers, cookie)
+    end
+
+    def extract_cookie(string)
+      return unless string
+      match = string.match(/clickstream=([^;]*)/)
+      match[1] if match && match.length > 1
+    end
+
+    def set_cookie(headers, cookie)
+      expires = Time.now+60*60
+      cookie = cookie.nil? ? {value: SecureRandom.uuid} : {value: cookie}
+      cookie[:expires] = expires
+      Rack::Utils.set_cookie_header!(headers, @cookie_name, cookie)
+      cookie[:value]
+    end
+
+    def insert_js(body, headers, sid, pid)
+      html = ''
+      body.each { |part| html += part }
+      body.close if body.respond_to?(:close)
+      str_filter_params = filter_params.map { |filter| filter.to_s }
+      if html.size > 0
+        script = "<script>(function(){var uri='#{client['ws']}', cid='#{client['clientId']}', sid='#{sid}', pid='#{pid}', paramsFilter = #{str_filter_params}; #{client['js']}})();</script>"
+        html += "\n" + script
+      end
+      headers['content-length'] = html.size.to_s
+      [html]
+    end
+
+    def log(env, message)
+      now = Time.now
+
+      logger = Clickstream.logger || env['rack.errors']
+
+      logger.write FORMAT % [
+          now.strftime('%d-%b-%Y %H:%M:%S'),
+          message,
+          env['REQUEST_METHOD'],
+          env['PATH_INFO'],
+          env['QUERY_STRING'].empty? ? '' : '?' + env['QUERY_STRING'],
+          env['HTTP_VERSION']
+      ]
+    end
+
+    def log_error(env, exception)
+      begin
+        logger = Clickstream.logger || env['rack.errors']
+        log env, "Error: " + exception.message
+        logger.write "#{exception.backtrace.join("\n")}\n"
+      end
+    end
+
+  end
+end

data/lib/clickstream/compressor.rb

@@ -0,0 +1,51 @@
+require 'stringio'
+require 'zlib'
+
+module Clickstream
+  class Compressor
+
+    def self.encoding_handled?(content_encoding)
+      %w(gzip deflate).include? content_encoding
+    end
+
+    def self.unzip(source, content_encoding)
+      case content_encoding
+        when 'gzip' then decompress(source)
+        when 'deflate' then inflate(source)
+        else source
+      end
+    end
+
+    def self.zip(source, accept_encoding)
+      if accept_encoding.match 'deflate'
+        deflate(source)
+      elsif  accept_encoding.match 'gzip'
+        compress(source)
+      else
+        source
+      end
+    end
+
+    # Compresses a string using gzip inspired by ActiveSupport::Gzip
+    def self.compress(source)
+      output = StringIO.new
+      gz = Zlib::GzipWriter.new(output)
+      gz.write(source)
+      gz.close
+      output.string
+    end
+
+    def self.deflate(source)
+      Zlib::Deflate.deflate(source)
+    end
+
+    def self.decompress(source)
+      Zlib::GzipReader.new(StringIO.new(source)).read
+    end
+
+    def self.inflate(source)
+      Zlib::Inflate.inflate(source.read)
+    end
+
+  end
+end

data/lib/clickstream/inspector.rb

@@ -0,0 +1,78 @@
+require 'rack/request'
+require 'rack/response'
+require 'socket'
+
+module Clickstream
+  class Inspector
+
+    def initialize(api_key, api_uri, crawlers, capture_crawlers, filter_params)
+      @client = Clickstream::APIClient.new(api_key, api_uri)
+      @crawlers, @capture_crawlers, @filter_params = crawlers, capture_crawlers, filter_params
+      @hostname = Socket.gethostname
+      @rg = Regexp.new(crawlers, Regexp::IGNORECASE)
+    end
+
+    def investigate(env, status, headers, body, start, stop, cookie, pid)
+      # Normalise request from env
+      request = Rack::Request.new(env)
+      # Don't capture bots traffic by default
+      return unless @capture_crawlers || !request.user_agent.match(@rg)
+      html = ''
+      # in case of gzipping has been done by the app
+      body.each { |part| html += Clickstream::Compressor.unzip(part, headers['Content-Encoding']) }
+      request_headers = {}
+      request.env.each { |key, value| request_headers[key.sub(/^HTTP_/, '').gsub(/_/, '-').downcase] = value if key.start_with? 'HTTP_'}
+      params = request.params.clone || {}
+      @filter_params.each {|param| params[param.to_s] = '[FILTERED]' if params[param.to_s]}
+      session_opts = request.session_options.clone || {}
+      session_opts.delete :secret
+      data = {
+          sid: cookie,
+          pid: pid,
+          hostname: @hostname,
+          filter_params: @filter_params,
+          request: {
+              params: params,
+              ip: request.ip,
+              user_agent: request.user_agent,
+              referer: request.referer,
+              method: request.request_method,
+              path: request.path, # script_name + path_info
+              fullpath: request.fullpath, # "#{path}?#{query_string}"
+              script_name: request.script_name,
+              path_info: request.path_info,
+              uri: request.env['REQUEST_URI'],
+              querystring: request.query_string,
+              scheme: request.scheme,
+              host: request.host,
+              port: request.port,
+              url: request.url, # base_url + fullpath
+              base_url: request.base_url, # scheme + host [+ port]
+              content_type: request.content_type,
+              content_charset: request.content_charset,
+              media_type: request.media_type,
+              media_type_params: request.media_type_params,
+              protocol: request.env['HTTP_VERSION'],
+              session: request.session,
+              session_options: session_opts,
+              cookies: request.cookies,
+              path_parameters: request.env['action_dispatch.request.path_parameters'],
+              headers: request_headers,
+              xhr: request.xhr?
+          },
+          response: {
+              status: status,
+              headers: headers,
+              size: html.length,
+              body: html,
+              start: start,
+              stop: stop,
+              time: stop-start
+          }
+      }
+      # Send data to API
+      @client.post_data data
+    end
+
+  end
+end

data/lib/clickstream/log_writer.rb

@@ -0,0 +1,25 @@
+require 'logger'
+
+module Clickstream
+
+  class LogWriter < Logger
+    def initialize(log = STDOUT)
+      super(log)
+      self.level = Logger::INFO
+      self.formatter = Simple.new
+      self
+    end
+
+    def write(message)
+      add Logger::INFO, message
+    end
+
+    class Simple < Logger::Formatter
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{message}"
+      end
+    end
+
+  end
+end

data/lib/clickstream/version.rb

@@ -0,0 +1,3 @@
+module Clickstream
+  VERSION = '0.3.0'
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: clickstream
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Jerome Touffe-Blin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: lorem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+description: 'A Ruby client library for ClickStream: a Customer Experience Management
+  tool'
+email: jtblin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- lib/clickstream/api_client.rb
+- lib/clickstream/capture.rb
+- lib/clickstream/compressor.rb
+- lib/clickstream/inspector.rb
+- lib/clickstream/log_writer.rb
+- lib/clickstream/version.rb
+- lib/clickstream.rb
+- LICENSE
+- README.md
+homepage: http://github.com/jtblin/clickstream-rb
+licenses:
+- BSD
+metadata: {}
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.10
+signing_key: 
+specification_version: 4
+summary: A Ruby client library for ClickStream
+test_files: []