columbo 0.1.5 → 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7e4dc0fa4f95aac130e637f09794b94404bf22e5
+  data.tar.gz: 64c82608f49ff6b51a6a60a054392d25c8ae93e4
+SHA512:
+  metadata.gz: 997eae7d6bf3e6e6595168508f3cc9d435b59b6f70bff1a47fc1c84f8aaec557ee3769bea9cda05486f94193a659c39204fb16962c2e759afc49fc40dc2633b9
+  data.tar.gz: b62b4e55bbae5cc25aaed09673e9ad5bec5ab5f7b38c35549d97cd6d5bde64825441349fb8abf61bac5b83e3ea2d4522a5fed62c26cf377e860edf2d71948b13

data/README.md

@@ -5,6 +5,12 @@ that captures users browsing sessions for Rack applications.
 
 Tribute to [Inspector Columbo](http://www.imdb.com/title/tt1466074/)
 
+## Disclaimer
+
+This is an alpha release, it is tested with Sinatra and Rails 3 only.
+The UI to explore sessions is in progress (ETA: 2013'Q4) therefore
+there is no point using this gem at the moment.
+
 ## Using with Rack application
 
 *Columbo* can be used with any Rack application,
@@ -16,8 +22,8 @@ simply require and use as follows:
     require 'columbo/capture'
     use Columbo::Capture, {
       capture: true,
-      bench: false,
-      mongo_uri: 'mongodb://user:password@mongodb_host:port/database',
+      bench: true,
+      api_key: 'your-private-api-key',
       logger: 'log/columbo.log'
     }
     run app
@@ -25,28 +31,35 @@ simply require and use as follows:
 ## Using with Rails 3
 
 In order to use, include the following in a Rails application
-*Gemfile* file:
+**Gemfile** file:
 
     gem 'columbo'
 
-*config/application.rb* file:
+**config/application.rb** file:
 
     require 'columbo/capture'
     config.middleware.insert 0, Columbo::Capture, {
       capture: Rails.env.production?,
-      bench: false,
-      mongo_uri: 'mongodb://user:password@mongodb_host:port/database',
-      logger: 'log/columbo.log'
+      api_key: 'your-private-api-key',
+      logger: 'log/columbo.log',
+      filter_uri: ['admin']
     }
 
 Check the Rack configuration:
 
     rake middleware
 
-## Disclaimer
+## Options
 
-This is an alpha release, it is tested with Sinatra and Rails 3 only.
-UI to explore sessions will be completed later (ETA: 2013'Q2).
+- `api_key`: the api key for authentication (mandatory),
+- `capture`: set to true to collect data, default `false`
+- `bench`: set to true to benchmark middleware overhead, default `false`
+- `logger`: file to write logs to, default `env['rack.errors']`
+- `capture_crawlers`: set to true to capture hits from crawlers, default `false`
+- `api_uri`: overwrite api endpoint uri
+- `crawlers`: overwrite crawlers user agent regex
+- `filter_params`: array of parameters to filter, for `Rails` default to `Rails.configuration.filter_parameters`
+- `filter_uri`: array of uri for which **not** to capture data
 
 ## Author
 
@@ -54,5 +67,5 @@ Jerome Touffe-Blin, [@jtblin](https://twitter.com/jtlbin), [http://www.linkedin.
 
 ## License
 
-Columbo is copyright 2012 Jerome Touffe-Blin and contributors. It is licensed under the BSD license. See the include LICENSE file for details.
+Columbo is copyright 2013 Jerome Touffe-Blin and contributors. It is licensed under the BSD license. See the include LICENSE file for details.
 

data/lib/columbo.rb

@@ -1,11 +1,11 @@
 require 'columbo/version'
-require 'columbo/db_client'
+require 'columbo/api_client'
+require 'columbo/capture'
 require 'columbo/inspector'
 require 'columbo/log_writer'
+require 'columbo/compressor'
 
 module Columbo
-  MONGO_COLLECTION = "hits".freeze
-
   def self.logger
     @logger
   end

data/lib/columbo/api_client.rb

@@ -0,0 +1,49 @@
+require 'net/http'
+require 'json'
+
+module Columbo
+  class APIClient
+
+    API_URI = "http://localhost:15080".freeze
+
+    def initialize(api_key, api_uri)
+      @api_key = api_key
+      @api_uri = api_uri || API_URI
+    end
+
+    def handshake
+      Thread.new do
+        headers = { "Content-Type" => "application/json; charset=utf-8" }
+
+        uri = URI(@api_uri + '/' + @api_key + '/handshake')
+        Net::HTTP.new(uri.host, uri.port).start do |http|
+          response = http.get(uri.request_uri, headers)
+          json = JSON.parse(response.body)
+          json.each { |k, v| yield k, v }
+        end
+      end
+    end
+
+    def post_data(hash)
+      headers = {
+          "Accept-Encoding" => "gzip, deflate",
+          "Content-Encoding" => "deflate",
+          "Content-Type" => "application/json; charset=utf-8"
+      }
+
+      zlib = Columbo::Compressor
+      json = hash.merge({api_key: @api_key}).to_json
+      payload = zlib.deflate(json)
+      uri = URI(@api_uri + '/' + @api_key + '/capture')
+
+      start = Time.now
+      Net::HTTP.new(uri.host, uri.port).start do |http|
+        response = http.post(uri.request_uri, payload, headers)
+        stop = Time.now
+        duration = ((stop-start) * 1000).round(3)
+        zlib.unzip(response.body, response['Content-Encoding']) + ' - Time: ' + duration.to_s + 'ms'
+      end
+    end
+
+  end
+end

data/lib/columbo/capture.rb

@@ -1,27 +1,40 @@
 require 'columbo'
 require 'rack/utils'
 require 'rack/logger'
+require 'securerandom'
 
 module Columbo
   class Capture
     include Rack::Utils
 
+    attr_reader :client, :filter_params, :filter_uri
+
     FORMAT = %{[Columbo #{Columbo::VERSION}] [%s] %s - %s "%s%s %s\n}
 
     def initialize(app, opts={})
       @app = app
       # Options
-      @capture          = opts[:capture]
+      @capture          = !!opts[:capture]
       @bench            = opts[:capture] && opts[:bench]
-      @capture_crawlers = opts[:capture_crawlers]
-      @crawlers         = opts[:crawlers] || "(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg|bot|crawler|spider|robot|crawling|facebook|w3c|coccoc|Daumoa)"
-      @mongo_uri        = opts[:mongo_uri]
+      capture_crawlers  = opts[:capture_crawlers]
+      crawlers          = opts[:crawlers] || "(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg|bot|crawler|spider|robot|crawling|facebook|w3c|coccoc|Daumoa|panopta)"
+      api_key           = opts[:api_key]
+      api_uri           = opts[:api_uri]
+      @filter_params    = opts[:filter_params] || []
+      @filter_uri       = opts[:filter_uri] || []
+
+      @cookie_name = 'clickstream.io'
+      filter_params.concat(Rails.configuration.filter_parameters || []) if defined?(Rails)
 
       Columbo.logger = opts[:logger] if opts[:logger]
 
-      raise ArgumentError, 'mongo URI missing.' if @mongo_uri.nil?
+      raise ArgumentError, 'API key missing.' if api_key.nil?
 
-      @inspector = Columbo::Inspector.new @mongo_uri
+      @inspector = Columbo::Inspector.new api_key, api_uri, crawlers, capture_crawlers, filter_params
+      @cookie_regex = Regexp.new "#{@cookie_name}="
+
+      @client = {}
+      Columbo::APIClient.new(api_key, api_uri).handshake { |k, v| @client[k] = v}
     end
 
     def call(env)
@@ -35,30 +48,36 @@ module Columbo
 
       start = Time.now if @bench
 
-      headers = HeaderHash.new(headers) # Is it required?
+      headers = HeaderHash.new(headers)
 
-      if !STATUS_WITH_NO_ENTITY_BODY.include?(status) &&
-          !headers['transfer-encoding'] &&
-          headers['content-type'] &&
-          headers['content-type'].include?("text/html")
+      if @capture && !STATUS_WITH_NO_ENTITY_BODY.include?(status) && !headers['transfer-encoding'] && headers['content-type'] && (
+        headers['content-type'].include?('text/html') || headers['content-type'].include?('application/json') ||
+            headers['content-type'].include?('application/xml') || headers['content-type'].include?('text/javascript') ||
+            headers['content-type'].include?('text/plain')
+      ) && !filtered_uri?(env['REQUEST_URI'])
 
-        Thread.abort_on_exception = true
+        cookie = session_cookie(env, headers)
+        pid = SecureRandom.uuid
+        body = response.clone
 
+        Thread.abort_on_exception = false
         Thread.new do
           begin
-            @inspector.investigate env, status, headers, response, start_processing, stop_processing, @crawlers, @capture_crawlers
+            result = @inspector.investigate env, status, headers, body, start_processing, stop_processing, cookie, pid
+            log env, result
           rescue Exception => e
             log_error env, e
           end
-        end if @capture
+        end
 
+        response = insert_js(response, headers, cookie, pid) if headers['content-type'].include?('text/html') #&& headers['content-length'].to_i > 0
       end
 
       if @bench
         stop = Time.now
         duration = ((stop-start) * 1000).round(3)
-        log(env, "Time: #{duration}ms")
         headers['Columbo'] = "version #{Columbo::VERSION}, time #{duration}ms"
+        Thread.new { log(env, "Time: #{duration}ms") }
       end
 
       [status, headers, response]
@@ -66,6 +85,42 @@ module Columbo
 
     private
 
+    def filtered_uri?(uri)
+      filter_uri.select {|filter| uri.match filter}.size > 0
+    end
+
+    def session_cookie(env, headers)
+      cookie = extract_cookie(env['HTTP_COOKIE'])
+      set_cookie(headers, cookie)
+    end
+
+    def extract_cookie(string)
+      return unless string
+      match = string.match(/columbo=([^;]*)/)
+      match[1] if match && match.length > 1
+    end
+
+    def set_cookie(headers, cookie)
+      expires = Time.now+60*60
+      cookie = cookie.nil? ? {value: SecureRandom.uuid} : {value: cookie}
+      cookie[:expires] = expires
+      Rack::Utils.set_cookie_header!(headers, @cookie_name, cookie)
+      cookie[:value]
+    end
+
+    def insert_js(body, headers, sid, pid)
+      html = ''
+      body.each { |part| html += part }
+      body.close if body.respond_to?(:close)
+      str_filter_params = filter_params.map { |filter| filter.to_s }
+      if html.size > 0
+        script = "<script>(function(){var uri='#{client['ws']}', cid='#{client['clientId']}', sid='#{sid}', pid='#{pid}', paramsFilter = #{str_filter_params}; #{client['js']}})();</script>"
+        html += "\n" + script
+      end
+      headers['content-length'] = html.size.to_s
+      [html]
+    end
+
     def log(env, message)
       now = Time.now
 

data/lib/columbo/compressor.rb

@@ -0,0 +1,51 @@
+require 'stringio'
+require 'zlib'
+
+module Columbo
+  class Compressor
+
+    def self.encoding_handled?(content_encoding)
+      %w(gzip deflate).include? content_encoding
+    end
+
+    def self.unzip(source, content_encoding)
+      case content_encoding
+        when 'gzip' then decompress(source)
+        when 'deflate' then inflate(source)
+        else source
+      end
+    end
+
+    def self.zip(source, accept_encoding)
+      if accept_encoding.match 'deflate'
+        deflate(source)
+      elsif  accept_encoding.match 'gzip'
+        compress(source)
+      else
+        source
+      end
+    end
+
+    # Compresses a string using gzip inspired by ActiveSupport::Gzip
+    def self.compress(source)
+      output = StringIO.new
+      gz = Zlib::GzipWriter.new(output)
+      gz.write(source)
+      gz.close
+      output.string
+    end
+
+    def self.deflate(source)
+      Zlib::Deflate.deflate(source)
+    end
+
+    def self.decompress(source)
+      Zlib::GzipReader.new(StringIO.new(source)).read
+    end
+
+    def self.inflate(source)
+      Zlib::Inflate.inflate(source.read)
+    end
+
+  end
+end

data/lib/columbo/inspector.rb

@@ -1,88 +1,77 @@
 require 'rack/request'
 require 'rack/response'
-require 'nokogiri'
-require 'html_mini'
+require 'socket'
 
 module Columbo
   class Inspector
 
-    def initialize(mongo_uri)
-      @mongo_uri = mongo_uri
+    def initialize(api_key, api_uri, crawlers, capture_crawlers, filter_params)
+      @client = Columbo::APIClient.new(api_key, api_uri)
+      @crawlers, @capture_crawlers, @filter_params = crawlers, capture_crawlers, filter_params
+      @hostname = Socket.gethostname
+      @rg = Regexp.new(crawlers, Regexp::IGNORECASE)
     end
 
-    def investigate(env, status, headers, body, start, stop, crawlers, capture_crawlers)
-      # Lazy connection to MongoDB
-      client = Columbo::DbClient.new @mongo_uri
+    def investigate(env, status, headers, body, start, stop, cookie, pid)
       # Normalise request from env
       request = Rack::Request.new(env)
       # Don't capture bots traffic by default
-      rg = Regexp.new(crawlers, Regexp::IGNORECASE)
-      return if request.user_agent.match(rg) && !capture_crawlers
+      return unless @capture_crawlers || !request.user_agent.match(@rg)
       html = ''
-      body.each { |part| html += part }
-      # Retrieve plain text body for full text search
-      text, title = to_plain_text(html)
-      # Get request headers
+      # in case of gzipping has been done by the app
+      body.each { |part| html += Columbo::Compressor.unzip(part, headers['Content-Encoding']) }
       request_headers = {}
-      request.env.each { |key, value| request_headers[key.sub(/^HTTP_/, '').downcase] = value if key.start_with? 'HTTP_'}
+      request.env.each { |key, value| request_headers[key.sub(/^HTTP_/, '').gsub(/_/, '-').downcase] = value if key.start_with? 'HTTP_'}
+      params = request.params.clone || {}
+      @filter_params.each {|param| params[param.to_s] = '[FILTERED]' if params[param.to_s]}
+      session_opts = request.session_options.clone || {}
+      session_opts.delete :secret
       data = {
+          sid: cookie,
+          pid: pid,
+          hostname: @hostname,
+          filter_params: @filter_params,
           request: {
-              params: request.params,
-              remote_ip: request.ip,
+              params: params,
+              ip: request.ip,
               user_agent: request.user_agent,
-              method: request.env['REQUEST_METHOD'],
+              referer: request.referer,
+              method: request.request_method,
+              path: request.path, # script_name + path_info
+              fullpath: request.fullpath, # "#{path}?#{query_string}"
+              script_name: request.script_name,
+              path_info: request.path_info,
               uri: request.env['REQUEST_URI'],
-              script: request.env['SCRIPT_NAME'],
-              path: request.env['PATH_INFO'],
-              query_string: request.env['QUERY_STRING'],
-              protocol: request.env['rack.url_scheme'],
-              server_name: request.env['SERVER_NAME'],
-              server_port: request.env['SERVER_PORT'],
-              http: request.env['SERVER_PROTOCOL'],
-              session: request.env['rack.session'],
-              cookie: request.env['rack.request.cookie_hash'],
+              querystring: request.query_string,
+              scheme: request.scheme,
+              host: request.host,
+              port: request.port,
+              url: request.url, # base_url + fullpath
+              base_url: request.base_url, # scheme + host [+ port]
+              content_type: request.content_type,
+              content_charset: request.content_charset,
+              media_type: request.media_type,
+              media_type_params: request.media_type_params,
+              protocol: request.env['HTTP_VERSION'],
+              session: request.session,
+              session_options: session_opts,
+              cookies: request.cookies,
               path_parameters: request.env['action_dispatch.request.path_parameters'],
-              headers: request_headers
+              headers: request_headers,
+              xhr: request.xhr?
           },
-          status: status,
-          headers: headers,
-          size: html.length,
-          body: HtmlMini.minify(html),
-          text: text,
-          title: title,
-          start: start,
-          stop: stop,
-          time: stop-start
+          response: {
+              status: status,
+              headers: headers,
+              size: html.length,
+              body: html,
+              start: start,
+              stop: stop,
+              time: stop-start
+          }
       }
-      # Insert data in MongoDB
-      client.insert sanitize(data)
-    end
-
-    def to_plain_text(html)
-      html_doc = Nokogiri::HTML(html)
-      html_doc.xpath('//script').each {|node| node.remove}
-      html_doc.xpath('//style').each {|node| node.remove}
-      text = ''
-      html_doc.xpath('//body').each {|node| text += node.text.gsub(/\s{2,}/, ' ')}
-      title_tag = html_doc.xpath('//title').first
-      title = title_tag.nil? ? nil : title_tag.text
-      [text, title]
-    end
-    
-    private
-
-    def sanitize(data)
-      Hash[
-        data.map do |key, value|
-          value = sanitize(value) if value.is_a? Hash
-          # replace $ and . in keys by Unicode full width equivalent
-          # http://docs.mongodb.org/manual/faq/developers/#faq-dollar-sign-escaping
-          key = key.gsub('.', 'U+FF0E').gsub('$', 'U+FF04') if key.is_a? String
-          # transform symbol into string to avoid auto transformation into $symbol
-          value = value.to_s if value.is_a? Symbol
-          [key, value]
-        end
-      ]
+      # Send data to API
+      @client.post_data data
     end
 
   end

data/lib/columbo/version.rb

@@ -1,3 +1,3 @@
 module Columbo
-  VERSION = '0.1.5'
+  VERSION = '0.2.1'
 end

metadata

@@ -1,113 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: columbo
 version: !ruby/object:Gem::Version
-  version: 0.1.5
-  prerelease: 
+  version: 0.2.1
 platform: ruby
 authors:
 - Jerome Touffe-Blin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-14 00:00:00.000000000 Z
+date: 2013-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
 - !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.5.9
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.5.9
-- !ruby/object:Gem::Dependency
-  name: html_mini
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.0.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.0.2
-- !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.6.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.6.1
-- !ruby/object:Gem::Dependency
-  name: bson_ext
+  name: sinatra
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.1
-  type: :runtime
+        version: 1.3.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
-  name: sinatra
+  name: lorem
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 0.1.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.3.0
-description: ! 'A Ruby client library for Columbo: a Customer Experience Management
+        version: 0.1.2
+description: 'A Ruby client library for Columbo: a Customer Experience Management
   tool'
 email: jtblin@gmail.com
 executables: []
@@ -116,7 +61,9 @@ extra_rdoc_files:
 - LICENSE
 - README.md
 files:
+- lib/columbo/api_client.rb
 - lib/columbo/capture.rb
+- lib/columbo/compressor.rb
 - lib/columbo/db_client.rb
 - lib/columbo/inspector.rb
 - lib/columbo/log_writer.rb
@@ -127,27 +74,26 @@ files:
 - README.md
 homepage: http://github.com/jtblin/columbo-rb
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options:
 - --charset=UTF-8
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.1.10
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A Ruby client library for Inspector Columbo
 test_files: []