turnstile-rb 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3cd7be07aee113fee488b2839dc8eadc611a3be257e6c37a7de50618e55fce94
+  data.tar.gz: f13265ed097e7bb4f4625ed6fcf7da9e45208f33e382c6af0df7157345af3efa
+SHA512:
+  metadata.gz: d68205747142bb284446c06221046a8746c7cf301e4d0306624872fc9ecc8a443386325a00581a22add14816822ac08fa98356ae36246bad49e1080001f04f02
+  data.tar.gz: 3f84e37343fd99ae8bed3fad5182b5f66c716d945db21e2c1d8be27258a1b97fd26aa5f8696a1f52226ad3a07b303b56ff5ccdf678bef8f2089c8b9dd128253f

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+.DS_Store
+coverate
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,22 @@
+rvm:
+- 2.2.7
+- 2.3.4
+- 2.4.1
+services:
+- redis-server
+env:
+  global:
+    - CC_TEST_REPORTER_ID=d17e91219ca5c271ca9bdd9eb447611f1be6b750abe8e4bc48146f5dec1154d4
+sudo: false
+language: ruby
+cache: bundler
+before_install: gem install bundler -v 1.15.4
+before_script:
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - ./cc-test-reporter before-build
+script:
+  - bundle exec rspec
+after_script:
+  - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
+

data/Gemfile

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in turnstile.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+#^syntax detection
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec' do
+  watch(%r{^lib/(.+)\.rb$}) { "spec" }
+  watch(%r{^spec/.+_spec\.rb$})
+  watch('spec/spec_helper.rb')  { "spec" }
+  watch(%r{spec/support/.*}) { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Wanelo, Inc
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,197 @@
+[![Build Status](https://travis-ci.org/kigster/turnstile.svg?branch=master)](https://travis-ci.org/kigster/turnstile)
+[![Test Coverage](https://codeclimate.com/github/wanelo/turnstile/badges/coverage.svg)](https://codeclimate.com/github/kigster/turnstile/coverage)
+[![Code Climate](https://codeclimate.com/github/wanelo/turnstile/badges/gpa.svg)](https://codeclimate.com/github/kigster/turnstile)
+[![Issue Count](https://codeclimate.com/github/wanelo/turnstile/badges/issue_count.svg)](https://codeclimate.com/github/kigster/turnstile)
+
+# Turnstile
+
+The goal of this gem is to provide near real time tracking and reporting on the number of users currently online and accessing given application.  It requires that the reporting layer is able to uniquely identify each user and provide a unique identifier.  It may also optionally assign another dimension to the users accessing, such as, for example, _platform_ -- which in our case denotes how the user is accessing our application: from desktop browser, iOS app, Android app, mobile web, etc.  But any other partitioning schemee can be used, or none at all.
+
+The gem uses (and depends on) a [Redis](http://redis.io/) instance in order to keep track of _unique_ users, and can operate in **online* mode (tracking users from a Rack Middleware) or **offline**, by taling a file log.
+
+## Installation
+
+Add this line to your application's Gemfile (Note that another gem with a competing name is on RubyGems, so you **must** specify the path below):
+
+    gem 'turnstile-rb'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install turnstile
+
+## Usage
+
+### Tracking 
+
+Turnstile contains two primary parts: data collection and reporting.  
+
+Data collection may happen in two way:
+
+1. Synchronously — in real time — i.e. from a web request
+2. Or asynchronously — by "tailing" the logs on your servers
+
+Synchronous tracking is more accurate, supports sampling, but introduces a run-time dependency into your application middleware stack that might not be desirable.
+
+Asynchronous tracking has a slight initial setup overhead, but has zero run-time overhead, as the data collection happens outside of the web request.
+
+#### Real Time Tracking API
+
+With Real Time tracking you can use sampling to _estimate_ the number of online users. 
+
+ * To do sampling, use the ```Turnstile::Tracker#track``` method 
+ * To store and analyze 100% of your data use ```Turnstile::Adapter#add```. 
+
+**Example:**
+
+```ruby
+@turnstile = Turnstile::Adapter.new
+
+user_id   = 12345
+platform  = 'desktop'
+ip        = '224.247.12.4'
+
+# register user
+@turnstile.add(user_id, platform, ip)
+```
+
+Without any further calls to ```track()``` method for this particular user/platform/ip combination, the user  is considered _online_ for 60 seconds.  Each subsequent call to ```track()``` resets the TTL.
+
+#### Offline Log Parsing by "Tailing Logs"
+
+If adding latency to a web request is not desirable, another option is to run Turnstile ```turnstile``` process as a daemon on each application server. The daemon "tails" the `production.log` file (or any other file), while scanning for lines matching a particular configurable pattern, and then extracting `user id`, `IP` and `platform` based on another configurable regular expression setting.
+
+##### Data Structure
+
+The logging approach expects that you print a special token into your log file, which contains three fields separated by a delimiter:
+
+ * `x-turnstile` is a hardcoded string used in finding this token,
+ * `platform` — ideally a short string, such as 'desktop', 'ios', 'android', etc
+ * `remote IP` — is the IP address of the request
+ * `user_id` — can be encoded, i.e. digest of user_id)
+ 
+for example, a token such as this:
+
+```
+x-turnstile:desktop:125.4.5.13:3456
+```
+
+is colon-separated, and easily extractable from the log.
+
+##### Log File Formats
+
+Turnstile supports two primary formats:
+
+ 1. JSON format, where each **log line** contains the following JSON fields:
+ 
+      ```json
+      { "user_id"     :17344742,
+        "platform"    :"iphone",
+        "session_id"  :"4eKMZJ4nggzvkix29zpS", 
+        "ip_address"  :"70.210.128.241",
+        .... }
+      ```
+ 
+ 2. Plain text format, where lines are space delimited, and the token is one of the fields of your log line, itself delimited using a configurable character.
+ 
+      ```
+      2017-10-06 11:03:21 x-turnstile|desktop|124.5.4.3|234324 GET /...
+      ```
+
+You can specify the file format using the `-t | --file-type` switch. 
+
+Possible values are:
+
+ * `json_formatted`
+ * `pipe_delimited`
+ * `colon_delimited`
+ * `comma_delimited`
+ 
+You can also pass the token delimiter on the command line, `-D | --delimiter "," ` in which case the `delimited` file type is used, with your custom delimiter. 
+
+> NOTE: Default format is **`pipe_delimited`**.
+
+### Examples
+
+For example:
+
+```bash
+> gem install turnstile
+
+> turnstile -v -f log/production/log -t json_formatted | \
+    tee -a /var/log/turnstile.log
+
+> turnstile -v -f log/production/log -h 127.0.0.1 -p 6432 | \
+    tee -a /var/log/turnstile.log
+
+2014-04-12 05:16:41 -0700: updater:flush        - nothing to flush, sleeping 6s..
+2014-04-12 05:16:41 -0700: updater:queue        - nothing in the queue, sleeping 5s...
+2014-04-12 05:16:41 -0700: log-reader           - starting to tail file log....
+2014-04-12 05:16:46 -0700: updater:queue        - nothing in the queue, sleeping 5s...
+2014-04-12 05:16:53 -0700: updater:flush        - nothing to flush, sleeping 6s..
+2014-04-12 05:16:56 -0700: updater:queue        - (     0.65ms) caching [746] keys locally
+2014-04-12 05:16:59 -0700: updater:flush        - (    91.73ms) flushing cache with [602] keys
+2014-04-12 05:17:05 -0700: updater:flush        - nothing to flush, sleeping 6s..
+^Ctrl-C
+```
+
+Note that ideally you should run ```turnstile``` on all app servers, for completeness, and because
+this does not incur any additional cost for the application (as user tracking is happening outside web request).
+
+### Reporting
+
+Once the tracking information is sent, the data can be queried.  
+
+If you used sampling, then you should query using ```Turnstile::Observer``` class that provides 
+exprapolation of the results based on sample size configuration.
+```ruby
+# Return data for sampled users and the summary 
+Turnstile::Observer.new.stats
+# => { stats: { total: 3, platforms: 2 }, users: [ { uid: 1, platform: 'desktop', ip: '123.2.4.54' }, ... ]
+```
+If you did not use sampling, you can get some answers from the ```Turnstile::Adapter``` class:
+```ruby
+Turntstile::Adapter.new.fetch
+# => [ { uid: 213, :platform: 'desktop', '123.2.4.54' }, { uid: 215, ... } ]
+```
+You can also request an aggregate results, suitable for sending to graphing systems or displaying on a dashboard:
+```ruby
+Turntstile::Adapter.new.aggregate
+# => { 'desktop' => 234, 'ios' => 3214, ...,  'total' => 4566 }
+```
+
+## Circonus NAD Integration
+
+We use Circonus to collect and graph data. You can use ```turnstile```
+to dump the current aggregate statistics from redis to standard output,
+which is a tab-delimited format consumable by the nad daemon.
+
+(below output is formatted to show tabs as aligned for readability).
+
+```ruby
+> turnstile --summary
+
+turnstile.iphone	   n      383
+turnstile.ipad       n	    34
+turnstile.android    n      108
+turnstile.ipod_touch n      34
+turnstile.unknown    n      36
+turnstile.total      n      595
+```
+
+## TODO:
+
+* Allow users of the gem to easier customize log reader to fit their own custom log files
+* Export configuration into a YAML file and load from there by defaul
+* Refactor commands to have a single ```turnstile``` CLI with sub-commands ```watch``` and ```report```.
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/turnstile/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/turnstile

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+require 'rubygems'
+require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
+require 'turnstile'
+require 'optparse'
+require 'colored2'
+
+options = {}
+ARGV << '-?' if ARGV.empty?
+
+Colored2.disable! unless STDOUT.tty?
+
+DESCRIPTION = <<-EOF
+Turnstile consists of two components:
+
+   1. A daemon process started with #{'turnstile -f <log-file> [ --daemon ]'.bold.green}
+      This process should run on every server that generates a log 
+      file, and they should all connect to the same Redis server. 
+
+   2. Query command, ran as #{'turnstile --summary'.bold.green}
+
+In the first form, turnstile can tail a given log file, 
+parsing and tallying and periodically syncing to the master 
+Redis server.
+
+When the query is invoked, data from all reporters is shown on STDOUT.
+EOF
+
+OptionParser.new do |opts|
+  opts.banner = "Usage:\n".bold.magenta +
+    "    turnstile -f <file> [ --daemon ] [options]\n".green +
+    "    turnstile --summary [options]\n".green
+
+  opts.separator "Description:".bold.magenta
+  opts.separator '    ' + DESCRIPTION.gsub(/\n/, "\n   ")
+
+  opts.separator "Log File Specification:".bold.magenta
+  opts.on('-f', '--file FILE', 'File to monitor') do |file|
+    options[:file] = file
+  end
+  opts.on('-t', '--file-type TYPE',
+          'Either: json_formatted, pipe_delimited,',
+          'or comma_delimited (default).') do |type|
+    options[:filetype] = type
+  end
+  opts.on('-D', '--delimiter CHAR',
+          'Forces "delimited" file type, and uses ',
+          'the character in the argument as the delimiter') do |v|
+    options[:delimiter] = v
+  end
+  opts.separator "\nRedis Server:".bold.magenta
+  opts.on('-h', '--redis-host HOST', 'Redis server host') do |host|
+    Turnstile.config.redis_host = host
+  end
+  opts.on('-p', '--redis-port PORT', 'Redis server port') do |port|
+    Turnstile.config.redis_port = port
+  end
+  opts.on('-n', '--redis-db DB', 'Redis server db') do |db|
+    Turnstile.config.redis_db = db
+  end
+  opts.separator "\nMode of Operation:".bold.magenta
+  opts.on('-d', '--daemonize', 'Daemonize to watch the logs') do |v|
+    options[:daemonize] = true
+  end
+  opts.on('-s', '--summary', 'Print current stats (using NAD format) and exit') do |v|
+    options[:summary] = true
+  end
+  opts.separator "\nTiming Adjustments:".bold.magenta
+  opts.on('-b', '--buffer-interval INTERVAL', 'Buffer for this many seconds') do |v|
+    options[:buffer_interval] = v.to_i
+  end
+  opts.on('-i', '--flush-interval INTERVAL', 'Flush then sleep for this many seconds') do |v|
+    options[:flush_interval] = v.to_i
+  end
+  opts.separator "\nMiscellaneous:".bold.magenta
+  opts.on('-v', '--verbose', 'Print status to stdout') do |v|
+    options[:debug] = true
+  end
+  opts.on_tail('-?', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+if options[:summary]
+  STDOUT.puts Turnstile::Nad.new.data
+else
+  Turnstile::Collector::Runner.new(options).run
+end
+

data/lib/turnstile.rb

@@ -0,0 +1,19 @@
+require 'turnstile/version'
+require 'turnstile/configuration'
+require 'turnstile/sampler'
+require 'turnstile/adapter'
+require 'turnstile/tracker'
+require 'turnstile/observer'
+require 'turnstile/logger'
+require 'turnstile/collector'
+require 'turnstile/nad'
+
+module Turnstile
+  def self.configure(&block)
+    @configuration = Turnstile::Configuration.new.configure(&block)
+  end
+
+  def self.config
+    @configuration ||= Turnstile::Configuration.new
+  end
+end

data/lib/turnstile/adapter.rb

@@ -0,0 +1,60 @@
+require 'redis'
+require 'timeout'
+
+module Turnstile
+  class Adapter
+    attr_accessor :redis
+    include Timeout
+
+    def initialize
+      self.redis = ::Redis.new(host: config.redis.host,
+                               port: config.redis.port,
+                               db:   config.redis.db)
+    end
+
+    def add(uid, platform, ip)
+      key = compose_key(uid, platform, ip)
+      timeout(config.redis.timeout) do
+        redis.setex(key, config.activity_interval, 1)
+      end
+    rescue StandardError => e
+      Turnstile::Logger.log "exception while writing to redis: #{e.inspect}"
+    end
+
+    def fetch
+      redis.keys('t:*').map do |key|
+        fields = key.split(':')
+        {
+          uid:      fields[1],
+          platform: fields[2],
+          ip:       fields[3],
+        }
+      end
+    end
+
+    def aggregate
+      redis.keys('t:*').inject({}) { |hash, key| increment_platform(hash, key) }.tap do |h|
+        h['total'] = h.values.inject(&:+) || 0
+      end
+    end
+
+    private
+
+    def increment_platform(hash, key)
+      platform       = key.split(':')[2]
+      hash[platform] ||= 0
+      hash[platform] += 1
+      hash
+    end
+
+    def config
+      Turnstile.config
+    end
+
+
+    def compose_key(uid, platform = nil, ip = nil)
+      "t:#{uid}:#{platform}:#{ip}"
+    end
+
+  end
+end