redtastic 0.2.1 → 0.2.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NGQzNDc2Y2UwNTk1NThlMTNjZDUxMGFlOGQyZGFiZjVlYTFmYzJmYQ==
+    MGFlYTFiMjY1ZmY5NDI3ZjQ5YWJmM2UzMTMyYzFhNGY1NThlNmFiZg==
   data.tar.gz: !binary |-
-    OGMxYTVjM2Q4NDJjMDNjNGFhNDIwY2NmOTUwMTEyNmQ2N2VjODcyYw==
+    ZmM3MzRmN2UyZWRmOGYzOTkwM2I4NDg1NzcxNWRiNDZlODQ2NmI3MA==
 SHA512:
   metadata.gz: !binary |-
-    YjAxNjc4YzhkYWZmY2EwZjUyNjk4ZDk3YWVlOWEwODBiY2JiZDE5OTYzNTJl
-    MGRlYmQyMDg3OGZiYWQ0NGJhMWVhNzViZDExZjA4OWI2MWNhNTdhNWE0NDZh
-    M2IyYTBjMmY2MjQ2Mjg1MjM3MDExZDE5OWI0MTZhMTA0ZWZkMmU=
+    MjJlMTI2OWQ4YjZiMjg2ZDBhZWYwMGI1NmNiNTI0MGI4NjViMjc2YTc5OWVk
+    ZGJiYjAzODJmN2E2Y2UxNGJiYWM2ZGNkYWYxNTAwZjFjOGZhZmRkMmQxNmE4
+    Njg3NGNkOWQwNjQwNDlhNjA0MmRjNDU2YTFhNzg0NjM2NDA3ZTA=
   data.tar.gz: !binary |-
-    NTYwM2Q1OTdhM2Y4YjJkODg3NzY0MTE5Mzg5OThiM2FlYjM2YzY4MmQ4YmQ2
-    ZTM4Zjg1ODgxZmE5ZTcwNGQ0YTNlNTE2YzgxOGViN2Q4NjUyNjcyNjYyNTJk
-    NTVkNjI3YTU2N2FkNzc2NGY0ZjMzYmY2NjMxZmEyMmM0NzA1MjI=
+    NTIzMTBjOWY5NWNlNGJkMjUxMGNmYjk2NjQ2YzU5OTRmZTQ4OWEwNDRmODkx
+    NmRkNWZkOTE1MTJiMmVmMmM3MjNiMTNjNDAzNzhhZWQyMDA5YzhiYWMzNTA4
+    MTExZTI4OGRkYjQzNGU3ZGIyYzc4MDZkYzEyOTgxOTI0NzVhOTc=

data/.gitignore

@@ -0,0 +1,10 @@
+.ruby-gemset
+.ruby-version
+.DS_Store
+.rspec
+.env
+.env.*
+log/*
+tmp/*
+!.keep
+coverage

data/.rubocop.yml

@@ -0,0 +1,17 @@
+Documentation:
+  Enabled: false
+
+Encoding:
+  Enabled: false
+
+HandleExceptions:
+  Enabled: false
+
+LineLength:
+  Max: 120
+
+NumericLiterals:
+  Enabled: false
+
+MethodLength:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,17 @@
+---
+language: ruby
+
+cache:
+  - bundler
+
+rvm:
+  - 2.0.0
+
+services:
+  - redis-server
+
+script: "bundle exec rspec"
+
+env:
+  global:
+    REDIS_PORT=6379

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in napa.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,84 @@
+PATH
+  remote: .
+  specs:
+    redtastic (0.2.2)
+      activesupport
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.0.2)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    ast (1.1.0)
+    atomic (1.1.14)
+    coderay (1.1.0)
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
+    diff-lcs (1.2.5)
+    docile (1.1.1)
+    dotenv (0.9.0)
+    git (1.2.6)
+    i18n (0.6.9)
+    method_source (0.8.2)
+    mime-types (2.0)
+    minitest (4.7.5)
+    multi_json (1.8.4)
+    parser (2.1.4)
+      ast (~> 1.1)
+      slop (~> 3.4, >= 3.4.5)
+    powerpack (0.0.9)
+    pry (0.9.12.4)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rainbow (1.99.1)
+    redis (3.0.7)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    rubocop (0.16.0)
+      parser (~> 2.1)
+      powerpack (~> 0.0.6)
+      rainbow (>= 1.1.4)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slop (3.4.7)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
+    thor (0.18.1)
+    thread_safe (0.1.3)
+      atomic
+    tins (0.13.1)
+    tzinfo (0.3.38)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  coveralls
+  dotenv
+  git
+  pry
+  redtastic!
+  rspec
+  rubocop
+  simplecov

data/README.md

@@ -0,0 +1,275 @@
+Redtastic [![Build Status](https://travis-ci.org/bellycard/redtastic.png?branch=master)](https://travis-ci.org/bellycard/redtastic) [![Coverage Status](https://coveralls.io/repos/bellycard/redtastic/badge.png?branch=master)](https://coveralls.io/r/bellycard/redtastic?branch=master)
+========
+
+Redtastic!  Why?  Because using Redis for analytics is fantastic!
+
+Redtastic provides a interface for storing, retriveing, and aggregating time intervalled data.  Applications of Redtastic include developing snappy dashboards containing daily / monthly / yearly counts over time.  Additionally Redtastic allows for the "mashing-up" of different statistics, allowing the drilling down of data into specific subgroups (such as "Number of unique customers who are also male, android users...etc").
+
+Redtastic is backed by [Redis](http://redis.io) - so it's super fast.
+
+Installation
+------------
+
+```
+$ gem install redtastic
+```
+
+or in your **Gemfile**
+
+``` ruby
+gem 'redtastic'
+```
+
+and run:
+
+```
+$ bundle install
+```
+
+Then initialize Redtastic in your application & connect it with a redis instance:
+
+``` ruby
+$redis = Redis.new
+Redtastic::Connection.establish_connection($redis, 'namespace')
+```
+\* *specifying a namespace is optional and is used to avoid key collisions if multiple applications are using the same instance of Redis.*
+
+Usage
+-----
+
+### Defining a Redtastic Model
+
+First, create a Redtastic Model:
+
+``` ruby
+class Checkins < Redtastic::Model
+  type :counter
+  resolution :days
+end
+```
+
+The class must inherit from Redtastic::Model and provide some attributes:
+* **type:** *Required*.  The data type of the model.  Valid values include:
+  * :counter
+  * :unique (more on types [below](https://github.com/bellycard/Redtastic#Redtastic-types)).
+* **resolution:** *Optional*.  The degree of fidelity you would like to store this model at.  Valid values include:
+  * :days
+  * :weeks
+  * :months
+  * :years
+  * nil
+
+\* *Note that methods requesting results at a higher resolution than that of the model will not work.  Using a lower resolution will result in less memory utilization and will generally see faster query response times.*
+
+### Using Redtastic Models
+
+Incrementing / decrementing counters:
+
+``` ruby
+Checkins.increment(id: 1, timestamp: '2014-01-01')
+Checkins.decrement(id: 1, timestamp: '2014-01-01')
+```
+
+Find a value of a counter for a single / day / week / month / year:
+
+``` ruby
+Checkins.find(id: 1, year: 2014, month: 1, day: 5)  # Day
+Checkins.find(id: 1, year: 2014, week:  2)          # Week
+Checkins.find(id: 1, year: 2014, month: 1)          # Month
+Checkins.find(id: 1, year: 2014)                    # Year
+```
+
+Find the aggregate total over a dataspan:
+
+``` ruby
+Checkins.aggregate(id: 1, start_date: '2014-01-01', end_date: '2014-01-05')
+```
+
+Get the aggregate total + data points for each date at the specified interval:
+
+``` ruby
+Checkins.aggregate(id: 1, start_date: '2014-01-01', end_date: '2014-01-05', interval: :days)
+```
+
+### Multiple Ids
+
+The above methods also have support for multiple ids.
+
+Incrementing / decrementing multiple ids in one request:
+
+``` ruby
+Checkins.increment(id: [1001, 1002, 2003], timestamp: '2014-01-01')
+Checkins.decrement(id: [1001, 1002, 2003], timestamp: '2014-01-01')
+```
+
+Find for mutiple ids at once:
+
+``` ruby
+Checkins.find(id: [1001, 1002, 2003], year: 2014, month: 1, day: 5)
+```
+
+Aggregations across mutiple ids can be quite powerful:
+
+``` ruby
+Checkins.aggregate(id: [1001, 1002, 2003], start_date: '2014-01-01', end_date: '2014-01-05')
+```
+
+As well as aggregating across mutiple ids w/ data points at the specified interval:
+
+``` ruby
+Checkins.aggregate(id: [1001, 1002, 2003], start_date: '2013-01-01', end_date: '2014-01-05', interval: :days)
+```
+
+### Redtastic Types
+
+#### Counters
+
+Counters are just what they appear to be - counters of things.  Examples of using counters is shown in the previous two sections.
+
+#### Unique Counters
+
+Unique counters are used when an event with the same unique_id should not be counted twice. A general example of this could be a counter for the number of users that visited a place. In this case the "id" parameter would represent the id of the place and the unique_id would be the users id.
+
+**Examples**
+
+Incrementing / Decrementing (adding / removing a unique_id from a set for a particular id / time):
+``` ruby
+Customers.increment(id: 1, timestamp: '2014-01-05', unique_id: 1000)
+Customers.decrement(id: 1, timestamp: '2014-01-05', unique_id: 1000)
+```
+
+Find:
+``` ruby
+Customers.find(id: 1, year: 2014, month: 1, day: 5, unique_id: 1000) # Returns true or false
+```
+
+Find the aggregate total over a datespan (this would only return the *unique* aggregate total):
+``` ruby
+Customers.aggregate(id: 1, start_date: '2014-01-01', end_date: '2014-01-05')
+```
+
+Find the aggregate total + data points for each point at a specified interval (again, this returns not only the unique aggregate total, but also the unique total for each interval data point ~ being each day / week / month / year...etc)
+``` ruby
+Customers.aggregate(id: 1, start_date: '2014-01-01', end_date: '2014-01-05', interval: :days)
+```
+
+Unique counters also support querying mutiple ids.  For example, we can find the unique aggregate totals across multiple ids by doing:
+``` ruby
+Customers.aggregate(id: [1,2,3], start_date: '2014-01-01', end_date: '2014-01-05', interval: :days)
+```
+
+#### Attributes
+
+Attributes are unique counters that are not associated with an id, and can be thought of as a type of "global" group.  This can be mashed up with other unique counters that are associated with ids to give the same result as if they were associated with that id.  The main advantages to using this technique are:
+
+* Save a tremendous amount of memory by not storing this data, for ever resolution interval, for every id
+* Easier to update / maintain / rebuilding data is much quicker ( instead of having to update at every interval / id, you can just update it once at the "global" level)
+
+This is best explained with the example below.
+
+Say you have a unique counter "Customers", and a unique couner "Males".  Instead of storing them both at the id & daily level we can get the number of males / day / id with the following:
+
+```ruby
+class Customers < Redtastic::Model
+  type :unique
+  resolution :days
+end
+
+class Males < Redtastic::Model
+  type :unique
+end
+```
+
+then to mash up Customers against Males, just use the attributes parameter:
+
+```ruby
+Customers.aggregate(id: 1, start_date: '2014-01-01', end_date: '204-01-09', attrbiutes: [:males])
+```
+
+You can even mash up multiple attributes.  Suppose I want to see all the customers who are Male and Android Users.  First add the AndroidUsers class:
+
+```ruby
+class AndroidUsers < Redtastic::Model
+  type :unique
+end
+```
+
+then just add that into the query:
+
+```ruby
+Customers.aggregate(id: 1, start_date: '2014-01-01', end_date: '2014-01-09', attributes: [:males, :android_users])
+```
+
+and just like every other example, attributes can be used in aggregations accross multiple ids:
+
+```ruby
+Customers.aggregate(id: [1,2,3], start_date: '2014-01-01', end_date: '2014-01-09', attributes: [:males, :android_users])
+```
+
+All the methods available to unique counters can be used for unique counters acting as global attributes, with a few simplifications.  Obviously, if it does not have a resolution and is not associated with an id, then there is no need to pass those parameters into any of those.
+
+For example, adding / removing a unique_id to a global attribute set:
+
+```ruby
+Males.increment(unique_id: 1000)
+Males.decrement(unique_id: 1000)
+```
+
+or seeing if a unique_id is in a global set:
+
+```ruby
+Males.find(unique_id: 1000)
+```
+
+### Misc
+
+#### Script Manager
+
+Redtastic also provides access to the *Redtastic::ScriptManager* class which it uses internally to pre-load & provide an interface to running Lua Scripts on Redis.  Although it is used by Redtastic to run its own scripts, anybody can use it to run their own custom scripts defined in their application:
+
+Create a script: *./lib/scripts/hello.lua*
+``` lua
+return 'hello'
+```
+Tell ScriptManager to pre-load your scripts (after initializing Redtastic)
+``` ruby
+Redtastic::ScriptManager.load_scripts('./lib/scripts')
+```
+
+Now you can easily use your script anywhere in your application:
+``` ruby
+puts Redtastic::ScriptManager.hello # prints 'hello'
+```
+
+with every script having the ability to accept parameters for the KEYS & ARGV arrays:
+```ruby
+  keys = []
+  argv = []
+  Redtastic::ScriptManager.hello(keys, argv)
+```
+
+Performance
+-----------
+
+Contributing
+------------
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+TODOS
+-----
+* Set elapsed expiration times for each resolution of a model (ie. keys of resolution days expire in 1 year, months expire in 2 years...etc).
+* For large, multi-id aggregations, set batch size & do aggregations in serveral batches rather than all in one lua run to prevent long running lua scripts from blocking any other redis operation.
+* Support for hourly resolutions
+
+
+
+
+
+
+
+

data/lib/redtastic/connection.rb

@@ -0,0 +1,14 @@
+module Redtastic
+  class Connection
+    class << self
+      attr_accessor :namespace
+      attr_accessor :redis
+
+      def establish_connection(connection, namespace = nil)
+        @redis      = connection
+        @namespace  = namespace
+        Redtastic::ScriptManager.load_scripts(File.join(File.dirname(__FILE__),'/scripts'))
+      end
+    end
+  end
+end