km-db 0.2.1 → 0.3.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a1d53c84fa85914d141e481f75759df0a9cdaad0
+  data.tar.gz: 58605a95792bfd91bc5b3877b1460836cc883086
+SHA512:
+  metadata.gz: 5dfb47a0f99adb83c288792df7fbe08b37474048844e966f604180da8c5885715e6aa7a04c866b50e257df5f3db322546697bfba9b11400342a2a33690a6a461
+  data.tar.gz: 1b14b242a34d432516402b47e7ac1e7db70b82afe1f7761aaa23a43915226f99f47cc7d21a06d2a9ad417a0fff843fd44c784fb20d64d87d6aff25538728419e

data/.gitignore

@@ -0,0 +1,7 @@
+data/
+tmp/
+pkg/
+tags
+.DS_Store
+.bundle
+

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/Gemfile

@@ -1,6 +1,4 @@
-source :gemcutter
+source ENV.fetch('GEM_SOURCE', 'https://rubygems.org')
 
-gem 'progressbar'
-
-# Specify your gem's dependencies in km.gemspec
 gemspec
+

data/Gemfile.lock

@@ -1,25 +1,175 @@
 PATH
   remote: .
   specs:
-    km-db (0.2.1)
-      activerecord (~> 2.3.12)
+    km-db (0.3.2)
+      activerecord (~> 4.1)
       andand
-      parallel
+      fog
+      foreman
+      mysql2
+      oj
       progressbar
-      yajl-ruby
+      resque
+      resque-lock
 
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
-    activerecord (2.3.18)
-      activesupport (= 2.3.18)
-    activesupport (2.3.18)
+    CFPropertyList (2.3.1)
+    activemodel (4.2.2)
+      activesupport (= 4.2.2)
+      builder (~> 3.1)
+    activerecord (4.2.2)
+      activemodel (= 4.2.2)
+      activesupport (= 4.2.2)
+      arel (~> 6.0)
+    activesupport (4.2.2)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
     andand (1.3.3)
+    arel (6.0.0)
+    builder (3.2.2)
+    coderay (1.1.0)
     diff-lcs (1.1.3)
-    json (1.7.7)
-    parallel (0.6.3)
-    progressbar (0.20.0)
-    rake (10.0.3)
+    excon (0.45.3)
+    fission (0.5.0)
+      CFPropertyList (~> 2.2)
+    fog (1.31.0)
+      fog-atmos
+      fog-aws (~> 0.0)
+      fog-brightbox (~> 0.4)
+      fog-core (~> 1.30)
+      fog-ecloud
+      fog-google (>= 0.0.2)
+      fog-json
+      fog-local
+      fog-powerdns (>= 0.1.1)
+      fog-profitbricks
+      fog-radosgw (>= 0.0.2)
+      fog-riakcs
+      fog-sakuracloud (>= 0.0.4)
+      fog-serverlove
+      fog-softlayer
+      fog-storm_on_demand
+      fog-terremark
+      fog-vmfusion
+      fog-voxel
+      fog-xml (~> 0.1.1)
+      ipaddress (~> 0.5)
+      nokogiri (~> 1.5, >= 1.5.11)
+    fog-atmos (0.1.0)
+      fog-core
+      fog-xml
+    fog-aws (0.5.0)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-brightbox (0.7.1)
+      fog-core (~> 1.22)
+      fog-json
+      inflecto (~> 0.0.2)
+    fog-core (1.31.1)
+      builder
+      excon (~> 0.45)
+      formatador (~> 0.2)
+      mime-types
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+    fog-ecloud (0.1.3)
+      fog-core
+      fog-xml
+    fog-google (0.0.5)
+      fog-core
+      fog-json
+      fog-xml
+    fog-json (1.0.2)
+      fog-core (~> 1.0)
+      multi_json (~> 1.10)
+    fog-local (0.2.1)
+      fog-core (~> 1.27)
+    fog-powerdns (0.1.1)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+    fog-profitbricks (0.0.3)
+      fog-core
+      fog-xml
+      nokogiri
+    fog-radosgw (0.0.4)
+      fog-core (>= 1.21.0)
+      fog-json
+      fog-xml (>= 0.0.1)
+    fog-riakcs (0.1.0)
+      fog-core
+      fog-json
+      fog-xml
+    fog-sakuracloud (1.0.1)
+      fog-core
+      fog-json
+    fog-serverlove (0.1.2)
+      fog-core
+      fog-json
+    fog-softlayer (0.4.6)
+      fog-core
+      fog-json
+    fog-storm_on_demand (0.1.1)
+      fog-core
+      fog-json
+    fog-terremark (0.1.0)
+      fog-core
+      fog-xml
+    fog-vmfusion (0.1.0)
+      fission
+      fog-core
+    fog-voxel (0.1.0)
+      fog-core
+      fog-xml
+    fog-xml (0.1.2)
+      fog-core
+      nokogiri (~> 1.5, >= 1.5.11)
+    foreman (0.78.0)
+      thor (~> 0.19.1)
+    formatador (0.2.5)
+    i18n (0.7.0)
+    inflecto (0.0.2)
+    ipaddress (0.8.0)
+    json (1.8.3)
+    method_source (0.8.2)
+    mime-types (2.6.1)
+    mini_portile (0.6.2)
+    minitest (5.7.0)
+    mono_logger (1.1.0)
+    multi_json (1.11.1)
+    mysql2 (0.3.18)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oj (2.12.9)
+    progressbar (0.21.0)
+    pry (0.10.0)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.2)
+    rack-protection (1.5.3)
+      rack
+    rake (10.3.2)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    resque (1.25.2)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    resque-lock (1.1.0)
     rspec (2.4.0)
       rspec-core (~> 2.4.0)
       rspec-expectations (~> 2.4.0)
@@ -28,19 +178,28 @@ GEM
     rspec-expectations (2.4.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.4.0)
-    sqlite3 (1.3.7)
-    sqlite3-ruby (1.3.3)
-      sqlite3 (>= 1.3.3)
-    yajl-ruby (1.1.0)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    slop (3.5.0)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tilt (2.0.1)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (>= 1.0.0)
-  json
+  bundler
   km-db!
-  progressbar
+  pry
   rake
   rspec (~> 2.4.0)
-  sqlite3-ruby
+
+BUNDLED WITH
+   1.10.4

data/Procfile

@@ -0,0 +1,2 @@
+web: bundle exec rackup 
+workers: foreman start  -f Procfile.work -c worker=$RESQUE_WORKERS

data/Procfile.work

@@ -0,0 +1 @@
+resque: bundle exec bin/kmdb-work

data/README.md

@@ -0,0 +1,186 @@
+# KMDB
+
+The `km-db` gem should be useful to KissMetrics (KM) users.
+Its aim is to efficiently process data obtained with KM's "Data Export" feature.
+
+Its main feature is to import dumps directly from S3 into a SQL database,
+optimized for typical queries (in particular, partitioned along the time
+dimension).
+
+Once imported, you can run complex queries against your visit history, for
+instance run multivariate analysis.
+
+Beware though, KM data can be huge, and processing it is taxing!
+
+
+## Installing
+
+If you want to run "just" KM-DB, you might want to just use [the
+app](https://github.com/HouseTrip/km-db-app).
+
+Otherwise, add this to your Gemfile if you're using Bundler:
+
+    gem 'km-db'
+
+### Configuration
+
+KMDB is configured through environment variables. We recommend storing this
+settings in a `.env` file if running locally, and using [foreman]() to start
+KMDB commands with the environment set.
+
+
+### Preparing your database
+
+KMDB requires a MySQL database (to store events, properties, etc) and a Redis
+store running (to store batch jobs and cache data).
+
+Set the following:
+
+- `DATABASE_URL` (required), e.g. `mysql2://km_db_test@localhost/km_db_test`
+- `KMDB_REDIS_URL` [localhost], e.g. `redis://localhost/14`
+
+Then run:
+
+    $ kmdb-flush
+
+to prepare your database.
+
+
+### Optimizing your database
+
+If your dataset is large (over 1 million events), KMDB can [partition]() your
+database, i.e. transparently split large tables into smaller buckets of
+continuous time periods.
+
+Set the following:
+
+- `KMDB_MIN_DATE` (required), e.g. '2014-01-01'
+- `KMDB_MAX_DATE` (required), e.g. '2016-01-01'
+- `KMDB_DAYS_PER_PARTITION` (required), e.g. '7'
+
+Then run:
+
+    $ kmdb-partition
+
+Notes:
+
+- MySQL only supports up to 1024 partitions.
+- You shoud aim for less than 1 million events per partitions for performance.
+- You should run this _before_ importing data, but it's possible to re-run it.
+  The `MIN_DATE` will be ignored, and partitions will be added up to the new
+  `MAX_DATE` (if larger).
+
+
+## Importing data
+
+KMDB will fetch JSON files form the S3 bucket where you instructed KissMetrics
+to back up your data, parse them, and store information in the database.
+
+It does so using [resque]() for high parallelism of the import process; in our
+experience, it's perfectly possible to import 100GB of data in a few hours.
+
+Set the following: 
+
+- `RESQUE_WORKERS` (1), number of worker nodes.
+- `KMDB_MIN_REVISION` (optional, default 1), first KissMetrics revision file you want to import.
+- `KMDB_REVISION_LOOKAHEAD` (10), how many revision files to check after the last known one
+- `KMDB_BATCH_SIZE` (100), how many events to process per batch (advisory, may
+  be higher as an entire second's worth of events will always be processed in one
+  batch to preserve ordering).
+- `AWS_BUCKET` (required), the name of the S3 bucket where the data is stored.
+- `AWS_ACCESS_KEY_ID` (required).
+- `AWS_SECRET_ACCESS_KEY` (required).
+
+
+### Ignoring some users
+
+You may want to ignore all events and properties for certain users, for instance
+the administrative users of your site (or employees).
+
+Simply add their identities to the `ignored_users` table before import. 
+
+
+### Whitelisting events
+
+It's typical to have some noisy and/or shorter-lived events sent to KissMetrics,
+e.g. for testing purposes or for temporary monitoring.
+
+Should you only want to import certain events, add their names to the
+`whitelisted_events` table before starting import.
+
+If the table is left empty, all events will be imported.
+
+
+### Dealiasing users
+
+When KissMetrics finds a way to tie two user identities as being a single actual
+user, it stores an "aliasing" event.
+KMDB de-aliases users automatically during import, and will store all events and
+properties against a single user identity (one that's numeric if any, otherwise
+the lexicographically lowest).
+
+
+## Using imported data
+
+### Using SQL directly
+
+KMDB tries to stay close to the KissMetrics data, leaving you to interpret it.
+As such, the main tables are unsurprisingly `events` and `properties`.
+
+Here's a summary of the data model:
+
+`events` has one row for each imported event:
+
+| **events** |
+|------------|
+| id         |
+| t          | the event timestamp             |
+| n          | reference to the event name     |
+| user_id    | reference to the user           |
+
+`properties` has one row for each property ever set on events or users
+
+| **properties** |
+|----------------|
+| id             |
+| t              | timestamp at which the property was set  |
+| key            | reference to the property name           |
+| value          | value (string)                           |
+| user_id        | reference to the user                    |
+| event_id       | reference to the event (may be NULL)     |
+
+`events.n` and `properties.key` reference the `id` column of the `keys` table;
+this is done for performance reasons (event and property names are only stored
+once):
+
+| **keys** |
+|----------|
+| id       |
+| string   |
+
+KMDB also keeps the original user identities around in `users`, although you'll
+probably never need them:
+
+| **users** |
+|-----------|
+| id        |
+| name      | the identity given by KissMetrics |
+
+as well as all aliasing events:
+
+| **aliases** |
+|-------------|
+| id          |
+| name1       |
+| name2       |
+
+
+### Using ActiveRecord
+
+The `KMDB` module exposes four `ActiveRecord` classes:
+`Event`, `Property`, `User` are the main domain objects.
+
+`Key` is used to intern strings (event and property names) for performance.
+
+Please consult the source of these models for details.
+

data/Rakefile

@@ -3,3 +3,4 @@ require 'rspec/core/rake_task'
 Bundler::GemHelper.install_tasks
 
 RSpec::Core::RakeTask.new(:spec)
+

data/bin/kmdb-flush

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+require 'kmdb'
+require 'kmdb/redis'
+
+KMDB.connect.migrate
+
+KMDB.transaction do |c|
+  %w(aliases dumpfiles events properties users).each do |table|
+   c.execute "TRUNCATE TABLE #{table}"
+  end
+end
+
+KMDB::Redis.connection.flushdb

data/bin/kmdb-import

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+=begin
+    
+  Import KM events from the raw dumps.
+
+=end
+require 'kmdb'
+require 'kmdb/resque'
+require 'kmdb/jobs/find_files'
+require 'kmdb/jobs/list_files'
+
+KMDB::Resque.enqueue(KMDB::Jobs::FindFiles)
+

data/bin/kmdb-partition

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+require 'kmdb'
+require 'kmdb/models/event'
+require 'kmdb/models/alias'
+require 'kmdb/models/property'
+require 'kmdb/services/partitioner'
+
+KMDB.connect.migrate
+
+module KMDB
+  [Event, Property, Alias].each do |model|
+    Services::Partitioner.new(model: model).run
+  end
+end
+

data/bin/kmdb-pool

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+exec(
+  'foreman', 'start',
+  '--procfile=%s' % File.expand_path('../../Procfile.work', __FILE__),
+  '--formation=resque=%s' % ENV.fetch('RESQUE_WORKERS', 1)
+)
+

data/bin/kmdb-realias

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+=begin
+    
+  Import KM events from the raw dumps.
+
+=end
+require 'kmdb'
+require 'kmdb/resque'
+require 'kmdb/jobs/redo_unaliasing'
+
+KMDB::Resque.enqueue(KMDB::Jobs::RedoUnaliasing, Date.today.to_s)
+

data/bin/kmdb-ui

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+exec('rackup',
+  '--port', ENV.fetch('PORT'),
+  File.expand_path('../../config.ru', __FILE__)
+)

data/bin/kmdb-work

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'kmdb'
+require 'kmdb/resque'
+
+# suppress a silly warning
+require 'i18n'
+I18n.enforce_available_locales = false
+
+# load all jobs
+Dir[File.expand_path('../../lib/kmdb/jobs/*.rb', __FILE__)].each do |job|
+  require job
+end
+
+
+KMDB.connect
+KMDB::Resque.work
+