km-db 0.2.1

data/Gemfile

@@ -0,0 +1,6 @@
+source :gemcutter
+
+gem 'progressbar'
+
+# Specify your gem's dependencies in km.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,46 @@
+PATH
+  remote: .
+  specs:
+    km-db (0.2.1)
+      activerecord (~> 2.3.12)
+      andand
+      parallel
+      progressbar
+      yajl-ruby
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activerecord (2.3.18)
+      activesupport (= 2.3.18)
+    activesupport (2.3.18)
+    andand (1.3.3)
+    diff-lcs (1.1.3)
+    json (1.7.7)
+    parallel (0.6.3)
+    progressbar (0.20.0)
+    rake (10.0.3)
+    rspec (2.4.0)
+      rspec-core (~> 2.4.0)
+      rspec-expectations (~> 2.4.0)
+      rspec-mocks (~> 2.4.0)
+    rspec-core (2.4.0)
+    rspec-expectations (2.4.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.4.0)
+    sqlite3 (1.3.7)
+    sqlite3-ruby (1.3.3)
+      sqlite3 (>= 1.3.3)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.0.0)
+  json
+  km-db!
+  progressbar
+  rake
+  rspec (~> 2.4.0)
+  sqlite3-ruby

data/README.markdown

@@ -0,0 +1,91 @@
+The `km-db` gem should be useful to KissMetrics (KM) users.
+Its aim is to efficiently process data obtained with KM's "Data Export" feature.
+
+It is meant to :
+
+* import KM event dumps into a SQL database (preferably MySQL / PostgreSQL)
+* quickly process KM event dumps
+
+Once imported, you can run complex queries against your visit history, for instance run multivariate analysis.
+
+Beware though, KM data can be huge, and processing it is taxing !
+
+
+Installing
+----------
+
+Add this to your Gemfile if you're using Bundler:
+
+    gem 'km-db', :git => 'git://github.com/HouseTrip/km-db.git'
+
+
+Importing data
+--------------
+
+Running reports on raw logs can be less effective than running against a (relational) database.
+`km-db` provides a `km_db_import` executable. Run it with:
+
+    $ bundle exec km_db_import <data-dump-directory>…
+
+By default, you events will be imported in `test.db`, a SQLite database.
+
+You can create `km_db.yml` or `config/km_db.yml` to have it import using another adapter, for instance:
+
+    ---- km_db.yml ----
+    adapter:  mysql2
+    database: km_events
+    user:     root
+
+Remember to add `sqlite3-ruby` or `mysql2` to your Gemfile.
+
+
+Using imported data
+-------------------
+
+The `KMDB` module exposes four `ActiveRecord` classes:
+`Event`, `Property`, `User` are the main domain objects.
+`Key` is used to intern strings (event and property names) for performance.
+
+### Finding events and properties
+
+All visits during Jan. 2012:
+
+    KMDB::Event.before('2012-02-1').after('2012-01-01').named('visited site').by_date
+
+All of a user's visit:
+
+    KMDB::User.last.events.named('visited site')
+
+A user's referers:
+    
+    KMDB::User.last.properties.named('referer').map(&:value)
+
+Load some properties with events (uses a left join by default):
+
+    KMDB::User.last.events.with_properties('a prop', 'another prop').map(&:another_prop)
+
+Note that many more complex queries will require building SQL queries directly.
+
+
+Processing data
+---------------
+
+You don't have to import to filter your data.
+
+The two classes you're looking for are `KMDB::Parser` and `KMDB::ParallelParser`.
+The latter runs your filter task on all available CPUs, using the `parallel` gem.
+
+The following example counts the number of *aliasing* events in all JSON files under `dumps/`:
+
+    require 'rubygems'
+    require 'kmdb'
+
+    counter = 0
+    parser = KMDB::Parser.new
+    parser.add_filter do |text,event|
+        counter += 1 if event['_p2']
+    end
+    parser.run('dumps/')
+    puts counter
+
+Note that it will not work with `ParallelParser`, as the `counter` variable will be different for each process.

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+Bundler::GemHelper.install_tasks
+
+RSpec::Core::RakeTask.new(:spec)

data/bin/km_db_import

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+=begin
+    
+  Import KM events from the raw dumps.
+
+=end
+
+require 'rubygems'
+require 'kmdb'
+
+if KMDB::Event.connection.class.to_s =~ /(mysql|pgsql)/i
+  parser_class = KMDB::ParallelParser
+else
+  parser_class = KMDB::Parser
+end
+
+parser = parser_class.new(:resume  => 'import', 
+                          :verbose => true)
+
+# import events by category
+parser.add_filter { |text, event|
+  if event['_p2']
+    KMDB::User.alias! event['_p'], event['_p2']
+  elsif event['_n']
+    KMDB::Event.record event
+  else
+    KMDB::Property.set event
+  end
+  
+  event
+}
+
+parser.run(ARGV)                     # heavy lifting here
+KMDB::Event.connection.reconnect!  # reconnect to database (breaks because of processes forking off)
+KMDB::User.resolve_alias_chains!   # detect and filter alias chains
+KMDB::Key.fix_duplicates!          # remove key duplicates

data/km-db.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path("../lib/kmdb/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "km-db"
+  s.version     = KMDB::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["HouseTrip"]
+  s.email       = ["jtl@housetrip.com"]
+  s.homepage    = "https://github.com/housetrip/km-db"
+  s.summary     = "Process KISSmetrics data dumps"
+  s.description = "Process KISSmetrics data dumps"
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  s.add_development_dependency "rspec", "~> 2.4.0"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "json"
+  s.add_development_dependency "sqlite3-ruby"
+  
+  s.add_dependency "yajl-ruby"
+  s.add_dependency "progressbar"
+  s.add_dependency "parallel"
+  s.add_dependency "andand"
+  s.add_dependency "activerecord", "~> 2.3.12"
+
+  s.files        = `git ls-files`.split("\n")
+  s.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
+  s.require_path = 'lib'
+end

data/lib/kmdb/belongs_to_user.rb

@@ -0,0 +1,15 @@
+module KMDB
+  module BelongsToUser
+    def self.included(mod)
+      mod.class_eval do
+        belongs_to :user,  :class_name => 'KMDB::User'
+        validates_presence_of :user
+
+        named_scope :user_is, lambda { |user| 
+          user.kind_of?(User) or raise TypeError.new("Not a kind of User")
+          { :conditions => { :user_id => user.id } }
+        }
+      end
+    end
+  end
+end

data/lib/kmdb/custom_record.rb

@@ -0,0 +1,54 @@
+=begin
+
+  Base class for KM data.
+  Connect to a secondary database to store events, users, & properties.
+
+  FIXME: the database connection is hard-coded for now.
+
+=end
+
+require 'active_record'
+require 'erb'
+require 'yaml'
+require 'kmdb/migration'
+
+
+module KMDB
+  class CustomRecord < ActiveRecord::Base
+    DefaultConfig = {
+      'adapter'  => 'sqlite3',
+      'database' => "test.db"
+    }
+
+    def self.disable_index
+      connection.execute %Q{
+        ALTER TABLE `#{table_name}` DISABLE KEYS;
+      }
+    end
+
+    def self.enable_index
+      connection.execute %Q{
+        ALTER TABLE `#{table_name}` ENABLE KEYS;
+      }
+    end
+
+    def self.find_or_create(options)
+      find(:first, :conditions => options) || create(options)
+    end
+
+    def self.connect_to_km_db!
+      config = DefaultConfig.dup
+      ['km_db.yml', 'config/km_db.yml'].each do |config_path|
+        next unless File.exist?(config_path)
+        config.merge! YAML.load(ERB.new(File.open(config_path).read).result)
+        break
+      end
+      establish_connection(config)
+
+      unless connection.table_exists?('events')
+        SetupEventsDatabase.up
+        self.reset_column_information
+      end
+    end
+  end
+end

data/lib/kmdb/dumpfile.rb

@@ -0,0 +1,23 @@
+require 'kmdb/custom_record'
+
+module KMDB
+  class Dumpfile < CustomRecord
+    set_table_name "dumpfiles"
+
+    validates_presence_of :offset
+    validates_presence_of :path
+
+    def set(offset)
+      update_attributes!(:offset => offset)
+    end
+
+    def offset
+      attributes['offset'] || 0
+    end
+
+    def self.get(pathname, job = nil)
+      job ||= 'nil'
+      find_or_create(:path => pathname.cleanpath.to_s, :job => job)
+    end
+  end
+end

data/lib/kmdb/event.rb

@@ -0,0 +1,39 @@
+require 'kmdb/custom_record'
+require 'kmdb/belongs_to_user'
+require 'kmdb/has_properties'
+
+module KMDB
+  class Event < CustomRecord
+    include BelongsToUser
+    include HasProperties
+
+    set_table_name "events"
+
+    named_scope :before, lambda { |date| { :conditions => ["`#{table_name}`.`t` < ?", date] } }
+    named_scope :after,  lambda { |date| { :conditions => ["`#{table_name}`.`t` > ?", date] } }
+
+    named_scope :named, lambda { |name| { :conditions => { :n => KMDB::Key.get(name) } } }
+
+    named_scope :by_date, lambda { { :order => "`#{table_name}`.`t` ASC" } }
+
+    # return value of property
+    def prop(name)
+      properties.named(name).first.andand.value
+    end
+
+    def name
+      KMDB::Key.find(n).value
+    end
+
+    def self.record(hash)
+      user_name = hash.delete('_p')
+      user ||= User.get(user_name)
+      raise UserError.new "User missing for '#{user_name}'" unless user.present?
+
+      stamp = Time.at hash.delete('_t')
+      key = Key.get hash.delete('_n')
+      event = create(:t => stamp, :n => key, :user => user)
+      Property.set(hash, stamp, user, event)
+    end
+  end
+end

data/lib/kmdb/has_properties.rb

@@ -0,0 +1,33 @@
+=begin
+
+  KMDB::HasProperties --
+
+  Trait shared by Event and User.
+
+=end
+
+module KMDB
+  module HasProperties
+    def self.included(mod)
+      mod.class_eval do
+        has_many   :properties, :class_name => 'KMDB::Property'
+
+        named_scope :with_properties, lambda { |*props|
+          direction = props.delete(:exclude_missing) ? 'INNER' : 'LEFT'
+          prop_table = Property.table_name
+          selects = ["`#{table_name}`.*"]
+          joins = []
+          props.each_with_index { |prop,k|
+            temp_name = "#{prop_table}_#{k}"
+            selects << "`#{temp_name}`.`value` AS `#{prop.split.join('_')}`"
+            joins << sanitize_sql_array([%Q{
+              #{direction} JOIN `properties` AS `#{temp_name}`
+              ON `#{table_name}`.id = `#{temp_name}`.event_id 
+              AND `#{temp_name}`.`key` = ?}, KMDB::Key.get(prop)])
+          }
+          { :select => selects.join(', '), :joins => joins.join("\n") }
+        }
+      end
+    end
+  end
+end

data/lib/kmdb/key.rb

@@ -0,0 +1,56 @@
+=begin
+
+  Map strings (event and property names) to unique integers (Key#id) for performance
+
+=end
+
+require 'kmdb/custom_record'
+
+module KMDB
+  class Key < CustomRecord
+    set_table_name "keys"
+
+    has_many :events,     :foreign_key => :n,   :class_name => 'KMDB::Event',    :dependent => :delete_all
+    has_many :properties, :foreign_key => :key, :class_name => 'KMDB::Property', :dependent => :delete_all
+
+    named_scope :has_duplicate, lambda {
+      {
+        :select => "id, string, COUNT(id) AS quantity",
+        :group => :string, :having => "quantity > 1"
+      }
+    }
+
+    def self.get(string)
+      @cache ||= {}
+      @cache[string] ||= get_uncached(string)
+    end
+
+    # Replace each duplicate key ID with its most-used variant
+    def self.fix_duplicates!
+      has_duplicate.map(&:string).each do |string|
+        all_keys = find(:all, :conditions => { :string => string })
+
+        # sort keys by usage
+        all_ids = all_keys.map { |key|
+          [key.id, Event.named(key.id).count + Property.named(key.id).count]
+        }.sort { |k1,k2|
+          k1.second <=> k2.second
+        }.map { |k|
+          k.first
+        }
+        id_to_keep = all_ids.pop
+        $stderr.write "Fixing key '#{string}' #{all_ids.inspect} -> #{id_to_keep.inspect}\n"
+        Event.update_all({ :n => id_to_keep }, ["`events`.`n` IN (?)", all_ids])
+        Property.update_all({ :key => id_to_keep }, ["`properties`.`key` IN (?)", all_ids])
+        Key.delete_all(["id IN (?)", all_ids])
+      end
+    end
+
+  private
+
+    def self.get_uncached(string)
+      string.size <= MaxStringSize or raise "String is too long"
+      find_or_create(:string => string).id
+    end
+  end
+end

data/lib/kmdb/migration.rb

@@ -0,0 +1,63 @@
+=begin
+  
+  Setup a custom database for KissMetrics tracking events.
+
+=end
+
+require 'active_record'
+
+module KMDB
+  class SetupEventsDatabase < ActiveRecord::Migration
+    def self.connection
+      CustomRecord.connection
+    end
+
+    def self.up
+      create_table :events do |t|
+        t.integer  :user_id
+        t.integer  :n
+        t.datetime :t
+      end
+      add_index :events, [:n]
+      add_index :events, [:user_id]
+
+
+      create_table :keys do |t|
+        t.string :string, :limit => MaxStringSize
+      end
+      add_index :keys, [:string]
+
+      create_table :properties do |t|
+        t.integer  :user_id
+        t.integer  :event_id
+        t.integer  :key
+        t.string   :value,   :limit => 64
+        t.datetime :t
+      end
+      add_index :properties, [:key]
+      add_index :properties, [:user_id]
+      add_index :properties, [:event_id]
+
+      create_table :users do |t|
+        t.string  :name, :limit => 48
+        t.integer :alias_id
+      end
+      add_index :users, [:name]
+
+      create_table :dumpfiles do |t|
+        t.string  :path
+        t.string  :job
+        t.integer :offset
+      end
+      add_index :dumpfiles, [:path]
+
+    end
+
+    def self.down
+      drop_table :events
+      drop_table :properties
+      drop_table :users
+      drop_table :aliases
+    end
+  end
+end

data/lib/kmdb/parallel_parser.rb

@@ -0,0 +1,85 @@
+require 'kmdb/parser'
+require 'parallel'
+
+module KMDB
+  class ParallelParser < Parser
+    
+    def initialize(options = {})
+      super(options)
+      @worker_count = options.delete(:workers) || Parallel.processor_count
+    end
+
+    def run(argv)
+      @pipe_rd, @pipe_wr = IO.pipe
+
+      inputs = list_files_in(argv)
+      total_bytes = total_size_of_files(inputs)
+      log "total bytes : #{total_bytes}"
+      total_bytes -= inputs.map { |p| Dumpfile.get(p, @resume_job) }.compact.map(&:offset).sum
+      log "left to process : #{total_bytes}"
+
+      # Start workers
+      log "Using #{@worker_count} workers."
+      Process.fork do
+        @pipe_rd.close
+        Parallel.each(inputs, :in_processes => @worker_count) do |input|
+          KMDB::Event.connection.reconnect!
+          log "Worker #{Process.pid} starting #{input}"
+          $0 = "worker: #{input}"
+          process_events_in_file(input)
+          log "Worker #{Process.pid} done"
+          true
+        end
+      end
+
+      # Start gatherer
+      $0 = "gatherer: #{$0}"
+      @pipe_wr.close
+      byte_counter = 0
+      log "Starting gatherer, total bytes: #{total_bytes}"
+      progress = ProgressBar.new("-" * 20, total_bytes)
+      while line = @pipe_rd.gets
+        if line =~ /^OK (\d+)$/
+          byte_counter += $1.to_i
+          progress.set byte_counter
+        elsif line =~ /^FILE (.*)$/
+          progress.title = $1
+        else
+          log "Unparsed line: '#{line}'"
+        end
+      end
+      progress.finish
+      log "Total bytes processed: #{byte_counter}"
+      Process.waitall
+    end
+
+  private
+
+    def process_events_in_file(pathname)
+      pathname.open do |input|
+        processed_bytes = 0
+        if @resume_job
+          dumpfile = Dumpfile.get(pathname, @resume_job)
+          log "Starting file #{pathname} from offset #{dumpfile.offset}"
+          input.seek(dumpfile.offset)
+        end
+        line_number = 0
+        @pipe_wr.write "FILE #{pathname.basename}\n"
+        while line = input.gets
+          line_number += 1
+          processed_bytes += line.size
+
+          process_event(line)
+          dumpfile.set(input.tell)
+
+          if processed_bytes > 100_000
+            @pipe_wr.write "OK #{processed_bytes}\n"
+            processed_bytes = 0
+          end
+        end
+        @pipe_wr.write "OK #{processed_bytes}\n"
+      end
+    end
+
+  end
+end

data/lib/kmdb/parser.rb

@@ -0,0 +1,143 @@
+require 'yajl/json_gem'
+require 'pathname'
+require 'progressbar'
+require 'pstore'
+
+module KMDB
+  class Parser
+    class ProgressBar < ::ProgressBar
+      attr_writer :title
+    end
+    
+    attr :resume_job
+    attr :verbose
+    attr :abort_on_error
+
+    def initialize(options = {})
+      @processed_bytes = nil
+      @total_bytes = nil
+      @exclude_regexps = []
+      @include_regexps = []
+      @filters = []
+      @verbose        = options.delete(:verbose)
+      @resume_job     = options.delete(:resume)
+      @abort_on_error = options.delete(:abort_on_error)
+
+      if @resume_job && @verbose && Dumpfile.count > 0
+        log "Using restart information"
+      end
+    end
+
+    def exclude(regexp)
+      @exclude_regexps << regexp
+      self
+    end
+
+    def only(regexp)
+      @include_regexps << regexp
+      self
+    end
+
+    def add_filter(&block)
+      @filters << block
+      self
+    end
+
+    def run(argv)
+      inputs = list_files_in(argv)
+      total_bytes = total_size_of_files(inputs)
+      log "total bytes : #{total_bytes}"
+      total_bytes -= inputs.map { |p| Dumpfile.get(p, @resume_job) }.compact.map(&:offset).sum
+      log "left to process : #{total_bytes}"
+      
+      @processed_bytes = 0
+      @progress = ProgressBar.new("-" * 20, total_bytes)
+      @progress.long_running if @progress.respond_to?(:long_running)
+      
+      inputs.sort.each do |input|
+        process_events_in_file(input)
+      end
+
+      @progress.finish
+    end
+
+  private
+
+    def log(message)
+      $stderr.write(message + "\n") if @verbose
+    end
+
+    def process_event(text)
+      return if @exclude_regexps.any? { |re| text =~ re }
+      return unless @include_regexps.all? { |re| text =~ re }
+
+      # filter strange utf-8 encoding/escaping found in KM dumps   
+      if text =~ /\\30[3-5]\\[0-9]{3}/
+        begin
+          text = eval("%Q(#{text})") 
+        rescue SyntaxError => e
+          log "Syntax error in: #{text}"
+          raise e if @abort_on_error
+        end
+      end
+
+      begin
+        data = JSON.parse(text)
+      rescue JSON::ParserError => e
+        log "Warning, JSON parse error in: #{text}"
+        raise e if @abort_on_error
+        return
+      end
+
+      if data.nil?
+        log "Warning, JSON parse failed in: #{text}"
+        return
+      end
+
+      @filters.each do |filter|
+        data = filter.call(text, data) or break
+      end
+    end
+
+    def process_events_in_file(pathname)
+      pathname.open do |input|
+        @progress.title = pathname.basename.to_s
+        if @resume_job
+          dumpfile = Dumpfile.get(pathname, @resume_job)
+          log "Starting file #{pathname} from offset #{dumpfile.offset}"
+          input.seek(dumpfile.offset)
+        end
+        line_number = 0
+        while line = input.gets
+          @processed_bytes += line.size
+          @progress.set @processed_bytes if line_number % 100 == 0
+          line_number += 1
+
+          process_event(line)
+          dumpfile.set(input.tell) if @resume_job
+        end
+      end
+    end
+
+    def total_size_of_files(inputs)
+      inputs.map { |c| c.stat.size }.inject(0) { |a,b| a+b }
+    end
+
+    def list_files_in_directory(directory)
+      input_fns = []
+      directory.find do |input_pn|
+        input_pn.to_s =~ /\.json$/ or next
+        input_fns << input_pn
+      end
+      input_fns.sort
+    end
+
+    def list_files_in(argv)
+      argv.map { |arg| Pathname.new(arg) }.map { |pn|
+        pn.exist? and pn or raise "No such file or directory '#{pn}'"
+      }.map { |pn|
+        pn.directory? ? list_files_in_directory(pn) : pn
+      }.flatten
+    end
+  end
+end

data/lib/kmdb/property.rb

@@ -0,0 +1,33 @@
+require 'kmdb/belongs_to_user'
+
+module KMDB
+  class Property < CustomRecord
+    include BelongsToUser
+
+    set_table_name "properties"
+    belongs_to :event, :class_name => 'KMDB::Event'
+
+    default_scope :order => 't DESC'
+    named_scope :named, lambda { |name| { :conditions => { :key => KMDB::Key.get(name) } } }
+
+    def self.set(hash, stamp=nil, user=nil, event=nil)
+      user_name = hash.delete('_p')
+      user ||= User.get(user_name)
+      raise UserError.new "User missing for '#{user_name}'" unless user.present?
+
+      event_id = event ? event.id : nil
+      stamp = Time.at hash.delete('_t') || stamp
+
+      return if hash.empty?
+      sql_insert = "INSERT INTO `#{table_name}` (`t`,`user_id`,`event_id`,`key`,`value`) VALUES "
+      sql_values = []
+
+      hash.each_pair do |prop_name,value|
+        key = Key.get(prop_name)
+        sql_values << sanitize_sql_array(["(?,?,?,?,?)", stamp,user.id,event_id,key,value])
+      end
+
+      connection.execute(sql_insert + sql_values.join(","))
+    end
+  end
+end

data/lib/kmdb/user.rb

@@ -0,0 +1,83 @@
+require 'kmdb/has_properties'
+
+module KMDB
+  class User < CustomRecord
+    include HasProperties
+
+    set_table_name "users"
+
+    has_many :events,     :class_name => 'KMDB::Event'
+    belongs_to :alias,    :class_name => 'KMDB::User' 
+      # points to the aliased user. if set, no properties/events should belong to this user
+
+    validates_presence_of   :name
+    validates_uniqueness_of :name
+
+    named_scope :named, lambda { |name| { :conditions => { :name => name } } }
+
+    named_scope :duplicates, lambda {{
+      :select => "id, COUNT(id) AS quantity", :group => :name, :having => "quantity > 1"
+    }}
+
+    # return (latest) value of property
+    def prop(name)
+      properties.named(name).first.andand.value
+    end
+
+    # mark this user as aliasing another
+    def aliases!(other)
+      [Property,Event].each do |model|
+        model.user_is(self).update_all({:user_id => other.id})
+      end
+      self.update_attributes!(:alias => other)
+    end
+
+    # return the user named `name` (creating it if necessary)
+    # if `name` is an alias, return the original user
+    def self.get(name)
+      user = named(name).first || create(:name => name)
+      user = user.alias while user.alias
+      return user
+    end
+
+
+    # mark the two names as pointing to the same user
+    def self.alias!(name1, name2)
+      u1 = get(name1)
+      u2 = get(name2)
+      $stderr.write "Warning: user '#{user.name}' has an alias\n" if u1.alias
+      $stderr.write "Warning: user '#{user.name}' has an alias\n" if u2.alias
+      
+      # nothing to do if both names already point to the same user
+      return if u1 == u2  
+
+      u2.aliases! u1
+    end
+
+
+    # duplication can occur during parallel imports because we're not running transactionally.
+    def self.fix_duplicates!
+      duplicates.map(&:name).each do |name|
+        named(name).all.tap do |all_users|
+          kept_user = all_users.pop
+          all_users.each do |user|
+            user.aliases! kept_user
+            user.destroy
+          end
+        end
+      end
+    end
+
+
+    # detect alias chains
+    def self.resolve_alias_chains!
+      find(:all, :joins => :alias, :conditions => 'aliases_users.alias_id IS NOT NULL').each do |user|
+        user = find(user.id)
+        origin = find(user.alias_id)
+        origin = origin.alias while origin.alias # go up the chain
+        $stderr.write "Aliasing #{user.name} -> #{origin.name}\n"
+        user.aliases!(origin)
+      end
+    end
+  end
+end

data/lib/kmdb/user_error.rb

@@ -0,0 +1,2 @@
+class KMDB::UserError < Exception
+end

data/lib/kmdb/version.rb

@@ -0,0 +1,4 @@
+module KMDB
+  VERSION = "0.2.1"
+end
+

data/lib/kmdb.rb

@@ -0,0 +1,10 @@
+module KMDB
+  MaxStringSize = 48
+
+  %w(key user property event user_error dumpfile parser parallel_parser).each do |mod|
+    require "kmdb/#{mod}"
+  end
+
+  # Connect to an alternate database when the module is loaded
+  CustomRecord.connect_to_km_db!
+end

metadata

@@ -0,0 +1,234 @@
+--- !ruby/object:Gem::Specification 
+name: km-db
+version: !ruby/object:Gem::Version 
+  hash: 21
+  prerelease: 
+  segments: 
+  - 0
+  - 2
+  - 1
+  version: 0.2.1
+platform: ruby
+authors: 
+- HouseTrip
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-03-23 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  prerelease: false
+  name: bundler
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 2
+        - 4
+        - 0
+        version: 2.4.0
+  prerelease: false
+  name: rspec
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: rake
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: json
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: sqlite3-ruby
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: yajl-ruby
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: progressbar
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: parallel
+  type: :runtime
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  name: andand
+  type: :runtime
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 2
+        - 3
+        - 12
+        version: 2.3.12
+  prerelease: false
+  name: activerecord
+  type: :runtime
+  version_requirements: *id010
+description: Process KISSmetrics data dumps
+email: 
+- jtl@housetrip.com
+executables: 
+- km_db_import
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- README.markdown
+- Rakefile
+- bin/km_db_import
+- km-db.gemspec
+- lib/kmdb.rb
+- lib/kmdb/belongs_to_user.rb
+- lib/kmdb/custom_record.rb
+- lib/kmdb/dumpfile.rb
+- lib/kmdb/event.rb
+- lib/kmdb/has_properties.rb
+- lib/kmdb/key.rb
+- lib/kmdb/migration.rb
+- lib/kmdb/parallel_parser.rb
+- lib/kmdb/parser.rb
+- lib/kmdb/property.rb
+- lib/kmdb/user.rb
+- lib/kmdb/user_error.rb
+- lib/kmdb/version.rb
+has_rdoc: true
+homepage: https://github.com/housetrip/km-db
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.9.5
+signing_key: 
+specification_version: 3
+summary: Process KISSmetrics data dumps
+test_files: []
+