km-db 0.2.1 → 0.3.2

metadata

@@ -1,234 +1,284 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: km-db
-version: !ruby/object:Gem::Version 
-  hash: 21
-  prerelease: 
-  segments: 
-  - 0
-  - 2
-  - 1
-  version: 0.2.1
+version: !ruby/object:Gem::Version
+  version: 0.3.2
 platform: ruby
-authors: 
-- HouseTrip
+authors:
+- Julien Letessier
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-03-23 00:00:00 +00:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
-  prerelease: false
+date: 2015-06-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 31
-        segments: 
-        - 2
-        - 4
-        - 0
-        version: 2.4.0
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.0
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+- !ruby/object:Gem::Dependency
   name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  prerelease: false
-  name: json
-  type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
-  name: sqlite3-ruby
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
-  name: yajl-ruby
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  prerelease: false
-  name: parallel
-  type: :runtime
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: andand
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id009
-- !ruby/object:Gem::Dependency 
-  requirement: &id010 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 2
-        - 3
-        - 12
-        version: 2.3.12
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
   type: :runtime
-  version_requirements: *id010
-description: Process KISSmetrics data dumps
-email: 
-- jtl@housetrip.com
-executables: 
-- km_db_import
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque-lock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: foreman
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- julien.letessier@gmail.com
+executables:
+- kmdb-flush
+- kmdb-import
+- kmdb-partition
+- kmdb-pool
+- kmdb-realias
+- kmdb-ui
+- kmdb-work
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
+- ".gitignore"
+- ".ruby-version"
 - Gemfile
 - Gemfile.lock
-- README.markdown
+- Procfile
+- Procfile.work
+- README.md
 - Rakefile
-- bin/km_db_import
+- bin/kmdb-flush
+- bin/kmdb-import
+- bin/kmdb-partition
+- bin/kmdb-pool
+- bin/kmdb-realias
+- bin/kmdb-ui
+- bin/kmdb-work
+- config.ru
+- config/amazon-rds-ca-cert.pem
 - km-db.gemspec
 - lib/kmdb.rb
-- lib/kmdb/belongs_to_user.rb
-- lib/kmdb/custom_record.rb
-- lib/kmdb/dumpfile.rb
-- lib/kmdb/event.rb
-- lib/kmdb/has_properties.rb
-- lib/kmdb/key.rb
-- lib/kmdb/migration.rb
-- lib/kmdb/parallel_parser.rb
+- lib/kmdb/concerns/belongs_to_user.rb
+- lib/kmdb/concerns/has_properties.rb
+- lib/kmdb/jobs/find_files.rb
+- lib/kmdb/jobs/list_files.rb
+- lib/kmdb/jobs/locked.rb
+- lib/kmdb/jobs/parse_file.rb
+- lib/kmdb/jobs/record_batch.rb
+- lib/kmdb/jobs/redo_unaliasing.rb
+- lib/kmdb/jobs/unalias_user.rb
+- lib/kmdb/migrations/01_kmdb_initial.rb
+- lib/kmdb/migrations/02_kmdb_partitions.rb
+- lib/kmdb/migrations/03_kmdb_blacklist.rb
+- lib/kmdb/models/alias.rb
+- lib/kmdb/models/blacklisted_property.rb
+- lib/kmdb/models/custom_record.rb
+- lib/kmdb/models/dumpfile.rb
+- lib/kmdb/models/event.rb
+- lib/kmdb/models/event_batch.rb
+- lib/kmdb/models/global_uid.rb
+- lib/kmdb/models/ignored_user.rb
+- lib/kmdb/models/json_file.rb
+- lib/kmdb/models/key.rb
+- lib/kmdb/models/property.rb
+- lib/kmdb/models/s3_object.rb
+- lib/kmdb/models/user.rb
+- lib/kmdb/models/whitelisted_event.rb
 - lib/kmdb/parser.rb
-- lib/kmdb/property.rb
-- lib/kmdb/user.rb
+- lib/kmdb/redis.rb
+- lib/kmdb/resque.rb
+- lib/kmdb/s3_bucket.rb
+- lib/kmdb/services/partitioner.rb
 - lib/kmdb/user_error.rb
 - lib/kmdb/version.rb
-has_rdoc: true
 homepage: https://github.com/housetrip/km-db
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 23
-      segments: 
-      - 1
-      - 3
-      - 6
-      version: 1.3.6
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.9.5
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Process KISSmetrics data dumps
 test_files: []
-

data/README.markdown

@@ -1,91 +0,0 @@
-The `km-db` gem should be useful to KissMetrics (KM) users.
-Its aim is to efficiently process data obtained with KM's "Data Export" feature.
-
-It is meant to :
-
-* import KM event dumps into a SQL database (preferably MySQL / PostgreSQL)
-* quickly process KM event dumps
-
-Once imported, you can run complex queries against your visit history, for instance run multivariate analysis.
-
-Beware though, KM data can be huge, and processing it is taxing !
-
-
-Installing
-----------
-
-Add this to your Gemfile if you're using Bundler:
-
-    gem 'km-db', :git => 'git://github.com/HouseTrip/km-db.git'
-
-
-Importing data
---------------
-
-Running reports on raw logs can be less effective than running against a (relational) database.
-`km-db` provides a `km_db_import` executable. Run it with:
-
-    $ bundle exec km_db_import <data-dump-directory>…
-
-By default, you events will be imported in `test.db`, a SQLite database.
-
-You can create `km_db.yml` or `config/km_db.yml` to have it import using another adapter, for instance:
-
-    ---- km_db.yml ----
-    adapter:  mysql2
-    database: km_events
-    user:     root
-
-Remember to add `sqlite3-ruby` or `mysql2` to your Gemfile.
-
-
-Using imported data
--------------------
-
-The `KMDB` module exposes four `ActiveRecord` classes:
-`Event`, `Property`, `User` are the main domain objects.
-`Key` is used to intern strings (event and property names) for performance.
-
-### Finding events and properties
-
-All visits during Jan. 2012:
-
-    KMDB::Event.before('2012-02-1').after('2012-01-01').named('visited site').by_date
-
-All of a user's visit:
-
-    KMDB::User.last.events.named('visited site')
-
-A user's referers:
-    
-    KMDB::User.last.properties.named('referer').map(&:value)
-
-Load some properties with events (uses a left join by default):
-
-    KMDB::User.last.events.with_properties('a prop', 'another prop').map(&:another_prop)
-
-Note that many more complex queries will require building SQL queries directly.
-
-
-Processing data
----------------
-
-You don't have to import to filter your data.
-
-The two classes you're looking for are `KMDB::Parser` and `KMDB::ParallelParser`.
-The latter runs your filter task on all available CPUs, using the `parallel` gem.
-
-The following example counts the number of *aliasing* events in all JSON files under `dumps/`:
-
-    require 'rubygems'
-    require 'kmdb'
-
-    counter = 0
-    parser = KMDB::Parser.new
-    parser.add_filter do |text,event|
-        counter += 1 if event['_p2']
-    end
-    parser.run('dumps/')
-    puts counter
-
-Note that it will not work with `ParallelParser`, as the `counter` variable will be different for each process.

data/bin/km_db_import

@@ -1,36 +0,0 @@
-#!/usr/bin/env ruby
-=begin
-    
-  Import KM events from the raw dumps.
-
-=end
-
-require 'rubygems'
-require 'kmdb'
-
-if KMDB::Event.connection.class.to_s =~ /(mysql|pgsql)/i
-  parser_class = KMDB::ParallelParser
-else
-  parser_class = KMDB::Parser
-end
-
-parser = parser_class.new(:resume  => 'import', 
-                          :verbose => true)
-
-# import events by category
-parser.add_filter { |text, event|
-  if event['_p2']
-    KMDB::User.alias! event['_p'], event['_p2']
-  elsif event['_n']
-    KMDB::Event.record event
-  else
-    KMDB::Property.set event
-  end
-  
-  event
-}
-
-parser.run(ARGV)                     # heavy lifting here
-KMDB::Event.connection.reconnect!  # reconnect to database (breaks because of processes forking off)
-KMDB::User.resolve_alias_chains!   # detect and filter alias chains
-KMDB::Key.fix_duplicates!          # remove key duplicates

data/lib/kmdb/custom_record.rb

@@ -1,54 +0,0 @@
-=begin
-
-  Base class for KM data.
-  Connect to a secondary database to store events, users, & properties.
-
-  FIXME: the database connection is hard-coded for now.
-
-=end
-
-require 'active_record'
-require 'erb'
-require 'yaml'
-require 'kmdb/migration'
-
-
-module KMDB
-  class CustomRecord < ActiveRecord::Base
-    DefaultConfig = {
-      'adapter'  => 'sqlite3',
-      'database' => "test.db"
-    }
-
-    def self.disable_index
-      connection.execute %Q{
-        ALTER TABLE `#{table_name}` DISABLE KEYS;
-      }
-    end
-
-    def self.enable_index
-      connection.execute %Q{
-        ALTER TABLE `#{table_name}` ENABLE KEYS;
-      }
-    end
-
-    def self.find_or_create(options)
-      find(:first, :conditions => options) || create(options)
-    end
-
-    def self.connect_to_km_db!
-      config = DefaultConfig.dup
-      ['km_db.yml', 'config/km_db.yml'].each do |config_path|
-        next unless File.exist?(config_path)
-        config.merge! YAML.load(ERB.new(File.open(config_path).read).result)
-        break
-      end
-      establish_connection(config)
-
-      unless connection.table_exists?('events')
-        SetupEventsDatabase.up
-        self.reset_column_information
-      end
-    end
-  end
-end

data/lib/kmdb/dumpfile.rb

@@ -1,23 +0,0 @@
-require 'kmdb/custom_record'
-
-module KMDB
-  class Dumpfile < CustomRecord
-    set_table_name "dumpfiles"
-
-    validates_presence_of :offset
-    validates_presence_of :path
-
-    def set(offset)
-      update_attributes!(:offset => offset)
-    end
-
-    def offset
-      attributes['offset'] || 0
-    end
-
-    def self.get(pathname, job = nil)
-      job ||= 'nil'
-      find_or_create(:path => pathname.cleanpath.to_s, :job => job)
-    end
-  end
-end