km-db 0.2.1 → 0.3.2

data/lib/kmdb/jobs/unalias_user.rb

@@ -0,0 +1,32 @@
+require 'kmdb'
+require 'kmdb/models/user'
+require 'kmdb/models/event'
+require 'kmdb/models/property'
+require 'kmdb/jobs/locked'
+
+module KMDB
+  module Jobs
+    # Removes all references to a user alias
+    class UnaliasUser < Locked
+      @queue = :medium
+
+      def self.perform(name1, name2)
+        new(name1, name2).work
+      end
+
+      def initialize(name1, name2)
+        @user  = User.where(name: name1).first
+        @alias = User.where(name: name2).first
+      end
+      
+      def work
+        return unless @user && @alias
+        [Property, Event].each do |model|
+          model.where(user_id: @alias.id).update_all(user_id: @user.id)
+        end
+        @alias.destroy
+      end
+    end
+  end
+end
+

data/lib/kmdb/migrations/01_kmdb_initial.rb

@@ -0,0 +1,78 @@
+=begin
+  
+  Setup a custom database for KissMetrics tracking events.
+
+=end
+require 'active_record'
+require 'kmdb'
+
+class KmdbInitial < ActiveRecord::Migration
+
+  def up
+    execute "SET character_set_database = 'utf8'"
+
+    create_table :events do |t|
+      t.integer  :user_id
+      t.integer  :n
+      t.datetime :t
+    end
+    add_index :events, [:n],          using: :hash
+    add_index :events, [:user_id],    using: :hash
+    add_index :events, [:user_id, :n, :t]
+
+
+    create_table :keys do |t|
+      t.string :string
+    end
+    add_index :keys, [:string],         using: :hash, unique: true
+
+    create_table :properties do |t|
+      t.integer  :user_id
+      t.integer  :event_id
+      t.integer  :key
+      t.datetime :t
+      t.string   :value
+    end
+    add_index :properties, [:key],      using: :hash
+    add_index :properties, [:user_id],  using: :hash
+    add_index :properties, [:event_id], using: :hash
+    add_index :properties, [:user_id, :key, :t]
+
+    create_table :users do |t|
+      t.string  :name, limit: 48
+    end
+    add_index :users, [:name],          using: :hash, unique: true
+
+    create_table :ignored_users do |t|
+      t.string :name, limit: 48
+    end
+
+    create_table :whitelisted_events do |t|
+      t.string :name
+    end
+
+    create_table :aliases do |t|
+      t.string   :name1, limit: 48
+      t.string   :name2, limit: 48
+      t.datetime :t
+    end
+    # add_index :aliases, [:name1, :name2], using: :hash, unique: true
+    add_index :aliases, [:name1], using: :hash
+    add_index :aliases, [:name2], using: :hash
+
+    create_table :dumpfiles do |t|
+      t.integer :revision
+      t.integer :length
+      t.integer :offset
+      t.timestamps
+    end
+    add_index :dumpfiles, [:revision]
+  end
+
+  def down
+    drop_table :events
+    drop_table :properties
+    drop_table :users
+    drop_table :aliases
+  end
+end

data/lib/kmdb/migrations/02_kmdb_partitions.rb

@@ -0,0 +1,28 @@
+=begin
+  
+  Setup events and properties for partitioning
+
+=end
+require 'active_record'
+require 'kmdb'
+
+class KmdbPartitions < ActiveRecord::Migration
+
+  def up
+    %w(events properties aliases).each do |t|
+      sql = %Q{
+        ALTER TABLE #{t} CHANGE id id BIGINT NOT NULL
+        ALTER TABLE #{t} DROP PRIMARY KEY
+        CREATE UNIQUE INDEX index_events_partition ON #{t} (t, id)
+        CREATE INDEX index_events_id ON #{t} (id)
+        ALTER TABLE #{t} CHANGE id id BIGINT NOT NULL AUTO_INCREMENT
+        ALTER TABLE #{t} PARTITION BY RANGE COLUMNS (t) (PARTITION pLast VALUES LESS THAN MAXVALUE)
+      }
+      sql.strip.split(/\n/).each { |stmt| execute stmt.strip }
+    end
+  end
+
+  def down
+  end
+end
+

data/lib/kmdb/migrations/03_kmdb_blacklist.rb

@@ -0,0 +1,20 @@
+=begin
+  
+  Setup events and properties for partitioning
+
+=end
+require 'active_record'
+require 'kmdb'
+
+class KmdbBlacklist < ActiveRecord::Migration
+
+  def up
+    create_table :blacklisted_properties do |t|
+      t.string :name
+    end
+  end
+
+  def down
+  end
+end
+

data/lib/kmdb/models/alias.rb

@@ -0,0 +1,36 @@
+require 'kmdb/models/custom_record'
+
+module KMDB
+  # Aliasing events, ie. pairs of user names that are considered the same actual user.
+  class Alias < ActiveRecord::Base
+    self.primary_key = :id
+
+    module ClassMethods
+      def record(name1, name2, stamp)
+        retries ||= 5
+        name1, name2 = _sorted(name2, name1)
+        where(name1: name1, name2: name2).first || create!(name1: name1, name2: name2, t: Time.at(stamp))
+      rescue ActiveRecord::RecordNotUnique
+        retry unless (retries -= 1).zero?
+        raise
+      end
+
+      private
+
+      # always the "lowest" name first, with preference to numeric names
+      def _sorted(name1, name2)
+        if name1 =~ /^[0-9]+$/
+          [name1, name2]
+        elsif name2 =~ /^[0-9]+$/
+          [name2, name1]
+        elsif name2 < name1
+          [name2, name1]
+        else
+          [name1, name2]
+        end
+      end
+    end
+    extend ClassMethods
+  end
+end
+

data/lib/kmdb/models/blacklisted_property.rb

@@ -0,0 +1,20 @@
+require 'kmdb'
+require 'active_record'
+require 'set'
+
+module KMDB
+  class BlacklistedProperty < ActiveRecord::Base
+    module ClassMethods
+      def include?(name)
+        _data.include?(name)
+      end
+
+      private
+
+      def _data
+        @_data ||= Set.new(pluck(:name))
+      end
+    end
+    extend ClassMethods
+  end
+end

data/lib/kmdb/models/custom_record.rb

@@ -0,0 +1,53 @@
+=begin
+
+  Base class for KM data.
+  Connect to a secondary database to store events, users, & properties.
+
+  FIXME: the database connection is hard-coded for now.
+
+=end
+
+require 'active_record'
+require 'erb'
+require 'yaml'
+
+
+module KMDB
+  module CustomRecord
+    def self.included(by)
+      by.extend ClassMethods
+    end
+
+    module ClassMethods
+      def disable_index
+        connection.execute %Q{
+          ALTER TABLE `#{table_name}` DISABLE KEYS;
+        }
+      end
+
+      def enable_index
+        connection.execute %Q{
+          ALTER TABLE `#{table_name}` ENABLE KEYS;
+        }
+      end
+
+      def find_or_create(options)
+        retries ||= 5
+        where(options).first || create!(options)
+      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+        $stderr.write("could not create #{self.name} with #{options.inspect}, retrying (#{retries})}\n")
+        retry unless (retries -= 1).zero?
+        raise
+      end
+
+      def commit(tid)
+        where(tid: tid).update_all(tid: nil)
+      end
+
+      def clear_uncommitted
+        # TODO: this needs to be protected by a global lock
+        where('tid IS NOT NULL').delete_all
+      end
+    end
+  end
+end

data/lib/kmdb/models/dumpfile.rb

@@ -0,0 +1,33 @@
+require 'kmdb/models/custom_record'
+require 'kmdb/models/json_file'
+
+module KMDB
+  # Remembers which JSON files where imported, and up to which point.
+  class Dumpfile < ActiveRecord::Base
+    include CustomRecord
+
+    validates_presence_of :offset
+    validates_presence_of :revision
+
+    def set(offset)
+      update_attributes!(offset: offset)
+    end
+
+    def offset
+      attributes['offset'] || 0
+    end
+
+    def file
+      JsonFile.new(revision)
+    end
+
+    def complete?
+      return if offset.nil? || length.nil?
+      offset >= length
+    end
+
+    def self.get(revision)
+      find_or_create(revision: revision)
+    end
+  end
+end

data/lib/kmdb/models/event.rb

@@ -0,0 +1,56 @@
+require 'kmdb/models/custom_record'
+require 'kmdb/concerns/belongs_to_user'
+require 'kmdb/concerns/has_properties'
+require 'kmdb/models/global_uid'
+require 'kmdb/user_error'
+
+module KMDB
+  class Event < ActiveRecord::Base
+    self.primary_key = :id
+
+    include CustomRecord
+    include BelongsToUser
+    include HasProperties
+
+    scope :before, lambda { |date| where("`#{table_name}`.`t` < ?", date) }
+    scope :after,  lambda { |date| where("`#{table_name}`.`t` > ?", date) }
+
+    scope :named, lambda { |name| where(n: KMDB::Key.get(name)) }
+
+    scope :by_date, lambda { order("`#{table_name}`.`t` ASC") }
+
+    # return value of property
+    def prop(name)
+      if p = properties.named(name).first
+        p.value
+      else
+        nil
+      end
+    end
+
+    def name
+      KMDB::Key.find(n).string
+    end
+
+    def self.sql_for(hash)
+      user_name = hash.delete('_p')
+      user = User.find_or_create(name: user_name)
+      raise UserError.new "User missing for '#{user_name}'" unless user.present?
+
+      stamp = Time.at hash.delete('_t')
+      key = Key.get hash.delete('_n').scrub
+
+      event_id = GlobalUID.get(:event)
+      event_sql = sanitize_sql_array(["(?,?,?,?)", event_id, stamp, key, user.id])
+      properties_sql = Property.sql_for(hash, stamp: stamp, user: user, event_id: event_id)
+      
+      yield event_sql, properties_sql
+    end
+
+    def self.mass_create(values_strings)
+      return if values_strings.empty?
+      sql_insert = "INSERT INTO `#{table_name}` (`id`, `t`, `n`, `user_id`) VALUES\n"
+      connection.execute(sql_insert + values_strings.join(",\n"))
+    end
+  end
+end

data/lib/kmdb/models/event_batch.rb

@@ -0,0 +1,72 @@
+require 'kmdb'
+require 'kmdb/redis'
+require 'kmdb/models/global_uid'
+require 'zlib'
+require 'digest'
+
+module KMDB
+  # Models a list of events, in chronological order,
+  # spanning entire seconds.
+  # Abstracts (compressed) storage in Redis.
+  # This effectively acts as a write cache.
+  class EventBatch
+    # provide either and Array (when creating a batch)
+    # or and encoded String (when loading)
+    def initialize(data, id:nil)
+      case data
+      when Array
+        @events = data
+      when String
+        @encoded = data
+        @id = id
+      else
+        raise ArgumentError
+      end
+    end
+
+    def save!
+      _check_redis_space!
+      redis.set(id, _encoded)
+      self
+    end
+
+    def delete
+      redis.del(id)
+    end
+
+    def self.find(id)
+      encoded = redis.get(id)
+      return if encoded.nil?
+      new(encoded, id: id)
+    end
+
+    def events
+      @events ||= Marshal.load(Zlib.inflate(@encoded))
+    end
+
+    def id
+      @id ||= GlobalUID.get('batches').to_s
+    end
+
+    private
+
+    def _encoded
+      @encoded ||= Zlib.deflate(Marshal.dump(@events), 3)
+    end
+
+    # raise exception if space in Redis is getting low
+    def _check_redis_space!
+      # FIXME: not implemented
+      # raise RuntimeError.new('low Redis storage space')
+      nil
+    end
+
+    module SharedMethods
+      def redis
+        @_redis ||= KMDB::Redis.namespaced('kmdb:batches')
+      end
+    end
+    include SharedMethods
+    extend SharedMethods
+  end
+end

data/lib/kmdb/models/global_uid.rb

@@ -0,0 +1,42 @@
+require 'kmdb'
+require 'kmdb/redis'
+
+module KMDB
+  # Efficiently generate cross-process globally unique IDs
+  # pernamespace, using Redis.
+  # IDs start at 1 and increment monotonically; each client is handed Ids in
+  # batches of 100.
+  class GlobalUID
+
+    def self.get(ns = 'value')
+      @instances ||= {}
+      @instances[ns] ||= new(ns)
+      @instances[ns].get
+    end
+
+    def initialize(ns)
+      @ns = ns
+      @major = nil
+      @minor = nil
+    end
+
+    def get
+      if @major.nil? || @minor > BATCH_SIZE
+        @major = _redis.incr(@ns) % (1 << 48)
+        @minor = 1
+      end
+
+      uid = (@major-1) * BATCH_SIZE + @minor
+      @minor += 1
+      return uid
+    end
+
+    private
+
+    BATCH_SIZE = 100
+
+    def _redis
+      @@_redis ||= Redis.namespaced('kmdb:globaluid:v2')
+    end
+  end
+end

data/lib/kmdb/models/ignored_user.rb

@@ -0,0 +1,20 @@
+require 'kmdb'
+require 'active_record'
+require 'set'
+
+module KMDB
+  class IgnoredUser < ActiveRecord::Base
+    module ClassMethods
+      def include?(name)
+        _data.include?(name)
+      end
+
+      private
+
+      def _data
+        @_data ||= Set.new(pluck(:name))
+      end
+    end
+    extend ClassMethods
+  end
+end

data/lib/kmdb/models/json_file.rb

@@ -0,0 +1,56 @@
+require 'kmdb'
+require 'kmdb/models/dumpfile'
+require 'kmdb/models/s3_object'
+
+module KMDB
+  # Models one of KissMetrics's JSON dumps.
+  # Supports caching them from S3 and finding the latest one.
+  class JsonFile
+    attr_reader :revision
+
+    def initialize(revision)
+      @revision = revision
+    end
+    
+    # Yields an IO object for this file
+    def open(&block)
+      _cached.open('r') do |io|
+        metadata.update_attributes!(length: io.size)
+        yield io
+      end
+      _flush_cache if _should_flush?
+    end
+
+    def exist?
+      _cached_path.exist? || _s3object.exist?
+    end
+
+    def metadata
+      @metadata ||= Dumpfile.get(revision)
+    end
+
+    private
+
+    def _should_flush?
+      !! (ENV.fetch('KMDB_KEEP_FILES', 'YES') !~ /YES/)
+    end
+
+    def _flush_cache
+      _cached_path.delete if _cached_path.exist?
+    end
+
+    def _cached
+      return _cached_path if _cached_path.exist?
+      _s3object.download(_cached_path)
+      _cached_path
+    end
+
+    def _cached_path
+      @_cached_path ||= Pathname.new("tmp/#{revision}.json")
+    end
+
+    def _s3object
+      @_s3object ||= S3Object.new("revisions/#{revision}.json")
+    end
+  end
+end

data/lib/kmdb/models/key.rb

@@ -0,0 +1,28 @@
+require 'kmdb/models/custom_record'
+
+module KMDB
+  # Map strings (event and property names) to unique integers (Key#id) for performance
+  class Key < ActiveRecord::Base
+    include CustomRecord
+
+    MAX_SIZE = 255
+
+    has_many :events,     foreign_key: :n,   class_name: 'KMDB::Event',    dependent: :delete_all
+    has_many :properties, foreign_key: :key, class_name: 'KMDB::Property', dependent: :delete_all
+
+
+    def self.get(string)
+      @cache ||= {}
+      @cache[string] ||= get_uncached(string)
+    end
+
+    # Replace each duplicate key ID with its most-used variant
+
+  private
+
+    def self.get_uncached(string)
+      string.size <= MAX_SIZE or raise "String is too long"
+      find_or_create(string: string).id
+    end
+  end
+end

data/lib/kmdb/models/property.rb

@@ -0,0 +1,44 @@
+require 'kmdb/models/custom_record'
+require 'kmdb/concerns/belongs_to_user'
+require 'kmdb/models/blacklisted_property'
+require 'kmdb/user_error'
+
+module KMDB
+  class Property < ActiveRecord::Base
+    self.primary_key = :id
+
+    include CustomRecord
+    include BelongsToUser
+
+    belongs_to :event, class_name: 'KMDB::Event'
+
+    scope :named, lambda { |name| where(key: KMDB::Key.get(name)) }
+
+    def self.sql_for(hash, stamp: nil, user: nil, event_id: nil)
+      user_name = hash.delete('_p')
+      user ||= User.find_or_create(name: user_name)
+      raise UserError.new "User missing for '#{user_name}'" unless user.present?
+
+      stamp = Time.at hash.delete('_t') || stamp
+      return if hash.empty?
+
+      sql_values = []
+
+      hash.each_pair do |prop_name,value|
+        next if BlacklistedProperty.include?(prop_name)
+        key = Key.get(prop_name)
+        value = value[0...255].scrub
+        sql_values << sanitize_sql_array(['(?,?,?,?,?)', stamp,user.id,event_id,key,value])
+      end
+
+      return if sql_values.empty?
+      sql_values.join(",\n")
+    end
+
+    def self.mass_create(values_strings)
+      return if values_strings.empty?
+      sql_insert = "INSERT INTO `#{table_name}` (`t`,`user_id`,`event_id`,`key`,`value`) VALUES\n"
+      connection.execute(sql_insert + values_strings.join(",\n"))
+    end
+  end
+end

data/lib/kmdb/models/s3_object.rb

@@ -0,0 +1,54 @@
+require 'kmdb'
+require 'kmdb/s3_bucket'
+require 'tempfile'
+
+module KMDB
+  class S3Object
+    def initialize(path)
+      @path   = path
+    end
+
+    def exist?
+      !!_file
+    end
+
+    def download(target)
+      raise "JSON file for revision #{@revision} not found" unless exist?
+      _log "downloading"
+      system 'curl', '-o', _tempfile.path, '--silent', _file.url(_expiry)
+      raise "Download failed for #{@path}" unless $?.success?
+      target.parent.mkpath
+      File.rename(_tempfile.path, target.to_s)
+      true
+    end
+
+    private
+
+    def _file
+      @_file ||= begin
+        _log "checking for existence"
+        S3Bucket.instance.files.head(@path)
+      end
+    end
+
+    def _expiry
+      Time.now.utc.to_i + 600
+    end
+
+    def _tempfile
+      @_tempfile ||= begin
+        Pathname.new('tmp').mkpath
+        Tempfile.new('kmdb', 'tmp')
+      end
+    end
+
+    def _tempdir
+      Pathname.new('tmp/').mk
+    end
+
+    def _log(message)
+      $stderr.write("s3 #{@path}: #{message}\n")
+    end
+  end
+end
+