km-db 0.2.1 → 0.3.2

data/lib/kmdb/event.rb

@@ -1,39 +0,0 @@
-require 'kmdb/custom_record'
-require 'kmdb/belongs_to_user'
-require 'kmdb/has_properties'
-
-module KMDB
-  class Event < CustomRecord
-    include BelongsToUser
-    include HasProperties
-
-    set_table_name "events"
-
-    named_scope :before, lambda { |date| { :conditions => ["`#{table_name}`.`t` < ?", date] } }
-    named_scope :after,  lambda { |date| { :conditions => ["`#{table_name}`.`t` > ?", date] } }
-
-    named_scope :named, lambda { |name| { :conditions => { :n => KMDB::Key.get(name) } } }
-
-    named_scope :by_date, lambda { { :order => "`#{table_name}`.`t` ASC" } }
-
-    # return value of property
-    def prop(name)
-      properties.named(name).first.andand.value
-    end
-
-    def name
-      KMDB::Key.find(n).value
-    end
-
-    def self.record(hash)
-      user_name = hash.delete('_p')
-      user ||= User.get(user_name)
-      raise UserError.new "User missing for '#{user_name}'" unless user.present?
-
-      stamp = Time.at hash.delete('_t')
-      key = Key.get hash.delete('_n')
-      event = create(:t => stamp, :n => key, :user => user)
-      Property.set(hash, stamp, user, event)
-    end
-  end
-end

data/lib/kmdb/has_properties.rb

@@ -1,33 +0,0 @@
-=begin
-
-  KMDB::HasProperties --
-
-  Trait shared by Event and User.
-
-=end
-
-module KMDB
-  module HasProperties
-    def self.included(mod)
-      mod.class_eval do
-        has_many   :properties, :class_name => 'KMDB::Property'
-
-        named_scope :with_properties, lambda { |*props|
-          direction = props.delete(:exclude_missing) ? 'INNER' : 'LEFT'
-          prop_table = Property.table_name
-          selects = ["`#{table_name}`.*"]
-          joins = []
-          props.each_with_index { |prop,k|
-            temp_name = "#{prop_table}_#{k}"
-            selects << "`#{temp_name}`.`value` AS `#{prop.split.join('_')}`"
-            joins << sanitize_sql_array([%Q{
-              #{direction} JOIN `properties` AS `#{temp_name}`
-              ON `#{table_name}`.id = `#{temp_name}`.event_id 
-              AND `#{temp_name}`.`key` = ?}, KMDB::Key.get(prop)])
-          }
-          { :select => selects.join(', '), :joins => joins.join("\n") }
-        }
-      end
-    end
-  end
-end

data/lib/kmdb/key.rb

@@ -1,56 +0,0 @@
-=begin
-
-  Map strings (event and property names) to unique integers (Key#id) for performance
-
-=end
-
-require 'kmdb/custom_record'
-
-module KMDB
-  class Key < CustomRecord
-    set_table_name "keys"
-
-    has_many :events,     :foreign_key => :n,   :class_name => 'KMDB::Event',    :dependent => :delete_all
-    has_many :properties, :foreign_key => :key, :class_name => 'KMDB::Property', :dependent => :delete_all
-
-    named_scope :has_duplicate, lambda {
-      {
-        :select => "id, string, COUNT(id) AS quantity",
-        :group => :string, :having => "quantity > 1"
-      }
-    }
-
-    def self.get(string)
-      @cache ||= {}
-      @cache[string] ||= get_uncached(string)
-    end
-
-    # Replace each duplicate key ID with its most-used variant
-    def self.fix_duplicates!
-      has_duplicate.map(&:string).each do |string|
-        all_keys = find(:all, :conditions => { :string => string })
-
-        # sort keys by usage
-        all_ids = all_keys.map { |key|
-          [key.id, Event.named(key.id).count + Property.named(key.id).count]
-        }.sort { |k1,k2|
-          k1.second <=> k2.second
-        }.map { |k|
-          k.first
-        }
-        id_to_keep = all_ids.pop
-        $stderr.write "Fixing key '#{string}' #{all_ids.inspect} -> #{id_to_keep.inspect}\n"
-        Event.update_all({ :n => id_to_keep }, ["`events`.`n` IN (?)", all_ids])
-        Property.update_all({ :key => id_to_keep }, ["`properties`.`key` IN (?)", all_ids])
-        Key.delete_all(["id IN (?)", all_ids])
-      end
-    end
-
-  private
-
-    def self.get_uncached(string)
-      string.size <= MaxStringSize or raise "String is too long"
-      find_or_create(:string => string).id
-    end
-  end
-end

data/lib/kmdb/migration.rb

@@ -1,63 +0,0 @@
-=begin
-  
-  Setup a custom database for KissMetrics tracking events.
-
-=end
-
-require 'active_record'
-
-module KMDB
-  class SetupEventsDatabase < ActiveRecord::Migration
-    def self.connection
-      CustomRecord.connection
-    end
-
-    def self.up
-      create_table :events do |t|
-        t.integer  :user_id
-        t.integer  :n
-        t.datetime :t
-      end
-      add_index :events, [:n]
-      add_index :events, [:user_id]
-
-
-      create_table :keys do |t|
-        t.string :string, :limit => MaxStringSize
-      end
-      add_index :keys, [:string]
-
-      create_table :properties do |t|
-        t.integer  :user_id
-        t.integer  :event_id
-        t.integer  :key
-        t.string   :value,   :limit => 64
-        t.datetime :t
-      end
-      add_index :properties, [:key]
-      add_index :properties, [:user_id]
-      add_index :properties, [:event_id]
-
-      create_table :users do |t|
-        t.string  :name, :limit => 48
-        t.integer :alias_id
-      end
-      add_index :users, [:name]
-
-      create_table :dumpfiles do |t|
-        t.string  :path
-        t.string  :job
-        t.integer :offset
-      end
-      add_index :dumpfiles, [:path]
-
-    end
-
-    def self.down
-      drop_table :events
-      drop_table :properties
-      drop_table :users
-      drop_table :aliases
-    end
-  end
-end

data/lib/kmdb/parallel_parser.rb

@@ -1,85 +0,0 @@
-require 'kmdb/parser'
-require 'parallel'
-
-module KMDB
-  class ParallelParser < Parser
-    
-    def initialize(options = {})
-      super(options)
-      @worker_count = options.delete(:workers) || Parallel.processor_count
-    end
-
-    def run(argv)
-      @pipe_rd, @pipe_wr = IO.pipe
-
-      inputs = list_files_in(argv)
-      total_bytes = total_size_of_files(inputs)
-      log "total bytes : #{total_bytes}"
-      total_bytes -= inputs.map { |p| Dumpfile.get(p, @resume_job) }.compact.map(&:offset).sum
-      log "left to process : #{total_bytes}"
-
-      # Start workers
-      log "Using #{@worker_count} workers."
-      Process.fork do
-        @pipe_rd.close
-        Parallel.each(inputs, :in_processes => @worker_count) do |input|
-          KMDB::Event.connection.reconnect!
-          log "Worker #{Process.pid} starting #{input}"
-          $0 = "worker: #{input}"
-          process_events_in_file(input)
-          log "Worker #{Process.pid} done"
-          true
-        end
-      end
-
-      # Start gatherer
-      $0 = "gatherer: #{$0}"
-      @pipe_wr.close
-      byte_counter = 0
-      log "Starting gatherer, total bytes: #{total_bytes}"
-      progress = ProgressBar.new("-" * 20, total_bytes)
-      while line = @pipe_rd.gets
-        if line =~ /^OK (\d+)$/
-          byte_counter += $1.to_i
-          progress.set byte_counter
-        elsif line =~ /^FILE (.*)$/
-          progress.title = $1
-        else
-          log "Unparsed line: '#{line}'"
-        end
-      end
-      progress.finish
-      log "Total bytes processed: #{byte_counter}"
-      Process.waitall
-    end
-
-  private
-
-    def process_events_in_file(pathname)
-      pathname.open do |input|
-        processed_bytes = 0
-        if @resume_job
-          dumpfile = Dumpfile.get(pathname, @resume_job)
-          log "Starting file #{pathname} from offset #{dumpfile.offset}"
-          input.seek(dumpfile.offset)
-        end
-        line_number = 0
-        @pipe_wr.write "FILE #{pathname.basename}\n"
-        while line = input.gets
-          line_number += 1
-          processed_bytes += line.size
-
-          process_event(line)
-          dumpfile.set(input.tell)
-
-          if processed_bytes > 100_000
-            @pipe_wr.write "OK #{processed_bytes}\n"
-            processed_bytes = 0
-          end
-        end
-        @pipe_wr.write "OK #{processed_bytes}\n"
-      end
-    end
-
-  end
-end

data/lib/kmdb/property.rb

@@ -1,33 +0,0 @@
-require 'kmdb/belongs_to_user'
-
-module KMDB
-  class Property < CustomRecord
-    include BelongsToUser
-
-    set_table_name "properties"
-    belongs_to :event, :class_name => 'KMDB::Event'
-
-    default_scope :order => 't DESC'
-    named_scope :named, lambda { |name| { :conditions => { :key => KMDB::Key.get(name) } } }
-
-    def self.set(hash, stamp=nil, user=nil, event=nil)
-      user_name = hash.delete('_p')
-      user ||= User.get(user_name)
-      raise UserError.new "User missing for '#{user_name}'" unless user.present?
-
-      event_id = event ? event.id : nil
-      stamp = Time.at hash.delete('_t') || stamp
-
-      return if hash.empty?
-      sql_insert = "INSERT INTO `#{table_name}` (`t`,`user_id`,`event_id`,`key`,`value`) VALUES "
-      sql_values = []
-
-      hash.each_pair do |prop_name,value|
-        key = Key.get(prop_name)
-        sql_values << sanitize_sql_array(["(?,?,?,?,?)", stamp,user.id,event_id,key,value])
-      end
-
-      connection.execute(sql_insert + sql_values.join(","))
-    end
-  end
-end

data/lib/kmdb/user.rb

@@ -1,83 +0,0 @@
-require 'kmdb/has_properties'
-
-module KMDB
-  class User < CustomRecord
-    include HasProperties
-
-    set_table_name "users"
-
-    has_many :events,     :class_name => 'KMDB::Event'
-    belongs_to :alias,    :class_name => 'KMDB::User' 
-      # points to the aliased user. if set, no properties/events should belong to this user
-
-    validates_presence_of   :name
-    validates_uniqueness_of :name
-
-    named_scope :named, lambda { |name| { :conditions => { :name => name } } }
-
-    named_scope :duplicates, lambda {{
-      :select => "id, COUNT(id) AS quantity", :group => :name, :having => "quantity > 1"
-    }}
-
-    # return (latest) value of property
-    def prop(name)
-      properties.named(name).first.andand.value
-    end
-
-    # mark this user as aliasing another
-    def aliases!(other)
-      [Property,Event].each do |model|
-        model.user_is(self).update_all({:user_id => other.id})
-      end
-      self.update_attributes!(:alias => other)
-    end
-
-    # return the user named `name` (creating it if necessary)
-    # if `name` is an alias, return the original user
-    def self.get(name)
-      user = named(name).first || create(:name => name)
-      user = user.alias while user.alias
-      return user
-    end
-
-
-    # mark the two names as pointing to the same user
-    def self.alias!(name1, name2)
-      u1 = get(name1)
-      u2 = get(name2)
-      $stderr.write "Warning: user '#{user.name}' has an alias\n" if u1.alias
-      $stderr.write "Warning: user '#{user.name}' has an alias\n" if u2.alias
-      
-      # nothing to do if both names already point to the same user
-      return if u1 == u2  
-
-      u2.aliases! u1
-    end
-
-
-    # duplication can occur during parallel imports because we're not running transactionally.
-    def self.fix_duplicates!
-      duplicates.map(&:name).each do |name|
-        named(name).all.tap do |all_users|
-          kept_user = all_users.pop
-          all_users.each do |user|
-            user.aliases! kept_user
-            user.destroy
-          end
-        end
-      end
-    end
-
-
-    # detect alias chains
-    def self.resolve_alias_chains!
-      find(:all, :joins => :alias, :conditions => 'aliases_users.alias_id IS NOT NULL').each do |user|
-        user = find(user.id)
-        origin = find(user.alias_id)
-        origin = origin.alias while origin.alias # go up the chain
-        $stderr.write "Aliasing #{user.name} -> #{origin.name}\n"
-        user.aliases!(origin)
-      end
-    end
-  end
-end