km-db 0.2.1 → 0.3.2

data/lib/kmdb/models/user.rb

@@ -0,0 +1,53 @@
+require 'kmdb/models/custom_record'
+require 'kmdb/concerns/has_properties'
+require 'kmdb/redis'
+
+module KMDB
+  class User < ActiveRecord::Base
+    include CustomRecord
+    include HasProperties
+
+    has_many :events,     class_name: 'KMDB::Event', inverse_of: :user
+      # points to the aliased user. if set, no properties/events should belong to this user
+
+    validates_presence_of   :name
+    
+    # it's bad practice, but this is (much) more efficiently done
+    # by the SQL engine:
+    # validates_uniqueness_of :name
+
+    scope :named, lambda { |name| where(name: name) }
+
+    # return (latest) value of property
+    def prop(name)
+      properties.named(name).first.andand.value
+    end
+
+    CACHE_EXPIRY = 3_600
+
+    def self.find_or_create(name:)
+      if raw = _redis.get(name)
+        Marshal.load(raw)
+      else
+        super.tap do |user|
+          _redis.set(name, Marshal.dump(user), ex: CACHE_EXPIRY)
+        end
+      end
+    end
+
+    def destroy
+      super
+      _redis.del(name)
+    end
+
+    private
+
+    def _redis(*args)
+      self.class._redis(*args)
+    end
+
+    def self._redis
+      @_redis ||= Redis.namespaced('kmdb:users')
+    end
+  end
+end

data/lib/kmdb/models/whitelisted_event.rb

@@ -0,0 +1,20 @@
+require 'kmdb'
+require 'active_record'
+require 'set'
+
+module KMDB
+  class WhitelistedEvent < ActiveRecord::Base
+    module ClassMethods
+      def include?(name)
+        _data.include?(name)
+      end
+
+      private
+
+      def _data
+        @_data ||= Set.new(pluck(:name))
+      end
+    end
+    extend ClassMethods
+  end
+end

data/lib/kmdb/parser.rb

@@ -1,7 +1,7 @@
-require 'yajl/json_gem'
 require 'pathname'
 require 'progressbar'
-require 'pstore'
+require 'oj'
+require 'kmdb/models/dumpfile'
 
 module KMDB
   class Parser
@@ -82,8 +82,8 @@ module KMDB
       end
 
       begin
-        data = JSON.parse(text)
-      rescue JSON::ParserError => e
+        data = Oj.load(text)
+      rescue Oj::ParseError => e
         log "Warning, JSON parse error in: #{text}"
         raise e if @abort_on_error
         return

data/lib/kmdb/redis.rb

@@ -0,0 +1,17 @@
+require 'redis'
+require 'redis-namespace'
+
+module KMDB
+  module Redis
+    module ModuleMethods
+      def connection
+        @@_connection ||= ::Redis.new(url: ENV.fetch('KMDB_REDIS_URL', 'localhost'))
+      end
+
+      def namespaced(ns)
+        ::Redis::Namespace.new(ns, redis: connection)
+      end
+    end
+    extend ModuleMethods
+  end
+end

data/lib/kmdb/resque.rb

@@ -0,0 +1,38 @@
+require 'resque'
+require 'kmdb/redis'
+
+module KMDB
+  module Resque
+    module ModuleMethods
+      def enqueue(*args)
+        _configure
+        ::Resque.enqueue(*args)
+      end
+
+      def work
+        _configure
+        ::Resque::Worker.new(:high, :medium, :low).tap do |w|
+          w.term_timeout   = 8
+          w.term_child     = true
+          w.log "starting worker"
+          w.work(5) # interval
+        end
+      end
+
+      def configure
+        _configure
+      end
+
+      private 
+      
+      def _configure
+        return if @configured
+        ::Resque.redis = Redis.connection
+        ::Resque.redis.namespace = ENV.fetch('KMDB_REDIS_NS', 'kmdb:resque')
+        ::Resque.logger.level = Logger::DEBUG
+        @configured = true
+      end
+    end
+    extend ModuleMethods
+  end
+end

data/lib/kmdb/s3_bucket.rb

@@ -0,0 +1,33 @@
+require 'kmdb'
+require 'fog'
+require 'singleton'
+
+module KMDB
+  class S3Bucket
+    include Singleton
+
+    def method_missing(method, *args, &block)
+      _directory.send(method, *args, &block)
+    end
+
+    def respond_to?(method, all = false)
+      _directory.respond_to(method, all)
+    end
+
+    private
+
+    def _directory
+      @_directory ||= _connection.directories.get(ENV.fetch('AWS_BUCKET'))
+    end
+    
+    def _connection
+      @_connection ||= Fog::Storage.new(
+        provider:              'AWS',
+        aws_access_key_id:     ENV.fetch('AWS_ACCESS_KEY_ID'),
+        aws_secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY')
+      )
+    end
+  end
+end
+
+

data/lib/kmdb/services/partitioner.rb

@@ -0,0 +1,65 @@
+require 'kmdb'
+
+module KMDB
+  module Services
+    class Partitioner
+      def initialize(model:, min_date: nil, max_date: nil, days_per_partition: nil)
+        @model              = model
+        @min_date           = min_date           || Date.parse(ENV.fetch('KMDB_MIN_DATE'))
+        @max_date           = max_date           || Date.parse(ENV.fetch('KMDB_MAX_DATE'))
+        @days_per_partition = days_per_partition || Integer(ENV.fetch('KMDB_DAYS_PER_PARTITION'))
+      end
+
+      def run
+        while true
+          last_limit = _get_last_limit || @min_date
+          break if last_limit > @max_date
+          next_limit = last_limit + @days_per_partition
+          _add_partition(next_limit)
+        end
+      end
+
+      private
+
+      def _get_last_limit
+        limit = _conn.select_value(%Q{
+          SELECT `partition_description` FROM information_schema.partitions
+          WHERE `table_schema` = '#{_database_name}'
+            AND `table_name` = '#{_table}'
+            AND `partition_description` <> 'MAXVALUE'
+          ORDER BY `partition_description` DESC LIMIT 1
+        })
+        limit ? Date.parse(limit) : nil
+      end
+
+      def _add_partition(date)
+        part_limit = date.strftime("'%F'")
+        part_name  = date.strftime('p%Y%m%d')
+        _log "adding partition up to #{part_limit} to #{_table}"
+        _conn.execute %Q{
+          ALTER TABLE #{_table} REORGANIZE PARTITION pLast INTO (
+            PARTITION #{part_name} VALUES LESS THAN (#{part_limit}),
+            PARTITION pLast VALUES LESS THAN MAXVALUE
+          )
+        }
+      end
+
+      def _table
+        @_table ||= @model.table_name
+      end
+
+      def _database_name
+        @_database_name ||= _conn.current_database
+      end
+
+      def _conn
+        @_conn ||= @model.connection
+      end
+
+      def _log(message)
+        $stderr.write("#{message}\n")
+      end
+
+    end
+  end
+end

data/lib/kmdb/version.rb

@@ -1,4 +1,4 @@
 module KMDB
-  VERSION = "0.2.1"
+  VERSION = "0.3.2"
 end
 

data/lib/kmdb.rb

@@ -1,10 +1,35 @@
+require 'active_record'
+
 module KMDB
-  MaxStringSize = 48
+  MIGRATIONS_DIR = Pathname(__FILE__).parent.join('kmdb/migrations').cleanpath.to_s
 
-  %w(key user property event user_error dumpfile parser parallel_parser).each do |mod|
-    require "kmdb/#{mod}"
-  end
+  module ModuleMethods
+    def env
+      ENV['RACK_ENV'] || ENV['KMDB_ENV'] || 'development'
+    end
+
+    def connect
+      url = ENV.fetch('DATABASE_URL')
+      puts url
+      ActiveRecord::Base.establish_connection(url)
 
-  # Connect to an alternate database when the module is loaded
-  CustomRecord.connect_to_km_db!
+      if ENV.fetch('KMDB_AR_LOG', 'NO') == 'YES'
+        ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT)
+      end
+      self
+    end
+
+    def migrate
+      ActiveRecord::Migration.verbose = true
+      ActiveRecord::Migrator.migrate MIGRATIONS_DIR
+      self
+    end
+
+    def transaction(&block)
+      ActiveRecord::Base.transaction do
+        yield ActiveRecord::Base.connection
+      end
+    end
+  end
+  extend ModuleMethods
 end