has_metrics 0.0.6 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 290cd9839b6158645e6952a21045665372b5d2fe
-  data.tar.gz: f26872bf500a652a99816716a8cbfbe3be85afc9
+  metadata.gz: f48a7598ad01f84ad88033cbd27f3db579471077
+  data.tar.gz: 8f2bc339c0a146c2c787e9d605c9097963c3dc69
 SHA512:
-  metadata.gz: 4ab5e73e945e0698c8fd82e620c64be6cb780d5b6c6ef46bce90d2b5539f252487cfa2f3ffb71fc801934c9715fd8dd5ac02255458fd781a337b07e525f6ebcb
-  data.tar.gz: aa6fb41dfd66fcaf7b541f3861b2237cc7e4d82faf9f0e5a92736cc41b334598ed399e3e0c3004c970686fe3a57c7f3261b52b60d40a351a53f13ffef99a7ddc
+  metadata.gz: 05f470ca3023d9c5ed3ebd1a2dd6f8dc3b3c6cbc2a20b96f9af249646b4afc7c72453e653b2a363165b8942d883afa899255615228b48d2b99e0270d809dd871
+  data.tar.gz: 4363c5d42770658fd5a197e36ebad47a495fe75915654dcd39becbe5c9e81a13bb3515e437a5de66d441d19fb70017c9db1100af62b88d8bc8d211715643ad06

data/has_metrics.gemspec

@@ -1,25 +1,37 @@
 # -*- encoding: utf-8 -*-
-require File.expand_path('../lib/has_metrics/version', __FILE__)
 
-Gem::Specification.new do |gem|
-  gem.name          = "has_metrics"
-  gem.version       = HasMetrics::VERSION
-  gem.authors       = ["Allan Grant"]
-  gem.email         = ["allan@allangrant.net"]
-  gem.description   = %q{Calculate metrics and store them in the DB.}
-  gem.summary       = %q{Calculate "metrics" (any expensive methods) on ActiveRecord entries and memoize them to an automagical table.}
-  gem.homepage      = "http://github.com/allangrant/has_metrics"
-  gem.license       = "MIT"
+Gem::Specification.new do |s|
+  s.name = "has_metrics"
+  s.version = "0.1.1"
 
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Allan Grant"]
+  s.date = "2013-08-12"
+  s.description = "Calculate metrics and store them in the DB."
+  s.email = ["allan@allangrant.net"]
+  s.files = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "has_metrics.gemspec", "lib/has_metrics.rb", "lib/has_metrics/metrics.rb", "lib/has_metrics/segmentation.rb", "lib/has_metrics/sql_capturer.rb", "lib/has_metrics/version.rb", "spec/metrics_spec.rb", "spec/segmentation_spec.rb", "spec/spec_helper.rb", "spec/support/active_record.rb"]
+  s.homepage = "http://github.com/allangrant/has_metrics"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.3"
+  s.summary = "Calculate \"metrics\" (any expensive methods) on ActiveRecord entries and memoize them to an automagical table."
+  s.test_files = ["spec/metrics_spec.rb", "spec/segmentation_spec.rb", "spec/spec_helper.rb", "spec/support/active_record.rb"]
 
-  gem.add_dependency "activerecord"
-  gem.add_development_dependency "rake"
-  gem.add_development_dependency "bundler"
-  # gem.add_development_dependency "shoulda"
-  # gem.add_development_dependency "sqlite3"
-  # gem.add_development_dependency "pry"
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<activerecord>, ["< 4.0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+    else
+      s.add_dependency(%q<activerecord>, ["< 4.0"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<activerecord>, ["< 4.0"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+  end
 end

data/lib/has_metrics/metrics.rb

@@ -25,7 +25,7 @@ module Metrics
       end
 
       def metrics
-        @metrics ||= self.class.metrics_class.find_or_create_by_id(id)
+        @metrics ||= self.class.metrics_class.find_or_initialize_by_id(id)
       end
     end
   end
@@ -35,22 +35,31 @@ module Metrics
     def metrics_class
       @metrics_class
     end
+
     def has_metric name, options={}, &block
+      options.merge!(single: block) if block
+      define_single_method(name, options) if options[:single]
+
+      metrics[name.to_sym] ||= {}
+      metrics[name.to_sym].merge!(options)
+      metrics_class.class_eval do
+        attr_accessible(name, "updated__#{name}__at")
+      end
+    end
+
+    def has_aggregate_metric(name, sql)
+      has_metric name, {}.merge(aggregate: sql)
+    end
+
+    def define_single_method(name, options)
       define_method name do |*args|
-        frequency = options[:every] || 20.hours
         previous_result = metrics.attributes[name.to_s] unless options[:every] == :always
         datestamp_column = "updated__#{name}__at"
-        datestamp = metrics.attributes[datestamp_column]
         force = [:force, true].include?(args[0])
-        case
-        when !force && previous_result && options[:once]
-          # Only calculate this metric once.  If it's not nil, reuse the old value.
-          previous_result
-        when !force && frequency.is_a?(Fixnum) && datestamp && datestamp > frequency.ago
-          # The metric was recently calculated and can be reused.
+        if !force && (options.has_key?(:aggregate) || options[:infer_aggregate])
           previous_result
         else
-          result = instance_exec(&block)
+          result = instance_exec(&options[:single])
           result = nil if result.is_a?(Float) && !result.finite?
           begin
             metrics.send "#{name}=", result
@@ -66,34 +75,32 @@ module Metrics
         end
       end
 
-      (@metrics ||= []) << name.to_sym
-      @metrics.uniq!
-
-      if respond_to?(:has_custom_order_by)  # TODO: carve out has_custom_order_by functionality into this gem
-        unless metrics_class == self
-          has_custom_order_by name do |column, order|
-            { :joins => :metrics, :order => "#{reflect_on_association(:metrics).table_name}.#{column} #{order}" }
-          end
-        end
-      end
-
-      if options[:type] && (options[:type].to_sym == :float)
-        (@float_metrics ||= []) << name.to_sym
+      if defined?(::NewRelic)
+        ::NewRelic::Agent.logger.debug "Installing instrumentation for #{name}"
+        add_transaction_tracer name, category: :task
       end
     end
 
     def metrics
-      @metrics
+      @metrics ||= {}
+    end
+
+    def single_only_metrics
+      metrics.select{ |metric, options| !options.has_key?(:aggregate) }
+    end
+
+    def aggregate_metrics
+      metrics.select{ |metric, options| options.has_key?(:aggregate) }
     end
 
     def metrics_column_type(column)
       case
+      when (metric = metrics.select { |metric, options| metric == column.to_sym && options[:type] }).any?
+        metric.values.first[:type]
       when (column.to_s =~ /^by_(.+)$/) && respond_to?(:segment_categories) && segment_categories.include?($1.to_sym) # TODO: carve out segementation functionality into this gem
         :string
       when (column.to_s =~ /_at$/)
         :datetime
-      when @float_metrics && @float_metrics.include?(column.to_sym)
-        :float
       else
         :integer
       end
@@ -101,35 +108,100 @@ module Metrics
 
     def update_all_metrics!(*args)
       metrics_class.migrate!
-      # start_time = Time.zone.now
-      # total = all.count
-      # if caller.find {|c| c =~ /irb_binding/} # When called from irb
-      #   puts "Updating all metrics on #{name}: #{metrics.join(', ')}"
-      #   puts "Updating #{total} records."
-      #   progress_bar = ProgressBar.new("Progress", total)
-      # end
+
+      return unless warmup = first
+
+      where("not exists(select id from #{metrics_class.table_name} where #{metrics_class.table_name}.id = #{table_name}.id)").each do |resource|
+        resource.metrics.save!
+      end
+
+      detected_aggregate_metrics, singular_metrics = collect_metrics(warmup)
+
+      bad_guesses = crank_detected_aggregate_metrics!(detected_aggregate_metrics)
+      
+      puts "#{bad_guesses.count} bad guesses found, scheduling as singular metrics." if bad_guesses.any?
+      singular_metrics = singular_metrics | bad_guesses
+
+      if singular_metrics.any?
+        crank_singular_metrics!(args, singular_metrics) 
+      end
+
+      crank_aggregate_metrics!
+
+      metrics
+    end
+
+    def collect_metrics(warmup)
+      detected_aggregate_metrics = {}
+      metrics.select{|m,o| o[:infer_aggregate] }.each do |metric_name, options|
+        next if options[:aggregate]
+        existing_logger = ActiveRecord::Base.logger
+        sql_capturer = ActiveRecord::Base.logger = SqlCapturer.new(existing_logger)
+        warmup.instance_exec(&options[:single])
+        ActiveRecord::Base.logger = existing_logger
+
+        unless sql_capturer.query.count != 1
+          subquery = sql_capturer.query.first.gsub(warmup.id.to_s, "#{metrics_class.table_name}.id")
+          unless subquery == %Q{SELECT "#{metrics_class.table_name}".* FROM "#{metrics_class.table_name}" WHERE "#{metrics_class.table_name}"."id" = #{metrics_class.table_name}.id LIMIT 1}
+            update_sql = %Q{
+              UPDATE #{metrics_class.table_name}
+                 SET #{metric_name} = (#{subquery}),
+                     updated__#{metric_name}__at = '#{Time.current.to_s(:db)}';
+              }
+              detected_aggregate_metrics[metric_name] = update_sql
+          end
+        end
+      end
+      return detected_aggregate_metrics, (metrics.keys - aggregate_metrics.keys - detected_aggregate_metrics.keys)
+    end
+
+    def crank_detected_aggregate_metrics!(detected_aggregate_metrics)
+      bad_guesses = []
+      detected_aggregate_metrics.each do |metric_name, update_sql|
+        begin
+          ActiveRecord::Base.connection.execute update_sql
+        rescue
+          bad_guesses << metric_name
+        end
+      end
+      bad_guesses
+    end
+
+    def crank_singular_metrics!(args, singular_metrics)
+      unless ENV['RAILS_ENV'] == 'test'
+        puts "Slow Metrics Found :: #{singular_metrics.count} \n"
+        puts " --  --  --  -- \n"
+        puts "Go implement their aggregate methods in #{self} to speed this up.\n"
+        puts singular_metrics.join(', ')
+        puts "\n ≈ ≈ ≈ ≈ ≈ ≈ ≈ "
+      end
+
       find_in_batches do |batch|
         metrics_class.transaction do
           batch.each do |record|
-            # puts "Updating record ##{record.id}: #{record}"
-            record.update_metrics!(*args)
+            singular_metrics.each do |singular_metric|
+              record.send(singular_metric, *args)
+            end
           end
         end
-        # progress_bar.inc if progress_bar
       end
-      # progress_bar.finish if progress_bar
-      # elapsed = Time.zone.now - start_time
-      # Notifier.deliver_simple_message('allan@curebit.com', '[CUREBIT] Metrics computation time', "Finished calculating #{metrics.count} metrics on #{total} #{name.underscore.humanize.downcase.pluralize} in #{elapsed/60} minutes (#{elapsed/total} sec per entry) (#{elapsed/(total*metrics.count)} sec per metric). \n\nMetrics calculated:\n\n#{metrics.join("\n")}")
-      metrics
+    end
+
+    def crank_aggregate_metrics!
+      aggregate_metrics.each do |metric_name, options|
+        ActiveRecord::Base.connection.execute options[:aggregate]
+        metrics_class.update_all "updated__#{metric_name}__at" => Time.current
+      end
     end
   end
   ### END CLASS METHODS, START INSTANCE METHODS
 
   def update_metrics!(*args)
-    self.class.metrics.each do |metric|
+    self.class.metrics.each do |metric, options|
       send(metric, *args)
     end
   end
+
   ### END INSTANCE METHODS
 
   ### Sets up a class like "SiteMetrics".  These are all CLASS methods:
@@ -139,11 +211,11 @@ module Metrics
     end
 
     def metrics_updated_at_columns
-      @object_class.metrics.map{|metric| "updated__#{metric}__at"}
+      @object_class.metrics.keys.map{|metric| "updated__#{metric}__at"}
     end
 
     def required_columns
-      @object_class.metrics.map(&:to_s) + metrics_updated_at_columns
+      @object_class.metrics.keys.map(&:to_s) + metrics_updated_at_columns
     end
 
     def missing_columns

data/lib/has_metrics/sql_capturer.rb

@@ -0,0 +1,29 @@
+class SqlCapturer
+  attr_accessor :query, :existing_logger
+
+  def initialize(existing_logger)
+    self.query = []
+    self.existing_logger = existing_logger
+    self
+  end
+
+  def error(string)
+    raise string
+  end
+
+  def debug?
+    true
+  end
+
+  def warn(message)
+    existing_logger.try(:warn, message)
+  end
+
+  def capture(log)
+    if select_statement = log.index("SELECT")
+      self.query << log[select_statement..-1].gsub(/\e\[(\d+)m/, '')
+    end
+    existing_logger.debug(log) if existing_logger
+  end
+  alias_method :debug, :capture
+end

data/lib/has_metrics/version.rb

@@ -1,3 +1,3 @@
 module HasMetrics
-  VERSION = "0.0.6"
+  VERSION = "0.0.9"
 end

data/lib/has_metrics.rb

@@ -1,10 +1,21 @@
 require "has_metrics/version"
 require "has_metrics/metrics"
 require "has_metrics/segmentation"
+require "has_metrics/sql_capturer"
 
 module HasMetrics
   def self.included(base)
     base.send :include, Metrics
     base.send :include, Segmentation
+
+    if defined?(::NewRelic)
+      base.send :include, ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      base.class_eval do
+        class << self
+          include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+          add_transaction_tracer :update_all_metrics!, category: :task
+        end
+      end
+    end
   end
-end
+end

data/spec/metrics_spec.rb

@@ -1,29 +1,65 @@
 require 'spec_helper'
 
-describe Metrics do
-  describe "defining metrics" do
-    let(:user) { User.create(:name => "Fuzz") }
+create_tables_for(:user)
 
-    before do
-      create_tables_for(:user)
+class Pet < ActiveRecord::Base
 
-      class User < ActiveRecord::Base
-        include Metrics
-        has_metric :name_length do
-          name.length
-        end
-      end
+end
 
-      User.update_all_metrics!
+class Identity < ActiveRecord::Base
+  has_one :user
+  has_many :activities, dependent: :destroy, foreign_key: :actor_id
+end
+
+class Activity < ActiveRecord::Base
+  belongs_to :actor, class_name: 'Identity'
+  attr_accessible :actor
+end
+
+class User < ActiveRecord::Base
+  attr_accessible :identity, :name
+  include Metrics
+  belongs_to :identity
+
+  has_many :pets, dependent: :destroy
+  has_many :activities, through: :identity
+
+  has_metric :name_length do
+    name.try(:length) || 0
+  end
+
+  has_metric :pets_count do
+    pets.count
+  end
+
+  has_metric :average_pet_weight, infer_aggregate: true do
+     pets.average(:weight)
+  end
+
+  has_metric :sent_activities do
+    activities.count
+  end
+end
+UserMetrics.migrate!
+User.update_all_metrics!
+
+
+describe Metrics do
+  describe "defining metrics" do
+    let(:user) { User.create(:name => "Fuzz") }
+    before { User.destroy_all }
+    after do
+      User.destroy_all
+      User.metrics.reject! {|m,o| m == :name_length_squared}
     end
 
     it "creates rows for the metrics" do
-      UserMetrics.columns.count.should == 3
+      UserMetrics.columns.count.should == 9
       User.has_metric :name_length_squared do
         name_length * name_length
       end
       User.update_all_metrics!
-      UserMetrics.columns.count.should == 5
+      UserMetrics.columns.count.should == 11
       user.name_length_squared.should == 16
     end
 
@@ -33,8 +69,6 @@ describe Metrics do
 
       user.name = "Bib"
 
-      # since 20 hours hasn't passed, the value is pulled from cache, not recalculated
-      user.name_length.should == 4
       # (true) forces it to recalculate right away
       user.name_length(true).should == 3
 
@@ -51,5 +85,52 @@ describe Metrics do
       User.update_all_metrics!
       UserMetrics.count(:group => :name_length).should == {4=>1}
     end
+
+    describe 'aggregate functions' do
+      before { User.create!(name: 'Goose').pets.create! weight: 265 }
+
+      it 'calls aggregate function alone' do
+        user.pets.create!(age: 1, weight: 2)
+        UserMetrics.any_instance.should_not_receive(:average_pet_weight=)
+        User.update_all_metrics!
+        expect(user.metrics.updated__average_pet_weight__at.to_i).to eql Time.current.to_i
+        expect(user.average_pet_weight).to eql 2
+      end
+
+      it 'arrives at the same value as the single instance calculation' do
+        user.pets.create!(age: 1, weight: 2)
+        user.pets.create!(age: 3, weight: 8)
+        User.update_all_metrics!
+        agg_result = user.average_pet_weight
+        single_result = user.average_pet_weight(true)
+        expect(agg_result.to_d).to eql single_result.to_d
+      end
+    end
+
+    describe 'collect_metrics' do
+      after { User.metrics.reject! {|k,v| k == :average_pet_age } }
+      it 'gives preferences to defined aggregate functions over detected ones' do
+        user.pets.create!(age: 1, weight: 2, age: 3)
+        User.has_metric :average_pet_age, single: -> { pets.average(:age) }, aggregate: 'SOME SQL'
+        UserMetrics.any_instance.should_not_receive(:average_pet_age=)
+        detected_aggregate_metrics, singular_metrics = User.collect_metrics(user)
+        expect(detected_aggregate_metrics.count).to eql 1
+        expect(singular_metrics.count).to eql 3
+        expect(User.aggregate_metrics.count).to eql 1
+      end
+    end
+
+    describe 'foreign key enable' do
+      it 'asdfsdf' do
+        User.create! # ensure user and identity don't have the same id
+        user = User.create!(name: 'bill', identity: Identity.create(thoughts: 'hurp'))
+        5.times do 
+          Activity.create!(actor: user.identity, type: 'wtf')
+        end
+        expect(user.metrics.sent_activities).to be_nil
+        User.update_all_metrics!
+        expect(user.metrics.reload.sent_activities).to eq(5)
+      end
+    end
   end
 end

data/spec/support/active_record.rb

@@ -4,7 +4,27 @@ ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
 
 def create_tables_for(model = :user)
   ActiveRecord::Migration.create_table "#{model}s", :force => true do |t|
-    t.string   "name"
+    t.string   :name
+    t.integer :identity_id
   end
   ActiveRecord::Migration.create_table "#{model}_metrics", :force => true
 end
+
+
+ActiveRecord::Migration.create_table :pets, :force => true do |t|
+  t.string :name
+  t.string :type
+  t.integer :user_id
+  t.integer :age
+  t.integer :weight
+end
+
+ActiveRecord::Migration.create_table :identities do |t|
+  t.string :thoughts
+end
+
+ActiveRecord::Migration.create_table :activities do |t|
+  t.integer :actor_id
+  t.string :type
+end
+

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: has_metrics
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.1
 platform: ruby
 authors:
 - Allan Grant
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-01 00:00:00.000000000 Z
+date: 2013-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Calculate metrics and store them in the DB.
@@ -59,7 +59,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE
 - README.md
@@ -68,6 +68,7 @@ files:
 - lib/has_metrics.rb
 - lib/has_metrics/metrics.rb
 - lib/has_metrics/segmentation.rb
+- lib/has_metrics/sql_capturer.rb
 - lib/has_metrics/version.rb
 - spec/metrics_spec.rb
 - spec/segmentation_spec.rb
@@ -83,17 +84,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Calculate "metrics" (any expensive methods) on ActiveRecord entries and memoize