formula_eval 0.0.1 → 0.1.0

data/Rakefile

@@ -11,6 +11,9 @@ begin
     gem.homepage = "http://github.com/mharris717/formula_eval"
     gem.authors = ["Mike Harris"]
     gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency 'mharris_ext'
+    gem.add_dependency 'nested_hash_tricks'
+    gem.add_dependency 'safe_eval'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.1.0

data/lib/formula_eval/calculating_collection.rb

@@ -1,55 +1,134 @@
-class CalculatingCollection
+
+
+class Hash
+  def each_with_str_key
+    each do |k,v|
+      yield(k.to_s,v)
+    end 
+  end
+  def nested_hash_set(*args)
+    args = args.flatten
+    KeyParts.with_parts(args[0..-2],:array => true) do |start,lst,mult|
+      start.each { |k| self[k] ||= {} }
+      self[lst] = args[-1]
+    end
+  end
+end
+
+class WrappingCollection
   attr_accessor :coll
   include FromHash
+  def method_missing(sym,*args,&b)
+    coll.send(sym,*args,&b)
+  end
+  def user_coll
+    coll.user_coll
+  end
+  fattr(:user_coll_last_calc_dt) do
+    user_coll.save! unless user_coll.last_calc_dt
+    user_coll.last_calc_dt
+  end
+end
+
+class CalculatingCollection < WrappingCollection
   fattr(:column_hash) do
-    res = {}
-    res[:zzz] = lambda { |doc| (doc['abc']||0).to_i + 7 }
-    res[:xyz] = FormulaEval.new(:formula => '=(abc||0)+8')
-    res[:percs] = FormulaEval.new(:formula => '=pension.perc')
-    res[:percs2] = FormulaEval.new(:formula => '=pension.perc * 2.0')
-    res['pension.just_perc'] = FormulaEval.new(:formula => '=perc')
-    res['pension.just_perc_plus'] = FormulaEval.new(:formula => '=perc+1')
-    res['pension.perc_plus_salary'] = FormulaEval.new(:formula => '=perc+salary')
-    res
+    {}
+  end
+  fattr(:constants_hash) do
+    {}
+  end
+  def add_constant_column(name,formula)
+    arr = eval(formula).to_a
+    constants_hash[name] = arr
+  end
+  def add_column(name,blk)
+    blk = FormulaEval.new(:formula => blk, :coll => coll) if blk.kind_of?(String)
+    self.column_hash[name.to_s] = blk
+  end
+  def cleaned_doc(doc)
+    column_hash.keys.each do |k|
+      doc.dot_set(k,nil)
+    end
+    doc
   end
-  def enriched_doc(doc)
-    column_hash.each do |col,blk|
-      # mylog 'enriched_doc', :col => col, :blk_class => blk.class, :doc => doc
-      if col.to_s.split('.').size == 1
+  def constants_enriched_doc(doc,ops)
+    constants_hash.each do |col,vals|
+      vals = vals.to_a
+      KeyParts.with_parts(col) do |start,lst,mult|
+        if mult
+          doc[start] ||= []
+          vals.each_with_index do |val,i|
+            field = "#{start}.#{i}.#{lst}"
+            doc.dot_set(field,val)
+          end
+        elsif ops[:row_index]
+          doc[col] = vals[ops[:row_index]]
+        end
+      end
+    end
+    doc
+  end
+  def calc_enriched_doc(doc,ops)
+    doc = cleaned_doc(doc)
+    column_hash.each_with_str_key do |col,blk|
+      if KeyParts.single?(col)
         val = blk.call(doc)
-        mylog 'dot_set',:col => col, :val => val, :doc => doc
-        doc.dot_set(col.to_s,val)  
+        doc.dot_set(col,val)  
       else
-        base = col.to_s.split('.')[0..-2].join(".")
-        obj = MultiEval.get_nested(doc,base)
-        val = blk.call(obj)
-        mylog 'dot_set',:col => col, :val => val, :doc => doc, :obj => obj
-        doc.dot_set(col.to_s,val)
+        doc.dot_set(col) do |obj|
+          multi = MultiEval.new(:objs => [obj,doc])
+          blk.call(multi)
+        end
       end
     end
     doc.to_unwrapped
   end
-  def save(doc,ops={})
-    mylog 'calc_save', :doc => doc, :doc_class => doc.class, :unwrapped => doc.to_unwrapped.class
-    coll.save(enriched_doc(doc),ops)
+  def enriched_doc(doc,other_ops={})
+    doc = constants_enriched_doc(doc,other_ops)
+    doc = calc_enriched_doc(doc,other_ops)
+  end
+  def keys
+    ks = column_hash.keys.map { |x| x.split('.').first }.uniq
+    coll.keys.make_last(ks)
+  end
+  def save(doc,ops={},other_ops={})
+    doc = enriched_doc(doc,other_ops)
+    doc.nested_hash_set '_admin','last_calc_dt',Time.now
+    
+    res = coll.save(doc,ops)
+    doc[:_id] = res
+    doc
   end
   def update_calcs!
-    find({},{:limit => 9999}).each do |row|
-      save(row)
+  end
+  def update_calcs_real!
+    constants_max = constants_hash.values.map { |x| x.to_a.size }.max || 0
+    find({},{:limit => 9999}).each_with_index do |row,i|
+      save(row,{},:row_index => i)
+    end
+    if constants_max > count
+      (count...constants_max).each do |i|
+        save({},{},:row_index => i)
+      end
     end
+  rescue => exp
+    mylog 'update_calcs', :trace => exp.backtrace.join("\n"), :message => exp.message
+    raise exp
   end
   def update_row(row_id,fields)
-    row = (row_id == 'NEW') ? {} : find_by_id(row_id)
-    raise "can't find row #{row_id} #{row_id.class} in coll #{name}.  Count is #{find.count} IDs are "+find.to_a.map { |x| x['_id'] }.inspect + "Trying to update with #{fields.inspect}" unless row
-    fields.each do |k,v|
-      row.dot_set(k,mongo_value(v))
-      row.delete(k) if v.blank?
-    end
-    row = enriched_doc(row)
-    save(row)
-    row
+    row = coll.update_row(row_id,fields)
+    mylog 'calc_update_row', :row => row, :coll => coll.class
+    row = save(row)
   end
-  def method_missing(sym,*args,&b)
-    coll.send(sym,*args,&b)
+  def needs_calc?(doc)
+    doc_dt = doc['_admin'].andand['last_calc_dt'] || Time.local(1970,1,1)
+    user_coll_last_calc_dt > doc_dt
+  end
+  def find(selector={},ops={})
+    res = coll.find(selector,ops)
+    if res.respond_to?(:each)
+      res.select { |doc| needs_calc?(doc) }.each { |doc| save(doc) }
+    end
+    res
   end
 end

data/lib/formula_eval/multi_eval.rb

@@ -27,9 +27,10 @@ class MultiEval
   end
   def method_missing(sym,*args,&b)
     objs.each do |obj|
-      if obj.respond_to?(sym)
+      if obj.smart_respond_to?(sym,args)
         res = obj.send(sym,*args,&b)
-        return MultiEval.new(:objs => [res]+without_obj(obj)) if res
+        #return MultiEval.new(:objs => [res]+without_obj(obj)) #if res
+        return res if res
       end
     end
     raise 'none respond'
@@ -48,9 +49,31 @@ class MultiEval
     objs.first * x
   end
   def self.get_nested(obj,method_str)
-    other = obj.instance_eval(method_str)
+    method_str = fix_str(method_str)
+    other = obj.to_wrapped.safe_instance_eval(method_str)
     mylog 'get_nested', :obj => obj, :method_str => method_str, :other => other
     arr = [other,obj]
     new(:objs => arr)
   end
+  def to_unwrapped
+    objs.first.to_unwrapped
+  end
+  def self.fix_str(str)
+    str.gsub(/\.(\d+)\./) { "._arrayindex_#{$1}." }.gsub(/\.(\d+)$/) { "._arrayindex_#{$1}" }
+  end
+end
+
+class Object
+  def smart_respond_to?(k,args)
+    sup = respond_to?(k)
+    if k.to_s == '[]'
+      return false if args.first.kind_of?(Numeric) && kind_of?(Hash)
+      return false if !args.first.kind_of?(Numeric) && kind_of?(Array)
+      return false if !args.first.kind_of?(Numeric) && kind_of?(Numeric)
+      return false if !args.first.kind_of?(Numeric) && kind_of?(String)
+      sup
+    else
+      sup
+    end
+  end
 end

data/lib/formula_eval/wrapper.rb

@@ -33,14 +33,52 @@ class HashWrapper
     obj[k.to_s] = v
   end
   def obj; hash; end
+  def to_wrapped
+    self
+  end
   def to_unwrapped
     hash.to_unwrapped
   end
   def respond_to?(sym)
     obj.keys.include?(sym.to_s)
   end
+  def delete(k)
+    obj.delete(k)
+  end
+  def kind_of?(x)
+    return true if x == Hash
+    super
+  end
+  def keys
+    obj.keys
+  end
+end
+
+class Array
+  def contains_all_hashes?
+    all? { |x| x.kind_of?(Hash) || x.kind_of?(OrderedHash) }
+  end
+end
+
+class Array
+  def method_missing(sym,*args,&b)
+    if sym.to_s =~ /_arrayindex_(\d+)/
+      self[$1.to_i].to_wrapped
+    else
+      super
+    end 
+  end
+end
+
+module ArrayMod
+  def [](i)
+    raise "tried to pass string #{i} to array [] #{inspect}" unless i.kind_of?(Fixnum)
+    super
+  end
 end
 
+Array.send(:include,ArrayMod)
+
 class ArrayWrapper
   attr_accessor :obj
   include FromHash
@@ -48,20 +86,34 @@ class ArrayWrapper
     map { |h| h[sym.to_s] }.select { |x| x }.flatten.to_wrapped
   end
   def method_missing(sym,*args,&b)
-    if obj.respond_to?(sym)
+    res = if obj.respond_to?(sym)
       obj.send(sym,*args,&b).to_wrapped
-    elsif contains_all_hashes?
+    elsif sym.to_s =~ /_arrayindex_(\d+)/
+      self[$1.to_i]
+    elsif obj.contains_all_hashes?
       hash_mm(sym)
     else
       obj.send(sym,*args,&b).to_wrapped
     end
+    res
+  end
+  def [](i)
+    raise "tried to pass string #{i} to array [] #{inspect}" unless i.kind_of?(Fixnum)
+    obj[i].to_wrapped
   end
   def *(arg)
     map { |x| x * arg }
   end
+  def to_wrapped
+    self
+  end
   def to_unwrapped
     obj.to_unwrapped
   end
+  def kind_of?(x)
+    return true if x == Array
+    super
+  end
 end
 
 class Array

data/lib/formula_eval.rb

@@ -1,15 +1,26 @@
 require 'mharris_ext'
-require 'active_support'
+require 'safe_eval'
 
 class Object
   attr_accessor :calculating_formula
 end
 
-
-  
+class WrappingProxy
+  attr_accessor :obj
+  include FromHash
+  def method_missing(sym,*args,&b)
+    WrappingProxy.new(:obj => obj.send(sym,*args,&b).to_wrapped)
+  end
+  def respond_to?(x)
+    obj.respond_to?(x)
+  end
+  def kind_of?(x)
+    obj.kind_of?(x)
+  end
+end
 
 class FormulaEval
-  attr_accessor :row, :formula, :row_index, :rows
+  attr_accessor :row, :formula, :row_index, :rows, :coll
   include FromHash
   def current_user
     $current_user
@@ -24,21 +35,25 @@ class FormulaEval
     elsif row.respond_to?('[]')
       wrapped_row[sym.to_s]
     else
+      puts "mm #{sym} #{args.inspect}"
       super
     end
   ensure
     # puts "Formula mm #{sym} #{args.inspect} #{res.inspect}"
   end
+  def fixed_formula
+    MultiEval.fix_str(formula)
+  end
   def safe_eval_result
-    instance_eval(formula).to_unwrapped
-  # rescue => exp
-  #   t = exp.backtrace.join("\n").gsub("/Users/mharris/.rvm/gems/ruby-1.9.1-p378/gems","gems").gsub("/Users/mharris/Code/smartlist/vendor/mongo_ui","mongo_ui")
-  #   t = t.gsub("/Users/mharris/Code/smartlist","smartlist")
-  #   mylog "formula_eval", :formula => formula, :row => row, :message => exp.message, :trace => t
-  #   "Error #{exp.message}"
+    safe_instance_eval(fixed_formula).to_unwrapped
   rescue => exp
+     t = exp.backtrace.join("\n").gsub("/Users/mharris/.rvm/gems/ruby-1.9.1-p378/gems","gems").gsub("/Users/mharris/Code/smartlist/vendor/mongo_ui","mongo_ui")
+     t = t.gsub("/Users/mharris/Code/smartlist","smartlist")
+     mylog "formula_eval", :formula => formula, :row => row, :message => exp.message, :trace => t
+     "Error #{exp.message}"
+  #rescue => exp
     # puts "error evaling #{formula} against #{row.inspect}, #{exp.message}"
-    raise exp
+    #raise exp
   end
   def result
     self.formula = formula[1..-1] if formula[0..0] == '='
@@ -66,15 +81,27 @@ class FormulaEval
   end
 end
 
-load File.dirname(__FILE__) + "/formula_eval/wrapper.rb"
-load File.dirname(__FILE__) + "/formula_eval/calculating_collection.rb"
-load File.dirname(__FILE__) + "/formula_eval/multi_eval.rb"
 
-def mylog(*args)
-  yield if block_given?
-  #puts args.inspect if args.first == 'enriched_doc' or args.first == 'dot_set'
+
+class FormulaEval
+  def self.load_subfiles!
+    load File.dirname(__FILE__) + "/formula_eval/wrapper.rb"
+    load File.dirname(__FILE__) + "/formula_eval/calculating_collection.rb"
+    load File.dirname(__FILE__) + "/formula_eval/multi_eval.rb"
+  end
+  def self.load_self!
+    load File.dirname(__FILE__) + "/formula_eval.rb"
+  end
+  def self.load_files!
+    load_subfiles!
+    load_self!
+  end
 end
 
+FormulaEval.load_subfiles!
+require 'nested_hash_tricks'
+
+
 class Object
   def klass
     self.class
@@ -90,79 +117,3 @@ class Object
   end
 end
 
-class Object
-  def dot_get(str)
-    str = str.split(".") if str.is_a?(String)
-    res = self
-    last_f = last_res = nil
-    str.each do |f|
-      if f.num? && !res.kind_of?(Array)
-        last_res[last_f] = res = []
-      end
-      last_res = res
-      if res.kind_of?(Array)
-        temp = res[f.safe_to_i]
-        if !temp
-          res << {}
-          temp = res.last
-          raise "can only add new row at end" unless res.size-1 == f.safe_to_i
-        end
-        res = temp
-      else
-        res = res[f]
-      end
-      last_f = f
-    end
-    res
-  end
-  def dot_set(str,val)
-    mylog 'dot_set', :str => str, :val => val, :self => self do
-      return self[str] = val if str.split(".").size == 1
-      strs = str.split(".")[0..-2]
-      lst = str.split(".")[-1]
-      obj = dot_get(strs)
-      return obj unless obj
-      #puts "dot_set, obj is #{obj.inspect}, str is #{str}, val is #{val}, lst is #{lst}"
-      obj.nested_set(lst,val)
-    end
-  end
-end
-
-class Object
-  def nested_set(k,v)
-    self[k] = v
-  end
-end
-
-class Array
-  def nested_set(k,v)
-    mylog 'dot_set', :context => 'nested', :klass => klass, :self => self, :k => k, :v => v do
-      each { |x| x.nested_set(k,v) } 
-    end
-  end
-end
-
-class String
-  def num?
-    size > 0 && self =~ /^[\d\.]*$/
-  end
-  def date?
-    matches = (self =~ /\/\d+\//) || (self =~ /-\d+-/)
-    matches2 = self =~ /^[ \d\-\/:]+$/
-    !!(matches && matches2 && Time.parse(self))
-  rescue
-    return false
-  end
-  def to_time
-    Time.parse(self)
-  end
-  def tmo
-    if num? 
-      to_f.tmo 
-    elsif blank?
-      nil
-    else
-      self
-    end
-  end
-end

data/spec/calculating_collection_spec.rb

@@ -6,8 +6,11 @@ describe "CalculatingCollection" do
   fattr(:formula) { FormulaEval.new(:row => row) }
   fattr(:column) { 'pension.perc_plus_salary' }
   fattr(:str) { '=perc*salary' }
+  fattr(:column_hash) { {column => FormulaEval.new(:formula => str)} }
+  fattr(:constants_hash) { {} }
   fattr(:calc) do
-    CalculatingCollection.new(:column_hash => {column => FormulaEval.new(:formula => str)})
+    CalculatingCollection.new(:column_hash => column_hash, :constants_hash => constants_hash, 
+                              :user_coll_last_calc_dt => Time.now, :coll => SilentMM.new)
   end
   fattr(:enriched) do
     calc.enriched_doc(wrapped_row)
@@ -23,4 +26,47 @@ describe "CalculatingCollection" do
     self.str = '=year+1'
     enriched['pension']['perc_plus_salary'].should == 2026
   end
+  it 'sub array' do
+    self.row['pension'] = [{'year' => 2025, 'perc' => 0.65},{'year' => 2026, 'perc' => 0.7}]
+    enriched['pension'][0]['perc_plus_salary'].should == 42000.0*0.65
+  end
+  it 'reverse uniq' do
+    a = [1,2,3,4,5,2,7]
+    a.reverse.uniq.reverse.should == [1,3,4,5,2,7]
+  end
+  it 'saving' do
+    calc.save(row)
+    row['pension']['perc_plus_salary'].should == 42000*0.65
+  end
+  describe 'constants' do
+    before do
+      self.row['pension'] = [{'year' => 2025, 'perc' => 0.65},{'year' => 2026, 'perc' => 0.7}]
+    end
+    it 'sub level' do
+      self.constants_hash = {'pension.one_five' => [1,2,3,4,5]}
+      calc.save(row)
+      row['pension'][0]['one_five'].should == 1
+      row['pension'].size.should == 5
+      row['pension'][3]['one_five'].should == 4
+    end
+    it 'top level' do
+      self.constants_hash = {'one_five' => [1,2,3,4,5]}
+      calc.save(row,{},:row_index => 2)
+      row['one_five'].should == 3
+    end
+  end
+  describe 'safe_eval' do
+    it 'shellout' do
+      SafeEval.with_level(4) do
+        self.str = "=`rm -r -f /code/fghgfhrthyhthyht`; perc*salary"
+        lambda { enriched }.should raise_error(SecurityError)
+      end
+    end
+    # it 'change' do
+    #   self.str = "=self.year = 42; perc*salary"
+    #   #lambda { enriched }.should raise_error(SecurityError)
+    #   raise enriched.inspect
+    # end
+  end
+  
 end

data/spec/formula_eval_spec.rb

@@ -20,11 +20,19 @@ class B
   end
 end
 
-def bt
-  raise 'foo'
-rescue => exp
-  puts exp.message
-  puts exp.backtrace.join("\n")
+class C
+  def c
+    3
+  end
+  def c_array
+    [{'d' => 4, 'e' => 5},{'d' => 6, 'e' => 7}]
+  end
+  def c_hash
+    {'d' => 4, 'e' => 5}
+  end
+  def [](k)
+    send(k)
+  end
 end
 
 describe "FormulaEval" do
@@ -37,9 +45,6 @@ describe "FormulaEval" do
   it 'double_year' do
     self.str = '=year*2'
     formula.result.should == 4020
-  end
-  it 'nested' do
-    
   end
   it 'double' do
     self.row = MultiEval.new(:objs => [A.new,B.new])
@@ -57,4 +62,38 @@ describe "FormulaEval" do
     self.str = '=year'
     formula.result.should == 2010
   end
+  it 'nested with array' do
+    self.row = MultiEval.get_nested(C.new,'c_array._arrayindex_0')
+    row.objs[0].obj.should == {'d' => 4, 'e' => 5}
+  end
+  it 'nested with array 2' do
+    MultiEval.fix_str('c_array.0').should == 'c_array._arrayindex_0'
+    MultiEval.fix_str('c_array.0.d').should == 'c_array._arrayindex_0.d'
+  end
+  it 'nested with array 3' do
+    self.row = MultiEval.get_nested(C.new,'c_array.0')
+    row.objs[0].obj.should == {'d' => 4, 'e' => 5}
+    self.str = '=c+c_array.0.d'
+    formula.result.should == 7
+  end
+  it 'nested with array 4' do
+    self.row = MultiEval.get_nested(C.new,'c_array.0')
+    self.str = '=c+d'
+    formula.result.should == 7
+  end
+  it 'nested with array 5' do
+    self.row = MultiEval.get_nested(C.new,'c_array.1')
+    self.str = '=c+d'
+    formula.result.should == 9
+  end
+  it 'nested with array' do
+    self.row = MultiEval.get_nested(C.new,'c_array._arrayindex_0')
+    #raise row.objs.inspect
+    self.str = '=c+d'
+    formula.result.should == 7
+  end
+  # it 'double 3' do
+  #   hash = {'year' => 2010, 'pension' => [{'start_year' => 2025, 'perc' => 0.65},{'start_year' => 2026, 'perc' => 0.7}]}
+  #   self.row = MultiEval.get_nested(Wrapper.new(hash),'pension')
+  # end
 end

data/spec/spec_helper.rb

@@ -7,3 +7,31 @@ require 'spec/autorun'
 Spec::Runner.configure do |config|
   
 end
+
+def mylog(*args)
+  yield if block_given?
+  #puts args.inspect if args.first == 'enriched_doc' or args.first == 'dot_set'
+end
+
+def bt
+  raise 'foo'
+rescue => exp
+  puts exp.message
+  puts exp.backtrace.join("\n")
+end
+
+class SilentMM
+  def method_missing(*args)
+  end
+end
+
+class String
+  def safe_to_i
+     num? ? to_i : (raise 'not num')
+  end
+end
+
+def debug_log(*args)
+end
+
+DEFAULT_SAFE_LEVEL = 0

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 0
   - 1
-  version: 0.0.1
+  - 0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Mike Harris
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-20 00:00:00 -04:00
+date: 2010-06-01 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -31,6 +31,42 @@ dependencies:
         version: 1.2.9
   type: :development
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mharris_ext
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: nested_hash_tricks
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: safe_eval
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: formula_eval
 email: mharris717@gmail.com
 executables: []