marty 0.5.12

data/lib/marty/data_row_processor.rb~

@@ -0,0 +1,206 @@
+class Marty::DataRowProcessor
+  attr_accessor :klass, :headers, :dt, :key_attrs, :hmap
+
+  EXCEL_START_DATE = Date.parse('1/1/1900')-2
+
+  MCFLY_COLUMNS = Set[
+                      "id",
+                      "group_id",
+                      "user_id",
+                      "created_dt",
+                      "obsoleted_dt",
+                      "o_user_id",
+                     ]
+
+  # Given a Mcfly class, return the set of attributes (excluding id)
+  # used to uniquely identify an instance.
+  def self.get_keys(klass)
+    raise "bad class arg #{klass}" unless
+      klass.is_a?(Class) && klass < ActiveRecord::Base
+
+    attrs = klass.const_get(:MCFLY_UNIQUENESS)
+
+    raise "class has no :MCFLY_UNIQUENESS" unless attrs
+
+    attrs = attrs[0..-2] + attrs.last.fetch(:scope, []) if
+      attrs.last.is_a?(Hash)
+    attrs -= [:obsoleted_dt]
+
+    raise "key list for #{klass} is empty" if attrs.empty?
+    attrs
+  end
+
+  def self.assoc_info(klass, a)
+    assoc_class = klass.reflect_on_association(a.to_sym).klass
+    keys = self.get_keys(assoc_class) rescue nil
+
+    assoc_keys = keys || [assoc_class.attribute_names.reject{|x| x=="id"}.first]
+
+    {assoc_keys: assoc_keys, assoc_class: assoc_class, mcfly: keys}
+  end
+
+  FLOAT_PAT = /^-?\d+(\.\d+)?$/
+
+  PATS = {
+    integer: /^-?\d+(\.0+)?$/,
+    float:   FLOAT_PAT,
+    decimal: FLOAT_PAT,
+  }
+
+  def convert(v, type)
+    pat = PATS[type]
+
+    raise "bad #{type} #{v.inspect}" if
+      v.is_a?(String) && pat && !(v =~ pat)
+
+    case type
+    when :boolean
+      case v.downcase
+      when "true" then true
+      when "false" then false
+      else raise "unknown boolean #{v}"
+      end
+    when :string, :text
+      v
+    when :integer
+      v.to_i
+    when :float
+      v.to_f
+    when :decimal
+      v.to_d
+    when :date
+      # Dates are kept as float in Google spreadsheets.  Need to
+      # convert them to dates. FIXME: 'infinity' as a date in
+      # Rails 3.2 appears to be broken. Setting a date field to
+      # 'infinity' sets it to nil.
+      v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
+        Mcfly.is_infinity(v) ? 'infinity' : v.to_date
+    when :datetime
+      Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
+    when :numrange, :int4range, :int8range
+      v.to_s
+    when :float_array, :json
+      JSON.parse Marty::DataExporter.decode_json(v)
+    else
+      raise "unknown type #{type} for #{v}"
+    end
+  end
+
+  def initialize(klass, headers, dt)
+    @klass     = klass
+    @headers   = headers
+    @dt        = dt
+    @key_attrs = self.class.get_keys(klass)
+
+    # # HACK: not sure why there's a nil at the end of headers sometimes
+    # headers.pop if headers[-1].nil?
+
+    raise "row headers have nil! #{headers.inspect}" unless headers.all?
+
+    associations = klass.reflect_on_all_associations.map(&:name)
+
+    cols = klass.columns.each_with_object({}) { |c, h|
+      h[c.name] = c
+    }
+
+    @hmap = headers.each_with_object({}) do
+      |a, h|
+      # handle klass__attr type headers generated by Netzke.  Just
+      # keeps klass since we should be able to find the key attr.
+      aclass = a.split('__').first
+
+      if associations.member?(aclass.to_sym)
+        h[a] = self.class.assoc_info(klass, aclass)
+        next
+      end
+
+      raise "unknown column #{a}" unless cols[a]
+
+      # for JSON fields in Rails 3.x type is nil, so use sql_type
+      type = cols[a].type || cols[a].sql_type
+      type = "#{type}_array" if cols[a].array
+      h[a] = type.to_sym
+    end
+  end
+
+  def create_or_update(row)
+    assoc_options = {}
+
+    options = row.each_with_object({}) do |(a, v), h|
+      # ignore Mcfly columns
+      next if MCFLY_COLUMNS.member? a
+
+      if hmap[a].is_a?(Hash)
+        assoc_options[a] = v
+      else
+        raise "bad col #{a} value #{v}, row: #{row}" unless hmap[a]
+
+        # if it's not a hash (association) then its a type symbol
+        h[a] = v && convert(v, hmap[a])
+      end
+    end
+
+    assoc_groups = assoc_options.keys.group_by {|x| x.split('__').first}
+
+    assoc_groups.each do |aclass, attrs|
+      # if group has only one attr and the attr is nil or AR obj, then
+      # we don't need to search.
+      if attrs.length == 1
+        a = attrs.first
+        v = assoc_options[a]
+
+        if v.nil? || v.is_a?(ActiveRecord::Base)
+          options["#{aclass}_id"] = v && v.id
+          next
+        end
+      end
+
+      srch = attrs.each_with_object({}) do |a, h|
+        v = assoc_options[a]
+        ac, aa = a.split('__')
+
+        aa ||= hmap[a][:assoc_keys].first
+
+        h[aa] = v
+        h[:obsoleted_dt] = 'infinity' if hmap[a][:mcfly]
+      end
+
+      srch_class = hmap[attrs.first][:assoc_class]
+
+      av = srch_class.where(srch).first
+
+      raise "#{aclass} not found #{srch}" unless av
+
+      options[ "#{aclass}_id" ] = av.id
+    end
+
+    find_options = options.select { |k,v| key_attrs.member? k.to_sym }
+
+    raise "invalid entry" if find_options.empty?
+
+    find_options['obsoleted_dt'] = 'infinity'
+
+    obj = klass.where(find_options).first || klass.new
+
+    options.each do
+      |k, v|
+      # For each attr, check to see if it's begin changed before
+      # setting it.  The AR obj.changed? doesn't work properly
+      # with array, JSON or lazy attrs.
+      obj.send("#{k}=", v) if obj.send(k) != v
+    end
+
+    # FIXME: obj.changed? doesn't work properly for timestamp
+    # fields in Rails 3.2. It evaluates to true even when datetime
+    # is not changed.  Caused by lack of awareness of timezones.
+    tag = obj.new_record? ? :create : (obj.changed? ? :update : :same)
+
+    raise "old created_dt >= current #{obj} #{obj.created_dt} #{dt}" if
+      (tag == :update) && !Mcfly.is_infinity(dt) && (obj.created_dt > dt)
+
+    obj.created_dt = dt unless tag == :same || Mcfly.is_infinity(dt)
+    obj.save!
+
+    [tag, obj.id]
+  end
+end

data/lib/marty/drop_folder_hook.rb~

@@ -0,0 +1,17 @@
+class Marty::DropFolderHook
+  def initialize(login)
+    @login = login
+  end
+
+  def run(res)
+    p 'H.'*10, res
+    drop_path = "/tmp/#{@login}"
+
+    begin
+      Dir.mkdir(drop_path) unless File.directory?(drop_path)
+    rescue => exc
+      Marty::Util.logger.error "Can't create 
+
+
+  end
+end

data/lib/marty/engine.rb

@@ -0,0 +1,10 @@
+module Marty
+  class Engine < ::Rails::Engine
+    isolate_namespace Marty
+
+    # generators add rspec tests
+    config.generators do |g|
+      g.test_framework :rspec, :view_specs => false
+    end
+  end
+end

data/lib/marty/folder_hook.rb~

@@ -0,0 +1,9 @@
+class Marty::FolderHook
+  def initialize
+  end
+
+  def run(res)
+    p 'H.'*10, res
+  end
+
+end

data/lib/marty/lazy_column_loader.rb

@@ -0,0 +1,57 @@
+# ATTRIBUTION NOTE: This module has been mostly copied from the
+# lazy_columns gem. The original code can be found at:
+# https://github.com/jorgemanrubia/lazy_columns
+
+module Marty
+  module LazyColumnLoader
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def lazy_load(*columns)
+        return unless table_exists?
+        columns = columns.collect(&:to_s)
+        exclude_columns_from_default_scope columns
+        define_lazy_accessors_for columns
+
+        # allow introspection of lazy-loaded column list
+        const_set(:LAZY_LOADED, columns)
+      end
+
+    private
+      def exclude_columns_from_default_scope(columns)
+        default_scope {
+          select((column_names - columns).map {
+                   |column_name|
+                   "#{table_name}.#{column_name}"
+                 })
+        }
+      end
+
+      def define_lazy_accessors_for(columns)
+        columns.each { |column| define_lazy_accessor_for column }
+      end
+
+      def define_lazy_accessor_for(column)
+        define_method column do
+          unless has_attribute?(column)
+            changes_before_reload = self.changes.clone
+            self.reload
+            changes_before_reload.each{
+              |attribute_name, values|
+              self.send("#{attribute_name}=", values[1])
+            }
+          end
+          read_attribute column
+        end
+      end
+    end
+  end
+end
+
+if ActiveRecord::Base.respond_to?(:lazy_load)
+  $stderr.puts "ERROR: Method `.lazy_load` already defined in " +
+    "`ActiveRecord::Base`. This is incompatible with LazyColumnLoader " +
+    "and the module will be disabled."
+else
+  ActiveRecord::Base.send :include, Marty::LazyColumnLoader
+end

data/lib/marty/lazy_column_loader.rb~

@@ -0,0 +1,47 @@
+module Marty
+  module LazyColumnLoader
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def lazy_load(*columns)
+        return unless table_exists?
+        columns = columns.collect(&:to_s)
+        exclude_columns_from_default_scope columns
+        define_lazy_accessors_for columns
+      end
+
+    private
+      def exclude_columns_from_default_scope(columns)
+        default_scope select((
+                              column_names - columns).map {
+                               |column_name|
+                               "#{table_name}.#{column_name}"
+                             })
+      end
+
+      def define_lazy_accessors_for(columns)
+        columns.each { |column| define_lazy_accessor_for column }
+      end
+
+      def define_lazy_accessor_for(column)
+        define_method column do
+          unless has_attribute?(column)
+            changes_before_reload = self.changes.clone
+            self.reload
+            changes_before_reload.each{|attribute_name, values|
+              self.send("#{attribute_name}=", values[1])}
+          end
+          read_attribute column
+        end
+      end
+    end
+  end
+end
+
+if ActiveRecord::Base.respond_to?(:lazy_load)
+  $stderr.puts "ERROR: Method `.lazy_load` already defined in " +
+    "`ActiveRecord::Base`. This is incompatible with LazyColumnLoader " +
+    "and the module will be disabled."
+else
+  ActiveRecord::Base.send :include, Marty::LazyColumnLoader
+end

data/lib/marty/mcfly_query.rb

@@ -0,0 +1,189 @@
+require 'mcfly'
+
+module Mcfly
+  module Model
+
+    def self.included(base)
+      base.send :extend, ClassMethods
+    end
+
+    module ClassMethods
+      def clear_lookup_cache!
+        @LOOKUP_CACHE.clear if @LOOKUP_CACHE
+      end
+
+      # Implements a VERY HACKY class-based caching mechanism for
+      # database lookup results.  Issues include: cached values are
+      # ActiveRecord objects.  Not sure if these should be shared
+      # across connections.  Query results can potentially be very
+      # large lists which we simply count as one item in the cache.
+      # Caching mechanism will result in large processes.  Caches are
+      # not sharable across different Ruby processes.
+      def cached_delorean_fn(name, options = {}, &block)
+        @LOOKUP_CACHE ||= {}
+
+        delorean_fn(name, options) do |ts, *args|
+          cache_key = [name, ts] + args.map{ |a|
+            a.is_a?(ActiveRecord::Base) ? a.id : a
+          } unless Mcfly.is_infinity(ts)
+
+          next @LOOKUP_CACHE[cache_key] if
+            cache_key && @LOOKUP_CACHE.has_key?(cache_key)
+
+          res = block.call(ts, *args)
+
+          if cache_key
+            # Cache has >1000 items, clear out the oldest 200.  FIXME:
+            # hard-coded, should be configurable.  Cache
+            # size/invalidation should be per lookup and not class.
+            # We're invalidating cache items simply based on age and
+            # not usage.  This is faster but not as fair.
+            if @LOOKUP_CACHE.count > 1000
+              @LOOKUP_CACHE.keys[0..200].each{|k| @LOOKUP_CACHE.delete(k)}
+            end
+            @LOOKUP_CACHE[cache_key] = res
+
+            # Since we're caching this object and don't want anyone
+            # changing it.  FIXME: ideally should freeze this object
+            # recursively.
+            res.freeze unless res.is_a?(ActiveRecord::Relation)
+          end
+          res
+        end
+      end
+
+      # FIXME: duplicate code from Mcfly's mcfly_lookup.
+      def cached_mcfly_lookup(name, options = {}, &block)
+        cached_delorean_fn(name, options) do |ts, *args|
+          raise "time cannot be nil" if ts.nil?
+
+          ts = Mcfly.normalize_infinity(ts)
+
+          where("obsoleted_dt >= ? AND created_dt < ?", ts, ts).scoping do
+            block.call(ts, *args)
+          end
+        end
+      end
+
+      # FIXME: for validation purposes, this mechanism should make
+      # sure that the allable attrs are not required.
+      def gen_mcfly_lookup(name, attrs, options={})
+        raise "bad options" unless options.is_a?(Hash)
+
+        # FIXME: mode should be sent later, not as a part of
+        # gen_mcfly_lookup.  i.e. we just generate the search and the
+        # mode is applied at runtime by delorean code.  That would
+        # allow lookups to be used in either mode dynamically.
+        mode = options.fetch(:mode, :first)
+
+        assoc = Set.new(self.reflect_on_all_associations.map(&:name))
+        attr_names = attrs.keys
+
+        allables = attrs.select {|k, v| v}
+
+        order = allables.keys.reverse.map { |k|
+          k = "#{k}_id" if assoc.member?(k)
+          "#{k} NULLS LAST"
+        }.join(", ")
+
+        qstr = attrs.map {|k, v|
+          k = "#{k}_id" if assoc.member?(k)
+          v ? "(#{k} = ? OR #{k} IS NULL)" : "(#{k} = ?)"
+        }.join(" AND ")
+
+        cached_mcfly_lookup(name, sig: attrs.length+1) do
+          |t, *attr_list|
+
+          attr_list_ids = attr_list.each_with_index.map {|x, i|
+            assoc.member?(attr_names[i]) ?
+            (attr_list[i] && attr_list[i].id) : attr_list[i]
+          }
+
+          q = self.where(qstr, *attr_list_ids)
+          q = q.order(order) unless order.empty?
+          mode = :to_a if mode == :all
+          mode ? q.send(mode) : q
+        end
+      end
+
+      ######################################################################
+
+      # Generates categorization lookups.  For instance,
+      # suppose we have the following in class GFee:
+      #
+      # gen_mcfly_lookup_cat :lookup_q,
+      # [:security_instrument,
+      #  'Gemini::SecurityInstrumentCategorization',
+      #  :g_fee_category],
+      # {
+      #   entity: true,
+      #   security_instrument: true,
+      #   coupon: true,
+      # },
+      # nil
+
+      # In the above case,
+      # rel_attr = :security_instrument
+      # cat_assoc_klass = Gemini::SecurityInstrumentCategorization
+      # cat_attr = :g_fee_category
+      # name = :lookup_q
+      # pc_name = :pc_lookup_q
+      # pc_attrs = {entity: true, security_instrument: true,
+      # g_fee_category: true, coupon: true}
+
+      def gen_mcfly_lookup_cat(name, catrel, attrs, options={})
+        rel_attr, cat_assoc_name, cat_attr = catrel
+
+        raise "#{rel_attr} should be mapped in attrs" if
+          attrs[rel_attr].nil?
+
+        cat_assoc_klass = cat_assoc_name.constantize
+
+        raise "need lookup method on #{cat_assoc_klass}" unless
+          cat_assoc_klass.respond_to? :lookup
+
+        # replace rel_attr with cat_attr in attrs
+        pc_attrs = attrs.each_with_object({}) {|(k, v), h|
+          h[k == rel_attr ? cat_attr : k] = v
+        }
+
+        pc_name = "pc_#{name}".to_sym
+        gen_mcfly_lookup(pc_name, pc_attrs, options)
+
+        lpi = attrs.keys.index rel_attr
+
+        raise "should not include #{cat_attr}" if
+          attrs.member?(cat_attr)
+
+        raise "need #{rel_attr} argument" unless lpi
+
+        delorean_fn(name, sig: attrs.length+1) do |ts, *args|
+          # Example: rel is a Gemini::SecurityInstrument instance.
+          rel = args[lpi]
+          raise "#{rel_attr} can't be nil" unless rel
+
+          # Assumes there's a mcfly :lookup function on
+          # cat_assoc_klass.
+          categorizing_obj = cat_assoc_klass.lookup(ts, rel)
+          raise "no categorization #{cat_assoc_klass} for #{rel}" unless
+            categorizing_obj
+
+          pc = categorizing_obj.send(cat_attr)
+          raise ("#{categorizing_obj} must have assoc." +
+                 " #{cat_attr}/#{rel.inspect}") unless pc
+
+          args[lpi] = pc
+          self.send(pc_name, ts, *args)
+        end
+      end
+
+    end
+  end
+end
+
+module Mcfly::Controller
+  # define mcfly user to be Flowscape's current_user.
+  def user_for_mcfly
+    find_current_user rescue nil
+  end
+end