marty 0.5.12

data/lib/marty/mcfly_query.rb~

@@ -0,0 +1,188 @@
+require 'mcfly'
+
+module Mcfly
+  module Model
+
+    def self.included(base)
+      base.send :extend, ClassMethods
+    end
+
+    module ClassMethods
+      def clear_cache
+        @LOOKUP_CACHE.clear if @LOOKUP_CACHE
+      end
+
+      # Implements a VERY HACKY class-based caching mechanism for
+      # database lookup results.  Issues include: cached values are
+      # ActiveRecord objects.  Not sure if these should be shared
+      # across connections.  Query results can potentially be very
+      # large lists which we simply count as one item in the cache.
+      # Caching mechanism will result in large processes.  Caches are
+      # not sharable across different Ruby processes.
+      def cached_delorean_fn(name, options = {}, &block)
+        @LOOKUP_CACHE ||= {}
+
+        delorean_fn(name, options) do |ts, *args|
+          cache_key = [name, ts] + args.map{ |a|
+            a.is_a?(ActiveRecord::Base) ? a.id : a
+          } unless Mcfly.is_infinity(ts)
+
+          next @LOOKUP_CACHE[cache_key] if
+            cache_key && @LOOKUP_CACHE.has_key?(cache_key)
+
+          res = block.call(ts, *args)
+
+          if cache_key
+            # Cache has >1000 items, clear out the oldest 200.  FIXME:
+            # hard-coded, should be configurable.  Cache
+            # size/invalidation should be per lookup and not class.
+            # We're invalidating cache items simply based on age and
+            # not usage.  This is faster but not as fair.
+            if @LOOKUP_CACHE.count > 1000
+              @LOOKUP_CACHE.keys[0..200].each{|k| @LOOKUP_CACHE.delete(k)}
+            end
+            @LOOKUP_CACHE[cache_key] = res
+
+            # Since we're caching this object and don't want anyone
+            # changing it.  FIXME: ideally should freeze this object
+            # recursively.
+            res.freeze unless res.is_a?(ActiveRecord::Relation)
+          end
+          res
+        end
+      end
+
+      # FIXME: duplicate code from Mcfly's mcfly_lookup.
+      def cached_mcfly_lookup(name, options = {}, &block)
+        cached_delorean_fn(name, options) do |ts, *args|
+          raise "time cannot be nil" if ts.nil?
+
+          ts = Mcfly.normalize_infinity(ts)
+
+          where("obsoleted_dt >= ? AND created_dt < ?", ts, ts).scoping do
+            block.call(ts, *args)
+          end
+        end
+      end
+
+      # FIXME: for validation purposes, this mechanism should make
+      # sure that the allable attrs are not required.
+      def gen_mcfly_lookup(name, attrs, options={})
+        raise "bad options" unless options.is_a?(Hash)
+
+        # FIXME: mode should be sent later, not as a part of
+        # gen_mcfly_lookup.  i.e. we just generate the search and the
+        # mode is applied at runtime by delorean code.  That would
+        # allow lookups to be used in either mode dynamically.
+        mode = options.fetch(:mode, :first)
+
+        assoc = Set.new(self.reflect_on_all_associations.map(&:name))
+        attr_names = attrs.keys
+
+        allables = attrs.select {|k, v| v}
+
+        order = allables.keys.reverse.map { |k|
+          k = "#{k}_id" if assoc.member?(k)
+          "#{k} NULLS LAST"
+        }.join(", ")
+
+        qstr = attrs.map {|k, v|
+          k = "#{k}_id" if assoc.member?(k)
+          v ? "(#{k} = ? OR #{k} IS NULL)" : "(#{k} = ?)"
+        }.join(" AND ")
+
+        cached_mcfly_lookup(name, sig: attrs.length+1) do
+          |t, *attr_list|
+
+          attr_list_ids = attr_list.each_with_index.map {|x, i|
+            assoc.member?(attr_names[i]) ?
+            (attr_list[i] && attr_list[i].id) : attr_list[i]
+          }
+
+          q = self.where(qstr, *attr_list_ids)
+          q = q.order(order) unless order.empty?
+          mode ? q.send(mode) : q
+        end
+      end
+
+      ######################################################################
+
+      # Generates Gemini categorization lookups.  For instance,
+      # suppose we have the following in class GFee:
+      #
+      # gen_mcfly_lookup_cat :lookup_q,
+      # [:security_instrument,
+      #  'Gemini::SecurityInstrumentCategorization',
+      #  :g_fee_category],
+      # {
+      #   entity: true,
+      #   security_instrument: true,
+      #   coupon: true,
+      # },
+      # nil
+
+      # In the above case,
+      # rel_attr = :security_instrument
+      # cat_assoc_klass = Gemini::SecurityInstrumentCategorization
+      # cat_attr = :g_fee_category
+      # name = :lookup_q
+      # pc_name = :pc_lookup_q
+      # pc_attrs = {entity: true, security_instrument: true,
+      # g_fee_category: true, coupon: true}
+
+      def gen_mcfly_lookup_cat(name, catrel, attrs, options={})
+        rel_attr, cat_assoc_name, cat_attr = catrel
+
+        raise "#{rel_attr} should be mapped in attrs" if
+          attrs[rel_attr].nil?
+
+        cat_assoc_klass = cat_assoc_name.constantize
+
+        raise "need lookup method on #{cat_assoc_klass}" unless
+          cat_assoc_klass.respond_to? :lookup
+
+        # replace rel_attr with cat_attr in attrs
+        pc_attrs = attrs.each_with_object({}) {|(k, v), h|
+          h[k == rel_attr ? cat_attr : k] = v
+        }
+
+        pc_name = "pc_#{name}".to_sym
+        gen_mcfly_lookup(pc_name, pc_attrs, options)
+
+        lpi = attrs.keys.index rel_attr
+
+        raise "should not include #{cat_attr}" if
+          attrs.member?(cat_attr)
+
+        raise "need #{rel_attr} argument" unless lpi
+
+        delorean_fn(name, sig: attrs.length+1) do |ts, *args|
+          # Example: rel is a Gemini::SecurityInstrument instance.
+          rel = args[lpi]
+          raise "#{rel_attr} can't be nil" unless rel
+
+          # Assumes there's a mcfly :lookup function on
+          # cat_assoc_klass.
+          categorizing_obj = cat_assoc_klass.lookup(ts, rel)
+          raise "no categorization #{cat_assoc_klass} for #{rel}" unless
+            categorizing_obj
+
+          pc = categorizing_obj.send(cat_attr)
+          raise ("#{categorizing_obj} must have assoc." +
+                 " #{cat_attr}/#{rel.inspect}") unless pc
+
+          args[lpi] = pc
+          self.send(pc_name, ts, *args)
+        end
+      end
+
+    end
+  end
+end
+
+module Mcfly::Controller
+  # define mcfly user to be Flowscape's current_user.
+  def user_for_mcfly
+    find_current_user rescue nil
+  end
+end

data/lib/marty/migrations.rb

@@ -0,0 +1,108 @@
+require 'digest/md5'
+
+module Marty::Migrations
+  def tb_prefix
+    "marty_"
+  end
+
+  def add_fk(from_table, to_table, options = {})
+    options[:column] ||= "#{to_table.to_s.singularize}_id"
+
+    from_table = "#{tb_prefix}#{from_table}" unless
+      from_table.to_s.start_with?(tb_prefix)
+
+    # FIXME: so hacky to specifically check for "marty_"
+    to_table = "#{tb_prefix}#{to_table}" unless
+      to_table.to_s.start_with?(tb_prefix) ||
+      to_table.to_s.start_with?("marty_")
+
+    add_foreign_key(from_table,
+                    to_table,
+                    fk_opts(from_table,
+                            to_table,
+                            options[:column]).update(options),
+                    )
+  end
+
+  # created_dt/obsoleted_dt need to be indexed since they appear in
+  # almost all queries.
+  MCFLY_INDEX_COLUMNS = [
+                         :created_dt,
+                         :obsoleted_dt,
+                        ]
+
+  def add_mcfly_index(tb, *attrs)
+    tb = "#{tb_prefix}#{tb}" unless
+      tb.to_s.start_with?(tb_prefix)
+
+    add_mcfly_attrs_index(tb, *attrs)
+
+    MCFLY_INDEX_COLUMNS.each { |a|
+      add_index tb.to_sym, a, index_opts(tb, a)
+    }
+  end
+
+  def add_mcfly_unique_index(klass)
+    raise "bad class arg #{klass}" unless
+      klass.is_a?(Class) && klass < ActiveRecord::Base
+
+    attrs = get_attrs(klass)
+
+    add_index(klass.table_name.to_sym,
+              attrs,
+              unique: true,
+              name: unique_index_name(klass)
+              ) unless index_exists?(klass.table_name.to_sym,
+                                     attrs,
+                                     name: unique_index_name(klass),
+                                     unique: true)
+
+  end
+
+  def remove_mcfly_unique_index(klass)
+    raise "bad class arg #{klass}" unless
+      klass.is_a?(Class) && klass < ActiveRecord::Base
+
+    attrs = get_attrs(klass)
+
+    remove_index(klass.table_name.to_sym,
+                 name: unique_index_name(klass)
+                 ) if index_exists?(klass.table_name.to_sym,
+                                    attrs,
+                                    name: unique_index_name(klass),
+                                    unique: true)
+  end
+
+private
+  def fk_opts(from, to, column)
+    name = "fk_#{from}_#{to}_#{column}"
+    if name.length > 63
+      s = Digest::MD5.hexdigest("#{to}_#{column}").slice(0..9)
+      name = "fk_#{from}_#{s}"
+    end
+    {name: name}
+  end
+
+  def index_opts(tb, a)
+    name = "index_#{tb}_on_#{a}"
+    name.length > 63 ? {
+      name: "index_#{tb}_#{Digest::MD5.hexdigest(a.to_s).slice(0..9)}",
+    } : {}
+  end
+
+  def add_mcfly_attrs_index(tb, *attrs)
+    attrs.each { |a|
+      options = index_opts(tb, a)
+      options[:order] = {a.to_sym => "NULLS LAST"}
+      add_index tb.to_sym, a, options
+    }
+  end
+
+  def unique_index_name(klass)
+    "unique_#{klass.table_name}"
+  end
+
+  def get_attrs(klass)
+    (Mcfly.mcfly_uniqueness(klass) + ['obsoleted_dt']).uniq
+  end
+end

data/lib/marty/migrations.rb~

@@ -0,0 +1,65 @@
+require 'digest/md5'
+
+module Marty::Migrations
+  def tb_prefix
+    "marty_"
+  end
+
+  def fk_opts(from, to, column)
+    name = "fk_#{from}_#{to}_#{column}"
+    if name.length > 63
+      s = Digest::MD5.hexdigest("#{to}_#{column}").slice(0..9)
+      name = "fk_#{from}_#{s}" 
+    end
+    {name: name}
+  end
+
+  def add_fk(from_table, to_table, options = {})
+    options[:column] ||= "#{to_table.to_s.singularize}_id"
+
+    from_table = "#{tb_prefix}#{from_table}" unless
+      from_table.to_s.start_with?(tb_prefix)
+
+    # FIXME: so hacky to specifically check for "marty_"
+    to_table = "#{tb_prefix}#{to_table}" unless
+      to_table.to_s.start_with?(tb_prefix) ||
+      to_table.to_s.start_with?("marty_")
+
+    add_foreign_key(from_table,
+                    to_table,
+                    fk_opts(from_table,
+                            to_table,
+                            options[:column]).update(options),
+                    )
+  end
+
+  def index_opts(tb, a)
+    name = "index_#{tb}_on_#{a}"
+    name.length > 63 ? {
+      name: "index_#{tb}_#{Digest::MD5.hexdigest(a.to_s).slice(0..9)}",
+    } : {}
+  end
+
+  # created_dt/obsoleted_dt need to be indexed since they appear in
+  # almost all queries.
+  MCFLY_INDEX_COLUMNS = [
+                         :created_dt,
+                         :obsoleted_dt,
+                        ]
+
+  def add_mcfly_index(tb, *attrs)
+    tb = "#{tb_prefix}#{tb}" unless
+      tb.to_s.start_with?(tb_prefix)
+
+    attrs.each { |a|
+      options = index_opts(tb, a)
+      options[:order] = {a.to_sym => "NULLS LAST"}
+      add_index tb.to_sym, a, options
+    }
+
+    MCFLY_INDEX_COLUMNS.each { |a|
+      add_index tb.to_sym, a, index_opts(tb, a)
+    }
+  end
+
+end

data/lib/marty/monkey.rb

@@ -0,0 +1,163 @@
+# Various monkey patches go here
+
+######################################################################
+
+require 'delorean_lang'
+
+# Very hacky to overwrite delorean's whitelist.  But, there's no
+# better way now.
+
+Delorean::RUBY_WHITELIST.
+  merge!({
+         })
+
+######################################################################
+
+# Be able to access Enums from Delorean
+class Delorean::BaseModule::BaseClass
+  class << self
+    alias_method :old_get_attr, :_get_attr
+
+    def _get_attr(obj, attr, _e)
+      if Marty::Enum === obj && !obj.respond_to?(attr)
+        obj[attr]
+      else
+        old_get_attr(obj, attr, _e)
+      end
+    end
+  end
+end
+
+######################################################################
+
+class Hash
+  # define addition on hashes -- useful in Delorean code.
+  def +(x)
+    self.merge(x)
+  end
+
+  # define hash slice (similar to node slice in Delorean)
+  def %(x)
+    x.each_with_object({}) { |k, h|
+      h[k] = self[k]
+    }
+  end
+end
+
+######################################################################
+
+require 'netzke-basepack'
+
+class Netzke::Base
+  # get root component session
+  def root_sess(component=nil)
+    component ||= self
+    component.parent ? root_sess(component.parent) : component.component_session
+  end
+end
+
+######################################################################
+
+# The following is a hack to get around ActiveRecord's broken handling
+# of PostgreSQL ranges.  Essentially, AR doesn't allow numranges to
+# exclude the range start e.g. anything like: "(1.1,2.2]".  This hack
+# turns off the casting of PostgreSQL ranges to ruby ranges. i.e. we
+# keep them as strings.
+
+require 'active_record/connection_adapters/postgresql_adapter'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module PostgreSQL
+      module OID
+        class Range
+          def cast_value(value)
+            super
+          end
+
+          def type_cast_for_database(value)
+            super
+          end
+        end
+      end
+    end
+  end
+end
+
+
+module ActiveRecord
+  module ConnectionAdapters
+    module PostgreSQL
+      module OID # :nodoc:
+        class Array
+
+          # In the 4.2.1 version of this code, under Mutable, the code
+          # checks for raw_old_value != type_cast_for_database(new_value)
+          #
+          # Since this is comparing db (string) version, we end up
+          # comparing "{1}"!="{1.0}" for float arrays. The following
+          # is a hack to check the new_value which is the ruby array.
+          # This could be problematic in other ways.  But, works for
+          # our purposes.  FIXME: In Rails 5.0 all this code has been
+          # changed and this should no longer be an issue.
+
+
+          def changed_in_place?(raw_old_value, new_value)
+            new_value != type_cast_from_database(raw_old_value)
+          end
+
+        end
+      end
+    end
+  end
+end
+
+######################################################################
+
+# Rails 4 doesn't handle 'infinity' datetime properly due to
+# in_time_zone conversion. Ergo this hack.
+
+class String
+  alias_method :old_in_time_zone, :in_time_zone
+
+  def in_time_zone(zone = ::Time.zone)
+    self == 'infinity' ? self : old_in_time_zone(zone)
+  end
+end
+
+######################################################################
+
+# Axlsx::sanitize modifies strings in worksheet definition -- this
+# doesn't work with Delorean's frozen strings.
+require 'axlsx'
+module Axlsx
+  def self.sanitize(str)
+    str.delete(CONTROL_CHARS)
+  end
+end
+
+######################################################################
+
+require 'netzke/basepack/data_adapters/active_record_adapter'
+module Netzke::Basepack::DataAdapters
+  class ActiveRecordAdapter < AbstractAdapter
+    # FIXME: another giant hack to handle lazy_load columns.
+    # Modified original count_records to call count('*') instead of
+    # count when lazy-loaded.  Otherwise, we run into issues with
+    # counting records in the default_scope placed by the lazy_load
+    # module.
+    def count_records(params, columns=[])
+
+      relation = @relation || get_relation(params)
+      columns.each do |c|
+        assoc, method = c[:name].split('__')
+        relation = relation.includes(assoc.to_sym).references(assoc.to_sym) if method
+      end
+
+      @model_class.const_defined?(:LAZY_LOADED) ? relation.count('*') :
+        relation.count
+    end
+
+  end
+end
+