marty 0.5.15 → 0.5.16

data/lib/marty/data_exporter.rb~

@@ -1,137 +0,0 @@
-require 'base64'
-require 'zlib'
-
-class Marty::DataExporter
-  # given an array of hashes, return set of all keys
-  def self.hash_array_keys(hl)
-    hl.each_with_object(Set.new) { |h, keys| keys.merge(h.keys) }
-  end
-
-  def self.hash_array_merge(hl, transpose)
-    # given a list of hashes hl, generates a merged hash.  The
-    # resulting hash contains a superset of all the hash keys.  The
-    # values are corresponding values from each hash in hl.
-    keys = hash_array_keys(hl)
-
-    if transpose
-      keys.each_with_object({}) { |k, rh|
-        rh[k] = hl.map { |h| h[k] }
-      }
-    else
-      [keys.to_a] + hl.map {|h| keys.map {|k| h[k]}}
-    end
-  end
-
-  def self.encode_json(s)
-    Base64.strict_encode64 Zlib.deflate(s)
-  end
-
-  def self.decode_json(s)
-    Zlib.inflate Base64.strict_decode64(s)
-  end
-
-  def self.to_csv(obj, config=nil)
-    obj = [obj] unless obj.respond_to? :map
-
-    config ||= {}
-
-    # if all array items are hashes, we merge them
-    obj = hash_array_merge(obj, config["transpose"]) if
-      obj.is_a?(Array) && obj.map {|x| x.is_a? Hash}.all?
-
-    # symbolize config keys as expected by CSV.generate
-    conf = config.each_with_object({}) { |(k,v), h|
-      h[k.to_sym] = v unless k.to_s == "transpose"
-    }
-
-    # FIXME: very hacky to default row_sep to CRLF
-    conf[:row_sep] ||= "\r\n"
-
-    # FIXME: the following is ridiculously complex. We have different
-    # data paths for hashes and arrays.  Also, arrays can turn into
-    # hashes is all their items are hashes!  We map to complex objects
-    # to JSON when inside hashes, but not arrays. Really need to
-    # rethink this.  Probably should have separate functions for
-    # to_csv for hash and arrays.
-
-    if obj.is_a? Hash
-      csv_string = CSV.generate(conf) do |csv|
-        obj.each do |x|
-          csv << x.flatten(1).map(&:to_s)
-        end
-      end
-    else
-      csv_string = CSV.generate(conf) do |csv|
-        obj.each do |x|
-          x = [x] unless x.respond_to? :map
-          csv << x.map { |v|
-            v.is_a?(Array) || v.is_a?(Hash) ? encode_json(v.to_json) : v.to_s
-          }
-        end
-      end
-    end
-  end
-
-  def self.class_info(klass)
-    @class_info ||= {}
-
-    return @class_info[klass] if @class_info[klass]
-
-    associations = klass.reflect_on_all_associations.map(&:name)
-
-    @class_info[klass] = {
-      cols:
-      klass.columns.map(&:name) - Marty::DataRowProcessor::MCFLY_COLUMNS.to_a,
-
-      assoc:
-      associations.each_with_object({}) { |a, h|
-        h["#{a}_id"] = Marty::DataRowProcessor.assoc_info(klass, a)
-      },
-    }
-  end
-
-  def self.export_attr(obj, c, info)
-    v = obj.send(c.to_sym)
-    assoc_info = info[:assoc][c] unless v.nil?
-    return [v] unless assoc_info
-
-    assoc_obj = assoc_info[:assoc_class].find(v)
-
-    assoc_info[:assoc_keys].map {|k| assoc_obj.send(k.to_sym)}
-  end
-
-  def self.export_header_attr(c, info)
-    assoc_info = info[:assoc][c]
-    return c unless assoc_info
-
-    # remove _id
-    c = c[0..-4]
-
-    assoc_keys = assoc_info[:assoc_keys]
-
-    # FIXME: this doesn't work if k is also an association.  Needs to
-    # be recursive.
-    assoc_keys.length > 1 ? assoc_keys.map {|k| "#{c}__#{k}"} : c
-  end
-
-  # Given a Mcfly klass, generate an export array.  Can potentially
-  # use up a lot of memory if the result set is large.
-  def self.do_export(ts, klass, sort_field=nil)
-    info = class_info(klass)
-
-    # strip _id from assoc fields
-    header = [ info[:cols].map {|c| export_header_attr(c, info)}.flatten(1) ]
-
-    query = klass
-
-    # is it Mcfly?
-    if (klass.const_get(:MCFLY_UNIQUENESS) rescue nil)
-      ts = Mcfly.normalize_infinity(ts)
-      query = query.where("obsoleted_dt >= ? AND created_dt < ?", ts, ts)
-    end
-
-    header + query.
-      order(sort_field || :id).
-      map {|obj| info[:cols].map {|c| export_attr(obj, c, info)}.flatten(1)}
-  end
-end

data/lib/marty/data_importer.rb~

@@ -1,114 +0,0 @@
-module Marty
-  class DataImporterError < StandardError
-    attr_reader :lines
-
-    def initialize(message, lines)
-      super(message)
-      @lines = lines
-    end
-  end
-
-  class DataImporter
-    # perform cleaning and do_import and summarize its results
-    def self.do_import_summary(klass,
-                               data,
-                               dt='infinity',
-                               cleaner_function=nil,
-                               validation_function=nil,
-                               col_sep="\t",
-                               allow_dups=false
-                               )
-
-      recs = self.do_import(klass,
-                            data,
-                            dt,
-                            cleaner_function,
-                            validation_function,
-                            col_sep,
-                            allow_dups,
-                            )
-
-      recs.each_with_object(Hash.new(0)) {|(op, id), h|
-        h[op] += 1
-      }
-    end
-
-    # Given a Mcfly klass and CSV data, import data into the database
-    # and report on affected rows.  Result is an array of tuples.
-    # Each tuple is associated with one data row and looks like [tag,
-    # id].  Tag is one of :same, :update, :create and "id" is the id
-    # of the affected row.
-    def self.do_import(klass,
-                       data,
-                       dt='infinity',
-                       cleaner_function=nil,
-                       validation_function=nil,
-                       col_sep="\t",
-                       allow_dups=false
-                       )
-
-      parsed = data.is_a?(Array) ? data :
-        CSV.new(data, headers: true, col_sep: col_sep)
-
-      klass.transaction do
-        cleaner_ids = cleaner_function ? klass.send(cleaner_function.to_sym) :
-          []
-
-        raise "bad cleaner function result" unless
-          cleaner_ids.all? {|id| id.is_a?(Fixnum) }
-
-        row_proc = nil
-        eline = 0
-
-        begin
-          res = parsed.each_with_index.map { |row, line|
-            eline = line
-
-            row_proc ||= Marty::DataRowProcessor.
-            new(klass,
-                row.respond_to?(:headers) ? row.headers : row.keys,
-                dt,
-                )
-            # skip lines which are all nil
-            next :blank if row.to_hash.values.none?
-
-            row_proc.create_or_update(row)
-          }
-        rescue => exc
-          # to find problems with the importer, comment out the rescue block
-          raise Marty::DataImporterError.new(exc.to_s, [eline])
-        end
-
-        ids = {}
-
-        # raise an error if record referenced more than once.
-        res.each_with_index do
-          |(op, id), line|
-          raise Marty::DataImporterError.
-            new("record referenced more than once", [ids[id], line]) if
-            op != :blank && ids.member?(id) && !allow_dups
-
-          ids[id] = line
-        end
-
-        begin
-          # Validate affected rows if necessary
-          klass.send(validation_function.to_sym, ids.keys) if
-            validation_function
-        rescue => exc
-          raise Marty::DataImporterError.new(exc.to_s, [])
-        end
-
-        remainder_ids = cleaner_ids - ids.keys
-
-        raise Marty::DataImporterError.
-          new("Missing import data. " +
-              "Please provide header line and at least one data line.", [1]) if
-          ids.keys.compact.count == 0
-
-        klass.delete(remainder_ids)
-        res + remainder_ids.map {|id| [:clean, id]}
-      end
-    end
-  end
-end

data/lib/marty/data_row_processor.rb~

@@ -1,206 +0,0 @@
-class Marty::DataRowProcessor
-  attr_accessor :klass, :headers, :dt, :key_attrs, :hmap
-
-  EXCEL_START_DATE = Date.parse('1/1/1900')-2
-
-  MCFLY_COLUMNS = Set[
-                      "id",
-                      "group_id",
-                      "user_id",
-                      "created_dt",
-                      "obsoleted_dt",
-                      "o_user_id",
-                     ]
-
-  # Given a Mcfly class, return the set of attributes (excluding id)
-  # used to uniquely identify an instance.
-  def self.get_keys(klass)
-    raise "bad class arg #{klass}" unless
-      klass.is_a?(Class) && klass < ActiveRecord::Base
-
-    attrs = klass.const_get(:MCFLY_UNIQUENESS)
-
-    raise "class has no :MCFLY_UNIQUENESS" unless attrs
-
-    attrs = attrs[0..-2] + attrs.last.fetch(:scope, []) if
-      attrs.last.is_a?(Hash)
-    attrs -= [:obsoleted_dt]
-
-    raise "key list for #{klass} is empty" if attrs.empty?
-    attrs
-  end
-
-  def self.assoc_info(klass, a)
-    assoc_class = klass.reflect_on_association(a.to_sym).klass
-    keys = self.get_keys(assoc_class) rescue nil
-
-    assoc_keys = keys || [assoc_class.attribute_names.reject{|x| x=="id"}.first]
-
-    {assoc_keys: assoc_keys, assoc_class: assoc_class, mcfly: keys}
-  end
-
-  FLOAT_PAT = /^-?\d+(\.\d+)?$/
-
-  PATS = {
-    integer: /^-?\d+(\.0+)?$/,
-    float:   FLOAT_PAT,
-    decimal: FLOAT_PAT,
-  }
-
-  def convert(v, type)
-    pat = PATS[type]
-
-    raise "bad #{type} #{v.inspect}" if
-      v.is_a?(String) && pat && !(v =~ pat)
-
-    case type
-    when :boolean
-      case v.downcase
-      when "true" then true
-      when "false" then false
-      else raise "unknown boolean #{v}"
-      end
-    when :string, :text
-      v
-    when :integer
-      v.to_i
-    when :float
-      v.to_f
-    when :decimal
-      v.to_d
-    when :date
-      # Dates are kept as float in Google spreadsheets.  Need to
-      # convert them to dates. FIXME: 'infinity' as a date in
-      # Rails 3.2 appears to be broken. Setting a date field to
-      # 'infinity' sets it to nil.
-      v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
-        Mcfly.is_infinity(v) ? 'infinity' : v.to_date
-    when :datetime
-      Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
-    when :numrange, :int4range, :int8range
-      v.to_s
-    when :float_array, :json
-      JSON.parse Marty::DataExporter.decode_json(v)
-    else
-      raise "unknown type #{type} for #{v}"
-    end
-  end
-
-  def initialize(klass, headers, dt)
-    @klass     = klass
-    @headers   = headers
-    @dt        = dt
-    @key_attrs = self.class.get_keys(klass)
-
-    # # HACK: not sure why there's a nil at the end of headers sometimes
-    # headers.pop if headers[-1].nil?
-
-    raise "row headers have nil! #{headers.inspect}" unless headers.all?
-
-    associations = klass.reflect_on_all_associations.map(&:name)
-
-    cols = klass.columns.each_with_object({}) { |c, h|
-      h[c.name] = c
-    }
-
-    @hmap = headers.each_with_object({}) do
-      |a, h|
-      # handle klass__attr type headers generated by Netzke.  Just
-      # keeps klass since we should be able to find the key attr.
-      aclass = a.split('__').first
-
-      if associations.member?(aclass.to_sym)
-        h[a] = self.class.assoc_info(klass, aclass)
-        next
-      end
-
-      raise "unknown column #{a}" unless cols[a]
-
-      # for JSON fields in Rails 3.x type is nil, so use sql_type
-      type = cols[a].type || cols[a].sql_type
-      type = "#{type}_array" if cols[a].array
-      h[a] = type.to_sym
-    end
-  end
-
-  def create_or_update(row)
-    assoc_options = {}
-
-    options = row.each_with_object({}) do |(a, v), h|
-      # ignore Mcfly columns
-      next if MCFLY_COLUMNS.member? a
-
-      if hmap[a].is_a?(Hash)
-        assoc_options[a] = v
-      else
-        raise "bad col #{a} value #{v}, row: #{row}" unless hmap[a]
-
-        # if it's not a hash (association) then its a type symbol
-        h[a] = v && convert(v, hmap[a])
-      end
-    end
-
-    assoc_groups = assoc_options.keys.group_by {|x| x.split('__').first}
-
-    assoc_groups.each do |aclass, attrs|
-      # if group has only one attr and the attr is nil or AR obj, then
-      # we don't need to search.
-      if attrs.length == 1
-        a = attrs.first
-        v = assoc_options[a]
-
-        if v.nil? || v.is_a?(ActiveRecord::Base)
-          options["#{aclass}_id"] = v && v.id
-          next
-        end
-      end
-
-      srch = attrs.each_with_object({}) do |a, h|
-        v = assoc_options[a]
-        ac, aa = a.split('__')
-
-        aa ||= hmap[a][:assoc_keys].first
-
-        h[aa] = v
-        h[:obsoleted_dt] = 'infinity' if hmap[a][:mcfly]
-      end
-
-      srch_class = hmap[attrs.first][:assoc_class]
-
-      av = srch_class.where(srch).first
-
-      raise "#{aclass} not found #{srch}" unless av
-
-      options[ "#{aclass}_id" ] = av.id
-    end
-
-    find_options = options.select { |k,v| key_attrs.member? k.to_sym }
-
-    raise "invalid entry" if find_options.empty?
-
-    find_options['obsoleted_dt'] = 'infinity'
-
-    obj = klass.where(find_options).first || klass.new
-
-    options.each do
-      |k, v|
-      # For each attr, check to see if it's begin changed before
-      # setting it.  The AR obj.changed? doesn't work properly
-      # with array, JSON or lazy attrs.
-      obj.send("#{k}=", v) if obj.send(k) != v
-    end
-
-    # FIXME: obj.changed? doesn't work properly for timestamp
-    # fields in Rails 3.2. It evaluates to true even when datetime
-    # is not changed.  Caused by lack of awareness of timezones.
-    tag = obj.new_record? ? :create : (obj.changed? ? :update : :same)
-
-    raise "old created_dt >= current #{obj} #{obj.created_dt} #{dt}" if
-      (tag == :update) && !Mcfly.is_infinity(dt) && (obj.created_dt > dt)
-
-    obj.created_dt = dt unless tag == :same || Mcfly.is_infinity(dt)
-    obj.save!
-
-    [tag, obj.id]
-  end
-end