marty 0.5.12

data/lib/marty/data_exporter.rb

@@ -0,0 +1,150 @@
+require 'base64'
+require 'zlib'
+require 'csv'
+require 'marty/data_conversion'
+
+class Marty::DataExporter
+  # given an array of hashes, return set of all keys
+  def self.hash_array_keys(hl)
+    hl.each_with_object(Set.new) { |h, keys| keys.merge(h.keys) }
+  end
+
+  def self.hash_array_merge(hl, transpose)
+    # given a list of hashes hl, generates a merged hash.  The
+    # resulting hash contains a superset of all the hash keys.  The
+    # values are corresponding values from each hash in hl.
+    # e.g. the following
+    #
+    # [{"a"=>1, "b"=>2}, {"a"=>11, "c"=>33}, {"a"=>1111, "b"=>222, "c"=>333}]
+    #
+    # maps to ...
+    #
+    # [["a", "b", "c"], [1, 2, nil], [11, nil, 33], [1111, 222, 333]]
+
+    keys = hash_array_keys(hl)
+
+    return keys.each_with_object({}) { |k, rh|
+        rh[k] = hl.map { |h| h[k] }
+    } if transpose
+
+    [keys.to_a] + hl.map {|h| keys.map {|k| h[k]}}
+  end
+
+  def self.encode_json(s)
+    Base64.strict_encode64 Zlib.deflate(s)
+  end
+
+  def self.decode_json(s)
+    Zlib.inflate Base64.strict_decode64(s)
+  end
+
+  def self.to_csv(obj, config=nil)
+    obj = [obj] unless obj.respond_to? :map
+
+    config ||= {}
+
+    # if all array items are hashes, we merge them
+    obj = hash_array_merge(obj, config["transpose"]) if
+      obj.is_a?(Array) && obj.map {|x| x.is_a? Hash}.all?
+
+    # symbolize config keys as expected by CSV.generate
+    conf = config.each_with_object({}) { |(k,v), h|
+      h[k.to_sym] = v unless k.to_s == "transpose"
+    }
+
+    # FIXME: very hacky to default row_sep to CRLF
+    conf[:row_sep] ||= "\r\n"
+
+    # FIXME: the following is ridiculously complex. We have different
+    # data paths for hashes and arrays.  Also, arrays can turn into
+    # hashes is all their items are hashes!  We map to complex objects
+    # to JSON when inside hashes, but not arrays. Really need to
+    # rethink this.  Probably should have separate functions for
+    # to_csv for hash and arrays.
+
+    return CSV.generate(conf) do |csv|
+      obj.each do |x|
+        csv << x.flatten(1).map(&:to_s)
+      end
+    end if obj.is_a?(Hash)
+
+    CSV.generate(conf) do |csv|
+      obj.each do |x|
+        x = [x] unless x.respond_to? :map
+        csv << x.map { |v|
+          v.is_a?(Array) || v.is_a?(Hash) ? encode_json(v.to_json) : v.to_s
+        }
+      end
+    end
+  end
+
+  def self.export_attrs(klass, obj, attrs=nil)
+    col_types = Marty::DataConversion.col_types(klass)
+
+    (attrs || col_types.keys).map do
+      |c|
+
+      v = obj.send(c.to_sym)
+
+      c = c.to_s
+      type = col_types[c]
+
+      # return [value] if not assoc or nil
+      next [v] if v.nil? || !type.is_a?(Hash)
+
+      assoc_keys  = type[:assoc_keys]
+      assoc_class = type[:assoc_class]
+      assoc_obj   = assoc_class.find(v)
+
+      # FIXME: this recursion will fail if a reference which then
+      # makes sub-references is nil.  To handle this, we'd need to
+      # create the export structure first.
+      export_attrs(assoc_class, assoc_obj, assoc_keys).flatten(1)
+    end
+  end
+
+  def self.export_headers(klass, attrs=nil)
+    col_types = Marty::DataConversion.col_types(klass)
+
+    (attrs || col_types.keys).map do
+      |c|
+
+      c = c.to_s
+      type = col_types[c]
+
+      next c unless type.is_a?(Hash)
+
+      # remove _id
+      c = c[0..-4]
+
+      assoc_keys = type[:assoc_keys]
+
+      # if association has a single key, just use col name
+      next c if assoc_keys.length == 1
+
+      assoc_class = type[:assoc_class]
+
+      export_headers(assoc_class, assoc_keys).map {|k| "#{c}__#{k}"}
+    end
+  end
+
+  # Given a Mcfly klass, generate an export array.  Can potentially
+  # use up a lot of memory if the result set is large.
+  def self.do_export(ts, klass, sort_field=nil)
+    query = klass
+
+    if Mcfly.has_mcfly?(klass)
+      ts = Mcfly.normalize_infinity(ts)
+      query = query.where("obsoleted_dt >= ? AND created_dt < ?", ts, ts)
+    end
+
+    do_export_query_result(klass, query.order(sort_field || :id))
+  end
+
+  def self.do_export_query_result(klass, qres)
+    # strip _id from assoc fields
+    header = [ export_headers(klass).flatten ]
+
+    header + qres.map {|obj| export_attrs(klass, obj).flatten(1)}
+  end
+end

data/lib/marty/data_exporter.rb~

@@ -0,0 +1,137 @@
+require 'base64'
+require 'zlib'
+
+class Marty::DataExporter
+  # given an array of hashes, return set of all keys
+  def self.hash_array_keys(hl)
+    hl.each_with_object(Set.new) { |h, keys| keys.merge(h.keys) }
+  end
+
+  def self.hash_array_merge(hl, transpose)
+    # given a list of hashes hl, generates a merged hash.  The
+    # resulting hash contains a superset of all the hash keys.  The
+    # values are corresponding values from each hash in hl.
+    keys = hash_array_keys(hl)
+
+    if transpose
+      keys.each_with_object({}) { |k, rh|
+        rh[k] = hl.map { |h| h[k] }
+      }
+    else
+      [keys.to_a] + hl.map {|h| keys.map {|k| h[k]}}
+    end
+  end
+
+  def self.encode_json(s)
+    Base64.strict_encode64 Zlib.deflate(s)
+  end
+
+  def self.decode_json(s)
+    Zlib.inflate Base64.strict_decode64(s)
+  end
+
+  def self.to_csv(obj, config=nil)
+    obj = [obj] unless obj.respond_to? :map
+
+    config ||= {}
+
+    # if all array items are hashes, we merge them
+    obj = hash_array_merge(obj, config["transpose"]) if
+      obj.is_a?(Array) && obj.map {|x| x.is_a? Hash}.all?
+
+    # symbolize config keys as expected by CSV.generate
+    conf = config.each_with_object({}) { |(k,v), h|
+      h[k.to_sym] = v unless k.to_s == "transpose"
+    }
+
+    # FIXME: very hacky to default row_sep to CRLF
+    conf[:row_sep] ||= "\r\n"
+
+    # FIXME: the following is ridiculously complex. We have different
+    # data paths for hashes and arrays.  Also, arrays can turn into
+    # hashes is all their items are hashes!  We map to complex objects
+    # to JSON when inside hashes, but not arrays. Really need to
+    # rethink this.  Probably should have separate functions for
+    # to_csv for hash and arrays.
+
+    if obj.is_a? Hash
+      csv_string = CSV.generate(conf) do |csv|
+        obj.each do |x|
+          csv << x.flatten(1).map(&:to_s)
+        end
+      end
+    else
+      csv_string = CSV.generate(conf) do |csv|
+        obj.each do |x|
+          x = [x] unless x.respond_to? :map
+          csv << x.map { |v|
+            v.is_a?(Array) || v.is_a?(Hash) ? encode_json(v.to_json) : v.to_s
+          }
+        end
+      end
+    end
+  end
+
+  def self.class_info(klass)
+    @class_info ||= {}
+
+    return @class_info[klass] if @class_info[klass]
+
+    associations = klass.reflect_on_all_associations.map(&:name)
+
+    @class_info[klass] = {
+      cols:
+      klass.columns.map(&:name) - Marty::DataRowProcessor::MCFLY_COLUMNS.to_a,
+
+      assoc:
+      associations.each_with_object({}) { |a, h|
+        h["#{a}_id"] = Marty::DataRowProcessor.assoc_info(klass, a)
+      },
+    }
+  end
+
+  def self.export_attr(obj, c, info)
+    v = obj.send(c.to_sym)
+    assoc_info = info[:assoc][c] unless v.nil?
+    return [v] unless assoc_info
+
+    assoc_obj = assoc_info[:assoc_class].find(v)
+
+    assoc_info[:assoc_keys].map {|k| assoc_obj.send(k.to_sym)}
+  end
+
+  def self.export_header_attr(c, info)
+    assoc_info = info[:assoc][c]
+    return c unless assoc_info
+
+    # remove _id
+    c = c[0..-4]
+
+    assoc_keys = assoc_info[:assoc_keys]
+
+    # FIXME: this doesn't work if k is also an association.  Needs to
+    # be recursive.
+    assoc_keys.length > 1 ? assoc_keys.map {|k| "#{c}__#{k}"} : c
+  end
+
+  # Given a Mcfly klass, generate an export array.  Can potentially
+  # use up a lot of memory if the result set is large.
+  def self.do_export(ts, klass, sort_field=nil)
+    info = class_info(klass)
+
+    # strip _id from assoc fields
+    header = [ info[:cols].map {|c| export_header_attr(c, info)}.flatten(1) ]
+
+    query = klass
+
+    # is it Mcfly?
+    if (klass.const_get(:MCFLY_UNIQUENESS) rescue nil)
+      ts = Mcfly.normalize_infinity(ts)
+      query = query.where("obsoleted_dt >= ? AND created_dt < ?", ts, ts)
+    end
+
+    header + query.
+      order(sort_field || :id).
+      map {|obj| info[:cols].map {|c| export_attr(obj, c, info)}.flatten(1)}
+  end
+end

data/lib/marty/data_importer.rb

@@ -0,0 +1,122 @@
+require 'csv'
+require 'marty/data_conversion'
+
+module Marty
+  class DataImporterError < StandardError
+    attr_reader :lines
+
+    def initialize(message, lines)
+      msg = lines && lines.respond_to?(:join) ?
+      "#{message} - lines: #{lines.join(',')}" : message
+
+      super(msg)
+      @lines = lines
+    end
+  end
+
+  class DataImporter
+    # perform cleaning and do_import and summarize its results
+    def self.do_import_summary(klass,
+                               data,
+                               dt                  = 'infinity',
+                               cleaner_function    = nil,
+                               validation_function = nil,
+                               col_sep             = "\t",
+                               allow_dups          = false,
+                               preprocess_function = nil
+                               )
+
+      recs = self.do_import(klass,
+                            data,
+                            dt,
+                            cleaner_function,
+                            validation_function,
+                            col_sep,
+                            allow_dups,
+                            preprocess_function,
+                            )
+
+      recs.each_with_object(Hash.new(0)) {|(op, id), h|
+        h[op] += 1
+      }
+    end
+
+    # Given a Mcfly klass and CSV data, import data into the database
+    # and report on affected rows.  Result is an array of tuples.
+    # Each tuple is associated with one data row and looks like [tag,
+    # id].  Tag is one of :same, :update, :create and "id" is the id
+    # of the affected row.
+    def self.do_import(klass,
+                       data,
+                       dt                  = 'infinity',
+                       cleaner_function    = nil,
+                       validation_function = nil,
+                       col_sep             = "\t",
+                       allow_dups          = false,
+                       preprocess_function = nil
+                       )
+
+      parsed = data.is_a?(Array) ? data :
+        CSV.new(data, headers: true, col_sep: col_sep)
+
+      # run preprocessor
+      parsed = klass.send(preprocess_function.to_sym, parsed) if
+        preprocess_function
+
+      klass.transaction do
+        cleaner_ids = cleaner_function ? klass.send(cleaner_function.to_sym) :
+          []
+
+        raise "bad cleaner function result" unless
+          cleaner_ids.all? {|id| id.is_a?(Fixnum) }
+
+        eline = 0
+
+        begin
+          res = parsed.each_with_index.map do
+            |row, line|
+            eline = line
+
+            # skip lines which are all nil
+            next :blank if row.to_hash.values.none?
+
+            Marty::DataConversion.create_or_update(klass, row, dt)
+          end
+        rescue => exc
+          # to find problems with the importer, comment out the rescue block
+          raise Marty::DataImporterError.new(exc.to_s, [eline])
+        end
+
+        ids = {}
+
+        # raise an error if record referenced more than once.
+        res.each_with_index do
+          |(op, id), line|
+          raise Marty::DataImporterError.
+            new("record referenced more than once", [ids[id], line]) if
+            op != :blank && ids.member?(id) && !allow_dups
+
+          ids[id] = line
+        end
+
+        begin
+          # Validate affected rows if necessary
+          klass.send(validation_function.to_sym, ids.keys) if
+            validation_function
+        rescue => exc
+          raise Marty::DataImporterError.new(exc.to_s, [])
+        end
+
+        remainder_ids = cleaner_ids - ids.keys
+
+        raise Marty::DataImporterError.
+          new("Missing import data. " +
+              "Please provide header line and at least one data line.", [1]) if
+          ids.keys.compact.count == 0
+
+        klass.delete(remainder_ids)
+        res + remainder_ids.map {|id| [:clean, id]}
+      end
+    end
+  end
+end

data/lib/marty/data_importer.rb~

@@ -0,0 +1,114 @@
+module Marty
+  class DataImporterError < StandardError
+    attr_reader :lines
+
+    def initialize(message, lines)
+      super(message)
+      @lines = lines
+    end
+  end
+
+  class DataImporter
+    # perform cleaning and do_import and summarize its results
+    def self.do_import_summary(klass,
+                               data,
+                               dt='infinity',
+                               cleaner_function=nil,
+                               validation_function=nil,
+                               col_sep="\t",
+                               allow_dups=false
+                               )
+
+      recs = self.do_import(klass,
+                            data,
+                            dt,
+                            cleaner_function,
+                            validation_function,
+                            col_sep,
+                            allow_dups,
+                            )
+
+      recs.each_with_object(Hash.new(0)) {|(op, id), h|
+        h[op] += 1
+      }
+    end
+
+    # Given a Mcfly klass and CSV data, import data into the database
+    # and report on affected rows.  Result is an array of tuples.
+    # Each tuple is associated with one data row and looks like [tag,
+    # id].  Tag is one of :same, :update, :create and "id" is the id
+    # of the affected row.
+    def self.do_import(klass,
+                       data,
+                       dt='infinity',
+                       cleaner_function=nil,
+                       validation_function=nil,
+                       col_sep="\t",
+                       allow_dups=false
+                       )
+
+      parsed = data.is_a?(Array) ? data :
+        CSV.new(data, headers: true, col_sep: col_sep)
+
+      klass.transaction do
+        cleaner_ids = cleaner_function ? klass.send(cleaner_function.to_sym) :
+          []
+
+        raise "bad cleaner function result" unless
+          cleaner_ids.all? {|id| id.is_a?(Fixnum) }
+
+        row_proc = nil
+        eline = 0
+
+        begin
+          res = parsed.each_with_index.map { |row, line|
+            eline = line
+
+            row_proc ||= Marty::DataRowProcessor.
+            new(klass,
+                row.respond_to?(:headers) ? row.headers : row.keys,
+                dt,
+                )
+            # skip lines which are all nil
+            next :blank if row.to_hash.values.none?
+
+            row_proc.create_or_update(row)
+          }
+        rescue => exc
+          # to find problems with the importer, comment out the rescue block
+          raise Marty::DataImporterError.new(exc.to_s, [eline])
+        end
+
+        ids = {}
+
+        # raise an error if record referenced more than once.
+        res.each_with_index do
+          |(op, id), line|
+          raise Marty::DataImporterError.
+            new("record referenced more than once", [ids[id], line]) if
+            op != :blank && ids.member?(id) && !allow_dups
+
+          ids[id] = line
+        end
+
+        begin
+          # Validate affected rows if necessary
+          klass.send(validation_function.to_sym, ids.keys) if
+            validation_function
+        rescue => exc
+          raise Marty::DataImporterError.new(exc.to_s, [])
+        end
+
+        remainder_ids = cleaner_ids - ids.keys
+
+        raise Marty::DataImporterError.
+          new("Missing import data. " +
+              "Please provide header line and at least one data line.", [1]) if
+          ids.keys.compact.count == 0
+
+        klass.delete(remainder_ids)
+        res + remainder_ids.map {|id| [:clean, id]}
+      end
+    end
+  end
+end