marty 2.5.2 → 2.5.4

data/lib/marty/data_conversion.rb

@@ -1,5 +1,5 @@
 class Marty::DataConversion
-  EXCEL_START_DATE = Date.parse('1/1/1900')-2
+  EXCEL_START_DATE = Date.parse('1/1/1900') - 2
 
   FLOAT_PAT = /^-?\d+(\.\d+)?$/
 
@@ -40,8 +40,8 @@ class Marty::DataConversion
     case type
     when :boolean
       case v.to_s.downcase
-      when "true",  "1", "y", "t" then true
-      when "false", "0", "n", "f" then false
+      when 'true',  '1', 'y', 't' then true
+      when 'false', '0', 'n', 'f' then false
       else raise "unknown boolean: #{v.inspect}"
       end
     when :string, :text, :enum
@@ -58,13 +58,13 @@ class Marty::DataConversion
       begin
         v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
           Mcfly.is_infinity(v) ? 'infinity' : v.to_date
-      rescue => exc
+      rescue StandardError => exc
         raise "date conversion failed for #{v.inspect}}"
       end
     when :datetime
       begin
         Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
-      rescue => exc
+      rescue StandardError => exc
         raise "datetime conversion failed for #{v.inspect}}"
       end
     when :numrange, :int4range, :int8range
@@ -81,11 +81,13 @@ class Marty::DataConversion
 
   def self.assoc_keys(klass)
     return Mcfly.mcfly_uniqueness(klass) if Mcfly.has_mcfly?(klass)
+
     # FIXME: very hacky -- picks 1st non-id attr as the association
     # key for regular (non-mcfly) AR models which don't have
     # MARTY_IMPORT_UNIQUENESS.
     klass.const_get(:MARTY_IMPORT_UNIQUENESS) rescue [
-    klass.column_names.reject{|x| x=="id"}.first.to_sym]
+      klass.column_names.reject { |x| x == 'id' }.first.to_sym
+    ]
   end
 
   @@associations = {}
@@ -95,8 +97,7 @@ class Marty::DataConversion
     # enables find/import of its database records
 
     @@associations[klass] ||= klass.reflect_on_all_associations.
-      each_with_object({}) do
-      |assoc, h|
+      each_with_object({}) do |assoc, h|
 
       h[assoc.name.to_s] = {
         assoc_keys:  assoc_keys(assoc.klass),
@@ -119,9 +120,7 @@ class Marty::DataConversion
     # build profile for ActiveRecord non-assoc columns -- used to
     # find/import of klass database records.
 
-    @@col_types[klass] ||= klass.columns.each_with_object({}) do
-      |col, h|
-
+    @@col_types[klass] ||= klass.columns.each_with_object({}) do |col, h|
       assoc ||= associations(klass)
       acols ||= assoc_cols(klass)
 
@@ -153,12 +152,12 @@ class Marty::DataConversion
 
     raise "no key_attrs for #{klass}" unless key_attrs
 
-    find_options = options.select { |k,v| key_attrs.member? k.to_sym }
+    find_options = options.select { |k, v| key_attrs.member? k.to_sym }
 
     raise "no keys for #{klass} -- #{options}" if find_options.empty?
 
     q = klass.where(find_options)
-    q = q.where("obsoleted_dt >= ? AND created_dt < ?", dt, dt) if
+    q = q.where('obsoleted_dt >= ? AND created_dt < ?', dt, dt) if
        dt && Mcfly.has_mcfly?(klass)
 
     # q.count is almost always 0 or 1 => hopefully it's not too slow on PG.
@@ -176,20 +175,17 @@ class Marty::DataConversion
     ctypes = col_types(klass)
     assoc  = associations(klass)
 
-    raise "bad row (extra columns?) -- #{row}" if row.has_key?(nil)
+    raise "bad row (extra columns?) -- #{row}" if row.key?(nil)
 
-    key_groups = row.keys.group_by {|x| x.to_s.split('__').first}
+    key_groups = row.keys.group_by { |x| x.to_s.split('__').first }
 
     # FIXME: map all empty string values to nil --- this means that
     # user can't import empty strings -- Perhaps, mapping "" -> nil
     # should be optional?
-    row = row.each_with_object({}) {
-      |(k,v), h|
+    row = row.each_with_object({}) do |(k, v), h|
       h[k.to_s] = v == '' ? nil : v
-    }
-    key_groups.each_with_object({}) do
-      |(ga, g), h|
-
+    end
+    key_groups.each_with_object({}) do |(ga, g), h|
       # find the association's details
       ai = assoc[ga]
 
@@ -234,7 +230,7 @@ class Marty::DataConversion
 
         # If it's an Enum, use the faster cached looked mechanism
         if Marty::Enum === srch_class
-          h[fk] = srch_class[ v ].id
+          h[fk] = srch_class[v].id
           next
         end
       end
@@ -244,9 +240,7 @@ class Marty::DataConversion
 
       # build a new row map for this association, we need to convert
       # it and search for it.
-      arow = g.each_with_object({}) do
-        |k, h|
-
+      arow = g.each_with_object({}) do |k, h|
         # Some old exports don't provide full assoc__attr column names
         # (e.g. 'xxx_name').  Instead the columns are just named by
         # assoc (e.g. 'xxx').
@@ -263,7 +257,6 @@ class Marty::DataConversion
 
       h[fk] = o_arow.id
     end
-
   end
 
   ######################################################################
@@ -279,8 +272,7 @@ class Marty::DataConversion
 
     obj ||= klass.new
 
-    c_row.each do
-      |k, v|
+    c_row.each do |k, v|
       # For each attr, check to see if it's begin changed before
       # setting it.  The AR obj.changed? doesn't work properly
       # with array, JSON or lazy attrs.

data/lib/marty/data_exporter.rb

@@ -18,11 +18,11 @@ class Marty::DataExporter
 
     keys = hash_array_keys(hl)
 
-    return keys.each_with_object({}) { |k, rh|
+    return keys.each_with_object({}) do |k, rh|
         rh[k] = hl.map { |h| h[k] }
-    } if transpose
+    end if transpose
 
-    [keys.to_a] + hl.map {|h| keys.map {|k| h[k]}}
+    [keys.to_a] + hl.map { |h| keys.map { |k| h[k] } }
   end
 
   def self.encode_json(s)
@@ -33,19 +33,19 @@ class Marty::DataExporter
     Zlib.inflate Base64.strict_decode64(s)
   end
 
-  def self.to_csv(obj, config=nil)
+  def self.to_csv(obj, config = nil)
     obj = [obj] unless obj.respond_to? :map
 
     config ||= {}
 
     # if all array items are hashes, we merge them
-    obj = hash_array_merge(obj, config["transpose"]) if
-      obj.is_a?(Array) && obj.all? {|x| x.is_a? Hash}
+    obj = hash_array_merge(obj, config['transpose']) if
+      obj.is_a?(Array) && obj.all? { |x| x.is_a? Hash }
 
     # symbolize config keys as expected by CSV.generate
-    conf = config.each_with_object({}) { |(k,v), h|
-      h[k.to_sym] = v unless k.to_s == "transpose"
-    }
+    conf = config.each_with_object({}) do |(k, v), h|
+      h[k.to_sym] = v unless k.to_s == 'transpose'
+    end
 
     # FIXME: very hacky to default row_sep to CRLF
     conf[:row_sep] ||= "\r\n"
@@ -69,7 +69,7 @@ class Marty::DataExporter
     CSV.generate(conf) do |csv|
       obj.each do |x|
         x = [x] unless x.respond_to? :map
-        csv << x.map { |v|
+        csv << x.map do |v|
           case v
           when Array, Hash
             readable ? v.to_json : encode_json(v.to_json)
@@ -78,25 +78,24 @@ class Marty::DataExporter
           else
             v.to_s
           end
-        }
+        end
       end
     end
   end
 
   def self.get_attrs_in_order(klass, attrs)
     return attrs unless klass.const_defined?(:EXPORT_ORDER)
+
     klass::EXPORT_ORDER.select { |attr| attrs.include?(attr) }
   end
 
-  def self.export_attrs(klass, obj, attrs=nil, exclude_attrs=[])
+  def self.export_attrs(klass, obj, attrs = nil, exclude_attrs = [])
     col_types = Marty::DataConversion.col_types(klass)
 
     attr_list_raw = (attrs || col_types.keys).map(&:to_s) - exclude_attrs
     attr_list = get_attrs_in_order(klass, attr_list_raw)
 
-    attr_list.map do
-      |c|
-
+    attr_list.map do |c|
       v = obj.send(c.to_sym)
       type = col_types[c]
 
@@ -104,7 +103,7 @@ class Marty::DataExporter
       next [v] if !type.is_a?(Hash)
 
       # no child row, return nils for each field
-      next [nil]*type[:assoc_keys].count if v.nil?
+      next [nil] * type[:assoc_keys].count if v.nil?
 
       assoc_keys  = type[:assoc_keys]
       assoc_class = type[:assoc_class]
@@ -117,15 +116,13 @@ class Marty::DataExporter
     end
   end
 
-  def self.export_headers(klass, attrs=nil, exclude_attrs=[])
+  def self.export_headers(klass, attrs = nil, exclude_attrs = [])
     col_types = Marty::DataConversion.col_types(klass)
 
     attr_list_raw = (attrs || col_types.keys).map(&:to_s) - exclude_attrs
     attr_list = get_attrs_in_order(klass, attr_list_raw)
 
-    attr_list.map do
-      |c|
-
+    attr_list.map do |c|
       type = col_types[c]
 
       next c unless type.is_a?(Hash)
@@ -139,30 +136,30 @@ class Marty::DataExporter
 
       assoc_class = type[:assoc_class]
 
-      export_headers(assoc_class, assoc_keys).map {|k| "#{c}__#{k}"}
+      export_headers(assoc_class, assoc_keys).map { |k| "#{c}__#{k}" }
     end
   end
 
   # Given a Mcfly klass, generate an export array.  Can potentially
   # use up a lot of memory if the result set is large.
-  def self.do_export(ts, klass, sort_field=nil, exclude_attrs=[])
+  def self.do_export(ts, klass, sort_field = nil, exclude_attrs = [])
     query = klass
 
     if Mcfly.has_mcfly?(klass)
       ts = Mcfly.normalize_infinity(ts)
-      query = query.where("obsoleted_dt >= ? AND created_dt < ?", ts, ts)
+      query = query.where('obsoleted_dt >= ? AND created_dt < ?', ts, ts)
     end
 
     do_export_query_result(klass, query.order(sort_field || :id), exclude_attrs)
   end
 
-  def self.do_export_query_result(klass, qres, exclude_attrs=[])
+  def self.do_export_query_result(klass, qres, exclude_attrs = [])
     # strip _id from assoc fields
-    header = [ export_headers(klass, nil, exclude_attrs).flatten ]
+    header = [export_headers(klass, nil, exclude_attrs).flatten]
 
-    header + qres.map {|obj|
+    header + qres.map do |obj|
       export_attrs(klass, obj, nil, exclude_attrs).flatten(1)
-    }
+    end
   end
 
   # Export a single object to hash -- FIXME: inefficient
@@ -171,6 +168,6 @@ class Marty::DataExporter
     klass = obj.class
     headers = export_headers(klass)
     rec = export_attrs(klass, obj).flatten
-    Hash[ headers.zip(rec) ]
+    Hash[headers.zip(rec)]
   end
 end

data/lib/marty/data_importer.rb

@@ -20,21 +20,21 @@ module Marty
                                col_sep             = "\t",
                                allow_dups          = false,
                                preprocess_function = nil
-                               )
-
-      recs = self.do_import(klass,
-                            data,
-                            dt,
-                            cleaner_function,
-                            validation_function,
-                            col_sep,
-                            allow_dups,
-                            preprocess_function,
-                            )
-
-      recs.each_with_object(Hash.new(0)) {|(op, id), h|
+                              )
+
+      recs = do_import(klass,
+                       data,
+                       dt,
+                       cleaner_function,
+                       validation_function,
+                       col_sep,
+                       allow_dups,
+                       preprocess_function,
+                      )
+
+      recs.each_with_object(Hash.new(0)) do |(op, id), h|
         h[op] += 1
-      }
+      end
     end
 
     # Given a Mcfly klass and CSV data, import data into the database
@@ -50,7 +50,7 @@ module Marty
                        col_sep             = "\t",
                        allow_dups          = false,
                        preprocess_function = nil
-                       )
+                      )
 
       parsed = data.is_a?(Array) ? data :
         CSV.new(data, headers: true, col_sep: col_sep)
@@ -63,14 +63,13 @@ module Marty
         cleaner_ids = cleaner_function ? klass.send(cleaner_function.to_sym) :
           []
 
-        raise "bad cleaner function result" unless
-          cleaner_ids.all? {|id| id.is_a?(Integer) }
+        raise 'bad cleaner function result' unless
+          cleaner_ids.all? { |id| id.is_a?(Integer) }
 
         eline = 0
 
         begin
-          res = parsed.each_with_index.map do
-            |row, line|
+          res = parsed.each_with_index.map do |row, line|
             eline = line
 
             # skip lines which are all nil
@@ -78,17 +77,16 @@ module Marty
 
             Marty::DataConversion.create_or_update(klass, row, dt)
           end
-        rescue => exc
+        rescue StandardError => exc
           # to find problems with the importer, comment out the rescue block
           raise Error.new(exc.to_s, [eline])
         end
 
         ids = {}
         # raise an error if record referenced more than once.
-        res.each_with_index do
-          |(op, id), line|
+        res.each_with_index do |(op, id), line|
           raise Error.
-            new("record referenced more than once", [ids[id], line]) if
+            new('record referenced more than once', [ids[id], line]) if
             op != :blank && ids.member?(id) && !allow_dups
 
           ids[id] = line
@@ -98,19 +96,19 @@ module Marty
           # Validate affected rows if necessary
           klass.send(validation_function.to_sym, ids.keys) if
             validation_function
-        rescue => exc
+        rescue StandardError => exc
           raise Error.new(exc.to_s, [])
         end
 
         remainder_ids = cleaner_ids - ids.keys
 
         raise Error.
-          new("Missing import data. " +
-              "Please provide header line and at least one data line.", [1]) if
+          new('Missing import data. ' +
+              'Please provide header line and at least one data line.', [1]) if
           ids.keys.compact.count == 0
 
         klass.delete(remainder_ids)
-        res + remainder_ids.map {|id| [:clean, id]}
+        res + remainder_ids.map { |id| [:clean, id] }
       end
     end
   end

data/lib/marty/engine.rb

@@ -3,8 +3,7 @@ module Marty
     isolate_namespace Marty
 
     # eager load paths instead of autoload paths
-    config.eager_load_paths += ['lib', 'other'].map do
-      |dir|
+    config.eager_load_paths += ['lib', 'other'].map do |dir|
       File.expand_path("../../../#{dir}", __FILE__)
     end
 

data/lib/marty/json_schema.rb

@@ -1,17 +1,17 @@
 require 'json-schema'
 
 module Marty
-
   private
+
   class PgEnumAttribute < JSON::Schema::Attribute
-    def self.validate(curr_schema, data, frag, pro, validator, opt={})
+    def self.validate(curr_schema, data, frag, pro, validator, opt = {})
       values = nil
       path = '#/' + frag.join('/')
       begin
-        cs = curr_schema.schema["pg_enum"]
+        cs = curr_schema.schema['pg_enum']
         enum = cs.constantize
         values = enum::VALUES
-      rescue => e
+      rescue StandardError => e
         msg = "The property '#{path}': '#{cs}' is not a pg_enum class"
         validation_error(pro, msg, frag, curr_schema, self, opt[:record_errors])
       end
@@ -23,25 +23,25 @@ module Marty
   end
 
   class JsonSchema < JSON::Schema::Draft4
-    RAW_URI = "http://json-schema.org/marty-draft/schema#"
+    RAW_URI = 'http://json-schema.org/marty-draft/schema#'
 
     def initialize
       super
-      @attributes["pg_enum"] = PgEnumAttribute
-      @formats["date-time"]  = JSON::Schema::DateTimeFormat
-      @formats["date"]       = JSON::Schema::DateFormat
+      @attributes['pg_enum'] = PgEnumAttribute
+      @formats['date-time']  = JSON::Schema::DateTimeFormat
+      @formats['date']       = JSON::Schema::DateFormat
       @uri                   = JSON::Util::URI.parse(RAW_URI)
-      @names                 = ["marty-draft", RAW_URI]
+      @names                 = ['marty-draft', RAW_URI]
     end
 
-    JSON::Validator.register_validator(self.new)
+    JSON::Validator.register_validator(new)
 
     def self.get_numbers(schema)
       numbers = []
 
       # traverse the schema, if we find a type: number, add to numbers []
-      trav = lambda { |tree, key, path=[]|
-        return tree.each do|k, v|
+      trav = lambda { |tree, key, path = []|
+        return tree.each do |k, v|
           trav.call(v, k, path + [k])
         end if tree.is_a?(Hash)
         numbers << path[0..-2] if key == 'type' && tree == 'number'
@@ -50,16 +50,16 @@ module Marty
 
       # convert the array stuff [ie. "items", "properties"] to :array
       numbers.map do |num|
-        num.delete("properties")
-        num.map{|n| n=="items" ? :array : n}
+        num.delete('properties')
+        num.map { |n| n == 'items' ? :array : n }
       end
     end
 
     def self.fix_numbers(json, numbers)
-
       # follow path to drill into json
-      drill = lambda {|tree, path|
+      drill = lambda { |tree, path|
         return unless tree
+
         key = path.first
         val = val = tree.send(:[], key) unless key == :array
         if key == :array
@@ -70,7 +70,7 @@ module Marty
             end
           else
             # this is an array of object so continue to drill down
-            tree.each {|sub| drill.call(sub, path[1..-1])}
+            tree.each { |sub| drill.call(sub, path[1..-1]) }
           end
         elsif path.length == 1
           # fix a non array field
@@ -80,24 +80,22 @@ module Marty
           drill.call(val, path[1..-1])
         end
       }
-      numbers.each {|number| drill.call(json, number)}
+      numbers.each { |number| drill.call(json, number) }
     end
 
     def self.get_schema(tag, sname, node, attr)
-      begin
-        Marty::ScriptSet.new(tag).get_engine(sname+'Schemas').
+        Marty::ScriptSet.new(tag).get_engine(sname + 'Schemas').
           evaluate(node, attr, {})
-      rescue => e
+    rescue StandardError => e
         id = "#{sname}/#{node} attrs=#{attr}"
 
         # the schema DL might not exist at all, or might not define the attr
         # being requested
         sch_not_found = ['No such script', "undefined method `#{attr}__D'",
                          "node #{node} is undefined"]
-        msg = sch_not_found.detect{|msg| e.message.starts_with?(msg)} ?
+        msg = sch_not_found.detect { |msg| e.message.starts_with?(msg) } ?
                 'Schema not defined' : "Problem with schema: #{e.message}"
-        return "Schema error for #{id}: #{msg}"
-      end
+        "Schema error for #{id}: #{msg}"
     end
   end
 end