marty 0.5.12

data/db/migrate/095_create_marty_tags.rb~

@@ -0,0 +1,19 @@
+require 'marty/migrations'
+
+class CreateMartyPostings < McflyAppendOnlyMigration
+  include Marty::Migrations
+
+  def change
+    create_table :marty_postings do |t|
+      t.string :name, null: false
+      t.references :posting_type, null: false
+      t.boolean :is_test, null: false
+      t.string :comment, null: false
+    end
+
+    add_mcfly_index :marty_postings,
+    :name, :posting_type_id, :is_test
+
+    add_fk :postings, :posting_types
+  end
+end

data/db/migrate/096_add_user_roles_to_import_types.rb

@@ -0,0 +1,11 @@
+class AddUserRolesToImportTypes < ActiveRecord::Migration
+  include Marty::Migrations
+
+  def change
+    change_table :marty_import_types do |t|
+      t.references :role
+    end
+
+    add_fk :import_types, :roles
+  end
+end

data/db/migrate/097_drop_versions.rb

@@ -0,0 +1,9 @@
+class DropVersions < ActiveRecord::Migration
+  def self.up
+    drop_table :versions
+  end
+
+  def self.down
+    announce("No-op on DropVersions.down")
+  end
+end

data/db/migrate/098_create_marty_api_auths.rb

@@ -0,0 +1,20 @@
+require 'mcfly'
+
+class CreateMartyApiAuths < McflyMigration
+  include Marty::Migrations
+
+  def change
+    create_table :marty_api_auths do |t|
+      t.string :app_name,    null: false, limit: 255
+      t.string :api_key,     null: false, limit: 255
+      t.string :script_name, null: false, limit: 255
+    end
+
+    add_mcfly_unique_index(Marty::ApiAuth)
+    add_index :marty_api_auths, [:app_name,
+                                 :script_name,
+                                 :obsoleted_dt],
+    unique: true,
+    name: 'unique_marty_api_auths_2'
+  end
+end

data/db/seeds.rb

@@ -0,0 +1,48 @@
+# create system account if not there
+system_login = Rails.configuration.marty.system_account || 'marty'
+unless Marty::User.find_by_login(system_login)
+  user           = Marty::User.new
+  user.login     = system_login
+  user.firstname = system_login
+  user.lastname  = system_login
+  user.active    = true
+  user.save!
+end
+
+# FIXME: hacky -- globally changes whodunnit
+Mcfly.whodunnit = Marty::User.find_by_login(system_login)
+
+# Create all Marty roles from configuration
+(Rails.configuration.marty.roles || []).each do |role|
+  Marty::Role.create(name: role.to_s)
+end
+
+# Give system account all roles
+Marty::Role.all.map { |role|
+  ur = Marty::UserRole.new
+  ur.user = Mcfly.whodunnit
+  ur.role = role
+  ur.save
+}
+
+# FIXME: this is a hack.  It's needed due to the fact that 'NOW'
+# posting requires a PostingType.
+Marty::PostingType.create name: 'BASE'
+
+# Create NOW posting
+unless Marty::Posting.find_by_name('NOW')
+  sn                 = Marty::Posting.new
+  # FIXME: this is Gemini-specific
+  sn.posting_type_id = Marty::PostingType['BASE'].id
+  sn.comment         = '---'
+  sn.created_dt      = 'infinity'
+  sn.save!
+end
+
+# Create DEV tag
+unless Marty::Tag.find_by_name('DEV')
+  tag            = Marty::Tag.new
+  tag.comment    = '---'
+  tag.created_dt = 'infinity'
+  tag.save!
+end

data/lib/marty.rb

@@ -0,0 +1,18 @@
+# Do not change order of require, since there are some dependencies
+# Do not require 'marty/permissions' - it relies on Rails being loaded first
+require 'marty/railtie'
+require 'marty/engine'
+require 'marty/monkey'
+require 'marty/mcfly_query'
+require 'marty/util'
+require 'marty/migrations'
+require 'marty/data_exporter'
+require 'marty/xl.rb'
+require 'marty/data_conversion'
+require 'marty/data_importer'
+require 'marty/relation'
+require 'marty/promise_job'
+require 'marty/promise_proxy'
+require 'marty/content_handler'
+require 'marty/lazy_column_loader'
+require 'marty/version'

data/lib/marty.rb~

@@ -0,0 +1,13 @@
+require 'marty/engine'
+require 'marty/monkey'
+require 'marty/mcfly_query'
+require 'marty/util'
+require 'marty/migrations'
+require 'marty/data_exporter'
+require 'marty/xl.rb'
+require 'marty/data_importer'
+require 'marty/promise_job'
+require 'marty/promise_proxy'
+require 'marty/content_handler'
+require 'marty/lazy_column_loader'
+require 'marty/version'

data/lib/marty/content_handler.rb

@@ -0,0 +1,97 @@
+require 'csv'
+
+module Marty::ContentHandler
+  GEN_FORMATS = {
+    "csv"  => ['text/csv',                 'download'],
+    "zip"  => ['application/zip',          'download'],
+    "xlsx" => ['application/vnd.ms-excel', 'download'],
+    "html" => ['text/html',                'inline'],
+    "txt"  => ['text/plain',               'inline'],
+    "json" => ['application/json',         'download'],
+
+    # hacky: default format is JSON
+    nil    => ['application/json',         'download'],
+  }
+
+  def self.log_and_raise(err)
+    Marty::Util.logger.error err
+    raise err
+  end
+
+  def self.export(data, format, name)
+    begin
+      case format
+      when "csv"
+        # Somewhat hacky, if data is string => pass through as CSV.
+        # Should generalize to other data types, not just CSV.
+        res = data.is_a?(String) ? data : Marty::DataExporter.to_csv(data)
+      when "xlsx"
+        res = Marty::Xl.spreadsheet(data).to_stream.read
+      when "zip"
+        res = to_zip(data)
+      when nil, "json"
+        res, format = data.to_json, "json"
+      else
+        res, format = {error: "Unknown format: #{format}"}.to_json, "json"
+      end
+    rescue => exc
+      res, format =
+        {error: "Failed conversion #{format}: #{exc}"}.to_json, "json"
+    end
+
+    type, disposition = GEN_FORMATS[format]
+
+    return [res, type, disposition, "#{name}.#{format}"]
+  end
+
+  private
+
+  def self.sanitize_filename(filename)
+    filename.strip.
+      gsub(/[\\\/]/, '_').
+      gsub(/[^[:print:]]/, '_')
+  end
+
+  def self.uniq_filename(filename, fset)
+    (0..1000).each { |i|
+      post = i==0 ? "" : " (#{i})"
+      fn = filename + post
+      return fn unless fset.member? fn
+    }
+    filename
+  end
+
+  def self.to_zip_stream(stream, path, data)
+    fset = Set.new
+
+    data.each { |r|
+      title, format, result = r["title"], r["format"], r["result"]
+
+      log_and_raise "Result has no title" unless title
+      log_and_raise "Result has no result" unless result
+
+      if format == "zip"
+        to_zip_stream(stream, path + [title], result)
+        next
+      end
+
+      res_data, _type, _disposition, res_name = export(result, format, title)
+
+      filename = uniq_filename(sanitize_filename(res_name), fset)
+      fset.add filename
+
+      stream.put_next_entry((path + [filename]).join('/'))
+      stream.write res_data
+    }
+  end
+
+  def self.to_zip(data)
+    raise "Can't convert non-array data to zip format: #{data.class}" unless
+      data.is_a?(Array)
+
+    res = Zip::OutputStream.write_buffer do |stream|
+      to_zip_stream(stream, [], data)
+    end
+    res.string
+  end
+end

data/lib/marty/content_handler.rb~

@@ -0,0 +1,93 @@
+module Marty::ContentHandler
+  GEN_FORMATS = {
+    "csv" 	=> ['text/csv',			'download'],
+    "zip" 	=> ['application/zip',		'download'],
+    "xlsx" 	=> ['application/vnd.ms-excel',	'download'],
+    "html" 	=> ['text/html',		'inline'],
+    "txt"	=> ['text/plain',		'inline'],
+    "json"	=> ['application/json',		'download'],
+
+    # hacky: default format is JSON
+    nil		=> ['application/json',		'download'],
+  }
+
+  def self.log_and_raise(err)
+    Marty::Util.logger.error err
+    raise err
+  end
+
+  def self.export(data, format, name)
+    begin
+      case format
+      when "csv"
+        res = Marty::DataExporter.to_csv(data)
+      when "xlsx"
+        res = Marty::Xl.spreadsheet(data).to_stream.read
+      when "zip"
+        res = to_zip(data)
+      when nil, "json"
+        res, format = data.to_json, "json"
+      else
+        res, format = {error: "Unknown format: #{format}"}.to_json, "json"
+      end
+    rescue => exc
+      res, format =
+        {error: "Failed conversion #{format}: #{exc}"}.to_json, "json"
+    end
+
+    type, disposition = GEN_FORMATS[format]
+
+    return [res, type, disposition, "#{name}.#{format}"]
+  end
+
+  private
+
+  def self.sanitize_filename(filename)
+    filename.strip.
+      gsub(/[\\\/]/, '_').
+      gsub(/[^[:print:]]/, '_')
+  end
+
+  def self.uniq_filename(filename, fset)
+    (0..1000).each { |i|
+      post = i==0 ? "" : " (#{i})"
+      fn = filename + post
+      return fn unless fset.member? fn
+    }
+    filename
+  end
+
+  def self.to_zip_stream(stream, path, data)
+    fset = Set.new
+
+    data.each { |r|
+      title, format, result = r["title"], r["format"], r["result"]
+
+      log_and_raise "Result has no title" unless title
+      log_and_raise "Result has no result" unless result
+
+      if format == "zip"
+        to_zip_stream(stream, path + [title], result)
+        next
+      end
+
+      res_data, _type, _disposition, res_name = export(result, format, title)
+
+      filename = uniq_filename(sanitize_filename(res_name), fset)
+      fset.add filename
+
+      stream.put_next_entry((path + [filename]).join('/'))
+      stream.write res_data
+    }
+  end
+
+  def self.to_zip(data)
+    raise "Can't convert non-array data to zip format: #{data.class}" unless
+      data.is_a?(Array)
+
+    res = Zip::OutputStream.write_buffer do |stream|
+      to_zip_stream(stream, [], data)
+    end
+    res.string
+  end
+end

data/lib/marty/data_conversion.rb

@@ -0,0 +1,298 @@
+require 'mcfly'
+
+class Marty::DataConversion
+  EXCEL_START_DATE = Date.parse('1/1/1900')-2
+
+  FLOAT_PAT = /^-?\d+(\.\d+)?$/
+
+  PATS = {
+    integer: /^-?\d+(\.0+)?$/,
+    float:   FLOAT_PAT,
+    decimal: FLOAT_PAT,
+  }
+
+  # database types that can be converted to on import
+  DATABASE_TYPES = Set[
+    :boolean,
+    :string,
+    :text,
+    :integer,
+    :float,
+    :decimal,
+    :date,
+    :datetime,
+    :numrange,
+    :int4range,
+    :int8range,
+    :float_array,
+    :json,
+    :jsonb,
+  ]
+
+  def self.convert(v, type)
+    # Converts external data v (e.g. from a CSV, cut/paste) to
+    # ActiveRecord database data type.
+
+    pat = PATS[type]
+
+    raise "bad #{type} #{v.inspect}" if
+      v.is_a?(String) && pat && !(v =~ pat)
+
+    case type
+    when :boolean
+      case v.to_s.downcase
+      when "true",  "1", "y" then true
+      when "false", "0", "n" then false
+      else raise "unknown boolean #{v}"
+      end
+    when :string, :text
+      v
+    when :integer
+      v.to_i
+    when :float
+      v.to_f
+    when :decimal
+      v.to_d
+    when :date
+      # Dates are kept as float in Google spreadsheets.  Need to
+      # convert them to dates. FIXME: 'infinity' as a date in
+      # Rails 3.2 appears to be broken. Setting a date field to
+      # 'infinity' sets it to nil.
+      v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
+        Mcfly.is_infinity(v) ? 'infinity' : v.to_date
+    when :datetime
+      Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
+    when :numrange, :int4range, :int8range
+      v.to_s
+    when :float_array, :json, :jsonb
+      JSON.parse Marty::DataExporter.decode_json(v)
+    else
+      raise "unknown type #{type} for #{v}"
+    end
+  end
+
+  ######################################################################
+
+  def self.assoc_keys(klass)
+    return Mcfly.mcfly_uniqueness(klass) if Mcfly.has_mcfly?(klass)
+    # FIXME: very hacky -- picks 1st non-id attr as the association
+    # key for regular (non-mcfly) AR models which don't have
+    # MARTY_IMPORT_UNIQUENESS.
+    klass.const_get(:MARTY_IMPORT_UNIQUENESS) rescue [
+    klass.attribute_names.reject{|x| x=="id"}.first.to_sym]
+  end
+
+  @@associations = {}
+
+  def self.associations(klass)
+    # build a profile for ActiveRecord klass associations which
+    # enables find/import of its database records
+
+    @@associations[klass] ||= klass.reflect_on_all_associations.
+      each_with_object({}) do
+      |assoc, h|
+
+      h[assoc.name.to_s] = {
+        assoc_keys:  assoc_keys(assoc.klass),
+        assoc_class: assoc.klass,
+        foreign_key: assoc.foreign_key,
+      }
+    end
+  end
+
+  def self.assoc_cols(klass)
+    # array of klass association columns (e.g. ["xxx_id", ...])
+    associations(klass).values.map { |a| a[:foreign_key] }
+  end
+
+  ######################################################################
+
+  @@col_types = {}
+
+  def self.col_types(klass)
+    # build profile for ActiveRecord non-assoc columns -- used to
+    # find/import of klass database records.
+
+    @@col_types[klass] ||= klass.columns.each_with_object({}) do
+      |col, h|
+
+      assoc ||= associations(klass)
+      acols ||= assoc_cols(klass)
+
+      cn = col.name
+
+      # ignore mcfly cols
+      next if Mcfly::COLUMNS.member?(cn)
+
+      if acols.member?(cn)
+        h[cn] = assoc.values.detect { |a| a[:foreign_key] == cn }
+      else
+        # for JSON fields in Rails 3.x type is nil, so use sql_type
+        type = col.type || col.sql_type
+        type = "#{type}_array" if col.array
+        h[cn] = type.to_sym
+      end
+    end
+  end
+
+  def self.columns(klass)
+    # list of non-mcfly columns
+    col_types(klass).keys
+  end
+
+  ######################################################################
+
+  def self.find_row(klass, options, dt)
+    key_attrs = assoc_keys(klass)
+
+    raise "no key_attrs for #{klass}" unless key_attrs
+
+    find_options = options.select { |k,v| key_attrs.member? k.to_sym }
+
+    raise "no keys for #{klass} -- #{options}" if find_options.empty?
+
+    # unscope klass since we're sometimes sent lazy column classes
+    q = klass.unscoped.where(find_options)
+    q = q.where('obsoleted_dt >= ?', dt) if dt && Mcfly.has_mcfly?(klass)
+
+    # q.count is almost always 0 or 1 => hopefully it's not too slow on PG.
+    raise "too many results for: #{klass} -- #{options}" if q.count > 1
+
+    q.first
+  end
+
+  ######################################################################
+
+  def self.convert_row(klass, row, dt)
+    # Given row information from imports (usually csv row or hash),
+    # return a hash with fields converted into proper ruby types.
+
+    ctypes = col_types(klass)
+    assoc  = associations(klass)
+
+    raise "bad row (extra columns?) -- #{row}" if row.has_key?(nil)
+
+    key_groups = row.keys.group_by {|x| x.to_s.split('__').first}
+
+    # FIXME: map all empty string values to nil --- this means that
+    # user can't import empty strings -- Perhaps, mapping "" -> nil
+    # should be optional?
+    row = row.each_with_object({}) {
+      |(k,v), h|
+      h[k.to_s] = v == '' ? nil : v
+    }
+
+    key_groups.each_with_object({}) do
+      |(ga, g), h|
+
+      # find the association's details
+      ai = assoc[ga]
+
+      unless ai
+        raise "unexpected grouping for non assoc #{g}" unless g.length == 1
+
+        type = ctypes[ga]
+
+        raise "unknown column #{ga} in #{klass}" unless type
+
+        v = row[ga]
+
+        if v.nil?
+          h[ga] = nil
+        elsif Hash === type
+          # got an id for an association -- FIXME: perhaps this should
+          # not be allowed at all?
+          raise "#{type[:assoc_class].name} with id #{v} not found" unless
+            type[:assoc_class].find_by_id(v)
+
+          h[ga] = v
+        else
+          # not an association, so we need to convert
+          h[ga] = convert(v, type)
+        end
+        next
+      end
+
+      srch_class = ai[:assoc_class]
+      fk = "#{ga}_id"
+
+      if g.length == 1
+        # optimization for case where we have a 1-key association
+        v = row[g.first]
+
+        # If group has only one attr and the attr is nil or AR obj, then
+        # we don't need to search.
+        if v.nil? || v.is_a?(ActiveRecord::Base)
+          h[fk] = v && v.id
+          next
+        end
+
+        # If it's an Enum, use the faster cached looked mechanism
+        if Marty::Enum === srch_class
+          h[fk] = srch_class[ v ].id
+          next
+        end
+      end
+
+      # group size > 1 or not an Enum, so it must be an association
+      raise "expected an association for #{ga}" unless ai
+
+      # build a new row map for this association, we need to convert
+      # it and search for it.
+      arow = g.each_with_object({}) do
+        |k, h|
+
+        # Some old exports don't provide full assoc__attr column names
+        # (e.g. 'xxx_name').  Instead the columns are just named by
+        # assoc (e.g. 'xxx').
+        gname, ka = k.split('__', 2)
+
+        ka ||= ai[:assoc_keys][0].to_s
+        h[ka] = row[k]
+      end
+
+      c_arow = convert_row(srch_class, arow, dt)
+      o_arow = find_row(srch_class, c_arow, dt)
+
+      raise "obj not found: #{ai[:assoc_class]}, #{c_arow}, #{dt}" unless o_arow
+
+      h[fk] = o_arow.id
+    end
+
+  end
+
+  ######################################################################
+
+  def self.create_or_update(klass, row, dt)
+    # Given a row data (usually from import) try to find the
+    # associated DB row from the klass keys.  If found the row is
+    # updated using the dt datetime.  Otherwise, a new row is created
+    # with the provided row data.
+
+    c_row = convert_row(klass, row.to_hash, dt)
+    obj = find_row(klass, c_row, dt)
+
+    obj ||= klass.new
+
+    c_row.each do
+      |k, v|
+      # For each attr, check to see if it's begin changed before
+      # setting it.  The AR obj.changed? doesn't work properly
+      # with array, JSON or lazy attrs.
+      obj.send("#{k}=", v) if obj.send(k) != v
+    end
+
+    # FIXME: obj.changed? doesn't work properly for timestamp
+    # fields in Rails 3.2. It evaluates to true even when datetime
+    # is not changed.  Caused by lack of awareness of timezones.
+    tag = obj.new_record? ? :create : (obj.changed? ? :update : :same)
+
+    raise "old created_dt >= current #{obj} #{obj.created_dt} #{dt}" if
+      (tag == :update) && dt && !Mcfly.is_infinity(dt) && (obj.created_dt > dt)
+
+    obj.created_dt = dt unless tag == :same || Mcfly.is_infinity(dt) || !dt
+    obj.save!
+
+    [tag, obj.id]
+  end
+end