importu 0.1.0 → 0.2.0

data/lib/importu/duplicate_manager.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+require "set"
+
+require "importu/exceptions"
+
+# The duplicate manager provides support for recording records and objects
+# encountered during the import process. When records or objects have been
+# encountered previously, a Importu::DuplicateRecord exception is raised.
+class Importu::DuplicateManager
+
+  # Creates a new instance of the duplicate manager.
+  #
+  # @example
+  #   manager = Importu::DuplicateManager.new
+  #
+  # @param finder_fields [Array<Array<Symbol>>] A list of finder field
+  #   groups that should be used when checking if records are duplicates.
+  # @return [Importu::DuplicateManager]
+  #
+  # @api public
+  def initialize(finder_fields: [])
+    # Proc-based finder fields cannot be directly applied to records, as
+    # it requires looking up the corresponding object using the backend.
+    @finder_fields = finder_fields.reject {|fg| fg.respond_to?(:call) }
+
+    @encountered = Set.new
+  end
+
+  # Checks that the unique id of an object returned from the backend has not
+  # been encountered before. Raises a DuplicateError exception if the object
+  # has been encountered before, otherwise the object is marked as seen.
+  #
+  # @example
+  #   manager.check_object!(71)
+  #   manager.check_object!("0aefe55a-58bb-4a16-b873-ba3425e443bb")
+  #   manager.check_object!(71) # raises Importu::DuplicateManager
+  #
+  # @param unique_id [#eql, #hash] A unique object identifier that can be
+  #   compared against other object identifiers.
+  # @return [void]
+  # @raise [Importu::DuplicateRecord]
+  #
+  # @api public
+  def check_object!(unique_id)
+    return unless unique_id
+
+    result = @encountered.add?(_object_unique_id: unique_id)
+    duplicate_record! if result.nil?
+  end
+
+  # Checks that a conflicting record has not been encountered before. Uses
+  # the configured finder_fields to construct sets of key/value pairs that
+  # are considered unique enough to look up objects from the backend. Marks
+  # all of the key/value pairs as encountered if not seen before. Raises a
+  # DuplicateError exception if any were previously encountered.
+  #
+  # @example
+  #   manager.check_record!(record)
+  #   manager.check_record!(record) # raises Importu::DuplicateRecord
+  #
+  # @param record [Importu::Record]
+  # @return [void]
+  # @raise [Importu::DuplicateRecord]
+  #
+  # @api public
+  def check_record!(record)
+    results = @finder_fields.map do |field_group|
+      conditions = field_group.to_h {|f| [f, record.fetch(f)] }
+      @encountered.add?(conditions) ? :added : :duplicate
+    rescue KeyError
+      # Field group key not defined on record, always nil so invalid
+      :skipped
+    end
+
+    duplicate_record! if results.include?(:duplicate)
+  end
+
+  # Raises an Importu::DuplicateRecord exception
+  #
+  # @return [void] never returns
+  # @raise [Importu::DuplicateRecord]
+  #
+  # @api private
+  private def duplicate_record!
+    raise Importu::DuplicateRecord, "matches a previous record"
+  end
+
+end

data/lib/importu/exceptions.rb

@@ -1,34 +1,165 @@
+# frozen_string_literal: true
 module Importu
+  # Base class for all Importu exceptions.
+  #
+  # @api public
   class ImportuException < StandardError
+    # Returns the exception class name without module prefix.
+    #
+    # @return [String] the short class name
     def name
       self.class.name[/[^:]+$/]
     end
   end
 
+  # Raised when an importer definition is invalid.
+  #
+  # Common causes:
+  # - Referencing an undefined field in a converter
+  # - Using an unregistered converter type
+  #
+  # @api public
+  class InvalidDefinition < ImportuException; end
+
+  # Raised when source data cannot be parsed.
+  #
+  # Common causes:
+  # - Malformed CSV, JSON, or XML
+  # - Empty source file
+  # - Encoding issues
+  #
+  # @api public
   class InvalidInput < ImportuException; end
 
+  # Raised when a requested backend is not registered.
+  #
+  # @example
+  #   model "Book", backend: :nonexistent  # raises BackendNotRegistered
+  #
+  # @api public
+  class BackendNotRegistered < ImportuException; end
+
+  # Raised when no backend matches the configured model.
+  #
+  # This happens during auto-detection when no registered backend
+  # recognizes the model class, or when multiple backends match.
+  #
+  # @example Fix by specifying backend explicitly
+  #   model "Book", backend: :active_record
+  #
+  # @api public
+  class BackendMatchError < ImportuException; end
+
+  # Raised when fields cannot be assigned to the model.
+  #
+  # This happens when the model doesn't have setter methods for
+  # all the fields defined in the importer.
+  #
+  # @api public
+  class UnassignableFields < ImportuException; end
+
+  # Raised when a record fails validation during import.
+  #
+  # This is the base class for record-level errors. The import continues
+  # processing other records; failed records are tracked in the Summary.
+  #
+  # @see Importu::Summary#validation_errors
+  # @see Importu::Summary#itemized_errors
+  # @api public
   class InvalidRecord < ImportuException
+    # @return [Array<String>, nil] validation error messages
     attr_reader :validation_errors
 
-    def initialize(message = nil, validation_errors = nil)
+    # @return [String, nil] aggregation-friendly error message
+    attr_reader :normalized_message
+
+    # Creates a new InvalidRecord exception.
+    #
+    # @param message [String, nil] the error message
+    # @param validation_errors [Array<String>, nil] detailed validation errors
+    # @param normalized_message [String, nil] message for error aggregation
+    def initialize(message = nil, validation_errors = nil, normalized_message: nil)
       @validation_errors = validation_errors
+      @normalized_message = normalized_message
       super(message)
     end
   end
 
-  class FieldParseError < InvalidRecord; end
+  # Raised when a field value cannot be parsed or converted.
+  #
+  # Common causes:
+  # - Invalid date format: "not-a-date" for a date field
+  # - Invalid number: "abc" for an integer field
+  # - Invalid boolean: "maybe" for a boolean field
+  #
+  # @api public
+  class FieldParseError < InvalidRecord
+    # @return [Symbol] the name of the field that failed parsing
+    attr_reader :field_name
+
+    # Creates a new FieldParseError.
+    #
+    # @param field_name [Symbol] the field that failed
+    # @param message [String] description of the parse error
+    def initialize(field_name, message)
+      @field_name = field_name
+      @message = message
+      super(message)
+    end
+
+    # Returns the error message with field name prefix.
+    #
+    # @return [String] formatted error message
+    def to_s
+      "#{@field_name}: #{@message}"
+    end
+  end
+
+  # Raised when a record is a duplicate of one already processed.
+  #
+  # Duplicates are detected based on the find_by fields. If two records
+  # in the same import have matching finder field values, the second
+  # is marked as a duplicate.
+  #
+  # @see Importu::ConfigDSL#find_by
+  # @api public
   class DuplicateRecord < InvalidRecord; end
 
+  # Raised when a required field is missing from source data.
+  #
+  # By default, all fields are required. Use `required: false` to make
+  # a field optional.
+  #
+  # @example Making a field optional
+  #   field :notes, required: false
+  #
+  # @api public
   class MissingField < InvalidRecord
+    # @return [Hash] the field definition that was missing
     attr_reader :definition
 
-    def initialize(definition)
+    # @return [Array<String>, nil] fields that were available in source
+    attr_reader :available_fields
+
+    # Creates a new MissingField exception.
+    #
+    # @param definition [Hash] the field definition
+    # @param available_fields [Array<String>, nil] fields present in source data
+    def initialize(definition, available_fields: nil)
       @definition = definition
+      @available_fields = available_fields
     end
 
+    # Returns a helpful error message listing available fields.
+    #
+    # @return [String] the error message
     def message
       field = definition[:label] || definition[:name]
-      "missing field \"#{field}\" from source data"
+      msg = "missing field \"#{field}\" from source data"
+      if available_fields&.any?
+        msg += "; available fields: #{available_fields.join(", ")}"
+      end
+      msg
     end
   end
 end

data/lib/importu/importer.rb

@@ -1,118 +1,205 @@
-require 'active_record/errors'
-
+# frozen_string_literal: true
+require "importu/backends"
+require "importu/converters"
+require "importu/definition"
+require "importu/exceptions"
+require "importu/record"
+require "importu/summary"
+
+# The main class for defining and running imports.
+#
+# Subclass Importer to define your import specification, then instantiate
+# with a data source to process records.
+#
+# @example Define an importer
+#   class BookImporter < Importu::Importer
+#     # Define fields to extract from source data
+#     fields :title, :author
+#     field :isbn, label: "ISBN-10"
+#     field :pages, required: false, &convert_to(:integer)
+#
+#     # Connect to a model for persistence (optional)
+#     model "Book"
+#     allow_actions :create, :update
+#     find_by :isbn
+#   end
+#
+# @example Process records without persistence
+#   source = Importu::Sources::CSV.new("books.csv")
+#   importer = BookImporter.new(source)
+#
+#   importer.records.each do |record|
+#     puts "#{record[:title]} by #{record[:author]}"
+#   end
+#
+# @example Import with persistence
+#   source = Importu::Sources::CSV.new("books.csv")
+#   importer = BookImporter.new(source)
+#   summary = importer.import!
+#
+#   puts summary.result_msg
+#   # Total:     100
+#   # Created:   95
+#   # Updated:   3
+#   # Invalid:   2
+#   # Unchanged: 0
+#
+# @see Importu::ConfigDSL for all DSL methods
+# @see Importu::Summary for import results
+# @see Importu::Record for accessing record data
+# @api public
 class Importu::Importer
-  attr_reader :options, :infile, :outfile, :validation_errors
-  attr_reader :total, :invalid, :created, :updated, :unchanged
 
-  include Importu::Dsl
+  extend Importu::ConfigDSL
   include Importu::Converters
 
-  def initialize(infile, options = {})
-    @options = options
-    @total = @invalid = @created = @updated = @unchanged = 0
-    @validation_errors = Hash.new(0) # counter for each validation error
-
-    @infile = infile.respond_to?(:readline) ? infile : File.open(infile, 'rb')
-  end
-
-  def records
-    [].to_enum # implement in a subclass
+  # The data source used for generating records
+  #
+  # @return [#rows]
+  #
+  # @example
+  #   importer.source # => #<Importu::Backends::CSV: ...>
+  #
+  # @api public
+  attr_reader :source
+
+  # Creates a new instance of an importer.
+  #
+  # @example
+  #   Importu::Importer.new # => #<Importu::Importer: ...>
+  #
+  # @param source [#rows] The source to read data from.
+  # @param backend [#find, #unique_id, #create, #update] The backend to
+  #   persist records to.
+  # @param definition [Importu::Definition, nil] A definition/contract to
+  #   use for generating records and controlling the import.
+  # @return [Importu::Importer]
+  #
+  # @api public
+  def initialize(source, backend: nil, definition: nil)
+    @source = source
+    @backend = backend
+    @definition = definition || self.class
+    @context = Importu::ConverterContext.with_config(**config)
   end
 
-  def outfile
-    @outfile ||= Tempfile.new('import', Rails.root.join('tmp'), 'wb+')
+  # A registry of importer backends available for use.
+  #
+  # @example
+  #   Importu::Importer.backend_registry # => #<Importu::Backends: ...>
+  #
+  # @return [Importu::Backend]
+  #
+  # @api semipublic
+  def self.backend_registry
+    Importu::Backends.registry
   end
 
-  def import!(finder_scope = nil, &block)
-    # if a scope is passed in, that scope becomes the starting scope used by
-    # the finder, otherwise the model's default scope is used).
-
-    finder_scope ||= model_class.scoped
-    records.each {|r| import_record(r, finder_scope, &block) }
+  # A hash-based configuration of the definition used by the importer.
+  #
+  # @return [Hash]
+  #
+  # @example
+  #   importer.config # => { ... }
+  #
+  # @api semipublic
+  def config
+    @definition.config
   end
 
-  def result_msg
-    msg = <<-END.strip_heredoc
-      Total:     #{@total}
-      Created:   #{@created}
-      Updated:   #{@updated}
-      Invalid:   #{@invalid}
-      Unchanged: #{@unchanged}
-    END
-
-    if @validation_errors.any?
-      msg << "\nValidation Errors:\n"
-      msg << @validation_errors.map {|e,c| "  - #{e}: #{c}" }.join("\n")
+  # Reads data from the source and attempts to create or update records
+  # through the backend. A summary of results from the import, including
+  # any errors encountered will be returned.
+  #
+  # If you need a way to track the progress of an import as each record is
+  # added, a custom recorder can be provided that can hook into other parts
+  # of your system; the recorder's #record method is called after each record
+  # has been processed.
+  #
+  # @example
+  #   summary = importer.import!
+  #   summary.created # => 2
+  #
+  #   class CustomRecorder
+  #     def record(result, index: nil, errors: [])
+  #       puts "record: #{index}: #{result}"
+  #     end
+  #   end
+  #
+  #   importer.import!(CustomRecorder.new)
+  #   # (stdout) "record 0: created"
+  #   # (stdout) "record 1: unchanged"
+  #   # ...
+  #
+  # @param recorder [#record] An optional recorder to use instead of the
+  #   default summarizer. Must implement a #record method.
+  # @return [Importu::Summary, #record] a summary object, or the same object
+  # passed into the method.
+  #
+  # @api public
+  def import!(recorder = Importu::Summary.new)
+    backend = with_middleware(@backend || backend_from_config)
+
+    records.each.with_index do |record, idx|
+      import_record(backend, record, idx, recorder)
     end
-
-    msg
+    recorder
   end
 
-
-  protected
-
-  def model_class
-    @model_class ||= model.constantize
+  # An iterator of Importu::Record objects from the source data. Each call
+  # to the method returns a new iterator from the start.
+  #
+  # @return [Importu::Record::Iterator]
+  #
+  # @example
+  #   importer.records
+  #
+  # @api public
+  def records
+    Importu::Record::Iterator.new(@source.rows, **config)
   end
 
-  def import_record(record, finder_scope, &block)
-    begin
-      object = find(finder_scope, record) || model_class.new
-      action = object.new_record? ? :create : :update
-      check_duplicate(object) if action == :update
-
-      case ([action] - allowed_actions).first
-        when :create then raise Importu::InvalidRecord, "#{model} not found"
-        when :update then raise Importu::InvalidRecord, "existing #{model} found"
-      end
-
-      record.assign_to(object, action, &block)
-
-      case record.save!
-        when :created   then @created   += 1
-        when :updated   then @updated   += 1
-        when :unchanged then @unchanged += 1
-      end
-
-    rescue Importu::InvalidRecord => e
-      if errors = e.validation_errors
-        # convention: assume data-specific error messages put data inside parens, e.g. 'Dupe record found (sysnum 5489x)'
-        errors.each {|error| @validation_errors[error.gsub(/ *\([^)]+\)/,'')] += 1 }     
-      else
-        @validation_errors["#{e.name}: #{e.message}"] += 1
-      end
-
-      @invalid += 1
-      raise
-
-    ensure
-      @total += 1
-    end
+  # Looks for a backend that is compatible with the definition used for
+  # the importer.
+  #
+  # @return [#find, #unique_id, #create, #update]
+  # @raise [Importu::BackendMatchError] if a compatible backend could not be
+  #   found.
+  #
+  # @api private
+  private def backend_from_config
+    backend_class = self.class.backend_registry.from_config!(**config[:backend])
+    backend_class.new(**config[:backend])
   end
 
-  def find(scope, record)
-    # FIXME: find does not report if it finds more than one record matching
-    # the :find_by conditions passed in.  it just uses the first match for
-    # now.  what should be the correct behaviour?
-
-    field_groups = self.class.finder_fields or return
-    field_groups.each do |field_group|
-      if field_group.respond_to?(:call) # proc
-        object = scope.instance_exec(record, &field_group).first
-      else
-        conditions = Hash[field_group.map {|f| [f, record[f]]}]
-        object = scope.where(conditions).first
-      end
-
-      return object if object
-    end
-    nil
+  # Performs an import of a single record. Acts as a wrapper around behavior
+  # that interfaces with the backend.
+  #
+  # @return [void]
+  #
+  # @api private
+  private def import_record(backend, record, index, recorder)
+    object = backend.find(record)
+
+    result, _object = object.nil? \
+      ? backend.create(record)
+      : backend.update(record, object)
+
+    recorder.record(result, index: index)
+  rescue Importu::InvalidRecord => e
+    errors = e.validation_errors || ["#{e.name}: #{e.message}"]
+    recorder.record(:invalid, index: index, errors: errors)
   end
 
-  def check_duplicate(record)
-    return unless id = record.respond_to?(:id) && record.id
-    if ((@encountered||=Hash.new(0))[id] += 1) > 1
-      raise Importu::DuplicateRecord, 'matches a previously imported record'
+  # Wraps the configured backend with additional behaviors, such as duplicate
+  # detection.
+  #
+  # @return [#find, #unique_id, #create, #update]
+  #
+  # @api private
+  private def with_middleware(orig_backend)
+    Importu::Backends.middleware.inject(orig_backend) do |backend, middleware|
+      middleware.new(backend, **config[:backend])
     end
   end