importu 0.1.0 → 0.2.0

data/lib/importu/record.rb

@@ -1,123 +1,159 @@
-require 'active_support/core_ext/module/delegation'
-
+# frozen_string_literal: true
+
+require "forwardable"
+
+require "importu/converter_context"
+require "importu/exceptions"
+
+# Represents a single record from the import source.
+#
+# Records lazily convert field values on access and behave like hashes,
+# supporting standard hash methods like [], fetch, keys, values, and each.
+#
+# @example Iterating over records
+#   importer.records.each do |record|
+#     puts "#{record[:title]} by #{record[:author]}"
+#   end
+#
+# @example Accessing field values
+#   record[:title]        # => "The Ruby Programming Language"
+#   record.fetch(:author) # => "David Flanagan"
+#   record.keys           # => [:title, :author, :isbn]
+#
+# @example Converting to a plain hash
+#   record.to_hash  # => { title: "...", author: "...", isbn: "..." }
+#
+# @example Accessing raw source data
+#   record.data     # => { "Title" => "...", "Author" => "..." }
+#
+# @example Checking for conversion errors
+#   if record.valid?
+#     process(record.to_hash)
+#   else
+#     record.errors.each { |e| puts e.to_s }
+#   end
+#
+# @see Importu::Importer#records
+# @api public
 class Importu::Record
-  attr_reader :importer, :data, :raw_data
-
-  include Enumerable
-
-  delegate :keys, :values, :each, :[], :key?, :to => :record_hash
-  delegate :preprocessor, :postprocessor, :to => :importer
-  delegate :definitions, :converters, :to => :importer
-
-  def initialize(importer, data, raw_data)
-    @importer, @data, @raw_data = importer, data, raw_data
-  end
 
-  def record_hash
-    @record_hash ||= generate_record_hash
+  extend Forwardable
+
+  # The raw data from the source before conversion.
+  #
+  # @return [Hash] the raw source data
+  # @api public
+  attr_reader :data
+
+  # Creates a new record from source data.
+  #
+  # @param data [Hash] the raw source data
+  # @param context [Class] the converter context class
+  # @param fields [Hash] field definitions
+  # @api private
+  def initialize(data, context, fields:, **)
+    @data = data
+    @field_definitions = fields
+    @context = context.new(data)
+
+    @errors = []
   end
 
-  def to_hash
-    record_hash
-  end
-
-  def convert(name, type, options = {})
-    type, options = type[:to], type if type.kind_of?(Hash)
-    converter = type ? converters[type] : options[:converter] \
-      or raise "converter not found: #{type}"
-
-    # TODO: defining options in field definition is deprecated
-    definition = definitions[name] || {}
-    options = definition.merge(options)
-
-    begin
-      value = instance_exec(name, options, &converter)
-      value.nil? ? options[:default] : value
-
-    rescue Importu::MissingField => e
-      raise e if options[:required]
-      options[:default]
-
-    rescue ArgumentError => e
-      # conversion of field value most likely failed
-      raise Importu::FieldParseError, "#{name}: #{e.message}"
+  # Returns field names that can be assigned for the given action.
+  #
+  # @param action [Symbol] :create or :update
+  # @return [Array<Symbol>] assignable field names
+  # @api semipublic
+  def assignable_fields_for(action)
+    @field_definitions.each_with_object([]) do |(name, definition), acc|
+      if definition[action] == true && definition[:abstract] == false
+        acc << name
+      end
     end
   end
 
-  def field_value(name, options = {})
-    definition = definitions[name] \
-      or raise "importer field not defined: #{name}"
-
-    convert(name, nil, definition.merge(options))
+  # Returns any conversion errors encountered when processing fields.
+  #
+  # @return [Array<Importu::FieldParseError>] conversion errors
+  # @api public
+  def errors
+    ensure_record_hash
+    @errors
   end
 
-  def assign_to(object, action, &block)
-    @object, @action = object, action
-
-    instance_eval(&preprocessor) if preprocessor
-    instance_exec(object, record_hash, &block) if block
-
-    # filter out any fields we're not allowed to copy for this action
-    allowed_fields = definitions.select {|n,d| d[action] }.keys
-    concrete_fields = definitions.reject {|n,d| d[:abstract] }.keys
-    field_names = record_hash.keys & allowed_fields & concrete_fields
-
-    unsupported = field_names.reject {|n| object.respond_to?("#{n}=") }
-    if unsupported.any?
-      raise "model does not support assigning fields: #{unsupported.to_sentence}"
-    end
+  # Converts the record to a hash of field names to converted values.
+  #
+  # @return [Hash{Symbol => Object}] the converted field values
+  # @raise [Importu::InvalidRecord] if any field conversion errors occurred
+  # @api public
+  def to_hash
+    ensure_record_hash
 
-    (record_hash.keys & allowed_fields & concrete_fields).each do |name|
-      if object.respond_to?("#{name}=")
-        object.send("#{name}=", record_hash[name])
-      else
-      end
+    if errors.any?
+      raise Importu::InvalidRecord.new("field parse errors", errors)
+    else
+      @record_hash
     end
-
-    instance_eval(&postprocessor) if postprocessor
-
-    object
   end
 
-  def save!
-    return :unchanged unless @object.changed?
-
-    begin
-      @object.save!
-      case @action
-        when :create then :created
-        when :update then :updated
-      end
-
-    rescue ActiveRecord::RecordInvalid => e
-      error_msgs = @object.errors.map do |name,message|
-        name = definitions[name][:label] if definitions[name]
-        name == 'base' ? message : "#{name} #{message}"
-      end.join(', ')
-
-      raise Importu::InvalidRecord, error_msgs, @object.errors.full_messages
-    end
+  # Returns whether the record has any conversion errors.
+  #
+  # @return [Boolean] true if no errors, false otherwise
+  # @api public
+  def valid?
+    ensure_record_hash
+    errors.none?
   end
 
-
-  private
-
-  attr_reader :object, :action # needed for exposing to instance_eval'd blocks
-
-  alias_method :record, :record_hash
-
-  def generate_record_hash
-    definitions.inject({}) do |hash,(name,definition)|
-      hash[name.to_sym] = field_value(name)
-      hash
+  # @!method [](key)
+  #   Access a field value by name.
+  #   @param key [Symbol] the field name
+  #   @return [Object] the converted value
+  #
+  # @!method fetch(key, default = nil)
+  #   Access a field value with a default.
+  #   @param key [Symbol] the field name
+  #   @param default [Object] value to return if key not found
+  #   @return [Object] the converted value or default
+  #
+  # @!method keys
+  #   Returns all field names.
+  #   @return [Array<Symbol>] the field names
+  #
+  # @!method values
+  #   Returns all converted field values.
+  #   @return [Array<Object>] the field values
+  delegate (Hash.public_instance_methods - public_instance_methods) => :to_hash
+
+  private def ensure_record_hash
+    @record_hash ||= @field_definitions.each_with_object({}) do |(name, _), hash|
+      hash[name] = @context.field_value(name)
+    rescue Importu::FieldParseError => e
+      @errors << e
     end
   end
 
-  def method_missing(meth, *args, &block)
-    if converters[meth]
-      convert(args[0], meth, args[1]||{}) # convert(name, type, options)
-    else
-      super
+  # Iterates over source rows, yielding Record instances.
+  #
+  # @api semipublic
+  class Iterator < Enumerator
+    # Creates a new iterator over source rows.
+    #
+    # @param rows [Enumerator] the source rows to iterate
+    # @param converters [Hash] converter definitions
+    # @param fields [Hash] field definitions
+    # @api private
+    def initialize(rows, converters:, fields:, **)
+      context = Importu::ConverterContext.with_config(
+        converters: converters,
+        fields: fields,
+      )
+
+      super() do |yielder|
+        rows.each do |row|
+          yielder.yield Importu::Record.new(row, context, fields: fields)
+        end
+      end
     end
   end
 

data/lib/importu/sources/csv.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+require "csv"
+require "tempfile"
+
+require "importu/exceptions"
+require "importu/sources"
+
+# Parses CSV files as import source data.
+#
+# Each row becomes a hash with header names as keys. The CSV must have a
+# header row.
+#
+# @example Basic usage
+#   source = Importu::Sources::CSV.new("data.csv")
+#   source.rows.each { |row| puts row["name"] }
+#
+# @example From a string
+#   csv_data = "name,email\nAlice,alice@example.com"
+#   source = Importu::Sources::CSV.new(StringIO.new(csv_data))
+#
+# @example With semicolon delimiter
+#   source = Importu::Sources::CSV.new("data.csv", csv_options: { col_sep: ";" })
+#
+# @example With tab delimiter
+#   source = Importu::Sources::CSV.new("data.tsv", csv_options: { col_sep: "\t" })
+#
+# @example Common csv_options
+#   csv_options: {
+#     col_sep: ";",        # Column separator (default: ",")
+#     quote_char: "'",     # Quote character (default: '"')
+#     encoding: "UTF-8",   # File encoding
+#   }
+#
+# @see https://ruby-doc.org/stdlib/libdoc/csv/rdoc/CSV.html Ruby CSV documentation
+# @api public
+class Importu::Sources::CSV
+  # Creates a new CSV source.
+  #
+  # @param infile [String, IO] file path or IO object to read from
+  # @param csv_options [Hash] options passed to Ruby's CSV parser
+  # @raise [Importu::InvalidInput] if the CSV is malformed or empty
+  def initialize(infile, csv_options: {}, **)
+    @owns_handle = !infile.respond_to?(:readline)
+    @infile = @owns_handle ? File.open(infile, "rb") : infile
+
+    @csv_options = {
+      headers:        true,
+      return_headers: true,
+      write_headers:  true,
+      skip_blanks:    true,
+    }.merge(csv_options)
+
+    begin
+      @reader = ::CSV.new(@infile, **@csv_options)
+      @header = @reader.readline
+    rescue CSV::MalformedCSVError => e
+      raise Importu::InvalidInput, e.message
+    end
+
+    if @header.nil?
+      raise Importu::InvalidInput, "Empty document"
+    end
+  rescue StandardError
+    close
+    raise
+  end
+
+  # Closes the underlying file handle if opened by this source.
+  #
+  # Safe to call multiple times. Only closes handles that were opened
+  # by this source (not IO objects passed in).
+  #
+  # @return [void]
+  def close
+    return unless @owns_handle && @infile && !@infile.closed?
+    @infile.close
+  end
+
+  # Returns an enumerator that yields each row as a hash.
+  #
+  # @return [Enumerator<Hash>] rows with header names as keys
+  def rows
+    @infile.rewind
+    reader = ::CSV.new(@infile, **@csv_options)
+    Enumerator.new do |yielder|
+      reader.each {|row| yielder.yield(row.to_hash) unless row.header_row? }
+    end
+  end
+
+  # Generates a CSV file with error information appended.
+  #
+  # Creates a copy of the original data with an "_errors" column containing
+  # any validation errors for each row. Useful for returning to data providers.
+  #
+  # @param summary [Importu::Summary] the import summary containing errors
+  # @param only_errors [Boolean] if true, only include rows that had errors
+  # @return [Tempfile, nil] temp file with error data, or nil if no errors
+  def write_errors(summary, only_errors: false)
+    return unless summary.itemized_errors.any?
+
+    header = @header.fields | ["_errors"]
+    itemized_errors = summary.itemized_errors
+
+    Tempfile.new("import").tap do |file|
+      writer = CSV.new(file, **@csv_options)
+      writer << header
+
+      rows.each.with_index do |row, index|
+        errors = itemized_errors.key?(index) \
+          ? itemized_errors[index].join(", ")
+          : nil
+
+        if errors || !only_errors
+          writer << row.merge("_errors" => errors).values_at(*header)
+        end
+      end
+
+      file.rewind
+    end
+  end
+
+end

data/lib/importu/sources/json.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+require "json"
+require "tempfile"
+
+require "importu/exceptions"
+require "importu/sources"
+
+# Parses JSON files as import source data.
+#
+# The JSON must have an array as the root element. Each array element becomes
+# a row. The entire file is loaded into memory.
+#
+# @example Basic usage
+#   source = Importu::Sources::JSON.new("data.json")
+#   source.rows.each { |row| puts row["name"] }
+#
+# @example Expected JSON format
+#   # data.json
+#   [
+#     { "name": "Alice", "email": "alice@example.com" },
+#     { "name": "Bob", "email": "bob@example.com" }
+#   ]
+#
+# @example From a string
+#   json_data = '[{"name": "Alice"}, {"name": "Bob"}]'
+#   source = Importu::Sources::JSON.new(StringIO.new(json_data))
+#
+# @note The entire JSON file is loaded into memory. For very large files,
+#   consider using CSV or a streaming JSON parser.
+#
+# @api public
+class Importu::Sources::JSON
+  # Creates a new JSON source.
+  #
+  # @param infile [String, IO] file path or IO object to read from
+  # @raise [Importu::InvalidInput] if the JSON is malformed or empty
+  def initialize(infile, **)
+    owns_handle = !infile.respond_to?(:readline)
+    @infile = owns_handle ? File.open(infile, "rb") : infile
+
+    begin
+      @infile.rewind
+      @reader = ::JSON.parse(@infile.read)
+    rescue ::JSON::ParserError => e
+      raise Importu::InvalidInput, e.message
+    ensure
+      # JSON loads entire content into memory, so we can close immediately
+      @infile.close if owns_handle && @infile && !@infile.closed?
+    end
+
+    if @reader.nil?
+      raise Importu::InvalidInput, "Empty document"
+    end
+  end
+
+  # Closes any resources held by this source.
+  #
+  # For JSON sources, the file is already closed after initialization
+  # since the entire content is loaded into memory. This method is
+  # provided for API consistency with other sources.
+  #
+  # @return [void]
+  def close
+    # JSON source closes file immediately after reading into memory
+  end
+
+  # Returns an enumerator that yields each element as a hash.
+  #
+  # @return [Enumerator<Hash>] rows from the JSON array
+  def rows
+    Enumerator.new do |yielder|
+      @reader.each {|row| yielder.yield(row) }
+    end
+  end
+
+  # Generates a JSON file with error information appended.
+  #
+  # Creates a copy of the original data with an "_errors" key containing
+  # any validation errors for each row.
+  #
+  # @param summary [Importu::Summary] the import summary containing errors
+  # @param only_errors [Boolean] if true, only include rows that had errors
+  # @return [Tempfile, nil] temp file with error data, or nil if no errors
+  def write_errors(summary, only_errors: false)
+    return unless summary.itemized_errors.any?
+
+    itemized_errors = summary.itemized_errors
+    updated_rows = rows.each.with_index.with_object([]) do |(row, index), acc|
+      if itemized_errors.key?(index)
+        acc << row.merge("_errors" => itemized_errors[index].join(", "))
+      elsif only_errors
+        # Requested to only include rows with new errors, row has none
+      elsif row.key?("_errors")
+        acc << row.dup.tap {|r| r.delete("_errors") }
+      else
+        acc << row
+      end
+    end
+
+    Tempfile.new("import").tap do |file|
+      file.write(JSON.pretty_generate(updated_rows))
+      file.rewind
+    end
+  end
+
+end

data/lib/importu/sources/ruby.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+require "importu/sources"
+
+# Uses Ruby objects as import source data.
+#
+# Accepts an array of hashes or any enumerable that yields objects responding
+# to #to_hash. Hash keys should be strings to match other source formats.
+#
+# @example Basic usage
+#   data = [{ "name" => "Alice" }, { "name" => "Bob" }]
+#   source = Importu::Sources::Ruby.new(data)
+#   source.rows.each { |row| puts row["name"] }
+#
+# @api public
+class Importu::Sources::Ruby
+  # Creates a new Ruby source.
+  #
+  # @param data [Array<Hash>, Enumerable] objects that respond to #to_hash
+  def initialize(data, **)
+    @data = data
+  end
+
+  # Returns an enumerator that yields each element as a hash.
+  #
+  # @return [Enumerator<Hash>] rows from the data array
+  def rows
+    Enumerator.new do |yielder|
+      @data.each {|row| yielder.yield(row.to_hash) }
+    end
+  end
+
+  # Not implemented for Ruby source.
+  #
+  # @param summary [Importu::Summary] the import summary (unused)
+  # @param only_errors [Boolean] (unused)
+  # @return [nil] always returns nil
+  def write_errors(summary, only_errors: false); end
+
+  # No-op for Ruby source (no file handles to close).
+  #
+  # Provided for API consistency with file-based sources.
+  #
+  # @return [void]
+  def close; end
+
+end

data/lib/importu/sources/xml.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+require "nokogiri"
+require "tempfile"
+
+require "importu/exceptions"
+require "importu/sources"
+
+# Parses XML files as import source data.
+#
+# Requires an XPath expression to identify which elements represent records.
+# Each matching element becomes a row, with child elements and attributes
+# as fields.
+#
+# ## Field Extraction
+# For each matching element:
+# - XML attributes become fields (e.g., `<book id="123">` → `{ "id" => "123" }`)
+# - Child element text becomes fields (e.g., `<title>Ruby</title>` → `{ "title" => "Ruby" }`)
+#
+# @example Basic usage
+#   source = Importu::Sources::XML.new("data.xml", records_xpath: "//book")
+#   source.rows.each { |row| puts row["title"] }
+#
+# @example Expected XML format
+#   # data.xml
+#   <library>
+#     <book id="1">
+#       <title>The Ruby Way</title>
+#       <author>Hal Fulton</author>
+#     </book>
+#     <book id="2">
+#       <title>Programming Ruby</title>
+#       <author>Dave Thomas</author>
+#     </book>
+#   </library>
+#
+# @example Resulting rows
+#   # With records_xpath: "//book"
+#   { "id" => "1", "title" => "The Ruby Way", "author" => "Hal Fulton" }
+#   { "id" => "2", "title" => "Programming Ruby", "author" => "Dave Thomas" }
+#
+# @example Configure in importer
+#   class BookImporter < Importu::Importer
+#     source :xml, records_xpath: "//book"
+#   end
+#
+# @note Requires the nokogiri gem.
+# @api public
+class Importu::Sources::XML
+  # Creates a new XML source.
+  #
+  # @param infile [String, IO] file path or IO object to read from
+  # @param records_xpath [String] XPath expression to select record elements
+  # @raise [Importu::InvalidInput] if the XML is malformed or empty
+  def initialize(infile, records_xpath:, **)
+    @owns_handle = !infile.respond_to?(:readline)
+    @infile = @owns_handle ? File.open(infile, "rb") : infile
+    @records_xpath = records_xpath
+
+    if reader.root.nil?
+      raise Importu::InvalidInput, "Empty document"
+    elsif reader.errors.any?
+      raise Importu::InvalidInput, reader.errors.join("\n")
+    end
+  rescue StandardError
+    close
+    raise
+  end
+
+  # Closes the underlying file handle if opened by this source.
+  #
+  # Safe to call multiple times. Only closes handles that were opened
+  # by this source (not IO objects passed in).
+  #
+  # @return [void]
+  def close
+    return unless @owns_handle && @infile && !@infile.closed?
+    @infile.close
+  end
+
+  # Returns an enumerator that yields each matching element as a hash.
+  #
+  # Element attributes and child element text content become hash keys.
+  #
+  # @return [Enumerator<Hash>] rows from matching XML elements
+  def rows
+    Enumerator.new do |yielder|
+      reader.xpath(@records_xpath).each do |xml|
+        data = [
+          *xml.attribute_nodes.map {|a| [a.node_name, a.content] },
+          *xml.elements.map {|e| [e.name, e.content]},
+        ].to_h
+        yielder.yield(data)
+      end
+    end
+  end
+
+  # Generates an XML file with error information appended.
+  #
+  # Creates a copy of the original data with an "_errors" child element
+  # containing any validation errors for each record.
+  #
+  # @param summary [Importu::Summary] the import summary containing errors
+  # @param only_errors [Boolean] if true, only include records that had errors
+  # @return [Tempfile, nil] temp file with error data, or nil if no errors
+  def write_errors(summary, only_errors: false)
+    return unless summary.itemized_errors.any?
+
+    @infile.rewind
+    writer = Nokogiri::XML(@infile, &:nonet)
+    writer.xpath("//_errors").remove
+
+    itemized_errors = summary.itemized_errors
+    writer.xpath(@records_xpath).each_with_index do |xml, index|
+      if itemized_errors.key?(index)
+        node = Nokogiri::XML::Node.new "_errors", writer
+        node.content = itemized_errors[index].join(", ")
+        xml.add_child(node)
+      elsif only_errors
+        xml.remove
+      end
+    end
+
+    Tempfile.new("import").tap do |file|
+      file.write(writer)
+      file.rewind
+    end
+  end
+
+  private def reader
+    @reader ||= Nokogiri::XML(@infile, &:nonet)
+  end
+
+end

data/lib/importu/sources.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Namespace for data source parsers.
+#
+# Sources parse input data (CSV, JSON, XML, Ruby objects) and provide
+# an enumerator of row hashes for the importer to process.
+#
+# @see Importu::Sources::CSV
+# @see Importu::Sources::JSON
+# @see Importu::Sources::XML
+# @see Importu::Sources::Ruby
+module Importu::Sources
+end