importu 0.1.0 → 0.2.0

data/lib/importu/summary.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+# Records and aggregates results from an import.
+#
+# The Summary tracks counts for each result type (created, updated, unchanged,
+# invalid) and collects validation errors for reporting.
+#
+# @example Basic usage
+#   summary = importer.import!
+#
+#   puts "Processed #{summary.total} records"
+#   puts "Created: #{summary.created}, Updated: #{summary.updated}"
+#   puts "Failed: #{summary.invalid}"
+#
+# @example Checking for errors
+#   if summary.invalid > 0
+#     puts "Errors encountered:"
+#     summary.validation_errors.each do |message, count|
+#       puts "  #{message}: #{count} occurrences"
+#     end
+#   end
+#
+# @example Getting detailed errors by record
+#   summary.itemized_errors.each do |index, errors|
+#     puts "Record #{index} failed: #{errors.map(&:to_s).join(', ')}"
+#   end
+#
+# @example Human-readable output
+#   puts summary.result_msg
+#
+# @example Machine-readable output (for JSON APIs, etc.)
+#   summary.to_hash  # => { created: 5, updated: 2, ... }
+#
+# @see Importu::Importer#import!
+# @api public
+class Importu::Summary
+
+  # The number of times a :created result was recorded
+  #
+  # @example
+  #   summary.created # => 2
+  #
+  # @return [Integer]
+  #
+  # @api public
+  attr_reader :created
+
+  # The number of times an :invalid result was recorded
+  #
+  # @example
+  #   summary.invalid # => 3
+  #
+  # @return [Integer]
+  #
+  # @api public
+  attr_reader :invalid
+
+  # A hash of record indexes and the errors recorded for that record
+  #
+  # @example
+  #   summary.itemized_errors
+  #
+  #   {
+  #     0 => [
+  #       #<Importu::InvalidRecord: ...>,
+  #       #<Importu::InvalidRecord: ...>,
+  #     ],
+  #     3 => [
+  #       #<Importu::InvalidRecord: ...>
+  #     ]
+  #   }
+  #
+  # @return [Hash<Integer=>Array>] a hash of record indexes and the errors
+  #   recorded for that record. Each key represents the position of the
+  #   record in the source data and the value is a list of errors.
+  #   Errors will all be Importu::InvalidRecord exceptions or subclasses
+  #   that can be converted to a string using #to_s.
+  #
+  # @api public
+  attr_reader :itemized_errors
+
+  # The total number of times any result was recorded
+  #
+  # @example
+  #   summary.total # => 36
+  #
+  # @return [Integer]
+  #
+  # @api public
+  attr_reader :total
+
+  # The number of times an :unchanged result was recorded
+  #
+  # @example
+  #   summary.unchanged # => 30
+  #
+  # @return [Integer]
+  #
+  # @api public
+  attr_reader :unchanged
+
+  # The number of times an :updated result was recorded
+  #
+  # @example
+  #   summary.updated # => 1
+  #
+  # @return [Integer]
+  #
+  # @api public
+  attr_reader :updated
+
+  # A hash of error messages and the number of occurrences of each
+  #
+  # @example
+  #   summary.validation_errors
+  #
+  #   {
+  #     "description is required" => 3,
+  #     "title is too long" => 2
+  #   }
+  #
+  # @return [Hash<String=>Integer>] a hash of error messages, with each key
+  #   being an error message and the value representing the number of times
+  #   the error was recorded.
+  #
+  # @api public
+  attr_reader :validation_errors
+
+  # Creates a new instance of a summary. Generally, a new summary object
+  # would get created on each attempt to import data.
+  #
+  # @example
+  #   Summary.new # => Importu::Summary.new
+  #
+  # @api semipublic
+  def initialize
+    @total = @invalid = @created = @updated = @unchanged = 0
+    @validation_errors = Hash.new(0) # counter for each validation error
+
+    # Sparse array of error messages grouped by the index of the record.
+    # Should stay ordered by index because rows are processed sequentially
+    # and hashes preserve insertion order. Recorded errors without an index
+    # will be ignored. Index is 0-based from first record.
+    @itemized_errors = Hash.new {|h, idx| h[idx] = [] }
+  end
+
+  # Record the result of an import. The result may be used for aggregated
+  # statistics or, in the case of errors, a way to retrieve error messages
+  # associated with a record after the import has completed.
+  #
+  # @example
+  #   summary.record(:created, index: 4)
+  #   summary.record(:unchanged, index: 9)
+  #   summary.record(:invalid, index: 17, errors: [
+  #     Importu::InvalidRecord.new("contains non utf8 characters")
+  #   ])
+  #
+  # @param result [:created, :invalid, :unchanged, :updated] the result of
+  #   trying to import the record.
+  # @param index [Integer] A zero-indexed position of the record relative
+  #   to where it was read from the source data.
+  # @param errors [Array<Importu::InvalidRecord>] A list of errors
+  #   encountered while converting or importing the record.
+  # @return [void]
+  #
+  # @api semipublic
+  def record(result, index: nil, errors: [])
+    @total += 1
+
+    case result
+      when :created then @created += 1
+      when :updated then @updated += 1
+      when :unchanged then @unchanged += 1
+      when :invalid
+        @invalid += 1
+        record_errors(errors, index: index)
+    end
+  end
+
+  # An aggregated summary of results meant for human consumption, such
+  # as displaying in a terminal window. If any errors were encountered
+  # during the import, an aggregated list of error messages and the
+  # number of times each error was encountered will also be included.
+  #
+  # @example
+  #   puts summary.result_msg
+  #
+  #   Total:     36
+  #   Created:   2
+  #   Updated:   1
+  #   Invalid:   3
+  #   Unchanged: 30
+  #
+  #   Validation Errors:
+  #     - description is required: 3
+  #     - title is too long: 2
+  #
+  # @return [String] a human-readable aggregate summary of results suitable
+  #   for displaying in a terminal window.
+  #
+  # @api public
+  def result_msg
+    msg = <<-END.gsub(/^\s*/, "")
+      Total:     #{total}
+      Created:   #{created}
+      Updated:   #{updated}
+      Invalid:   #{invalid}
+      Unchanged: #{unchanged}
+    END
+
+    if validation_errors.any?
+      msg << "\nValidation Errors:\n"
+      msg << validation_errors.map {|e, c| "  - #{e}: #{c}" }.join("\n")
+    end
+
+    msg
+  end
+
+  # An aggregated summary of results that can be used by a custom formatter,
+  # or for any purpose by software interacting with this gem.
+  #
+  # @example
+  #   summary.to_hash
+  #
+  #   {
+  #     :created => 2,
+  #     :invalid => 3,
+  #     :total => 36,
+  #     :unchanged => 30,
+  #     :updated => 1,
+  #     :validation_errors => {
+  #       "description is required" => 3,
+  #       "title is too long" => 2
+  #     }
+  #   }
+  #
+  # @return [Hash<Symbol=>Integer,Hash>] a hash of
+  #   aggregated results. Top-level keys are always symbols with all values
+  #   being integers except for :validation_errors; :validation_errors
+  #   will always contain a nested hash of error messages and their counts,
+  #   with each error message keys being represented as a string.
+  #
+  # @api public
+  def to_hash
+    {
+      created: created,
+      invalid: invalid,
+      total: total,
+      unchanged: unchanged,
+      updated: updated,
+      validation_errors: validation_errors,
+    }
+  end
+
+  alias to_s result_msg
+
+  # Updates error attributes to include the newly encountered errors.
+  #
+  # @return [void]
+  #
+  # @api private
+  private def record_errors(errors, index: nil)
+    errors.each do |error|
+      # Use normalized_message if available (set by backends like ActiveRecord),
+      # otherwise fall back to stripping trailing parenthesized data for aggregation.
+      has_normalized = error.respond_to?(:normalized_message) && error.normalized_message
+      normalized_error = has_normalized \
+        ? error.normalized_message
+        : error.to_s.gsub(/ *\([^)]+\) *$/, "")
+
+      @validation_errors[normalized_error] += 1
+    end
+
+    @itemized_errors[index] += errors if index
+  end
+
+end

data/lib/importu/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
 module Importu
-  VERSION = '0.1.0'
+  # Current version of the Importu gem.
+  VERSION = "0.2.0"
 end

data/lib/importu.rb

@@ -1,12 +1,48 @@
+# frozen_string_literal: true
+
+# Importu is a Ruby gem for declarative data import.
+#
+# Define importers as readable contracts with fields, converters, and rules.
+# The gem handles parsing, converting, finding/creating records, and reporting.
+#
+# ## Quick Start
+#
+#     require "importu"
+#
+#     class BookImporter < Importu::Importer
+#       fields :title, :author, :isbn
+#     end
+#
+#     source = Importu::Sources::CSV.new("books.csv")
+#     importer = BookImporter.new(source)
+#     importer.records.each { |record| puts record[:title] }
+#
+# ## Key Classes
+#
+# - {Importu::Importer} - Base class for defining importers
+# - {Importu::Sources::CSV} - Parse CSV files
+# - {Importu::Sources::JSON} - Parse JSON files
+# - {Importu::Sources::XML} - Parse XML files
+# - {Importu::Summary} - Import results and error reporting
+# - {Importu::Record} - Individual record from source data
+#
+# ## DSL Reference
+#
+# See {Importu::ConfigDSL} for all importer configuration options:
+# fields, converters, model, find_by, allow_actions, and more.
+#
+# ## Built-in Converters
+#
+# See {Importu::Converters} for type conversion: boolean, date, datetime,
+# decimal, float, integer, string, trimmed.
+#
+# @api public
 module Importu; end
 
-require 'importu/core_ext'
+require "importu/importer"
+
+require "importu/sources/csv"
+require "importu/sources/json"
+require "importu/sources/xml"
 
-require 'importu/dsl'
-require 'importu/exceptions'
-require 'importu/converters'
-require 'importu/record'
-require 'importu/importer'
-require 'importu/importer/csv'
-require 'importu/importer/json'
-require 'importu/importer/xml'
+require "importu/backends/active_record" if defined?(::ActiveRecord)

data/spec/fixtures/books-duplicates/README.md

@@ -0,0 +1,7 @@
+Contains the original three records from the "books-valid" fixture, along with
+three additional records that share the same isbn10 value as valid records. The
+additional records are valid, but they appear in the same source file as other
+records sharing the same `find_by` key. Because they share the same values used
+to find the object, they will find an object with the same :id used earlier in
+the import. If an object's id matches an id used earlier in the import, the
+record is considered to be a duplicate.

data/spec/fixtures/books-duplicates/infile.csv

@@ -0,0 +1,7 @@
+"isbn10","title","author","release_date","pages"
+"0596516177","The Ruby Programming Language","David Flanagan and Yukihiro Matsumoto","Feb 1, 2008","0448"
+"1449355978","Computer Science Programming Basics in Ruby","Ophir Frieder, Gideon Frieder and David Grossman","1 May, 2013","188"
+"  0596523696  ","  Ruby Cookbook  ","  Lucas Carlson and Leonard Richardson  ","  2006-7-26  ","  910  "
+"0596516177","Duplicate/conflicting record 1","An author","2017-01-01","1"
+"0596516177","Duplicate/conflicting record 2","An author","2017-01-01","1"
+"1449355978","Duplicate/conflicting record 3","An author","2017-01-01","1"

data/spec/fixtures/books-duplicates/model.json

@@ -0,0 +1,23 @@
+[
+  {
+    "isbn10": "0596516177",
+    "title": "The Ruby Programming Language",
+    "authors": ["David Flanagan", "Yukihiro Matsumoto"],
+    "release_date": "2008-02-01",
+    "pages": 448
+  },
+  {
+    "isbn10": "1449355978",
+    "title": "Computer Science Programming Basics in Ruby",
+    "authors": ["Ophir Frieder", "Gideon Frieder", "David Grossman"],
+    "release_date": "2013-05-01",
+    "pages": 188
+  },
+  {
+    "isbn10": "0596523696",
+    "title": "Ruby Cookbook",
+    "authors": ["Lucas Carlson", "Leonard Richardson"],
+    "release_date": "2006-07-26",
+    "pages": 910
+  }
+]

data/spec/fixtures/books-duplicates/summary.json

@@ -0,0 +1,10 @@
+{
+  "created": 3,
+  "updated": 0,
+  "unchanged": 0,
+  "invalid": 3,
+  "total": 6,
+  "validation_errors": {
+    "DuplicateRecord: matches a previous record": 3
+  }
+}

data/spec/fixtures/books-valid/README.md

@@ -0,0 +1,13 @@
+Contains three records that are compatible with the `BookImporter`. They
+should all be created successfully on import.
+
+The following anomalies exist in the data:
+
+| Record | Field         | Formats  | Value           | Description |
+| _all_  | _all_         | xml      | n/a             | mixed xml attribute-element styles, should treat similarly |
+| 1      | release\_date | _all_    | "Feb 1, 2008"   | should detect date format |
+| 1      | pages         | _all_    | "0448"          | leading 0 should parse as decimal not octal |
+| 2      | release\_date | _all_    | "1 May, 2013"   | should detect date format |
+| 2      | pages         | json     | 188             | should handle value already being an integer |
+| 3      | _all_         | _all_    | "  value  "     | should trim surrounding whitespace |
+| 3      | release\_date | _all_    | "  2006-7-26  " | should handle missing leading '0' in date |

data/spec/fixtures/books-valid/infile.csv

@@ -0,0 +1,4 @@
+"isbn10","title","author","release_date","pages"
+"0596516177","The Ruby Programming Language","David Flanagan and Yukihiro Matsumoto","Feb 1, 2008","0448"
+"1449355978","Computer Science Programming Basics in Ruby","Ophir Frieder, Gideon Frieder and David Grossman","1 May, 2013","188"
+"  0596523696  ","  Ruby Cookbook  ","  Lucas Carlson and Leonard Richardson  ","  2006-7-26  ","  910  "

data/spec/fixtures/books-valid/infile.json

@@ -0,0 +1,23 @@
+[
+  {
+    "isbn10": "0596516177",
+    "title": "The Ruby Programming Language",
+    "author": "David Flanagan and Yukihiro Matsumoto",
+    "release_date": "Feb 1, 2008",
+    "pages": "0448"
+  },
+  {
+    "isbn10": "1449355978",
+    "title": "Computer Science Programming Basics in Ruby",
+    "author": "Ophir Frieder, Gideon Frieder and David Grossman",
+    "release_date": "1 May, 2013",
+    "pages": 188
+  },
+  {
+    "isbn10": "  0596523696  ",
+    "title": "  Ruby Cookbook  ",
+    "author": "  Lucas Carlson and Leonard Richardson  ",
+    "release_date": "  2006-7-26  ",
+    "pages": "  910  "
+  }
+]

data/spec/fixtures/books-valid/infile.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<books>
+  <!-- attribute-style record -->
+  <book isbn10="0596516177" title="The Ruby Programming Language" author="David Flanagan and Yukihiro Matsumoto" release_date="Feb 1, 2008" pages="0448" />
+
+  <!-- element style record -->
+  <book>
+    <isbn10>1449355978</isbn10>
+    <title>Computer Science Programming Basics in Ruby</title>
+    <author>Ophir Frieder, Gideon Frieder and David Grossman</author>
+    <release_date>1 May, 2013</release_date>
+    <pages>188</pages>
+  </book>
+
+  <!-- mixed style record -->
+  <book isbn10="  0596523696  " title="  Ruby Cookbook  " author="  Lucas Carlson and Leonard Richardson  ">
+    <release_date>  2006-7-26  </release_date>
+    <pages>  910  </pages>
+  </book>
+</books>

data/spec/fixtures/books-valid/model.json

@@ -0,0 +1,23 @@
+[
+  {
+    "isbn10": "0596516177",
+    "title": "The Ruby Programming Language",
+    "authors": ["David Flanagan", "Yukihiro Matsumoto"],
+    "release_date": "2008-02-01",
+    "pages": 448
+  },
+  {
+    "isbn10": "1449355978",
+    "title": "Computer Science Programming Basics in Ruby",
+    "authors": ["Ophir Frieder", "Gideon Frieder", "David Grossman"],
+    "release_date": "2013-05-01",
+    "pages": 188
+  },
+  {
+    "isbn10": "0596523696",
+    "title": "Ruby Cookbook",
+    "authors": ["Lucas Carlson", "Leonard Richardson"],
+    "release_date": "2006-07-26",
+    "pages": 910
+  }
+]

data/spec/fixtures/books-valid/record.json

@@ -0,0 +1,26 @@
+[
+  {
+    "isbn10": "0596516177",
+    "title": "The Ruby Programming Language",
+    "authors": ["David Flanagan", "Yukihiro Matsumoto"],
+    "release_date": "2008-02-01",
+    "pages": 448,
+    "by_matz": true
+  },
+  {
+    "isbn10": "1449355978",
+    "title": "Computer Science Programming Basics in Ruby",
+    "authors": ["Ophir Frieder", "Gideon Frieder", "David Grossman"],
+    "release_date": "2013-05-01",
+    "pages": 188,
+    "by_matz": false
+  },
+  {
+    "isbn10": "0596523696",
+    "title": "Ruby Cookbook",
+    "authors": ["Lucas Carlson", "Leonard Richardson"],
+    "release_date": "2006-07-26",
+    "pages": 910,
+    "by_matz": false
+  }
+]

data/spec/fixtures/books-valid/summary.json

@@ -0,0 +1,8 @@
+{
+  "created": 3,
+  "updated": 0,
+  "unchanged": 0,
+  "invalid": 0,
+  "total": 3,
+  "validation_errors": {}
+}

data/spec/fixtures/source-empty-records/infile.csv

@@ -0,0 +1,3 @@
+"isbn10","title","author","release_date","pages"
+,,,,
+,,,,

data/spec/fixtures/source-empty-records/infile.json

@@ -0,0 +1 @@
+[{},{}]

data/spec/fixtures/source-empty-records/infile.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<books>
+  <book />
+  <book></book>
+</books>

data/spec/fixtures/source-malformed/infile.csv

@@ -0,0 +1 @@
+"isbn10","title","author

data/spec/fixtures/source-malformed/infile.json

@@ -0,0 +1 @@
+[{},[],{

data/spec/fixtures/source-malformed/infile.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<boo

data/spec/fixtures/source-no-records/infile.csv

@@ -0,0 +1 @@
+"isbn10","title","author","release_date","pages"

data/spec/fixtures/source-no-records/infile.json

@@ -0,0 +1 @@
+[]

data/spec/fixtures/source-no-records/infile.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<books />