importu 0.1.0 → 0.2.0

data/spec/lib/importu/sources/importer_source_examples.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+require "importu/exceptions"
+require "importu/summary"
+
+# Implement the following in specs that use this example:
+#
+#   subject(:source) { described_class.new(input, **source_config) }
+#   let(:definition) { Class.new(Importu::Definition) }
+#   let(:source_config) { definition.config[:sources][:csv] }
+#
+RSpec.shared_examples "importer source" do |format, exclude: []|
+  describe "#initialize" do
+    unless exclude.include?(:empty_file)
+      context "when input file is blank" do
+        let(:input) { infile("source-empty-file", format) }
+
+        it "raises an InvalidInput exception" do
+          expect { source }.to raise_error Importu::InvalidInput
+        end
+      end
+    end
+
+    unless exclude.include?(:malformed)
+      context "when input file is malformed" do
+        let(:input) { infile("source-malformed", format) }
+
+        it "raises an InvalidInput exception" do
+          expect { source }.to raise_error Importu::InvalidInput
+        end
+      end
+    end
+  end
+
+  describe "#rows" do
+    unless exclude.include?(:no_records)
+      context "when input has no rows" do
+        let(:input) { infile("source-no-records", format) }
+
+        it "treats file as having 0 rows" do
+          expect(source.rows.count).to eq 0
+        end
+      end
+    end
+
+    unless exclude.include?(:empty_records)
+      context "when input contains empty rows" do
+        let(:input) { infile("source-empty-records", format) }
+
+        it "treats empty rows as existing (albeit invalid)" do
+          expect(source.rows.count).to eq 2
+        end
+      end
+    end
+
+    context "when input contains multiple valid rows" do
+      let(:input) { infile("books-valid", format) }
+
+      it "returns rows parsed from source data" do
+        expect(source.rows.count).to eq 3
+      end
+    end
+  end
+
+  describe "#write_errors" do
+    subject(:source) { described_class.new(infile("books-valid", format), **source_config) }
+
+    context "when there are no errors during import" do
+      let(:summary) { Importu::Summary.new }
+
+      it "doesn't try to generate a file, but returns nil" do
+        expect(source.write_errors(summary)).to be_nil
+      end
+    end
+
+    context "when there are errors during import" do
+      let(:summary) do
+        Importu::Summary.new.tap do |summary|
+          summary.record(:invalid, index: 1, errors: ["foo was invalid"])
+          summary.record(:invalid, index: 2, errors: ["foo was invalid", "bar was invalid"])
+        end
+      end
+
+      it "returns a rewoundfile handle/io" do
+        errfile = source.write_errors(summary)
+        expect(errfile).to respond_to(:readline)
+        expect(errfile.pos).to eq 0
+      end
+
+      it "records errors to '_errors' field with source data" do
+        errfile = source.write_errors(summary)
+        new_source = described_class.new(errfile, **source_config)
+
+        expect(new_source.rows.map {|r| r["_errors"] }).to eq [
+          nil,                                # 0
+          "foo was invalid",                  # 1
+          "foo was invalid, bar was invalid", # 2
+        ]
+
+        # Everything except "_errors" should match original input
+        expect(new_source.rows.map {|row| row.reject {|k, _v| k == "_errors" } })
+          .to eq source.rows.to_a
+      end
+
+      it "clears any existing '_errors' values in source data" do
+        errfile = source.write_errors(summary)
+        source2 = described_class.new(errfile, **source_config)
+
+        summary2 = Importu::Summary.new.tap do |summary|
+          summary.record(:invalid, index: 0, errors: ["baz was invalid"])
+        end
+
+        errfile2 = source2.write_errors(summary2)
+        source3 = described_class.new(errfile2, **source_config)
+
+        expect(source3.rows.map {|r| r["_errors"] }).to eq [
+          "baz was invalid", # 0
+          nil,               # 1
+          nil,               # 2
+        ]
+      end
+
+      it "does not affect original source data" do
+        expect { source.write_errors(summary) }
+          .to_not change { source.rows.to_a }
+      end
+
+      it "supports only writing records with errors" do
+        errfile = source.write_errors(summary, only_errors: true)
+        new_source = described_class.new(errfile, **source_config)
+        expect(new_source.rows.map {|r| r["_errors"] }).to eq [
+          # nil,                              # 0, excluded
+          "foo was invalid",                  # 1
+          "foo was invalid, bar was invalid", # 2
+        ]
+      end
+    end
+  end
+
+  describe "Importer#records (usable as a record source?)" do
+    subject(:importer) { BookImporter.new(source) }
+
+    context "when source data is valid" do
+      let(:input) { infile("books-valid", format) }
+
+      it "returns records parsed from source data" do
+        expected_record_json!("books-valid", importer.records)
+      end
+
+      it "returns same records on subsequent invocations (rewinds)" do
+        previous_count = importer.records.count
+        expect(importer.records.count).to eq previous_count
+      end
+    end
+  end
+
+  describe "Importer#import! (verifies entire flow)" do
+    subject(:importer) { importer_class.new(source) }
+    let(:importer_class) do
+      stub_const("Book", Class.new)
+
+      Class.new(BookImporter) do
+        model "Book", backend: :dummy
+      end
+    end
+
+    context "when source data is valid" do
+      let(:input) { infile("books-valid", format) }
+
+      it "returns a summary with expected results" do
+        summary = importer.import!
+        expected_summary_json!("books-valid", summary)
+      end
+    end
+  end
+end

data/spec/lib/importu/sources/json_spec.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+require "importu/definition"
+require "importu/sources/json"
+
+require_relative "importer_source_examples"
+
+RSpec.describe Importu::Sources::JSON do
+  it_behaves_like "importer source", :json do
+    subject(:source) { described_class.new(input, **source_config) }
+    let(:definition) { Class.new(Importu::Definition) }
+    let(:source_config) { definition.config[:sources][:json] }
+  end
+
+  describe "#initialize" do
+    subject(:source) { described_class.new(StringIO.new(data), **source_config) }
+    let(:definition) { Class.new(Importu::Definition) }
+    let(:source_config) { definition.config[:sources][:json] }
+
+    context "when document is JSON null" do
+      let(:data) { "null" }
+      it "raises InvalidInput with empty document message" do
+        expect { source }.to raise_error(Importu::InvalidInput, "Empty document")
+      end
+    end
+  end
+
+end

data/spec/lib/importu/sources/ruby_spec.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+require "importu/sources/ruby"
+require "importu/summary"
+
+RSpec.describe Importu::Sources::Ruby do
+  subject(:source) { described_class.new(data) }
+
+  describe "#initialize" do
+    let(:data) { [] }
+
+    it "accepts an array" do
+      expect { source }.not_to raise_error
+    end
+
+    it "accepts any enumerable" do
+      source = described_class.new([1, 2, 3].each)
+      expect(source).to be_a(described_class)
+    end
+  end
+
+  describe "#rows" do
+    context "with an array of hashes" do
+      let(:data) { [{ "name" => "Alice" }, { "name" => "Bob" }] }
+
+      it "returns an enumerator" do
+        expect(source.rows).to be_a(Enumerator)
+      end
+
+      it "yields each hash" do
+        expect(source.rows.to_a).to eq data
+      end
+
+      it "can be iterated multiple times" do
+        first_pass = source.rows.to_a
+        second_pass = source.rows.to_a
+        expect(second_pass).to eq first_pass
+      end
+    end
+
+    context "with objects responding to to_hash" do
+      let(:hashable_object) do
+        Class.new do
+          def initialize(name)
+            @name = name
+          end
+
+          def to_hash
+            { "name" => @name }
+          end
+        end
+      end
+
+      let(:data) { [hashable_object.new("Alice"), hashable_object.new("Bob")] }
+
+      it "converts each object via to_hash" do
+        expect(source.rows.to_a).to eq [{ "name" => "Alice" }, { "name" => "Bob" }]
+      end
+    end
+
+    context "with empty array" do
+      let(:data) { [] }
+
+      it "returns 0 rows" do
+        expect(source.rows.count).to eq 0
+      end
+    end
+
+    context "with symbol keys in hashes" do
+      let(:data) { [{ name: "Alice" }] }
+
+      # NOTE: The Ruby source calls to_hash which preserves symbol keys.
+      # However, importu field definitions use string keys by default.
+      # Users should ensure their hash keys match what fields expect.
+      it "preserves symbol keys (caller must ensure key type matches)" do
+        expect(source.rows.first.keys).to eq [:name]
+      end
+    end
+  end
+
+  describe "#write_errors" do
+    let(:data) { [{ "name" => "Alice" }] }
+
+    let(:summary) do
+      Importu::Summary.new.tap do |s|
+        s.record(:invalid, index: 0, errors: ["name was invalid"])
+      end
+    end
+
+    # The Ruby source has a stub implementation that does nothing.
+    # This is intentional - there's no sensible file output for in-memory data.
+    it "returns nil (stub implementation)" do
+      expect(source.write_errors(summary)).to be_nil
+    end
+
+    it "accepts only_errors option without error" do
+      expect { source.write_errors(summary, only_errors: true) }.not_to raise_error
+    end
+  end
+end

data/spec/lib/importu/sources/xml_spec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+require "importu/definition"
+require "importu/sources/xml"
+
+require_relative "importer_source_examples"
+
+RSpec.describe Importu::Sources::XML do
+  it_behaves_like "importer source", :xml do
+    subject(:source) { described_class.new(input, **source_config) }
+    let(:source_config) { definition.config[:sources][:xml] }
+
+    let(:definition) do
+      Class.new(Importu::Definition) { source :xml, records_xpath: "//book" }
+    end
+  end
+
+  describe "xpath edge cases" do
+    let(:xml_content) do
+      <<~XML
+        <?xml version="1.0"?>
+        <library>
+          <book><title>Red Dwarf</title></book>
+          <book><title>Better Than Life</title></book>
+        </library>
+      XML
+    end
+
+    let(:input) { StringIO.new(xml_content) }
+
+    context "when records_xpath matches nothing" do
+      subject(:source) { described_class.new(input, records_xpath: "//nonexistent") }
+
+      it "returns 0 rows" do
+        expect(source.rows.count).to eq 0
+      end
+    end
+
+    context "when records_xpath matches the root" do
+      subject(:source) { described_class.new(input, records_xpath: "/library") }
+
+      it "returns the root element as a single row" do
+        rows = source.rows.to_a
+        expect(rows.count).to eq 1
+      end
+    end
+
+    context "when records_xpath uses attributes" do
+      let(:xml_content) do
+        <<~XML
+          <?xml version="1.0"?>
+          <library>
+            <book isbn="123"><title>Red Dwarf</title></book>
+            <book isbn="456"><title>Better Than Life</title></book>
+          </library>
+        XML
+      end
+
+      subject(:source) { described_class.new(input, records_xpath: "//book") }
+
+      it "includes attributes in the row hash" do
+        rows = source.rows.to_a
+        expect(rows.first["isbn"]).to eq "123"
+        expect(rows.first["title"]).to eq "Red Dwarf"
+      end
+    end
+  end
+end

data/spec/lib/importu/summary_spec.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+require "importu/exceptions"
+require "importu/summary"
+
+RSpec.describe Importu::Summary do
+  subject(:summary) { Importu::Summary.new }
+
+  describe "#record" do
+    it "allows incrementing :created count" do
+      expect { summary.record(:created) }.to change { summary.created }.by(1)
+      expect { summary.record(:created) }.to change { summary.total }.by(1)
+    end
+
+    it "allows incrementing :updated count" do
+      expect { summary.record(:updated) }.to change { summary.updated }.by(1)
+      expect { summary.record(:updated) }.to change { summary.total }.by(1)
+    end
+
+    it "allows incrementing :unchanged count" do
+      expect { summary.record(:unchanged) }.to change { summary.unchanged }.by(1)
+      expect { summary.record(:unchanged) }.to change { summary.total }.by(1)
+    end
+
+    it "allows incrementing :invalid count" do
+      expect { summary.record(:invalid) }.to change { summary.invalid }.by(1)
+      expect { summary.record(:invalid) }.to change { summary.total }.by(1)
+    end
+
+    it "records aggregated errors associated with :invalid results" do
+      summary.record(:invalid, errors: ["foo was invalid", "bar was invalid"])
+      expect(summary.validation_errors.count).to eq 2
+    end
+
+    it "allows passing a record index to associate with errors" do
+      summary.record(:invalid, index: 71, errors: ["foo was invalid"])
+      summary.record(:invalid, index: 94, errors: ["bar was invalid"])
+      expect(summary.itemized_errors.keys).to eq [71, 94]
+    end
+  end
+
+  describe "#result_msg" do
+    it "includes a summary of recorded counts" do
+      5.times { summary.record(:created) }
+      4.times { summary.record(:updated) }
+      3.times { summary.record(:unchanged) }
+      2.times { summary.record(:invalid, errors: ["foo was invalid"]) }
+      1.times { summary.record(:invalid, errors: ["bar was invalid"]) }
+
+      expect(summary.result_msg).to match(/total:\s*15/i)
+      expect(summary.result_msg).to match(/created:\s*5/i)
+      expect(summary.result_msg).to match(/updated:\s*4/i)
+      expect(summary.result_msg).to match(/unchanged:\s*3/i)
+      expect(summary.result_msg).to match(/invalid:\s*3/i)
+    end
+
+    context "when there are no validation errors" do
+      it "does not include a breakdown of errors" do
+        expect(summary.result_msg).to_not match(/validation errors/i)
+      end
+    end
+
+    context "when there are validation errors" do
+      it "includes a breakdown of errors" do
+        3.times { summary.record(:invalid, errors: ["foo was invalid"]) }
+        2.times { summary.record(:invalid, errors: ["bar was invalid"]) }
+
+        expect(summary.result_msg).to match(/validation errors/i)
+        expect(summary.result_msg).to match(/foo was invalid:\s*3/i)
+        expect(summary.result_msg).to match(/bar was invalid:\s*2/i)
+      end
+    end
+  end
+
+  describe "#to_hash" do
+    it "returns summary counts as a hash" do
+      5.times { summary.record(:created) }
+      4.times { summary.record(:updated) }
+      3.times { summary.record(:unchanged) }
+      2.times { summary.record(:invalid, errors: ["foo was invalid"]) }
+      1.times { summary.record(:invalid, errors: ["bar was invalid"]) }
+
+      expect(summary.to_hash).to include({
+        created: 5,
+        invalid: 3,
+        unchanged: 3,
+        updated: 4,
+        total: 15,
+      })
+    end
+
+    context "when there are no validation errors" do
+      it "includes an empty hash of errors" do
+        expect(summary.to_hash[:validation_errors]).to eq({})
+      end
+    end
+
+    context "when there are validation errors" do
+      it "includes a breakdown of errors" do
+        2.times { summary.record(:invalid, errors: ["foo was invalid"]) }
+        1.times { summary.record(:invalid, errors: ["bar was invalid"]) }
+
+        expect(summary.to_hash[:validation_errors]).to eq({
+          "foo was invalid" => 2,
+          "bar was invalid" => 1,
+        })
+      end
+    end
+  end
+
+  describe "#to_s" do
+    it "returns same summary as #result_msg" do
+      expect(summary.to_s).to eq summary.result_msg
+    end
+  end
+
+  describe "#validation_errors" do
+    context "when no errors" do
+      it "returns an empty hash" do
+        expect(summary.validation_errors).to eq({})
+      end
+    end
+
+    context "when multiple errors" do
+      it "counts each occurrence of error message" do
+        summary.record(:invalid, errors: ["foo was invalid", "bar was invalid"])
+        summary.record(:invalid, errors: ["bar was invalid", "baz was invalid"])
+        expect(summary.validation_errors["foo was invalid"]).to eq 1
+        expect(summary.validation_errors["bar was invalid"]).to eq 2
+        expect(summary.validation_errors["baz was invalid"]).to eq 1
+      end
+    end
+
+    context "when errors contain data within parentheses" do
+      it "strips out parenthesis and data within if at end of error" do
+        summary.record(:invalid, errors: ["lost parrot (polly)"])
+        summary.record(:invalid, errors: ["lost parrot (cracker)"])
+        expect(summary.validation_errors["lost parrot"]).to eq 2
+      end
+    end
+
+    context "when error has a normalized_message" do
+      it "uses normalized_message for aggregation" do
+        error1 = Importu::InvalidRecord.new(
+          "isbn too short (was: abc)", nil, normalized_message: "isbn too short"
+        )
+        error2 = Importu::InvalidRecord.new(
+          "isbn too short (was: xyz)", nil, normalized_message: "isbn too short"
+        )
+
+        summary.record(:invalid, errors: [error1])
+        summary.record(:invalid, errors: [error2])
+        expect(summary.validation_errors["isbn too short"]).to eq 2
+      end
+    end
+  end
+
+  describe "#itemized_errors" do
+    context "when no errors" do
+      it "returns an empty hash" do
+        expect(summary.itemized_errors).to eq({})
+      end
+
+      it "ignores errors not associated with a record index" do
+        expect { summary.record(:invalid, errors: ["foo was invalid"]) }
+          .to_not change { summary.itemized_errors }
+        expect { summary.record(:invalid, index: nil, errors: ["foo was invalid"]) }
+          .to_not change { summary.itemized_errors }
+      end
+
+      it "groups error messages by record index" do
+        summary.record(:invalid, index: 9, errors: ["foo", "bar"])
+        summary.record(:invalid, index: 11, errors: ["foo"])
+        summary.record(:invalid, index: 11, errors: ["bar", "baz"])
+        summary.record(:invalid, index: 17, errors: ["bar"])
+        expect(summary.itemized_errors).to eq({
+           9 => ["foo", "bar"],
+          11 => ["foo", "bar", "baz"],
+          17 => ["bar"],
+        })
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,14 +1,98 @@
-ENV['RAILS_ENV'] ||= 'test'
+# frozen_string_literal: true
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 
-require 'importu'
-require 'factory_girl'
+require "simplecov"
+
+require "active_record" if Gem.loaded_specs.has_key?("activerecord")
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
-Dir[File.expand_path('../support/**/*.rb', __FILE__)].each {|f| require f }
+Dir[File.expand_path("support/**/*.rb", __dir__)].each {|f| require f }
 
 RSpec.configure do |config|
-  config.include FactoryGirl::Syntax::Methods
-end
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Only run certain specs when the relevant gems are available. This is to
+  # allow for testing integration with different frameworks/backends in
+  # isolation with the appraisal gem. If all gems were loaded simultaneously
+  # then we wouldn't know if we were using an ActiveSupport monkeypatched
+  # method (i.e. String#present?) that would break w/o ActiveSupport loaded.
+  config.filter_run_excluding active_record: !defined?(::ActiveRecord)
 
-FactoryGirl.find_definitions
+  config.include FixturesHelper
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+end

data/spec/support/active_record.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+if defined?(::ActiveRecord)
+  ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+
+  ActiveRecord::Schema.define do
+    self.verbose = false
+
+    create_table :books, force: true do |t|
+      t.string :title, null: false
+      t.string :authors, null: false
+      t.string :isbn10, null: false
+      t.integer :pages
+      t.date :release_date, null: false
+      t.timestamps null: false
+    end
+
+    add_index :books, :isbn10, unique: true
+  end
+
+end