importu 0.1.0 → 0.2.0

data/spec/lib/importu/importer_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+require "importu/definition"
+require "importu/importer"
+require "importu/sources/ruby"
+
+RSpec.describe Importu::Importer do
+  subject(:importer) { importer_class.new(source) }
+
+  let(:source) { Importu::Sources::Ruby.new(data) }
+
+  let(:data) do
+    [
+      { "animal" => "llama",    "name" => "Nathan",   "age" => "3" },
+      { "animal" => "aardvark", "name" => "Stella",   "age" => "2" },
+      { "animal" => "crow",     "name" => "Hamilton", "age" => "6" },
+    ]
+  end
+
+  let(:importer_class) do
+    Class.new(Importu::Importer) do
+      fields :animal, :name, :age, required: true
+      field :age, &convert_to(:integer)
+    end
+  end
+
+  describe "#config" do
+    it "returns the configuration of the import definition" do
+      expect(importer.config[:fields]).to include(:animal, :name, :age)
+    end
+
+    context "when a definition is specified at initialization" do
+      subject(:importer) { importer_class.new(source, definition: definition) }
+      let(:definition) do
+        Class.new(Importu::Definition) { fields :species, :extinction_date }
+      end
+
+      it "uses the definition from initialization" do
+        expect(importer.config[:fields]).to include(:species, :extinction_date)
+      end
+    end
+  end
+
+  describe "#import!" do
+    context "when a backend is specified at initialization" do
+      subject(:importer) { importer_class.new(source, backend: backend) }
+      let(:backend) { DummyBackend.new(**importer_class.config[:backend]) }
+
+      it "uses the backend from initialization" do
+        expect(Importu::Importer).to_not receive(:backend_registry)
+        expect(backend).to receive(:create).exactly(3).times.and_call_original
+        importer.import!
+      end
+    end
+
+    context "when a backend is not specified at initialization" do
+      it "tries to detect backend from the definition" do
+        expect(Importu::Importer.backend_registry)
+          .to receive(:from_config!).with(importer.config[:backend])
+          .and_return(DummyBackend)
+        importer.import!
+      end
+    end
+
+    context "when a custom recorder/summarizer is specified" do
+      subject(:importer) { importer_class.new(source, backend: backend) }
+      let(:backend) { DummyBackend.new(**importer_class.config[:backend]) }
+
+      it "records status of each record import" do
+        recorder = instance_double("Importu::Summary")
+        expect(recorder).to receive(:record).exactly(3).times
+
+        importer.import!(recorder)
+      end
+    end
+  end
+
+  describe "#records" do
+    it "returns record objects with conversions applied" do
+      expect(importer.records.map(&:to_hash)).to eq([
+        { animal: "llama",    name: "Nathan",   age: 3 },
+        { animal: "aardvark", name: "Stella",   age: 2 },
+        { animal: "crow",     name: "Hamilton", age: 6 },
+      ])
+    end
+  end
+
+  describe "#source" do
+    it "returns the source provided to importer at initialization" do
+      expect(importer.source).to eq source
+    end
+  end
+
+end

data/spec/lib/importu/integration_spec.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+require "importu/backends/active_record"
+require "importu/importer"
+require "importu/sources/csv"
+require "importu/sources/json"
+require "importu/sources/xml"
+
+# Integration tests that exercise the full import pipeline across different
+# source formats and scenarios. These serve as smoke tests and living
+# documentation of the gem's primary use cases.
+RSpec.describe "Integration", :active_record do
+  let!(:model) do
+    stub_const("Book", Class.new(ActiveRecord::Base) do
+      serialize :authors, type: Array
+      validates :title, :authors, :isbn10, :release_date, presence: true
+      validates :isbn10, length: { is: 10 }, uniqueness: true
+    end)
+  end
+
+  let(:importer_class) do
+    Class.new(BookImporter) do
+      model "Book"
+    end
+  end
+
+  around(:each) do |example|
+    require "database_cleaner-active_record"
+    DatabaseCleaner.cleaning { example.run }
+  end
+
+  describe "JSON import updating existing records" do
+    let(:source) { Importu::Sources::JSON.new(infile("books-valid", :json)) }
+    subject(:importer) { importer_class.new(source) }
+
+    context "when records already exist" do
+      before { importer.import! }
+
+      it "finds and updates existing records" do
+        summary = importer.import!
+        expect(summary.created).to eq 0
+        expect(summary.updated).to eq 0
+        expect(summary.unchanged).to eq 3
+      end
+
+      context "when data has changed" do
+        let(:modified_source) do
+          data = JSON.parse(File.read(infile("books-valid", :json)))
+          data[0]["title"] = "The Ruby Programming Language (2nd Edition)"
+          Importu::Sources::JSON.new(StringIO.new(JSON.generate(data)))
+        end
+
+        it "updates the changed record" do
+          modified_importer = importer_class.new(modified_source)
+          summary = modified_importer.import!
+
+          expect(summary.created).to eq 0
+          expect(summary.updated).to eq 1
+          expect(summary.unchanged).to eq 2
+
+          updated_book = Book.find_by(isbn10: "0596516177")
+          expect(updated_book.title).to eq "The Ruby Programming Language (2nd Edition)"
+        end
+      end
+    end
+  end
+
+  describe "XML import with duplicate detection" do
+    let(:source_config) do
+      Class.new(Importu::Definition) { source :xml, records_xpath: "//book" }
+        .config[:sources][:xml]
+    end
+
+    let(:xml_with_duplicates) do
+      <<~XML
+        <?xml version="1.0"?>
+        <books>
+          <book isbn10="0596516177" title="First" author="Author A" release_date="2008-01-01" pages="100" />
+          <book isbn10="1449355978" title="Second" author="Author B" release_date="2013-05-01" pages="200" />
+          <book isbn10="0596516177" title="Duplicate" author="Author C" release_date="2010-01-01" pages="300" />
+        </books>
+      XML
+    end
+
+    let(:source) { Importu::Sources::XML.new(StringIO.new(xml_with_duplicates), **source_config) }
+    subject(:importer) { importer_class.new(source) }
+
+    it "creates the first record and marks duplicate as invalid" do
+      summary = importer.import!
+
+      expect(summary.created).to eq 2
+      expect(summary.invalid).to eq 1
+      expect(summary.validation_errors.keys).to include(match(/duplicate/i))
+    end
+  end
+
+  describe "mixed create/update with error reporting" do
+    let(:source) { Importu::Sources::CSV.new(infile("books-valid", :csv)) }
+    subject(:importer) { importer_class.new(source) }
+
+    context "when some records exist and others are new" do
+      before do
+        # Create only the first book
+        Book.create!(
+          isbn10: "0596516177",
+          title: "Old Title",
+          authors: ["Old Author"],
+          release_date: Date.new(2008, 1, 1),
+          pages: 100
+        )
+      end
+
+      it "creates new records and updates existing ones" do
+        summary = importer.import!
+
+        expect(summary.created).to eq 2
+        expect(summary.updated).to eq 1
+        expect(summary.unchanged).to eq 0
+        expect(summary.invalid).to eq 0
+
+        expect(Book.count).to eq 3
+        updated_book = Book.find_by(isbn10: "0596516177")
+        expect(updated_book.title).to eq "The Ruby Programming Language"
+      end
+    end
+
+    context "when some records fail validation" do
+      let(:importer_class) do
+        Class.new(super()) do
+          # Force first record to have invalid ISBN (apply trimmed first to
+          # handle whitespace in source data)
+          field :isbn10 do
+            value = trimmed(:isbn10)
+            value == "0596516177" ? "bad" : value
+          end
+        end
+      end
+
+      it "reports validation errors while processing valid records" do
+        summary = importer.import!
+
+        expect(summary.created).to eq 2
+        expect(summary.invalid).to eq 1
+        expect(summary.validation_errors.keys).to include(match(/isbn10/i))
+      end
+    end
+
+    context "when actions are restricted" do
+      context "with only create allowed" do
+        let(:importer_class) do
+          Class.new(super()) { allow_actions :create }
+        end
+
+        before do
+          Book.create!(
+            isbn10: "0596516177",
+            title: "Existing",
+            authors: ["Author"],
+            release_date: Date.new(2008, 1, 1)
+          )
+        end
+
+        it "creates new records but marks updates as invalid" do
+          summary = importer.import!
+
+          expect(summary.created).to eq 2
+          expect(summary.invalid).to eq 1
+          expect(summary.validation_errors.keys).to include(match(/update.*not allowed/i))
+        end
+      end
+
+      context "with only update allowed" do
+        let(:importer_class) do
+          Class.new(super()) { allow_actions :update }
+        end
+
+        before do
+          Book.create!(
+            isbn10: "0596516177",
+            title: "Existing",
+            authors: ["Author"],
+            release_date: Date.new(2008, 1, 1)
+          )
+        end
+
+        it "updates existing records but marks creates as invalid" do
+          summary = importer.import!
+
+          expect(summary.created).to eq 0
+          expect(summary.updated).to eq 1
+          expect(summary.invalid).to eq 2
+          expect(summary.validation_errors.keys).to include(match(/create.*not allowed/i))
+        end
+      end
+    end
+  end
+
+  describe "import with before_save hooks" do
+    let(:source) { Importu::Sources::JSON.new(infile("books-valid", :json)) }
+
+    let(:importer_class) do
+      Class.new(super()) do
+        before_save do
+          object.title = object.title.upcase
+        end
+      end
+    end
+
+    subject(:importer) { importer_class.new(source) }
+
+    it "applies the hook to all records" do
+      importer.import!
+
+      Book.all.each do |book|
+        expect(book.title).to eq book.title.upcase
+      end
+    end
+  end
+end

data/spec/lib/importu/record_spec.rb

@@ -1,123 +1,173 @@
-require 'spec_helper'
+# frozen_string_literal: true
+require "spec_helper"
 
-describe Importu::Record do
-  subject(:record) { build(:importer_record) }
+require "importu/converter_context"
+require "importu/definition"
+require "importu/record"
 
-  it "includes Enumerable" do
-    record.should be_a_kind_of(Enumerable)
-  end
+RSpec.describe Importu::Record do
+  subject(:record) { Importu::Record.new(data, context, **definition.config) }
+  let(:context) { Importu::ConverterContext.with_config(**definition.config) }
 
-  describe "#importer" do
-    it "returns the importer used during construction" do
-      importer = build(:importer)
-      record = build(:importer_record, :importer => importer)
-      record.importer.should be importer
+  let(:definition) do
+    Class.new(Importu::Definition) do
+      field :pilot
+      field :balloons, &convert_to(:integer)
+      field :flying, &convert_to(:boolean)
     end
+  end
 
-    it "is delegated from #preprocessor" do
-      record.should delegate(:preprocessor).to(:importer)
-    end
+  let(:data) { { "pilot" => "Nena", "balloons" => "99", "flying" => "yes" } }
+
+  it "supports hash-like accessors" do
+    expect(record[:pilot]).to eq "Nena"
+    expect(record.fetch(:balloons)).to eq 99
+    expect { record.fetch(:color) }.to raise_error(KeyError)
+    expect(record.key?(:flying)).to be true
+    expect(record.key?(:color)).to be false
+    expect(record.keys).to eq [:pilot, :balloons, :flying]
+    expect(record.values).to eq ["Nena", 99, true]
+  end
+
+  it "supports enumerable behaviors" do
+    expect(record.each.with_index.to_a) # enumerable is composable (#w_index)
+      .to eq [[[:pilot, "Nena"], 0], [[:balloons, 99], 1], [[:flying, true], 2]]
 
-    it "is delegated from #postprocessor" do
-      record.should delegate(:postprocessor).to(:importer)
+    expect(record.reduce([]) {|acc, (k, _)| acc << k }).to eq record.keys
+  end
+
+  describe "#assignable_fields_for" do
+    it "returns all field names by default" do
+      expect(record.assignable_fields_for(:create))
+        .to eq [:pilot, :balloons, :flying]
     end
 
-    it "is delegated from #definitions" do
-      record.should delegate(:definitions).to(:importer)
+    context "when a field is allowed for create but not update" do
+      let(:definition) do
+        Class.new(super()) { field :pilot, create: true, update: false }
+      end
+
+      it "includes field for :create action but not :update action" do
+        expect(record.assignable_fields_for(:create)).to include(:pilot)
+        expect(record.assignable_fields_for(:update)).to_not include(:pilot)
+      end
     end
 
-    it "is delegated from #converters" do
-      record.should delegate(:converters).to(:importer)
+    context "when a field is marked as abstract" do
+      let(:definition) do
+        Class.new(super()) { field :balloons, abstract: true }
+      end
+
+      it "excludes the abstract field from list" do
+        expect(record.assignable_fields_for(:create)).to eq [:pilot, :flying]
+        expect(record.assignable_fields_for(:update)).to eq [:pilot, :flying]
+      end
     end
   end
 
   describe "#data" do
-    it "returns the data used during construction" do
-      data = { "foo" => "bar" }
-      record = build(:importer_record, :data => data)
-      record.data.should == data
+    it "returns data supplied during initialization" do
+      expect(record.data).to eq data
     end
   end
 
-  describe "#raw_data" do
-    it "returns the raw_data used during construction" do
-      raw_data = "this,is\tsome_raw_data\n"
-      record = build(:importer_record, :raw_data => raw_data)
-      record.raw_data.should == raw_data
+  describe "#errors" do
+    it "returns []" do
+      expect(record.errors).to eq []
     end
-  end
 
-  describe "#definitions" do
-    it "returns the definitions defined in importer on construction" do
-      importer = build(:importer)
-      importer.stub(:definitions => { :foo => :bar })
-      record = build(:importer_record, :importer => importer)
-      record.definitions.should be importer.definitions
+    context "when one or more values could not be converted" do
+      let(:data) { super().merge!("balloons" => "many", "flying" => "maybe") }
+
+      it "returns list of field parse errors for failed conversions" do
+        expect(record.errors.map(&:field_name))
+          .to match_array([:balloons, :flying])
+      end
     end
   end
 
-  describe "#record_hash" do
-    it "tries to generate a record hash on first access" do
-      expected = { :foo => 1, :bar => 2 }
-      record.should_receive(:generate_record_hash).and_return(expected)
-      record.record_hash.should eq expected
+  describe "#to_hash" do
+    it "returns data with field converters applied" do
+      expect(record.to_hash).to eq(pilot: "Nena", balloons: 99, flying: true)
     end
 
-    it "should not try to regenerate record hash no subsequent access" do
-      expected = { :foo => 1, :bar => 2 }
-      record.should_receive(:generate_record_hash).once.and_return(expected)
-      record.record_hash
-      record.record_hash.should eq expected
+    it "does not try to recovert the data each time (returns same has)" do
+      expect(record.to_hash.object_id).to eq record.to_hash.object_id
     end
 
-    it "is aliased from #to_hash" do
-      record.should_receive(:record_hash).and_return(:called)
-      record.to_hash.should == :called
-    end
+    context "when one or more values could not be converted" do
+      let(:data) { super().merge!("flying" => "maybe") }
 
-    it "is delegated from #keys" do
-      record.should delegate(:keys).to(:record_hash)
-    end
+      it "raises an InvalidRecord error with errors from field conversion" do
+        expect { record.to_hash }.to raise_error(Importu::InvalidRecord)
 
-    it "is delegated from #values" do
-      record.should delegate(:values).to(:record_hash)
+        begin
+          record.to_hash
+        rescue Importu::InvalidRecord => e
+          expect(e.validation_errors.count).to eq 1
+          expect(e.validation_errors.first.field_name).to eq :flying
+        end
+      end
     end
+  end
 
-    it "is delegated from #each" do
-      record.should delegate(:each).to(:record_hash)
+  describe "#valid?" do
+    context "when all values can be converted successfully" do
+      it "returns true" do
+        expect(record).to be_valid
+      end
     end
 
-    it "is delegated from #[]" do
-      record.should delegate(:[]).to(:record_hash)
+    context "when one or more values could not be converted" do
+      let(:data) { super().merge!("flying" => "maybe") }
+
+      it "returns false" do
+        expect(record).to_not be_valid
+      end
     end
+  end
 
-    it "is delegated from #key?" do
-      record.should delegate(:key?).to(:record_hash)
+end
+
+RSpec.describe Importu::Record::Iterator do
+  subject(:iterator) { described_class.new(rows, **definition.config) }
+
+  let(:definition) do
+    Class.new(Importu::Definition) do
+      fields :animal, :name, :age, required: true
+      field :age, &convert_to(:integer)
     end
+  end
 
-    describe "#convert" do
-      context "with a :default option" do
-        it "returns data value if data value not nil" do
-          record.converters[:clean] = proc { "value1" }
-          record.convert(:field1, :clean, :default => "foobar").should eq "value1"
-        end
+  let(:rows) do
+    [
+      { "animal" => "llama",    "name" => "Nathan",   "age" => "3" },
+      { "animal" => "aardvark", "name" => "Stella",   "age" => "2" },
+      { "animal" => "crow",     "name" => "Hamilton", "age" => "6" },
+    ]
+  end
 
-        it "returns default value if data value is nil" do
-          record.converters[:clean] = proc { nil }
-          record.convert(:field1, :clean, :default => "foobar").should eq "foobar"
-        end
+  it "returns the same number of records as source data" do
+    expect(iterator.count).to eq 3
+  end
 
-        it "returns default value if data field is missing and not required" do
-          record.converters[:clean] = proc { raise Importu::MissingField, "field1" }
-          record.convert(:field1, :clean, :default => "foobar").should eq "foobar"
-        end
+  it "returns record objects with conversions applied" do
+    expect(iterator.first.to_hash).to eq({
+      animal: "llama", name: "Nathan", age: 3
+    })
+  end
 
-        it "raises an exception if data field is missing and is required" do
-          record.converters[:clean] = proc { raise Importu::MissingField, "field1" }
-          expect { record.convert(:field1, :clean, :default => "foobar", :required => true) }.to raise_error(Importu::MissingField)
-        end
-      end
+  context "when one of the records is invalid" do
+    before { rows.first["age"] = "old" }
+
+    it "returns the invalid record" do
+      expect(iterator.first).to_not be_valid
     end
 
+    it "raises an exception when converted record value is accessed" do
+      expect { iterator.first["animal"] }
+        .to raise_error(Importu::InvalidRecord)
+    end
   end
+
 end

data/spec/lib/importu/sources/csv_spec.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+require "importu/definition"
+require "importu/sources/csv"
+
+require_relative "importer_source_examples"
+
+RSpec.describe Importu::Sources::CSV do
+  it_behaves_like "importer source", :csv do
+    subject(:source) { described_class.new(input, **source_config) }
+    let(:definition) { Class.new(Importu::Definition) }
+    let(:source_config) { definition.config[:sources][:csv] }
+  end
+
+  describe "#initialize" do
+    context "with custom csv options" do
+      let(:csv_options) { { skip_blanks: false } }
+      let(:data) { "foo\n\n\n" }
+
+      it "allows overriding csv options" do
+        original_source = described_class.new(StringIO.new(data))
+        source = described_class.new(StringIO.new(data), csv_options: csv_options)
+        expect(source.rows.count).to be > original_source.rows.count
+      end
+    end
+  end
+
+end