smart_csv_import 0.1.0

data/lib/smart_csv_import/processor/import_result_builder.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Processor
+    module ImportResultBuilder
+      module_function
+
+      def build_parse_errors(bad_rows)
+        bad_rows.map do |bad_row|
+          ParseError.new(
+            line_number: bad_row[:csv_line_number] || bad_row[:file_line_number],
+            raw_line: bad_row[:raw_logical_line].to_s.chomp,
+            error_message: bad_row[:error_message].to_s
+          )
+        end
+      end
+
+      def build_row_error_attributes(errors:, parse_errors:, import_id:)
+        now = Time.current
+        validation_attrs = errors.map do |err|
+          {
+            import_id: import_id,
+            row_number: err.row,
+            error_type: 'validation',
+            column_name: err.column.to_s,
+            messages: err.messages,
+            raw_line: nil,
+            error_message: nil,
+            created_at: now
+          }
+        end
+        parse_attrs = parse_errors.map do |pe|
+          {
+            import_id: import_id,
+            row_number: pe.line_number,
+            error_type: 'parse',
+            column_name: nil,
+            messages: [],
+            raw_line: pe.raw_line,
+            error_message: pe.error_message,
+            created_at: now
+          }
+        end
+        validation_attrs + parse_attrs
+      end
+
+      def build_blank_row_warnings(blank_count)
+        return [] if blank_count.zero?
+
+        noun = blank_count == 1 ? 'row' : 'rows'
+        verb = blank_count == 1 ? 'was' : 'were'
+        [RowWarning.new(row: 0, message: "#{blank_count} blank #{noun} #{verb} skipped", type: :blank_rows)]
+      end
+
+      def build_abort_warning(parse_error_count:, total:, bad_row_limit:)
+        ratio = total.positive? ? parse_error_count.to_f / total : 0
+        pct = (ratio * 100).round(1)
+        limit_pct = (bad_row_limit * 100).round(1)
+        RowWarning.new(
+          row: 0,
+          message: "Import aborted: #{pct}% of rows were malformed (limit: #{limit_pct}%)"
+        )
+      end
+
+      def final_result(imported:, failed:, errors:, parse_errors:, warnings:, header_mappings:, import_id:, dry_run:)
+        total = imported + failed + parse_errors.size
+        all_failed = failed + parse_errors.size
+
+        if dry_run
+          Result.dry_run(imported: imported, failed: failed, total: total, errors: errors,
+                         header_mappings: header_mappings, warnings: warnings, parse_errors: parse_errors)
+        elsif all_failed.positive?
+          Result.partial_failure(imported: imported, failed: failed, total: total, errors: errors,
+                                 header_mappings: header_mappings, import_id: import_id,
+                                 warnings: warnings, parse_errors: parse_errors)
+        else
+          Result.completed(imported: imported, failed: failed, total: total, errors: errors,
+                           header_mappings: header_mappings, import_id: import_id,
+                           warnings: warnings, parse_errors: parse_errors)
+        end
+      end
+
+      def abort_result(imported:, failed:, parse_errors:, warnings:, header_mappings:, import_id:, bad_row_limit:)
+        total = imported + failed + parse_errors.size
+        abort_warning = build_abort_warning(
+          parse_error_count: parse_errors.size, total: total, bad_row_limit: bad_row_limit
+        )
+
+        Result.partial_failure(
+          imported: imported, failed: failed, total: total, errors: [],
+          header_mappings: header_mappings, import_id: import_id,
+          warnings: [*warnings, abort_warning], parse_errors: parse_errors
+        )
+      end
+    end
+  end
+end

data/lib/smart_csv_import/processor/mapping_review_policy.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Processor
+    module MappingReviewPolicy
+      module_function
+
+      def serialize_mappings(mappings)
+        mappings.each_with_object({}) do |(header, result), acc|
+          acc[header] = result.respond_to?(:target_field) ? result.target_field.to_s : nil
+        end
+      end
+
+      def build_field_map(mappings)
+        mappings.each_with_object({}) do |(header, result), acc|
+          next unless result.respond_to?(:target_field) && result.matched?
+
+          acc[header] = result.target_field
+        end
+      end
+
+      def unmatched_headers(mappings)
+        mappings.select { |_, result| result.unmatched? }.keys
+      end
+
+      def build_proposed_mappings(mappings, confidence_threshold:)
+        mappings.each_with_object({}) do |(header, result), proposed|
+          proposed[header] = if result.unmatched?
+                               { field: nil, confidence: 0.0, status: :unmatched }
+                             elsif result.confidence < confidence_threshold
+                               { field: result.target_field, confidence: result.confidence, status: :low_confidence }
+                             else
+                               { field: result.target_field, confidence: result.confidence, status: :matched }
+                             end
+        end
+      end
+
+      def review_gate_triggered?(mappings, review_mode:, confidence_threshold:, required_fields:)
+        case review_mode
+        when :always
+          true
+        when :auto
+          auto_gate?(mappings, confidence_threshold)
+        when :skip
+          required_field_below_threshold?(mappings, confidence_threshold, required_fields)
+        end
+      end
+
+      def collect_warnings(mappings, duplicate_warning: nil)
+        base = duplicate_warning ? [duplicate_warning] : []
+        unmatched = unmatched_headers(mappings)
+        return base if unmatched.empty?
+
+        per_column = unmatched.map do |header|
+          UnmatchedColumnWarning.new(
+            column_name: header,
+            message: "Column '#{header}' was not imported — no matching field found"
+          )
+        end
+        [*base, *per_column, summary_warning(unmatched)]
+      end
+
+      def auto_gate?(mappings, confidence_threshold)
+        return true if mappings.values.any?(&:unmatched?)
+
+        mappings.values.any? do |r|
+          r.respond_to?(:confidence) && r.confidence < confidence_threshold
+        end
+      end
+
+      def required_field_below_threshold?(mappings, confidence_threshold, required_fields)
+        required_fields.any? do |field_name|
+          mapping = mappings.values.find do |result|
+            result.respond_to?(:target_field) && result.target_field == field_name
+          end
+          mapping.nil? || mapping.confidence < confidence_threshold
+        end
+      end
+
+      def summary_warning(unmatched)
+        noun = unmatched.size == 1 ? 'column' : 'columns'
+        verb = unmatched.size == 1 ? 'was' : 'were'
+        RowWarning.new(
+          row: 0,
+          message: "#{unmatched.size} #{noun} from your file #{verb} not imported: #{unmatched.join(', ')}"
+        )
+      end
+    end
+  end
+end

data/lib/smart_csv_import/processor/nil_cell_counter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Processor
+    module NilCellCounter
+      module_function
+
+      def count_cells(rows:, nil_values:)
+        return 0 if nil_values.blank?
+
+        rows.sum { |row| count_row(row, nil_values) }
+      end
+
+      def count_row(row, nil_values)
+        row.count { |cell| cell && nil_values.include?(cell.strip) }
+      end
+    end
+  end
+end

data/lib/smart_csv_import/processor/null_progress_callback.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Processor
+    class NullProgressCallback
+      def call(_payload)
+        nil
+      end
+    end
+  end
+end

data/lib/smart_csv_import/processor/row_processor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Processor
+    module RowProcessor
+      module_function
+
+      ChunkResult = Struct.new(:imported, :failed, :blank, :errors, :aborted_error, keyword_init: true)
+
+      def process_chunk(chunk, field_map:, form_class:, dry_run:)
+        blank_rows, data_rows = chunk.partition { |row| blank_row?(row) }
+        imported = 0
+        failed = 0
+        errors = []
+        aborted_error = nil
+
+        data_rows.each do |csv_row|
+          outcome = submit_row(csv_row, field_map: field_map, form_class: form_class, dry_run: dry_run)
+          case outcome.first
+          when :imported
+            imported += 1
+          when :failed
+            failed += 1
+            errors = [*errors, *outcome.last]
+          when :aborted
+            aborted_error = outcome.last
+            break
+          end
+        end
+
+        ChunkResult.new(
+          imported: imported, failed: failed, blank: blank_rows.size,
+          errors: errors, aborted_error: aborted_error
+        )
+      end
+
+      def submit_row(csv_row, field_map:, form_class:, dry_run:)
+        form = form_class.new(build_attributes(csv_row, field_map))
+        success = dry_run ? form.valid? : form.save
+        return [:imported] if success
+
+        [:failed, collect_row_errors(form, csv_row[:csv_line_number] || 0)]
+      rescue ActiveRecord::StatementInvalid => e
+        [:aborted, e]
+      end
+
+      def blank_row?(row)
+        row.except(:csv_line_number).values.all? { |v| v.nil? || v.to_s.empty? }
+      end
+
+      def build_attributes(csv_row, field_map)
+        field_map.each_with_object({}) do |(header, field_name), attrs|
+          next if header == :csv_line_number
+
+          attrs[field_name] = csv_row[header]
+        end
+      end
+
+      def collect_row_errors(form, row_number)
+        form.errors.group_by_attribute.each_with_object([]) do |(attr, row_errors), errs|
+          errs << RowError.new(
+            row: row_number,
+            column: attr.to_sym,
+            messages: row_errors.map(&:message)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/smart_csv_import/processor.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+require "csv"
+require "smarter_csv"
+
+module SmartCsvImport
+  class Processor
+    include Logging
+
+    FULL_LOAD_CHUNK_SIZE = 10_000_000
+
+    def initialize(file_path:, form_class:, mode: :sync, batch_size: SmartCsvImport.configuration.batch_size, dry_run: false, import: nil, confirmed_mappings: nil, on_progress: nil)
+      @file_path = file_path
+      @form_class = form_class
+      @mode = mode
+      @batch_size = batch_size
+      @dry_run = dry_run
+      @existing_import = import
+      @confirmed_mappings = confirmed_mappings
+      @on_progress = on_progress
+
+      validate_form_class!
+      validate_on_progress!
+
+      @on_progress ||= NullProgressCallback.new
+    end
+
+    def call
+      import = @existing_import || create_import
+      check_duplicate(import) unless @existing_import
+
+      return enqueue_async(import) if @mode == :async
+
+      process_csv(import)
+    end
+
+    private
+
+    def validate_form_class!
+      unless @form_class.respond_to?(:csv_fields) && @form_class.ancestors.include?(Matchable)
+        raise ConfigurationError, "form_class must include SmartCsvImport::Matchable"
+      end
+
+      unless @form_class.method_defined?(:save)
+        raise ConfigurationError, "form_class must define a #save method"
+      end
+    end
+
+    def create_import
+      stored_path = FileStorage.store(source_path: @file_path, import_type: @form_class.name)
+      file_hash = FileStorage.compute_hash(file_path: @file_path)
+
+      Import.create!(
+        import_type: @form_class.name,
+        original_filename: File.basename(@file_path),
+        file_path: stored_path,
+        file_hash: file_hash,
+        status: "processing"
+      )
+    end
+
+    def check_duplicate(import)
+      duplicate = Import.find_duplicate(file_hash: import.file_hash, import_type: import.import_type)
+      return unless duplicate && duplicate.id != import.id
+
+      @duplicate_warning = RowWarning.new(
+        row: 0,
+        message: "File appears to be a duplicate of import ##{duplicate.id}"
+      )
+    end
+
+    def enqueue_async(import)
+      ImportJob.perform_later(import.id, @form_class.name)
+      Result.queued(import_id: import.id)
+    end
+
+    def process_csv(import)
+      dup_header_warnings = CsvPreflightAnalyzer.duplicate_header_warnings(file_path: @file_path)
+      nil_match_count = CsvPreflightAnalyzer.count_nil_matches(
+        file_path: @file_path, nil_values: SmartCsvImport.configuration.nil_values_matching
+      )
+      nil_warnings = if nil_match_count > 0
+        noun = nil_match_count == 1 ? "cell" : "cells"
+        [RowWarning.new(row: 0, message: "#{nil_match_count} #{noun} contained Excel error markers and were treated as empty", type: :nil_cleaned)]
+      else
+        []
+      end
+
+      if @confirmed_mappings
+        field_map = @confirmed_mappings
+        serialized_mappings = field_map.transform_values(&:to_s)
+        import.update!(header_mappings: serialized_mappings)
+        return process_rows(import, field_map, [*dup_header_warnings, *nil_warnings])
+      end
+
+      mappings = run_matcher
+      serialized_mappings = MappingReviewPolicy.serialize_mappings(mappings)
+      import.update!(header_mappings: serialized_mappings)
+
+      warnings = MappingReviewPolicy.collect_warnings(mappings, duplicate_warning: @duplicate_warning)
+
+      threshold = SmartCsvImport.configuration.confidence_threshold
+
+      if MappingReviewPolicy.review_gate_triggered?(
+        mappings,
+        review_mode: SmartCsvImport.configuration.review_mode,
+        confidence_threshold: threshold,
+        required_fields: @form_class.required_csv_fields
+      )
+        import.update!(status: "mapping_review")
+        proposed = MappingReviewPolicy.build_proposed_mappings(mappings, confidence_threshold: threshold)
+        unmatched = MappingReviewPolicy.unmatched_headers(mappings)
+        return Result.review_required(
+          header_mappings: serialized_mappings,
+          import_id: import.id,
+          proposed_mappings: proposed,
+          unmatched_columns: unmatched,
+          warnings: [*dup_header_warnings, *nil_warnings, *warnings]
+        )
+      end
+
+      field_map = MappingReviewPolicy.build_field_map(mappings)
+      process_rows(import, field_map, [*dup_header_warnings, *nil_warnings, *warnings])
+    rescue StandardError => e
+      log_error("Processing failed: #{e.message}")
+      import.update!(status: "failed")
+      raise
+    end
+
+    def run_matcher
+      Matcher.new(
+        file_path: @file_path,
+        form_class: @form_class,
+        confidence_threshold: SmartCsvImport.configuration.confidence_threshold
+      ).call
+    end
+
+    def process_rows(import, field_map, warnings)
+      imported_count = 0
+      failed_count = 0
+      errors = []
+      blank_count = 0
+      aborted = false
+
+      reader = SmarterCSV::Reader.new(@file_path, smarter_csv_options.merge(chunk_size: effective_chunk_size))
+      reader.each_chunk do |chunk, chunk_index|
+        chunk_number = chunk_index + 1
+
+        if chunk_number > 1
+          bad_rows_so_far = reader.errors.fetch(:bad_rows, []).size
+          total_seen = imported_count + failed_count + bad_rows_so_far
+          if total_seen > 0 && bad_rows_so_far.to_f / total_seen > SmartCsvImport.configuration.bad_row_limit
+            aborted = true
+            break
+          end
+        end
+
+        chunk_result = RowProcessor.process_chunk(
+          chunk, field_map: field_map, form_class: @form_class, dry_run: @dry_run
+        )
+        imported_count += chunk_result.imported
+        failed_count += chunk_result.failed
+        blank_count += chunk_result.blank
+        errors = [*errors, *chunk_result.errors]
+
+        if chunk_result.aborted_error
+          log_error("Database error during chunk #{chunk_number}: #{chunk_result.aborted_error.message}")
+          chunk_parse_errors = ImportResultBuilder.build_parse_errors(reader.errors.fetch(:bad_rows, []))
+          persist_final_import(
+            import,
+            status: "partial_failure",
+            imported_count: imported_count,
+            failed_count: failed_count + chunk_parse_errors.size,
+            total_rows: imported_count + failed_count + chunk_parse_errors.size,
+            errors: errors, parse_errors: chunk_parse_errors
+          )
+          return Result.partial_failure(
+            imported: imported_count,
+            failed: failed_count,
+            total: imported_count + failed_count + chunk_parse_errors.size,
+            errors: errors,
+            header_mappings: import.header_mappings,
+            import_id: import.id,
+            warnings: [*warnings, *ImportResultBuilder.build_blank_row_warnings(blank_count)],
+            parse_errors: chunk_parse_errors
+          )
+        end
+
+        parse_errors_count = reader.errors.fetch(:bad_rows, []).size
+        import.update!(imported_count: imported_count, failed_count: failed_count + parse_errors_count)
+
+        fire_progress_callback(imported_count, failed_count, chunk_number)
+      end
+
+      parse_errors = ImportResultBuilder.build_parse_errors(reader.errors.fetch(:bad_rows, []))
+      all_warnings = [*warnings, *ImportResultBuilder.build_blank_row_warnings(blank_count)]
+
+      total = imported_count + failed_count + parse_errors.size
+      if !aborted && parse_errors.any? && total > 0
+        ratio = parse_errors.size.to_f / total
+        aborted = ratio > SmartCsvImport.configuration.bad_row_limit
+      end
+
+      return build_abort_result(import, imported_count, failed_count, errors, parse_errors, all_warnings) if aborted
+
+      build_final_result(import, imported_count, failed_count, errors, parse_errors, all_warnings)
+    end
+
+    def smarter_csv_options
+      {
+        strings_as_keys: true,
+        strip_whitespace: true,
+        keep_original_headers: true,
+        field_size_limit: SmartCsvImport.configuration.field_size_limit,
+        with_line_numbers: true,
+        on_bad_row: :collect,
+        collect_raw_lines: true,
+        convert_values_to_numeric: false,
+        remove_empty_values: false,
+        remove_empty_hashes: false,
+        invalid_byte_sequence: "",
+        force_utf8: true,
+        duplicate_header_suffix: "_",
+        nil_values_matching: SmartCsvImport.configuration.nil_values_regexp,
+        bad_row_limit: nil
+      }
+    end
+
+    def effective_chunk_size
+      SmartCsvImport.configuration.chunk_size || FULL_LOAD_CHUNK_SIZE
+    end
+
+    def build_abort_result(import, imported_count, failed_count, errors, parse_errors, warnings)
+      persist_final_import(
+        import, status: "failed",
+                imported_count: imported_count, failed_count: failed_count + parse_errors.size,
+                total_rows: imported_count + failed_count + parse_errors.size,
+                errors: errors, parse_errors: parse_errors
+      )
+      ImportResultBuilder.abort_result(
+        imported: imported_count, failed: failed_count, parse_errors: parse_errors,
+        warnings: warnings, header_mappings: import.header_mappings, import_id: import.id,
+        bad_row_limit: SmartCsvImport.configuration.bad_row_limit
+      )
+    end
+
+    def build_final_result(import, imported_count, failed_count, errors, parse_errors, warnings)
+      all_failed = failed_count + parse_errors.size
+      total = imported_count + failed_count + parse_errors.size
+      persist_final_import(
+        import, status: all_failed > 0 ? "partial_failure" : "completed",
+                imported_count: imported_count, failed_count: all_failed,
+                total_rows: total, errors: errors, parse_errors: parse_errors
+      )
+      ImportResultBuilder.final_result(
+        imported: imported_count, failed: failed_count, errors: errors,
+        parse_errors: parse_errors, warnings: warnings,
+        header_mappings: import.header_mappings, import_id: import.id, dry_run: @dry_run
+      )
+    end
+
+    def persist_final_import(import, status:, imported_count:, failed_count:, total_rows:, errors:, parse_errors:)
+      ImportRowError.transaction do
+        import.update!(
+          status: status, imported_count: imported_count, failed_count: failed_count,
+          total_rows: total_rows
+        )
+        attrs = ImportResultBuilder.build_row_error_attributes(
+          errors: errors, parse_errors: parse_errors, import_id: import.id
+        )
+        # rubocop:disable Rails/SkipsModelValidations
+        # Bulk insert is intentional — attributes built from pre-validated RowError/ParseError structs.
+        attrs.each_slice(1000) { |batch| ImportRowError.insert_all(batch) } if attrs.any?
+        # rubocop:enable Rails/SkipsModelValidations
+      end
+    end
+
+    def fire_progress_callback(imported_count, failed_count, chunk_number)
+      @on_progress.call({
+        imported: imported_count,
+        failed: failed_count,
+        chunk_number: chunk_number
+      })
+    end
+
+    def validate_on_progress!
+      return if @on_progress.nil?
+
+      unless @on_progress.respond_to?(:call)
+        raise ConfigurationError, "on_progress must respond to :call"
+      end
+    end
+  end
+end

data/lib/smart_csv_import/result.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  RowError = Struct.new(:row, :column, :messages, keyword_init: true)
+  RowWarning = Struct.new(:row, :message, :type, keyword_init: true)
+  ParseError = Struct.new(:line_number, :raw_line, :error_message, keyword_init: true)
+  UnmatchedColumnWarning = Struct.new(:column_name, :message, keyword_init: true)
+
+  class Result
+    attr_reader :status, :imported, :failed, :total, :errors,
+                :header_mappings, :import_id, :warnings, :parse_errors,
+                :proposed_mappings, :unmatched_columns
+
+    private_class_method :new
+
+    def self.completed(imported:, failed:, total:, errors:, header_mappings:, import_id: nil, warnings: [], parse_errors: [])
+      new(
+        status: :completed,
+        imported: imported,
+        failed: failed,
+        total: total,
+        errors: errors,
+        header_mappings: header_mappings,
+        import_id: import_id,
+        warnings: warnings,
+        parse_errors: parse_errors
+      )
+    end
+
+    def self.partial_failure(imported:, failed:, total:, errors:, header_mappings:, import_id: nil, warnings: [], parse_errors: [])
+      new(
+        status: :partial_failure,
+        imported: imported,
+        failed: failed,
+        total: total,
+        errors: errors,
+        header_mappings: header_mappings,
+        import_id: import_id,
+        warnings: warnings,
+        parse_errors: parse_errors
+      )
+    end
+
+    def self.review_required(header_mappings:, import_id: nil, proposed_mappings: {}, unmatched_columns: [], warnings: [])
+      new(
+        status: :review_required,
+        header_mappings: header_mappings,
+        import_id: import_id,
+        proposed_mappings: proposed_mappings,
+        unmatched_columns: unmatched_columns,
+        warnings: warnings
+      )
+    end
+
+    def self.queued(import_id:, **rest)
+      invalid_keys = rest.keys & %i[imported failed total]
+      raise ArgumentError, "queued result does not accept: #{invalid_keys.join(", ")}" if invalid_keys.any?
+
+      new(
+        status: :queued,
+        import_id: import_id
+      )
+    end
+
+    def self.dry_run(imported:, failed:, total:, errors:, header_mappings:, warnings: [], parse_errors: [])
+      new(
+        status: :dry_run,
+        imported: imported,
+        failed: failed,
+        total: total,
+        errors: errors,
+        header_mappings: header_mappings,
+        warnings: warnings,
+        parse_errors: parse_errors
+      )
+    end
+
+    def completed? = status == :completed
+    def partial_failure? = status == :partial_failure
+    def review_required? = status == :review_required
+    def queued? = status == :queued
+    def dry_run? = status == :dry_run
+    def success? = completed? && errors.empty?
+
+    private
+
+    def initialize(status:, imported: nil, failed: nil, total: nil, errors: [], header_mappings: {}, import_id: nil, warnings: [], parse_errors: [], proposed_mappings: {}, unmatched_columns: [])
+      @status = status
+      @imported = imported
+      @failed = failed
+      @total = total
+      @errors = errors
+      @header_mappings = header_mappings
+      @import_id = import_id
+      @warnings = warnings
+      @parse_errors = parse_errors
+      @proposed_mappings = proposed_mappings
+      @unmatched_columns = unmatched_columns
+    end
+  end
+end