smart_csv_import 0.1.0

data/lib/smart_csv_import/file_storage.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+
+module SmartCsvImport
+  module FileStorage
+    module_function
+
+    def store(source_path:, import_type:)
+      raise SmartCsvImport::Error, "Source file not found: #{source_path}" unless File.exist?(source_path)
+
+      destination_dir = File.join(SmartCsvImport.configuration.storage_path, import_type)
+      FileUtils.mkdir_p(destination_dir)
+
+      extension = File.extname(source_path)
+      basename = File.basename(source_path, extension)
+      timestamp = Time.current.strftime("%Y%m%d%H%M%S")
+      random_suffix = SecureRandom.hex(4)
+      destination_filename = "#{timestamp}_#{random_suffix}_#{basename}#{extension}"
+      destination_path = File.join(destination_dir, destination_filename)
+
+      FileUtils.cp(source_path, destination_path)
+
+      destination_path
+    end
+
+    def compute_hash(file_path:)
+      raise SmartCsvImport::Error, "File not found: #{file_path}" unless File.exist?(file_path)
+
+      Digest::SHA256.file(file_path).hexdigest
+    end
+  end
+end

data/lib/smart_csv_import/header_normalizer.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  module HeaderNormalizer
+    # Unambiguous abbreviations only — terms that reliably mean one thing in a
+    # business CSV context. This list is intentionally small and conservative.
+    #
+    # DO NOT add entries that could mean two different things depending on
+    # domain (e.g. "ext" = file extension or phone extension, "co" = company
+    # or county, "apt" = apartment or adjective, "sal" / "val" = names).
+    #
+    # This list is not meant to be comprehensive. The LLM fallback strategy
+    # handles the long tail of ambiguous and domain-specific abbreviations
+    # far better than any static dictionary can.
+    ABBREVIATIONS = {
+      # Personal
+      "dob"    => "date of birth",
+      "dod"    => "date of death",
+      "ssn"    => "social security number",
+      "nin"    => "national insurance number",
+      "dba"    => "doing business as",
+      # Contact
+      "tel"    => "telephone",
+      # Location
+      "addr"   => "address",
+      "zip"    => "zip code",
+      "ste"    => "suite",
+      # Organisation / HR
+      "dept"   => "department",
+      "mgr"    => "manager",
+      "emp"    => "employee",
+      "org"    => "organization",
+      "corp"   => "corporation",
+      # Quantities / identifiers
+      "qty"    => "quantity",
+      "amt"    => "amount",
+      "num"    => "number",
+      "ref"    => "reference",
+      "acct"   => "account",
+      # Finance
+      "bal"    => "balance",
+      "pmt"    => "payment",
+      "inv"    => "invoice",
+      # Misc
+      "desc"   => "description",
+      "info"   => "information",
+      "misc"   => "miscellaneous",
+    }.freeze
+
+    def self.normalize(header)
+      text = header.to_s
+
+      # Split camelCase and PascalCase: "CustomerDOB" → "Customer DOB"
+      text = text
+        .gsub(/([a-z])([A-Z])/, '\1 \2')
+        .gsub(/([A-Z]{2,})([A-Z][a-z])/, '\1 \2')
+
+      # Underscores, dashes, dots, slashes → spaces
+      text = text.tr("_./\\-", " ")
+
+      # Strip non-alphanumeric characters (removes #, *, (, ), etc.)
+      text = text.gsub(/[^a-zA-Z0-9\s]/, " ")
+
+      # Collapse whitespace
+      text = text.gsub(/\s+/, " ").strip
+
+      # Expand abbreviations — whole-word, case-insensitive
+      text = text.split(" ").map do |word|
+        ABBREVIATIONS[word.downcase] || word
+      end.join(" ")
+
+      # Final collapse in case expansions introduced extra spaces
+      text.gsub(/\s+/, " ").strip
+    end
+  end
+end

data/lib/smart_csv_import/logging.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  module Logging
+    private
+
+    def log_info(message)
+      tagged = "[SmartCsvImport::#{self.class.name.split("::").last}] #{message}"
+
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.info(tagged)
+      else
+        SmartCsvImport.configuration.logger.info(tagged)
+      end
+    end
+
+    def log_debug(message)
+      tagged = "[SmartCsvImport::#{self.class.name.split("::").last}] #{message}"
+
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.debug(tagged)
+      else
+        SmartCsvImport.configuration.logger.debug(tagged)
+      end
+    end
+
+    def log_error(message)
+      tagged = "[SmartCsvImport::#{self.class.name.split("::").last}] #{message}"
+
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.error(tagged)
+      else
+        SmartCsvImport.configuration.logger.error(tagged)
+      end
+    end
+  end
+end

data/lib/smart_csv_import/match_result.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class MatchResult
+    attr_reader :target_field, :confidence, :strategy_name
+
+    private_class_method :new
+
+    def self.matched(target_field:, confidence:, strategy_name:)
+      new(target_field: target_field, confidence: confidence, strategy_name: strategy_name)
+    end
+
+    def matched? = true
+    def unmatched? = false
+
+    private
+
+    def initialize(target_field:, confidence:, strategy_name:)
+      @target_field = target_field
+      @confidence = confidence
+      @strategy_name = strategy_name
+    end
+  end
+
+  class UnmatchedResult
+    attr_reader :csv_header, :attempted_strategies
+
+    def initialize(csv_header:, attempted_strategies:)
+      @csv_header = csv_header
+      @attempted_strategies = attempted_strategies
+    end
+
+    def matched? = false
+    def unmatched? = true
+  end
+end

data/lib/smart_csv_import/matchable.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  CsvFieldDefinition = Struct.new(:name, :description, :required, keyword_init: true)
+
+  module Matchable
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def csv_field(name, description:, required: false)
+        name = name.to_sym
+
+        raise ConfigurationError, "#{name} is not a declared attribute on #{self}" unless _has_attribute?(name)
+        raise ConfigurationError, "#{name} is already declared as a csv_field" if csv_fields.key?(name)
+        raise ConfigurationError, "description must be non-empty for #{name}" if description.to_s.strip.empty?
+
+        csv_fields[name] = CsvFieldDefinition.new(
+          name: name,
+          description: description,
+          required: required
+        )
+      end
+
+      def csv_fields
+        @csv_fields ||= {}
+      end
+
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@csv_fields, csv_fields.dup)
+      end
+
+      def required_csv_fields
+        csv_fields.select { |_, field| field.required }.keys
+      end
+
+      # Optional context that enriches the LLM matching prompt.
+      # Both act as getter (no arg) and setter (with arg):
+      #
+      #   csv_source  "ADP Workforce payroll export"
+      #   csv_context "HR platform for staffing agencies"
+      #
+      # Source describes where the CSV comes from (system/tool).
+      # Context describes the business domain of the importing app.
+      # Together they let the LLM disambiguate headers like "Cell" or "Sal"
+      # that are genuinely ambiguous without domain knowledge.
+      def csv_source(value = nil)
+        value ? @csv_source = value : @csv_source
+      end
+
+      def csv_context(value = nil)
+        value ? @csv_context = value : @csv_context
+      end
+
+      def matching_strategy
+        @matching_strategy
+      end
+
+      def matching_strategy=(strategy)
+        @matching_strategy = strategy
+      end
+
+      private
+
+      def _has_attribute?(name)
+        if respond_to?(:attribute_names)
+          attribute_names.map(&:to_sym).include?(name)
+        else
+          method_defined?(name) || private_method_defined?(name)
+        end
+      end
+    end
+  end
+end

data/lib/smart_csv_import/matcher.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "smarter_csv"
+
+module SmartCsvImport
+  class Matcher
+    include Logging
+    DATE_PATTERNS = [
+      /\A\d{4}-\d{2}-\d{2}\z/,
+      /\A\d{1,2}\/\d{1,2}\/\d{4}\z/,
+      /\A\d{2}-[A-Za-z]{3}-\d{4}\z/
+    ].freeze
+
+    PHONE_PATTERN = /\A[\d\s\-\(\)\+\.]{7,}\z/
+    EMAIL_PATTERN = /\A[^@\s]+@[^@\s]+\.[^@\s]+\z/
+
+    VALUE_BOOST = 0.05
+    VALUE_PENALTY = -0.10
+
+    def initialize(file_path:, form_class:, confidence_threshold: SmartCsvImport.configuration.confidence_threshold)
+      validate_form_class!(form_class)
+
+      @file_path = file_path
+      @form_class = form_class
+      @confidence_threshold = confidence_threshold
+    end
+
+    def call
+      validate_file!
+
+      parsed_rows = parse_csv
+      csv_headers = parsed_rows.first&.keys || []
+      @sample_rows = parsed_rows.first(SmartCsvImport.configuration.value_hint_rows)
+
+      log_info("Starting header matching for #{csv_headers.length} columns: #{csv_headers.join(", ")}")
+      log_info("Target fields: #{@form_class.csv_fields.keys.join(", ")}")
+
+      results = {}
+      attempted_strategies = []
+      remaining = csv_headers.dup
+
+      # Tier 1: Custom strategy from form_class
+      custom_strategy = @form_class.matching_strategy
+      if custom_strategy
+        tier_results = run_strategy(custom_strategy, remaining, attempted_strategies, "custom")
+        tier_results = with_value_hints(tier_results, @sample_rows, @form_class)
+        results, remaining = accept_matches(results, tier_results, remaining)
+      end
+
+      # Tier 2 and 3: Vector and LLM, ordered by SmartCsvImport.configuration.default_strategy
+      default_tier_strategies.each do |name, strategy|
+        tier_results = run_strategy(strategy, remaining, attempted_strategies, name)
+        tier_results = with_value_hints(tier_results, @sample_rows, @form_class)
+        results, remaining = accept_matches(results, tier_results, remaining)
+      end
+
+      # Remaining unresolved headers become UnmatchedResult
+      remaining.each do |header|
+        log_info("UNMATCHED: '#{header}' — tried: #{attempted_strategies.join(", ")}")
+        results[header] = UnmatchedResult.new(
+          csv_header: header,
+          attempted_strategies: attempted_strategies.dup
+        )
+      end
+
+      log_info("Matching complete: #{results.count { |_, r| r.matched? }} matched, #{results.count { |_, r| r.unmatched? }} unmatched")
+      results
+    end
+
+    private
+
+    def validate_form_class!(form_class)
+      return if form_class.respond_to?(:csv_fields) && form_class.ancestors.include?(Matchable)
+
+      raise ConfigurationError, "form_class must include SmartCsvImport::Matchable"
+    end
+
+    def default_tier_strategies
+      tiers = { "vector" => Strategies::Vector.new, "llm" => Strategies::Llm.new }
+      SmartCsvImport.configuration.default_strategy == :llm ? tiers.to_a.reverse : tiers.to_a
+    end
+
+    def validate_file!
+      raise Error, "CSV file not found: #{@file_path}" unless File.exist?(@file_path)
+    end
+
+    def parse_csv
+      rows = SmarterCSV.process(
+        @file_path,
+        strings_as_keys: true,
+        strip_whitespace: true,
+        keep_original_headers: true,
+        field_size_limit: SmartCsvImport.configuration.field_size_limit,
+        convert_values_to_numeric: false,
+        remove_empty_values: false,
+        remove_empty_hashes: false,
+        invalid_byte_sequence: "",
+        force_utf8: true,
+        duplicate_header_suffix: "_",
+        nil_values_matching: SmartCsvImport.configuration.nil_values_regexp
+      )
+
+      rows.reject { |row| row.except(:csv_line_number).values.all? { |v| v.nil? || v.to_s.empty? } }
+    end
+
+    def run_strategy(strategy, unresolved_headers, attempted_strategies, strategy_name)
+      return {} if unresolved_headers.empty?
+
+      log_info("Running #{strategy_name} strategy for #{unresolved_headers.length} headers: #{unresolved_headers.join(", ")}")
+      attempted_strategies << strategy_name
+
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      tier_results = strategy.match(csv_headers: unresolved_headers, form_class: @form_class, sample_rows: @sample_rows)
+      elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time).round(2)
+
+      if tier_results.is_a?(StrategyFailure)
+        log_error("Strategy #{strategy_name} errored (#{tier_results.reason}) after #{elapsed}s — continuing with next tier")
+        return {}
+      end
+
+      tier_results.each do |header, result|
+        if result&.matched?
+          log_info("  #{strategy_name}: '#{header}' → :#{result.target_field} (confidence: #{result.confidence})")
+        else
+          log_debug("  #{strategy_name}: '#{header}' → no match")
+        end
+      end
+      log_info("#{strategy_name} completed in #{elapsed}s — #{tier_results.count { |_, r| r&.matched? }} matches")
+
+      tier_results
+    end
+
+    def accept_matches(results, tier_results, remaining)
+      accepted = results.dup
+      still_remaining = remaining.dup
+
+      tier_results.each do |header, result|
+        next unless result&.matched?
+
+        if result.confidence >= @confidence_threshold
+          accepted[header] = result
+          still_remaining.delete(header)
+        else
+          log_info("  BELOW THRESHOLD: '#{header}' → :#{result.target_field} (#{result.confidence} < #{@confidence_threshold})")
+        end
+      end
+
+      [accepted, still_remaining]
+    end
+
+    def with_value_hints(tier_results, sample_rows, form_class)
+      return tier_results if sample_rows.empty?
+
+      tier_results.each_with_object({}) do |(header, result), adjusted|
+        unless result&.matched?
+          adjusted[header] = result
+          next
+        end
+
+        field_def = form_class.csv_fields[result.target_field]
+        sample_values = sample_rows.filter_map { |row| row[header] }
+        adjustment = field_def && sample_values.any? ? compute_value_hint(sample_values, field_def) : 0
+
+        if adjustment.zero?
+          adjusted[header] = result
+        else
+          adjusted[header] = MatchResult.matched(
+            target_field: result.target_field,
+            confidence: (result.confidence + adjustment).clamp(0.0, 1.0).round(4),
+            strategy_name: result.strategy_name
+          )
+        end
+      end
+    end
+
+    def compute_value_hint(sample_values, field_def)
+      description = field_def.description.downcase
+
+      if description.include?("date") || description.include?("birth")
+        date_match = sample_values.count { |v| DATE_PATTERNS.any? { |p| p.match?(v.to_s) } }
+        return date_match > 0 ? VALUE_BOOST : VALUE_PENALTY
+      end
+
+      if description.include?("email")
+        email_match = sample_values.count { |v| EMAIL_PATTERN.match?(v.to_s) }
+        return email_match > 0 ? VALUE_BOOST : VALUE_PENALTY
+      end
+
+      if description.include?("phone")
+        phone_match = sample_values.count { |v| PHONE_PATTERN.match?(v.to_s) }
+        return phone_match > 0 ? VALUE_BOOST : VALUE_PENALTY
+      end
+
+      0
+    end
+
+  end
+end

data/lib/smart_csv_import/normalizers/boolean_converter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  module Normalizers
+    class BooleanConverter
+      TRUTHY = Set.new(%w[true yes 1 t y]).freeze
+      FALSEY = Set.new(%w[false no 0 f n]).freeze
+
+      def call(value)
+        return nil if value.nil?
+        return value if value == true || value == false
+
+        normalized = value.to_s.strip.downcase
+        return nil if normalized.empty?
+
+        if TRUTHY.include?(normalized)
+          true
+        elsif FALSEY.include?(normalized)
+          false
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/smart_csv_import/normalizers/date_converter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "date"
+
+module SmartCsvImport
+  module Normalizers
+    class DateConverter
+      US_DATE = %r{\A(\d{1,2})/(\d{1,2})/(\d{4})\z}
+
+      def call(value)
+        return nil if value.nil?
+        return value if value.is_a?(Date)
+
+        str = value.to_s.strip
+        return nil if str.empty?
+
+        if (match = str.match(US_DATE))
+          month, day, year = match.captures.map(&:to_i)
+          Date.new(year, month, day)
+        else
+          Date.parse(str)
+        end
+      rescue Date::Error, ArgumentError
+        nil
+      end
+    end
+  end
+end

data/lib/smart_csv_import/notifications.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+
+module SmartCsvImport
+  module Notifications
+    def self.progress_notifier(import_id:)
+      ->(info) {
+        ActiveSupport::Notifications.instrument("smart_csv_import.import.progress", {
+          import_id: import_id,
+          **info
+        })
+      }
+    end
+  end
+end

data/lib/smart_csv_import/processor/csv_preflight_analyzer.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module SmartCsvImport
+  class Processor
+    module CsvPreflightAnalyzer
+      module_function
+
+      def duplicate_header_warnings(file_path:)
+        headers = read_headers(file_path)
+        return [] if headers.nil?
+
+        find_duplicates(headers).map { |dup| build_duplicate_warning(dup) }
+      rescue StandardError
+        []
+      end
+
+      def build_duplicate_warning(dup)
+        RowWarning.new(
+          row: 0,
+          message: "Duplicate column '#{dup[:original]}' — occurrence renamed to '#{dup[:renamed_to]}'",
+          type: :duplicate_header
+        )
+      end
+
+      def read_headers(file_path)
+        first_line = File.open(file_path, 'r:bom|utf-8:utf-8', invalid: :replace, undef: :replace, &:gets)
+        return nil unless first_line
+
+        CSV.parse_line(first_line)&.map { |h| h&.strip }
+      end
+
+      def count_nil_matches(file_path:, nil_values:)
+        return 0 if nil_values.nil? || (nil_values.respond_to?(:empty?) && nil_values.empty?)
+
+        count = 0
+        begin
+          each_data_row(file_path) { |row| count += NilCellCounter.count_row(row, nil_values) }
+        rescue CSV::MalformedCSVError
+          # Stop the scan and return the partial count — SmarterCSV surfaces the parse error downstream.
+        end
+        count
+      rescue Errno::ENOENT, Errno::EACCES => e
+        raise SmartCsvImport::Error, "Failed to read CSV for nil-value scan in #{File.basename(file_path)}: #{e.message}"
+      end
+
+      def find_duplicates(headers)
+        seen = {}
+        headers.each_with_object([]) do |header, duplicates|
+          next unless header
+
+          if seen[header]
+            seen[header] += 1
+            duplicates << { original: header, renamed_to: "#{header}_#{seen[header]}" }
+          else
+            seen[header] = 1
+          end
+        end
+      end
+
+      def each_data_row(file_path)
+        first_row = true
+        CSV.foreach(file_path, encoding: 'bom|utf-8:utf-8', invalid: :replace, undef: :replace) do |row|
+          if first_row
+            first_row = false
+            next
+          end
+          yield row
+        end
+      end
+    end
+  end
+end