smart_csv_import 0.1.0

data/app/jobs/smart_csv_import/import_job.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class ImportJob < ActiveJob::Base
+    queue_as :smart_csv_import
+
+    def perform(import_id, form_class_name)
+      import = Import.find(import_id)
+      form_class = form_class_name.constantize
+
+      Processor.new(
+        file_path: import.file_path,
+        form_class: form_class,
+        mode: :sync,
+        import: import
+      ).call
+    rescue StandardError => e
+      import&.update!(status: "failed")
+      raise
+    end
+  end
+end

data/app/models/smart_csv_import/import.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  # Parent resolved at class-definition time; set SmartCsvImport.base_model_class in an
+  # initializer before this model is autoloaded if you want a custom AR subclass.
+  class Import < SmartCsvImport.base_model_class
+    self.table_name = "smart_csv_import_imports"
+
+    STATUSES = %w[pending mapping_review processing completed partial_failure failed].freeze
+
+    has_many :row_errors,
+             -> { order(:row_number) },
+             class_name: "SmartCsvImport::ImportRowError",
+             foreign_key: :import_id,
+             inverse_of: :import,
+             dependent: :destroy
+
+    validates :import_type, presence: true
+    validates :original_filename, presence: true
+    validates :status, inclusion: { in: STATUSES }
+
+    STATUSES.each do |status_name|
+      define_method(:"#{status_name}?") do
+        status == status_name
+      end
+    end
+
+    def self.find_duplicate(file_hash:, import_type: nil)
+      return nil if file_hash.nil?
+
+      scope = where(file_hash: file_hash)
+      scope = scope.where(import_type: import_type) if import_type
+      scope.first
+    end
+  end
+end

data/app/models/smart_csv_import/import_row_error.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class ImportRowError < SmartCsvImport.base_model_class
+    self.table_name = 'smart_csv_import_import_row_errors'
+
+    ERROR_TYPES = %w[parse validation].freeze
+
+    belongs_to :import, class_name: 'SmartCsvImport::Import', inverse_of: :row_errors, optional: false
+
+    validates :error_type, inclusion: { in: ERROR_TYPES }
+    validates :row_number, presence: true
+
+    scope :validation_errors, -> { where(error_type: 'validation') }
+    scope :parse_errors,      -> { where(error_type: 'parse') }
+  end
+end

data/lib/generators/smart_csv_import/import/import_generator.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module SmartCsvImport
+  module Generators
+    class ImportGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      argument :model_name, type: :string, desc: "The model name (e.g., Employee)"
+      argument :fields, type: :array, default: [], desc: "Fields with optional types (e.g., first_name last_name:string email:string hire_date:date)"
+
+      desc "Generates a SmartCsvImport import form object with csv_field declarations"
+
+      def create_form_object
+        template "import_form.rb.tt", "app/forms/#{form_file_name}.rb"
+      end
+
+      def create_form_spec
+        template "import_form_spec.rb.tt", "spec/forms/#{form_file_name}_spec.rb"
+      end
+
+      private
+
+      def form_class_name
+        "#{model_name.camelize}ImportForm"
+      end
+
+      def form_file_name
+        "#{model_name.underscore}_import_form"
+      end
+
+      def parsed_fields
+        @parsed_fields ||= fields.map do |field_spec|
+          name, type = field_spec.split(":")
+          { name: name, type: type || "string" }
+        end
+      end
+
+      def field_names
+        parsed_fields.map { |f| f[:name] }
+      end
+
+      def email_fields
+        parsed_fields.select { |f| f[:name] =~ /email/ }.map { |f| f[:name] }
+      end
+    end
+  end
+end

data/lib/generators/smart_csv_import/import/templates/import_form.rb.tt

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class <%= form_class_name %>
+  include ActiveModel::API
+  include ActiveModel::Attributes
+  include SmartCsvImport::Matchable
+
+<% parsed_fields.each do |field| -%>
+  attribute :<%= field[:name] %>, :<%= field[:type] %>
+<% end -%>
+
+<% parsed_fields.each do |field| -%>
+  csv_field :<%= field[:name] %>, description: "<%= model_name.camelize %> <%= field[:name].humanize.downcase %>"
+<% end -%>
+
+  validates <%= field_names.map { |n| ":#{n}" }.join(", ") %>, presence: true
+<% email_fields.each do |email_field| -%>
+  validates :<%= email_field %>, format: { with: URI::MailTo::EMAIL_REGEXP }, allow_blank: true
+<% end -%>
+
+  def save
+    return false unless valid?
+
+    record = <%= model_name.camelize %>.new(
+<% parsed_fields.each_with_index do |field, index| -%>
+      <%= field[:name] %>: <%= field[:name] %><%= index < parsed_fields.length - 1 ? "," : "" %>
+<% end -%>
+    )
+
+    record.save
+  end
+end

data/lib/generators/smart_csv_import/import/templates/import_form_spec.rb.tt

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe <%= form_class_name %> do
+  describe ".csv_fields" do
+    it "declares all expected csv_fields" do
+      expected_fields = %i[<%= field_names.join(" ") %>]
+      expect(described_class.csv_fields.keys).to match_array(expected_fields)
+    end
+  end
+
+  describe "validations" do
+<% parsed_fields.each do |field| -%>
+    it "requires <%= field[:name] %>" do
+      form = described_class.new(<%= field[:name] %>: nil)
+      form.valid?
+      expect(form.errors[:<%= field[:name] %>]).to include("can't be blank")
+    end
+
+<% end -%>
+<% email_fields.each do |email_field| -%>
+    it "validates format of <%= email_field %>" do
+      form = described_class.new(<%= email_field %>: "invalid-email")
+      form.valid?
+      expect(form.errors[:<%= email_field %>]).to include("is invalid")
+    end
+
+<% end -%>
+  end
+
+  describe "#save" do
+    it "responds to save" do
+      form = described_class.new
+      expect(form).to respond_to(:save)
+    end
+  end
+end

data/lib/generators/smart_csv_import/install/install_generator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module SmartCsvImport
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Installs SmartCsvImport: initializer, migration, and storage directories"
+
+      def copy_initializer
+        template "initializer.rb.tt", "config/initializers/smart_csv_import.rb"
+      end
+
+      def create_migration
+        base_time = Time.now
+        template(
+          "create_smart_csv_import_imports.rb.tt",
+          "db/migrate/#{base_time.strftime('%Y%m%d%H%M%S')}_create_smart_csv_import_imports.rb"
+        )
+        template(
+          "create_smart_csv_import_import_row_errors.rb.tt",
+          "db/migrate/#{(base_time + 1).strftime('%Y%m%d%H%M%S')}_create_smart_csv_import_import_row_errors.rb"
+        )
+      end
+
+      def create_storage_directories
+        empty_directory "tmp/smart_csv_import/imports"
+        empty_directory "tmp/smart_csv_import/embeddings_cache"
+      end
+    end
+  end
+end

data/lib/generators/smart_csv_import/install/templates/create_smart_csv_import_import_row_errors.rb.tt

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateSmartCsvImportImportRowErrors < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :smart_csv_import_import_row_errors do |t|
+      t.references :import, null: false, foreign_key: { to_table: :smart_csv_import_imports, on_delete: :cascade }
+      t.integer :row_number, null: false
+      t.string :error_type, null: false
+      t.string :column_name
+      t.json :messages, default: []
+      t.text :raw_line
+      t.text :error_message
+      t.datetime :created_at, null: false
+    end
+
+    add_index :smart_csv_import_import_row_errors, [:import_id, :error_type, :row_number]
+  end
+end

data/lib/generators/smart_csv_import/install/templates/create_smart_csv_import_imports.rb.tt

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class CreateSmartCsvImportImports < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :smart_csv_import_imports do |t|
+      t.string :import_type, null: false
+      t.string :original_filename, null: false
+      t.string :file_path
+      t.string :file_hash
+      t.json :header_mappings, default: {}
+      t.integer :total_rows, default: 0
+      t.integer :imported_count, default: 0
+      t.integer :failed_count, default: 0
+      t.json :warnings, default: []
+      t.string :status, null: false, default: "pending"
+
+      t.timestamps
+    end
+
+    add_index :smart_csv_import_imports, :file_hash
+    add_index :smart_csv_import_imports, :status
+  end
+end

data/lib/generators/smart_csv_import/install/templates/initializer.rb.tt

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# SmartCsvImport uses RubyLLM for AI-powered header matching.
+# Configure your AI provider(s) below. At minimum, set one embedding model
+# (for vector matching) and one LLM model (for fallback matching).
+#
+# Recommended free-tier setup: Gemini embeddings + Anthropic LLM
+#
+# RubyLLM.configure do |config|
+#   # --- Gemini (embeddings) ---
+#   # Free tier available at https://aistudio.google.com/apikey
+#   # config.gemini_api_key = ENV["GEMINI_API_KEY"]
+#
+#   # --- Anthropic (LLM fallback) ---
+#   # config.anthropic_api_key = ENV["ANTHROPIC_API_KEY"]
+#
+#   # --- OpenAI (alternative for both) ---
+#   # config.openai_api_key = ENV["OPENAI_API_KEY"]
+# end
+
+SmartCsvImport.configure do |config|
+  # Path where uploaded CSV files are stored
+  # config.storage_path = "tmp/smart_csv_import"
+
+  # Number of rows to process per batch
+  # config.batch_size = 500
+
+  # Minimum confidence threshold for header matching (0.0 - 1.0)
+  # config.confidence_threshold = 0.80
+
+  # Which strategy tier runs first when no custom strategy is set on the form class.
+  # :vector (default) — vector similarity first, LLM as fallback.
+  # :llm — LLM first, vector as fallback.
+  # config.default_strategy = :vector
+
+  # LLM model for fallback header matching
+  # Anthropic: "claude-haiku-4-5-20251001"  (fast, cheap)
+  # OpenAI:    "gpt-4o-mini"
+  # config.llm_model = "claude-haiku-4-5-20251001"
+
+  # Embedding model for vector similarity matching
+  # Gemini: "gemini-embedding-001"  (free tier, recommended)
+  # OpenAI: "text-embedding-3-small"
+  # config.embedding_model = "gemini-embedding-001"
+
+  # Number of sample rows to use for value-based confidence hints
+  # config.value_hint_rows = 3
+
+  # Process imports asynchronously with ActiveJob
+  # config.async = false
+end

data/lib/generators/smart_csv_import/scaffold/scaffold_generator.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module SmartCsvImport
+  module Generators
+    class ScaffoldGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      argument :model_name, type: :string, desc: "The model name (e.g., Employee)"
+
+      desc "Generates SmartCsvImport import scaffold: controller, views, and routes"
+
+      def create_controller
+        template "controller.rb.tt", "app/controllers/#{imports_controller_file_name}.rb"
+      end
+
+      def create_views
+        template "new.html.erb.tt", "app/views/#{imports_name}/new.html.erb"
+        template "show.html.erb.tt", "app/views/#{imports_name}/show.html.erb"
+      end
+
+      def add_routes
+        route_line = "  resources :#{imports_name}, only: [:new, :create, :show]"
+
+        inject_into_file(
+          "config/routes.rb",
+          "#{route_line}\n",
+          after: "Rails.application.routes.draw do\n"
+        )
+      end
+
+      private
+
+      def model_class_name
+        model_name.camelize
+      end
+
+      def form_class_name
+        "#{model_class_name}ImportForm"
+      end
+
+      def imports_name
+        "#{model_name.underscore}_imports"
+      end
+
+      def imports_controller_class_name
+        "#{model_class_name}ImportsController"
+      end
+
+      def imports_controller_file_name
+        "#{model_name.underscore}_imports_controller"
+      end
+    end
+  end
+end

data/lib/generators/smart_csv_import/scaffold/templates/controller.rb.tt

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class <%= imports_controller_class_name %> < ApplicationController
+  def new
+    @import_form = <%= form_class_name %>.new
+  end
+
+  def create
+    uploaded_file = params[:file]
+
+    result = SmartCsvImport.process(
+      uploaded_file.tempfile.path,
+      form_class: <%= form_class_name %>
+    )
+
+    if result.review_required?
+      flash[:notice] = "Import requires header mapping review."
+    elsif result.completed?
+      flash[:notice] = "Imported #{result.imported} of #{result.total} rows."
+    elsif result.partial_failure?
+      flash[:alert] = "Imported #{result.imported} of #{result.total} rows with #{result.failed} failures."
+    end
+
+    redirect_to <%= imports_name.singularize %>_path(result.import_id)
+  rescue StandardError => e
+    flash[:alert] = "Import failed: #{e.message}"
+    redirect_to new_<%= imports_name.singularize %>_path
+  end
+
+  def show
+    @import = SmartCsvImport::Import.find(params[:id])
+  end
+end

data/lib/generators/smart_csv_import/scaffold/templates/new.html.erb.tt

@@ -0,0 +1,12 @@
+<h1>Import <%= model_class_name.pluralize %></h1>
+
+<%%= form_with url: <%= imports_name %>_path, method: :post, multipart: true do |form| %>
+  <div>
+    <%%= form.label :file, "Choose CSV file" %>
+    <%%= form.file_field :file, accept: ".csv" %>
+  </div>
+
+  <div>
+    <%%= form.submit "Upload and Import" %>
+  </div>
+<%% end %>

data/lib/generators/smart_csv_import/scaffold/templates/show.html.erb.tt

@@ -0,0 +1,59 @@
+<h1>Import Result</h1>
+
+<dl>
+  <dt>Status</dt>
+  <dd><%%= @import.status %></dd>
+
+  <dt>File</dt>
+  <dd><%%= @import.original_filename %></dd>
+
+  <dt>Total Rows</dt>
+  <dd><%%= @import.total_rows %></dd>
+
+  <dt>Imported</dt>
+  <dd><%%= @import.imported_count %></dd>
+
+  <dt>Failed</dt>
+  <dd><%%= @import.failed_count %></dd>
+</dl>
+
+<%% if @import.warnings.any? %>
+  <h2>Warnings</h2>
+  <ul>
+    <%% @import.warnings.each do |warning| %>
+      <li><%%= warning %></li>
+    <%% end %>
+  </ul>
+<%% end %>
+
+<%% if @import.row_errors.any? %>
+  <h2>Failed Rows</h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Row</th>
+        <th>Type</th>
+        <th>Column</th>
+        <th>Errors</th>
+      </tr>
+    </thead>
+    <tbody>
+      <%% @import.row_errors.each do |row_error| %>
+        <tr>
+          <td><%%= row_error.row_number %></td>
+          <td><%%= row_error.error_type %></td>
+          <td><%%= row_error.column_name %></td>
+          <td>
+            <%% if row_error.error_type == "parse" %>
+              <%%= row_error.error_message %>
+            <%% else %>
+              <%%= row_error.messages.join(", ") %>
+            <%% end %>
+          </td>
+        </tr>
+      <%% end %>
+    </tbody>
+  </table>
+<%% end %>
+
+<%%= link_to "New Import", new_<%= imports_name.singularize %>_path %>

data/lib/smart_csv_import/configuration.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Configuration
+    REVIEW_MODES = %i[auto always skip].freeze
+
+    attr_accessor :confidence_threshold,
+                  :batch_size,
+                  :storage_path,
+                  :default_strategy,
+                  :llm_model,
+                  :embedding_model,
+                  :async,
+                  :value_hint_rows,
+                  :field_size_limit,
+                  :bad_row_limit
+
+    attr_reader :review_mode, :logger, :nil_values_matching, :chunk_size
+
+    def initialize
+      @confidence_threshold = 0.80
+      @batch_size = 500
+      @storage_path = "tmp/smart_csv_import"
+      @default_strategy = :vector
+      @llm_model = "gpt-4o-mini"
+      @embedding_model = "text-embedding-3-small"
+      @async = false
+      @value_hint_rows = 5
+      @field_size_limit = 1_048_576
+      @bad_row_limit = 0.10
+      @chunk_size = 1000
+      @review_mode = :skip
+      @nil_values_matching = %w[#N/A #VALUE! #REF! #DIV/0! #NUM! #NAME? #NULL! NULL null N/A n/a nil Nil].freeze
+      @logger = Logger.new($stdout)
+    end
+
+    def logger=(logger)
+      if logger.nil?
+        @logger = Logger.new(nil)
+      elsif !logger.respond_to?(:info) || !logger.respond_to?(:debug) || !logger.respond_to?(:error)
+        raise ConfigurationError, "logger must respond to :info, :debug, and :error"
+      else
+        @logger = logger
+      end
+    end
+
+    def nil_values_regexp
+      return nil if nil_values_matching.nil? || nil_values_matching.empty?
+
+      Regexp.union(nil_values_matching.map { |v| /\A#{Regexp.escape(v)}\z/ })
+    end
+
+    def nil_values_matching=(value)
+      unless value.nil? || value.is_a?(Array)
+        raise ConfigurationError, "nil_values_matching must be an Array of strings or nil"
+      end
+
+      @nil_values_matching = value
+    end
+
+    def chunk_size=(value)
+      unless value.nil? || (value.is_a?(Integer) && value > 0)
+        raise ConfigurationError, "chunk_size must be a positive integer or nil"
+      end
+
+      @chunk_size = value
+    end
+
+    def review_mode=(value)
+      unless REVIEW_MODES.include?(value)
+        raise ConfigurationError, "review_mode must be one of: #{REVIEW_MODES.join(", ")}"
+      end
+
+      @review_mode = value
+    end
+  end
+end

data/lib/smart_csv_import/cosine_similarity.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  module CosineSimilarity
+    def self.call(vec_a, vec_b)
+      dot_product = vec_a.zip(vec_b).sum { |a, b| a * b }
+      magnitude_a = Math.sqrt(vec_a.sum { |v| v**2 })
+      magnitude_b = Math.sqrt(vec_b.sum { |v| v**2 })
+
+      return 0.0 if magnitude_a.zero? || magnitude_b.zero?
+
+      dot_product / (magnitude_a * magnitude_b)
+    end
+  end
+end

data/lib/smart_csv_import/engine.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module SmartCsvImport
+  class Engine < ::Rails::Engine
+    isolate_namespace SmartCsvImport
+
+    config.generators do |g|
+      g.test_framework :rspec
+      g.fixture_replacement :factory_bot, dir: "spec/factories"
+    end
+  end
+end

data/lib/smart_csv_import/failed_row_exporter.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module SmartCsvImport
+  class FailedRowExporter
+    def initialize(csv_path:, result: nil, import: nil)
+      raise ArgumentError, "provide either result: or import:" if result.nil? && import.nil?
+      raise ArgumentError, "provide only one of result: or import:" if result && import
+
+      @csv_path = csv_path
+      @failed_lines = if result
+                        errors_by_line_from_result(result)
+                      else
+                        errors_by_line_from_import(import)
+                      end
+    end
+
+    def call
+      return nil if @failed_lines.empty?
+
+      original_headers = read_original_headers
+      output_path = generate_output_path
+
+      write_failed_rows(original_headers, @failed_lines, output_path)
+
+      output_path
+    end
+
+    private
+
+    def errors_by_line_from_result(result)
+      result.errors.each_with_object({}) do |error, acc|
+        acc[error.row] ||= []
+        acc[error.row] << { column: error.column, messages: error.messages }
+      end
+    end
+
+    def errors_by_line_from_import(import)
+      import.row_errors.validation_errors.each_with_object({}) do |error, acc|
+        acc[error.row_number] ||= []
+        acc[error.row_number] << { column: error.column_name, messages: error.messages }
+      end
+    end
+
+    def read_original_headers
+      CSV.open(@csv_path, "r", &:readline)
+    end
+
+    def generate_output_path
+      output_dir = File.join(SmartCsvImport.configuration.storage_path, "failed_rows")
+      FileUtils.mkdir_p(output_dir)
+      timestamp = Time.current.strftime("%Y%m%d%H%M%S")
+      File.join(output_dir, "#{timestamp}_failed.csv")
+    end
+
+    def write_failed_rows(original_headers, failed_lines, output_path)
+      output_headers = [*original_headers, "_error"]
+
+      CSV.open(output_path, "w") do |csv|
+        csv << output_headers
+
+        CSV.foreach(@csv_path, headers: true).with_index(2) do |row, line_number|
+          next unless failed_lines.key?(line_number)
+
+          error_message = format_errors(failed_lines[line_number])
+          csv << [*row.fields, error_message]
+        end
+      end
+    end
+
+    def format_errors(errors)
+      errors.map do |error|
+        "#{error[:column]}: #{error[:messages].join(", ")}"
+      end.join("; ")
+    end
+  end
+end