prospector_engine 0.1.1

data/app/views/prospector/runs/show.html.erb

@@ -0,0 +1,111 @@
+<% if defined?(Turbo::StreamsFrom) %>
+  <%= turbo_stream_from @run %>
+<% end %>
+
+<div class="prospector-flex prospector-flex-between">
+  <h1 class="prospector-heading"><%= @run.label || "Run ##{@run.id}" %></h1>
+  <div class="prospector-flex prospector-gap-2">
+    <% if @run.cancellable? %>
+      <%= button_to "Cancel", run_cancellation_path(@run), method: :post, class: "prospector-btn prospector-btn-danger prospector-btn-sm" %>
+    <% end %>
+    <% if @run.retryable? %>
+      <%= button_to "Retry", run_retry_path(@run), method: :post, class: "prospector-btn prospector-btn-outline prospector-btn-sm" %>
+    <% end %>
+    <% if @run.restartable? %>
+      <%= button_to "Restart", run_restart_path(@run), method: :post, class: "prospector-btn prospector-btn-outline prospector-btn-sm" %>
+    <% end %>
+  </div>
+</div>
+
+<% if @run.error_messages.present? %>
+  <div class="prospector-card" style="border-left: 3px solid var(--p-danger); background: var(--p-danger-bg);">
+    <div class="prospector-subheading" style="color: var(--p-danger);">Error</div>
+    <pre style="font-family: var(--p-mono, monospace); font-size: 0.75rem; color: var(--p-text-secondary, #a1a1a8); white-space: pre-wrap; margin: 0;"><%= @run.error_messages %></pre>
+  </div>
+<% end %>
+
+<div class="prospector-stats">
+  <div class="prospector-stat">
+    <div class="prospector-stat-value"><%= @run.total_found %></div>
+    <div class="prospector-stat-label">Found</div>
+  </div>
+  <div class="prospector-stat">
+    <div class="prospector-stat-value"><%= @run.fetched_count %></div>
+    <div class="prospector-stat-label">Fetched</div>
+  </div>
+  <div class="prospector-stat">
+    <div class="prospector-stat-value"><%= @run.skipped_count %></div>
+    <div class="prospector-stat-label">Skipped</div>
+  </div>
+  <div class="prospector-stat">
+    <div class="prospector-stat-value"><span class="prospector-badge prospector-badge-<%= @run.status %>"><%= @run.status.titleize %></span></div>
+    <div class="prospector-stat-label">Status</div>
+  </div>
+</div>
+
+<% if @run.completed? || @run.classifying? %>
+  <div class="prospector-flex prospector-gap-2 prospector-mt-4" style="margin-bottom: 1rem;">
+    <%= button_to "Bulk Approve All", run_bulk_approval_path(@run), method: :post, class: "prospector-btn prospector-btn-primary prospector-btn-sm" %>
+    <%= button_to "Reclassify", run_reclassification_path(@run), method: :post, class: "prospector-btn prospector-btn-outline prospector-btn-sm" %>
+  </div>
+<% end %>
+
+<div class="prospector-tabs">
+  <% %w[pending approved rejected].each do |filter| %>
+    <% count = @status_counts[filter] || 0 %>
+    <%= link_to run_path(@run, filter: filter),
+        class: "prospector-tab #{@filter == filter ? 'prospector-tab-active' : ''}" do %>
+      <%= filter.titleize %> (<%= count %>)
+    <% end %>
+  <% end %>
+</div>
+
+<div class="prospector-card">
+  <table class="prospector-table">
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Address</th>
+        <th>Contact</th>
+        <th>Status</th>
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @candidates.each do |candidate| %>
+        <tr>
+          <td>
+            <strong><%= candidate.name %></strong>
+            <% if candidate.website.present? %>
+              <br><a href="<%= candidate.website %>" target="_blank" class="prospector-text-muted prospector-text-sm"><%= truncate(candidate.website, length: 40) %></a>
+            <% end %>
+          </td>
+          <td class="prospector-text-sm"><%= candidate.address %></td>
+          <td class="prospector-text-sm">
+            <% if candidate.email.present? %>
+              <%= mail_to candidate.email, candidate.email, class: "prospector-text-sm" %><br>
+            <% end %>
+            <% if candidate.has_contact_info? %>
+              <span class="prospector-text-muted"><%= Prospector::Candidate::SOCIAL_FIELDS.count { |f| candidate[f].present? } %> social</span>
+            <% end %>
+          </td>
+          <td><span class="prospector-badge prospector-badge-<%= candidate.status %>"><%= candidate.status.titleize %></span></td>
+          <td>
+            <div class="prospector-flex prospector-gap-2">
+              <% if candidate.pending? && candidate.approvable? %>
+                <%= button_to "Approve", run_candidate_path(@run, candidate, status: "approved", return_filter: @filter), method: :patch, class: "prospector-btn prospector-btn-primary prospector-btn-sm" %>
+              <% end %>
+              <% if candidate.pending? %>
+                <%= button_to "Reject", run_candidate_path(@run, candidate, status: "rejected", return_filter: @filter), method: :patch, class: "prospector-btn prospector-btn-danger prospector-btn-sm" %>
+              <% end %>
+              <% if candidate.rejected? %>
+                <%= button_to "Restore", run_candidate_path(@run, candidate, status: "pending", return_filter: @filter), method: :patch, class: "prospector-btn prospector-btn-outline prospector-btn-sm" %>
+              <% end %>
+              <%= link_to "Details", run_candidate_path(@run, candidate), class: "prospector-btn prospector-btn-outline prospector-btn-sm" %>
+            </div>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>

data/config/routes.rb

@@ -0,0 +1,15 @@
+Prospector::Engine.routes.draw do
+  resources :runs, only: [:index, :show, :new, :create] do
+    resource :retry, only: [:create], controller: "run_retries"
+    resource :restart, only: [:create], controller: "run_restarts"
+    resource :cancellation, only: [:create], controller: "run_cancellations"
+    resource :reclassification, only: [:create], controller: "run_reclassifications"
+    resource :bulk_approval, only: [:create], controller: "run_bulk_approvals"
+    resources :candidates, only: [:show, :update]
+  end
+
+  resources :keywords, only: [:index, :create, :update, :destroy]
+  resources :keyword_generations, only: [:create]
+
+  root to: "runs#index"
+end

data/db/prospector_schema.rb

@@ -0,0 +1,81 @@
+ActiveRecord::Schema.define do
+  create_table :prospector_runs, if_not_exists: true do |t|
+    t.string   :status,          null: false, default: "pending"
+    t.string   :source_adapter,  null: false
+    t.string   :geography_type,  null: false
+    t.jsonb    :geography_data,  null: false, default: {}
+    t.string   :categories,      array: true, default: []
+    t.string   :label
+    t.integer  :actor_id
+    t.integer  :total_found,     default: 0
+    t.integer  :fetched_count,   default: 0
+    t.integer  :skipped_count,   default: 0
+    t.integer  :error_count,     default: 0
+    t.text     :error_messages
+    t.jsonb    :metadata,        default: {}
+    t.datetime :started_at
+    t.datetime :completed_at
+    t.timestamps null: false
+
+    t.index :status
+    t.index :source_adapter
+    t.index :created_at
+  end
+
+  create_table :prospector_candidates, if_not_exists: true do |t|
+    t.references :run, null: false, foreign_key: { to_table: :prospector_runs }
+    t.string  :status,      null: false, default: "pending"
+    t.string  :name,        null: false
+    t.string  :address,     null: false
+    t.decimal :latitude,    precision: 10, scale: 6
+    t.decimal :longitude,   precision: 10, scale: 6
+    t.string  :phone_number
+    t.string  :website
+    t.string  :email
+    t.string  :facebook_url
+    t.string  :instagram_url
+    t.string  :linkedin_url
+    t.string  :tiktok_url
+    t.string  :youtube_url
+    t.text    :description
+    t.string  :category
+    t.jsonb   :hours_of_operation, default: {}
+    t.string  :source_uid,  null: false
+    t.jsonb   :source_data, default: {}
+    t.jsonb   :metadata,    default: {}
+    t.timestamps null: false
+
+    t.index [:run_id, :source_uid], unique: true
+    t.index [:run_id, :status]
+    t.index :status
+    t.index :source_uid
+  end
+
+  create_table :prospector_classification_runs, if_not_exists: true do |t|
+    t.references :run, null: false, foreign_key: { to_table: :prospector_runs }
+    t.integer  :actor_id
+    t.string   :ai_model,         null: false
+    t.string   :status,           null: false, default: "pending"
+    t.boolean  :include_approved,  null: false, default: false
+    t.jsonb    :results
+    t.text     :error_message
+    t.datetime :started_at
+    t.datetime :completed_at
+    t.timestamps null: false
+
+    t.index [:run_id, :status]
+  end
+
+  create_table :prospector_keywords, if_not_exists: true do |t|
+    t.string  :domain,   null: false
+    t.string  :category, null: false
+    t.string  :keyword,  null: false
+    t.string  :source,   null: false, default: "llm"
+    t.boolean :active,   null: false, default: true
+    t.jsonb   :metadata, default: {}
+    t.timestamps null: false
+
+    t.index [:domain, :category, :keyword], unique: true
+    t.index [:domain, :active]
+  end
+end

data/lib/generators/prospector/install/install_generator.rb

@@ -0,0 +1,31 @@
+module Prospector
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      def create_initializer
+        template "prospector.rb", "config/initializers/prospector.rb"
+      end
+
+      def create_migration
+        template "create_prospector_tables.rb",
+          "db/migrate/#{Time.now.strftime('%Y%m%d%H%M%S')}_create_prospector_tables.rb"
+      end
+
+      def mount_engine
+        route 'mount Prospector::Engine => "/prospector"'
+      end
+
+      def display_post_install
+        say ""
+        say "Prospector installed successfully!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Edit config/initializers/prospector.rb"
+        say "  2. Run: bin/rails db:migrate"
+        say "  3. Visit /prospector in your browser"
+        say ""
+      end
+    end
+  end
+end

data/lib/generators/prospector/install/templates/create_prospector_tables.rb

@@ -0,0 +1,83 @@
+class CreateProspectorTables < ActiveRecord::Migration[7.1]
+  def change
+    create_table :prospector_runs do |t|
+      t.string   :status,          null: false, default: "pending"
+      t.string   :source_adapter,  null: false
+      t.string   :geography_type,  null: false
+      t.jsonb    :geography_data,  null: false, default: {}
+      t.string   :categories,      array: true, default: []
+      t.string   :label
+      t.integer  :actor_id
+      t.integer  :total_found,     default: 0
+      t.integer  :fetched_count,   default: 0
+      t.integer  :skipped_count,   default: 0
+      t.integer  :error_count,     default: 0
+      t.text     :error_messages
+      t.jsonb    :metadata,        default: {}
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.timestamps null: false
+
+      t.index :status
+      t.index :source_adapter
+      t.index :created_at
+    end
+
+    create_table :prospector_candidates do |t|
+      t.references :run, null: false, foreign_key: { to_table: :prospector_runs }
+      t.string  :status,      null: false, default: "pending"
+      t.string  :name,        null: false
+      t.string  :address,     null: false
+      t.decimal :latitude,    precision: 10, scale: 6
+      t.decimal :longitude,   precision: 10, scale: 6
+      t.string  :phone_number
+      t.string  :website
+      t.string  :email
+      t.string  :facebook_url
+      t.string  :instagram_url
+      t.string  :linkedin_url
+      t.string  :tiktok_url
+      t.string  :youtube_url
+      t.text    :description
+      t.string  :category
+      t.jsonb   :hours_of_operation, default: {}
+      t.string  :source_uid,  null: false
+      t.jsonb   :source_data, default: {}
+      t.jsonb   :metadata,    default: {}
+      t.timestamps null: false
+
+      t.index [:run_id, :source_uid], unique: true
+      t.index [:run_id, :status]
+      t.index :status
+      t.index :source_uid
+    end
+
+    create_table :prospector_classification_runs do |t|
+      t.references :run, null: false, foreign_key: { to_table: :prospector_runs }
+      t.integer  :actor_id
+      t.string   :ai_model,         null: false
+      t.string   :status,           null: false, default: "pending"
+      t.boolean  :include_approved,  null: false, default: false
+      t.jsonb    :results
+      t.text     :error_message
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.timestamps null: false
+
+      t.index [:run_id, :status]
+    end
+
+    create_table :prospector_keywords do |t|
+      t.string  :domain,   null: false
+      t.string  :category, null: false
+      t.string  :keyword,  null: false
+      t.string  :source,   null: false, default: "llm"
+      t.boolean :active,   null: false, default: true
+      t.jsonb   :metadata, default: {}
+      t.timestamps null: false
+
+      t.index [:domain, :category, :keyword], unique: true
+      t.index [:domain, :active]
+    end
+  end
+end

data/lib/generators/prospector/install/templates/prospector.rb

@@ -0,0 +1,37 @@
+Prospector.configure do |config|
+  # Required: domain slug used for keyword generation and classification
+  config.domain = "your_domain"
+
+  # Required: block called when a candidate is approved
+  # The candidate is a Prospector::Candidate instance
+  config.on_approve do |candidate|
+    # Example:
+    # YourModel.create_from_prospector!(candidate)
+    raise "Configure Prospector.config.on_approve in config/initializers/prospector.rb"
+  end
+
+  # Required: admin authentication
+  config.authenticate_admin_with do |controller|
+    # Example:
+    # controller.current_user&.admin?
+    raise "Configure Prospector.config.authenticate_admin_with in config/initializers/prospector.rb"
+  end
+
+  # Required: classifier class (must inherit LlmClassifier::Classifier)
+  # Generate one with: rails g llm_classifier:classifier YourDomain category1 category2
+  # config.classifier = YourDomainClassifier
+
+  # Optional: default source adapter (default: :google_places)
+  # config.default_source = :google_places
+
+  # Optional: default AI model for classification (passed to classifier)
+  # config.default_classifier_model = "anthropic:claude-sonnet-4-20250514"
+
+  # Optional: duplicate check hook
+  # config.duplicate_check do |source_uid:, name:, **|
+  #   YourModel.exists?(["import_metadata->>'place_id' = ?", source_uid])
+  # end
+
+  # Optional: job queue name (default: :default)
+  # config.queue_name = :prospector
+end

data/lib/prospector/CLAUDE.md

@@ -0,0 +1,52 @@
+# Library Code
+
+## Sources (`sources/`)
+
+Pluggable adapter system for fetching business data from external APIs.
+
+### Interface
+
+All adapters extend `Sources::Base` and implement `fetch(geography:, keywords:)` returning `Array<Sources::Result>`.
+
+### Google Places Adapter (`sources/google_places/`)
+
+- `Adapter` - Main adapter class. Deduplicates by UID across keyword iterations. Routes to text search or nearby search based on geography type.
+- `Client` - HTTParty wrapper for Google Places API (New). Endpoints: searchText, searchNearby.
+- `UsAddressValidator` - Filters non-US addresses via state code regex.
+
+### Adding a New Source
+
+1. Create `sources/your_source/adapter.rb` extending `Sources::Base`
+2. Implement `fetch(geography:, keywords:)` returning `Array<Sources::Result>`
+3. Register in host app: `config.register_source(:your_source, YourSource::Adapter)`
+
+## Geography (`geography/`)
+
+Value objects representing search areas. All extend `Geography::Base`.
+
+- `MetroArea` - name + primary_state. Text search only.
+- `City` - city + state. Text search only.
+- `Coordinates` - lat/lng + radius_meters. Nearby search only.
+- `ZipCode` - zip code string. Text search only.
+- `BoundingBox` - NE/SW corners. Converted to center+radius for nearby search.
+
+Key methods: `to_query_string` (for text search), `to_coordinate_restriction` (for nearby search), `to_h`/`from_h` (serialization).
+
+## Pipeline (`pipeline/`)
+
+- `Orchestrator` - Coordinates the full fetch flow. Called by `FetchJob`. Collects keywords, calls adapter, normalizes results, deduplicates, creates candidates.
+- `Normalizer` - Parses formatted address strings into structured components (street, city, state, zip).
+
+## Classification (`classification/`)
+
+- `Runner` - Iterates unprocessed candidates, calls LLM via ruby_llm, stores results in candidate metadata. Auto-rejects candidates with empty categories.
+
+## Keywords (`keywords/`)
+
+- `Generator` - Checks DB for existing keywords first. On cache miss, calls LLM to generate keywords and persists them.
+
+## Other Files
+
+- `configuration.rb` - `Prospector.configure` DSL. Required keys: domain, on_approve, authenticate_admin_with.
+- `engine.rb` - Rails::Engine with isolate_namespace.
+- `error.rb` - Error hierarchy: Error, ConfigurationError, AdapterError, MissingApiKeyError, ClassificationError.

data/lib/prospector/classification/runner.rb

@@ -0,0 +1,105 @@
+module Prospector
+  module Classification
+    class Runner
+      def initialize(run, model: nil, include_approved: false, classification_run: nil)
+        @run = run
+        @model = model || Prospector.config.default_classifier_model
+        @include_approved = include_approved
+        @classification_run = classification_run
+        @processed = 0
+        @rejected = 0
+        @errors = 0
+      end
+
+      def perform
+        classify_candidates
+        finalize
+      end
+
+      private
+
+      def classify_candidates
+        candidates_to_classify.find_each do |candidate|
+          break if @run.reload.cancelled?
+
+          classify_candidate(candidate)
+        end
+      end
+
+      def candidates_to_classify
+        scope = @run.candidates
+        scope = @include_approved ? scope.where(status: %w[pending approved]) : scope.where(status: "pending")
+        scope.ai_unprocessed
+      end
+
+      def classify_candidate(candidate)
+        result = invoke_classifier(candidate)
+        store_result(candidate, result)
+        auto_reject_if_empty(candidate, result)
+        @processed += 1
+      rescue => e
+        candidate.update!(metadata: candidate.metadata.merge("llm_error" => e.message))
+        @errors += 1
+      end
+
+      def invoke_classifier(candidate)
+        classifier = resolve_classifier
+        input = build_input(candidate)
+        classifier.classify(input, model: @model)
+      end
+
+      def resolve_classifier
+        raw = Prospector.config.classifier
+        raise ConfigurationError, "Prospector.config.classifier must be set" unless raw
+
+        raw.is_a?(String) ? raw.constantize : raw
+      end
+
+      def build_input(candidate)
+        {
+          name: candidate.name,
+          address: candidate.address,
+          website: candidate.website,
+          description: candidate.description,
+          source_types: candidate.source_data.dig("types")&.join(", ")
+        }
+      end
+
+      def store_result(candidate, result)
+        candidate.update!(metadata: candidate.metadata.merge(
+          "llm_categories" => result.categories,
+          "llm_confidence" => result.confidence,
+          "llm_reasoning" => result.reasoning,
+          "llm_classified_at" => Time.current.iso8601,
+          "llm_model" => result.model,
+          "classification_run_id" => @classification_run&.id
+        ))
+      end
+
+      def auto_reject_if_empty(candidate, result)
+        return if result.categories.any?
+
+        candidate.reject!(reason: "no_relevant_categories")
+        @rejected += 1
+      end
+
+      def finalize
+        return finalize_classification_run("cancelled") if @run.reload.cancelled?
+
+        run_status = @processed.zero? && @errors.positive? ? "failed" : "completed"
+        @run.update!(status: run_status, completed_at: Time.current)
+        finalize_classification_run(run_status)
+      end
+
+      def finalize_classification_run(run_status)
+        return unless @classification_run
+
+        @classification_run.update!(
+          status: run_status,
+          completed_at: Time.current,
+          results: { "processed" => @processed, "rejected" => @rejected, "errors" => @errors }
+        )
+      end
+    end
+  end
+end

data/lib/prospector/configuration.rb

@@ -0,0 +1,56 @@
+module Prospector
+  class Configuration
+    attr_accessor :domain,
+                  :default_source,
+                  :default_classifier_model,
+                  :classifier,
+                  :turbo_enabled,
+                  :queue_name
+
+    attr_reader :sources
+
+    def initialize
+      @domain = nil
+      @default_source = :google_places
+      @default_classifier_model = "anthropic:claude-sonnet-4-20250514"
+      @classifier = nil
+      @turbo_enabled = defined?(Turbo::Broadcastable)
+      @queue_name = :default
+      @sources = { google_places: "Prospector::Sources::GooglePlaces::Adapter" }
+      @on_approve_block = nil
+      @authenticate_admin_block = nil
+      @duplicate_check_block = nil
+    end
+
+    def on_approve(&block)
+      @on_approve_block = block if block
+      @on_approve_block
+    end
+
+    def authenticate_admin_with(&block)
+      @authenticate_admin_block = block if block
+      @authenticate_admin_block
+    end
+
+    def duplicate_check(&block)
+      @duplicate_check_block = block if block
+      @duplicate_check_block
+    end
+
+    def register_source(key, adapter_class)
+      @sources[key.to_sym] = adapter_class.to_s
+    end
+
+    def resolve_source(key)
+      adapter = @sources[key.to_sym]
+      raise AdapterNotFoundError, "No source adapter registered for :#{key}" unless adapter
+
+      adapter.is_a?(String) ? adapter.constantize : adapter
+    end
+
+    def validate!
+      raise ConfigurationError, "Prospector.config.domain must be set" if domain.blank?
+      raise ConfigurationError, "Prospector.config.authenticate_admin_with must be set" unless @authenticate_admin_block
+    end
+  end
+end

data/lib/prospector/engine.rb

@@ -0,0 +1,18 @@
+module Prospector
+  class Engine < ::Rails::Engine
+    isolate_namespace Prospector
+
+    config.autoload_paths << File.expand_path("..", __dir__)
+    config.eager_load_paths << File.expand_path("..", __dir__)
+
+    initializer "prospector.configuration" do
+      Prospector.config.validate! if Rails.env.production?
+    end
+
+    initializer "prospector.assets" do |app|
+      if app.config.respond_to?(:assets)
+        app.config.assets.paths << root.join("app/assets/stylesheets")
+      end
+    end
+  end
+end