prospector_engine 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46798902589ee83ac3a63b7cf36fcb67b61d4a84fbf2ac07819b04a778439099
-  data.tar.gz: 87cb71c8d3c832156306184d31f9a7d3e8f45343e33bceb266ae3cc095a61130
+  metadata.gz: f6ff4fed78e9e8c81ec50623a7e2cbe0076b0f890f3c3fec3561551585de0513
+  data.tar.gz: 584620ecbee56ffdf94520638a5b7ab409e13c9fc264dd934a44d8e798bf4275
 SHA512:
-  metadata.gz: 68982b0d84db8931d2e691f365d0032a5bfbff4f0f9ea40fcafcabeab8e0783699bb8c6e202d2529c083fa639e22ac77aae961978ddb7b0663d4414f9f255868
-  data.tar.gz: 1876729cef3a17d3a09a717de8ef392c4e67a611b6d164d36b0b985e37dda846f16d3814273dabbfe551a01adae5d9b6999f577b88ce82a389bc102ba57d46de
+  metadata.gz: 7bbc93fdaca17a89d0ba229fbd23bba95eef79cd723b288ec872162eab5801bc9bc4fa0f93506245b1e7a796ded93523c0d237d61d5463c60c603644cb7450e6
+  data.tar.gz: ac7828b1928c11db81a56079fa6b65442a8b85404a214d147283e02a69c4f9109b3fed4568da47548e1e9b758dc415206339ef3e19abeecde10d4e9588d7ec09

data/README.md

@@ -1,20 +1,21 @@
 # Prospector
 
-A Rails engine for discovering businesses from multiple sources with AI-powered keyword generation and classification.
+A Rails engine for discovering businesses from multiple sources with AI-powered keyword generation, contact enrichment, and classification.
 
-Prospector handles the full discovery pipeline: generate search keywords with AI, fetch business listings from external APIs, classify results for domain relevance, and present an admin review interface -- all inside your existing Rails app.
+Prospector handles the full discovery pipeline: generate search keywords with AI, fetch business listings from external APIs, scrape websites for contact info, classify results for domain relevance, and present an admin review interface -- all inside your existing Rails app.
 
 ## Features
 
-- **Multi-source adapters** -- Ships with Google Places. Pluggable interface for adding Yelp, Bing, OpenStreetMap, or custom sources.
+- **Multi-source adapters** -- Ships with Google Places. Pluggable interface for adding Yelp, Bing, or custom sources.
+- **Contact enrichment** -- Automatically scrapes candidate websites for email and social links (Facebook, Instagram, LinkedIn, TikTok, YouTube). SSRF-protected with IP validation on every redirect hop.
 - **AI keyword generation** -- On-demand LLM-powered keyword generation for any business domain. Keywords are stored and reused across runs.
 - **AI classification** -- Automatically classifies discovered businesses for domain relevance. Non-relevant results are auto-rejected.
-- **Flexible geography** -- Search by metro area, city, coordinates + radius, ZIP code, or bounding box.
-- **Admin UI** -- Mountable admin interface with self-contained CSS. Review candidates, approve/reject, bulk approve, trigger reclassification.
+- **Flexible geography** -- Search by metro area, city, coordinates + radius, ZIP code, or bounding box. 42 preloaded US metro areas available as presets.
+- **Admin UI** -- Mountable admin interface with self-contained dark theme. Review candidates, approve/reject, bulk approve, trigger reclassification.
 - **Keyword management** -- Admin UI for viewing, adding, toggling, and AI-generating search keywords per category.
 - **Background jobs** -- Fetch, classify, and bulk approve jobs integrate with your existing queue (Solid Queue, Sidekiq, etc.).
-- **Turbo Streams** -- Real-time updates when Turbo is present. Gracefully degrades without it.
-- **Domain-agnostic** -- Works for motorcycle services, aviation operators, restaurants, or any business vertical.
+- **Turbo Streams** -- Real-time progress updates when Turbo is present. Gracefully degrades without it.
+- **Instrumentation** -- ActiveSupport::Notifications events for run status changes and candidate approvals.
 
 ## Requirements
 
@@ -51,11 +52,20 @@ Edit `config/initializers/prospector.rb`:
 
 ```ruby
 Prospector.configure do |config|
-  # Required: domain slug for keyword generation and classification context
+  # Required: domain slug for keyword generation and classification context.
   config.domain = "motorcycle_services"
 
-  # Required: called when an admin approves a candidate.
-  # Receives a Prospector::Candidate instance.
+  # Required: admin authentication.
+  config.authenticate_admin_with do |controller|
+    controller.current_user&.admin?
+  end
+
+  # Classifier class (must be set before classification runs).
+  # Can be a class constant or a string for lazy resolution.
+  config.classifier = MotorcycleClassifier
+  # config.classifier = "MotorcycleClassifier"
+
+  # Called when an admin approves a candidate (optional).
   # Runs AFTER the candidate status is committed (safe to enqueue jobs).
   config.on_approve do |candidate|
     data = candidate.normalized_data
@@ -67,26 +77,16 @@ Prospector.configure do |config|
       zip_code: data["zip_code"],
       phone_number: data["phone_number"],
       website: data["website"],
+      email: candidate.email,
       latitude: data["latitude"],
       longitude: data["longitude"],
       service_types: candidate.llm_categories,
-      import_source: "prospector"
+      facebook_url: candidate.facebook_url,
+      instagram_url: candidate.instagram_url
     )
   end
 
-  # Required: admin authentication.
-  # Receives the controller instance.
-  config.authenticate_admin_with do |controller|
-    controller.current_user&.admin?
-  end
-
-  # Required: classifier class (inherits LlmClassifier::Classifier).
-  # Defines categories, model, and classification rules for your domain.
-  # See "Defining a Classifier" below.
-  config.classifier = MotorcycleClassifier
-
   # Optional: check for duplicates before creating candidates.
-  # Return true to skip the candidate.
   config.duplicate_check do |source_uid:, name:, **|
     Business.exists?(["import_metadata->>'place_id' = ?", source_uid])
   end
@@ -94,14 +94,17 @@ Prospector.configure do |config|
   # Optional: default source adapter (default: :google_places)
   config.default_source = :google_places
 
-  # Optional: default AI model (passed to classifier when not overridden)
-  config.default_classifier_model = "anthropic:claude-sonnet-4-20250514"
+  # Optional: model override for classification and keyword generation (a bare ruby_llm model id).
+  # Defaults to nil: classification uses the classifier's `model`, keyword generation uses RubyLLM's default.
+  # config.default_classifier_model = "claude-opus-4-6"
 
   # Optional: job queue name (default: :default)
   config.queue_name = :prospector
 end
 ```
 
+**Note:** `validate!` only runs in production. In development, missing `domain` or `authenticate_admin_with` will cause runtime errors rather than startup failures.
+
 Set the required environment variables:
 
 ```bash
@@ -116,20 +119,21 @@ ANTHROPIC_API_KEY=your_anthropic_api_key  # or whatever ruby_llm needs
 Visit `/prospector` in your browser. The admin interface provides:
 
 - **Runs** -- Create, monitor, retry, restart, and cancel discovery runs
-- **Candidates** -- Review discovered businesses, approve or reject individually or in bulk
+- **Candidates** -- Review discovered businesses with contact info, approve or reject individually or in bulk
 - **Keywords** -- View, add, toggle, and AI-generate search keywords per category
 
 ### Creating a Run
 
 1. Click "New Run" in the admin UI
 2. Select a geography type (metro area, city, coordinates, ZIP code, or bounding box)
-3. Fill in the geography details
-4. Click "Start Run"
+3. For metro areas, optionally select from 42 preloaded US metros or enter a custom one
+4. Fill in the geography details
+5. Click "Start Run"
 
 The run progresses through these stages automatically:
 
 ```
-pending -> running (fetching from source) -> classifying (AI classification) -> completed
+pending -> running (fetch + enrich) -> classifying (AI) -> completed
 ```
 
 ### Defining a Classifier
@@ -143,9 +147,8 @@ class MotorcycleClassifier < LlmClassifier::Classifier
              :detailing, :training, :towing, :insurance, :touring,
              :clubs, :parts
 
-  model "anthropic:claude-sonnet-4-20250514"
+  model "claude-opus-4-6"
   multi_label true
-  require_categories true  # fail when no categories match (auto-reject)
 
   system_prompt <<~PROMPT
     You are classifying businesses for the motorcycle services domain.
@@ -164,13 +167,11 @@ class MotorcycleClassifier < LlmClassifier::Classifier
 end
 ```
 
-Then set it in your Prospector config:
+The classifier receives a hash with `name`, `address`, `website`, `description`, and `source_types` (comma-joined string) keys.
 
-```ruby
-config.classifier = MotorcycleClassifier
-```
+Use a bare model id that ruby_llm recognizes (`claude-opus-4-6`, not `anthropic:claude-opus-4-6`) and that supports structured outputs, since llm_classifier 0.3 sends every request with a JSON schema.
 
-The classifier receives a hash with `name`, `address`, `website`, `description`, and `source_types` keys. It returns an `LlmClassifier::Result` with `.categories`, `.confidence`, and `.reasoning`.
+Candidates the classifier assigns no categories are auto-rejected with reason `no_relevant_categories`. A failed classification (API error, unknown model, unparseable response) is recorded in `metadata["llm_error"]`, and the candidate stays pending so the next reclassification retries it. Leave `require_categories` off: with it on, llm_classifier reports "no categories" as a failure, so irrelevant businesses are counted as errors instead of being rejected.
 
 ### Programmatic Usage
 
@@ -195,13 +196,19 @@ Prospector::Keyword.create!(
 # Approve a candidate programmatically
 candidate = Prospector::Candidate.find(id)
 candidate.approve!  # fires the on_approve callback
+
+# Access enriched contact info
+candidate.email          # => "info@example.com"
+candidate.facebook_url   # => "https://facebook.com/example"
+candidate.instagram_url  # => "https://instagram.com/example"
 ```
 
 ### Geography Types
 
 ```ruby
-# Metro area (text search)
-Prospector::Geography::MetroArea.new(name: "San Francisco", primary_state: "CA")
+# Metro area (text search) -- 42 US metros available as presets
+Prospector::Geography::MetroArea.new(name: "San Francisco Bay Area", primary_state: "CA")
+Prospector::Geography::MetroArea::PRELOADED  # => [{name: "Atlanta", primary_state: "GA"}, ...]
 
 # City (text search)
 Prospector::Geography::City.new(city: "Austin", state: "TX")
@@ -226,8 +233,6 @@ class MyApp::YelpAdapter < Prospector::Sources::Base
   def self.adapter_key = "yelp"
 
   def fetch(geography:, keywords:)
-    # Call the Yelp API for each keyword + geography combination.
-    # Return an Array of Prospector::Sources::Result.
     keywords.flat_map do |keyword|
       search_yelp(keyword, geography).map do |biz|
         Prospector::Sources::Result.new(
@@ -260,8 +265,6 @@ Prospector.configure do |config|
 end
 ```
 
-Then select "Yelp" as the source when creating a run.
-
 ## Database Tables
 
 Prospector creates four tables, all prefixed with `prospector_`:
@@ -269,7 +272,7 @@ Prospector creates four tables, all prefixed with `prospector_`:
 | Table | Purpose |
 |-------|---------|
 | `prospector_runs` | Discovery runs with status, geography, and progress counters |
-| `prospector_candidates` | Discovered businesses pending review |
+| `prospector_candidates` | Discovered businesses with contact info (email, social links) pending review |
 | `prospector_classification_runs` | Tracks AI reclassification operations |
 | `prospector_keywords` | Stored search keywords per domain and category |
 
@@ -284,33 +287,52 @@ Prospector::Run (state machine)
   |     |-- Pipeline::Orchestrator
   |     |     |-- Pipeline::Normalizer (address parsing)
   |     |     |-- Duplicate checking (via config.duplicate_check)
-  |     |     '-- Creates Prospector::Candidate records
+  |     |     |-- Creates Prospector::Candidate records
+  |     |     '-- Enrichment::ContactScraper (email + social links)
   |     '-- Enqueues ClassifyJob
   |
   |-- ClassifyJob
   |     '-- Classification::Runner
-  |           |-- Calls LLM via ruby_llm
+  |           |-- Calls LLM via llm_classifier
   |           |-- Stores categories, confidence, reasoning in metadata
-  |           '-- Auto-rejects candidates with no relevant categories
+  |           |-- Auto-rejects candidates with no relevant categories
+  |           '-- Records failed classifications as llm_error, leaves them pending
   |
   '-- Admin UI
-        |-- Review candidates (approve / reject / restore)
-        |-- Bulk approve
-        '-- Trigger reclassification with different models
+        |-- Runs: create, monitor, retry, restart, cancel
+        |-- Candidates: review, approve, reject, restore, bulk approve
+        |-- Keywords: view, add, toggle active, AI-generate
+        '-- Reclassification with model override
 ```
 
 ## Run States
 
 ```
 pending --> running --> classifying --> completed
-               |                          |
-               v                          v
-            failed                    cancelled
-               ^                          ^
-               |__________________________|
-                    (retry / restart)
+    |          |            |
+    v          v            v
+             failed      cancelled
+               ^            ^
+               |____________|
+              (retry/restart)
 ```
 
+- `cancellable?` -- pending, running, or classifying
+- `retryable?` -- failed only
+- `restartable?` -- completed, failed, or cancelled
+- `restart!` destroys pending/rejected candidates and resets all counters
+- `reset_for_retry!` resets status and error fields only
+
+## Instrumentation
+
+Prospector emits ActiveSupport::Notifications events:
+
+| Event | Payload |
+|-------|---------|
+| `prospector.run.status_changed` | `run`, `status`, `previous_status` |
+| `prospector.candidate.approved` | `candidate`, `record` |
+| `prospector.candidate.rejected` | `candidate`, `reason` |
+
 ## Development
 
 The gem ships with a devcontainer for containerized development:
@@ -320,10 +342,10 @@ The gem ships with a devcontainer for containerized development:
 docker compose -f .devcontainer/compose.yaml up -d --build
 
 # Run tests (inside the container)
-docker exec -w /workspaces/prospector prospector-app-1 bundle exec rake test
+docker exec -w /workspaces/prospector_engine prospector_engine-app-1 bundle exec rake test
 
 # Run a single test file
-docker exec -w /workspaces/prospector prospector-app-1 bundle exec ruby -Itest test/models/run_test.rb
+docker exec -w /workspaces/prospector_engine prospector_engine-app-1 bundle exec ruby -Itest test/models/run_test.rb
 ```
 
 The test suite uses a dummy Rails app (`test/dummy/`) with PostgreSQL.

data/app/controllers/prospector/run_reclassifications_controller.rb

@@ -12,7 +12,7 @@ module Prospector
 
     def build_classification_run(run)
       run.classification_runs.new(
-        ai_model: params[:ai_model] || Prospector.config.default_classifier_model,
+        ai_model: params[:ai_model].presence || Prospector.config.resolved_classifier_model,
         include_approved: params[:include_approved] == "1",
         actor_id: respond_to?(:current_user) ? current_user&.id : nil
       )

data/lib/generators/prospector/install/templates/prospector.rb

@@ -24,8 +24,9 @@ Prospector.configure do |config|
   # Optional: default source adapter (default: :google_places)
   # config.default_source = :google_places
 
-  # Optional: default AI model for classification (passed to classifier)
-  # config.default_classifier_model = "anthropic:claude-sonnet-4-20250514"
+  # Optional: model override for classification and keyword generation (a bare ruby_llm model id)
+  # Defaults to nil: classification uses the classifier's `model`, keyword generation uses RubyLLM's default
+  # config.default_classifier_model = "claude-opus-4-6"
 
   # Optional: duplicate check hook
   # config.duplicate_check do |source_uid:, name:, **|

data/lib/prospector/CLAUDE.md

@@ -39,7 +39,7 @@ Key methods: `to_query_string` (for text search), `to_coordinate_restriction` (f
 
 ## Classification (`classification/`)
 
-- `Runner` - Iterates unprocessed candidates, calls LLM via ruby_llm, stores results in candidate metadata. Auto-rejects candidates with empty categories.
+- `Runner` - Iterates unprocessed candidates, calls the configured `LlmClassifier::Classifier`, stores results in candidate metadata. Auto-rejects candidates with empty categories. A failed `Result` (or raised error) is stored as `llm_error` and counted as an error, never a rejection; the candidate stays `ai_unprocessed` so it is retried.
 
 ## Keywords (`keywords/`)
 
@@ -47,6 +47,6 @@ Key methods: `to_query_string` (for text search), `to_coordinate_restriction` (f
 
 ## Other Files
 
-- `configuration.rb` - `Prospector.configure` DSL. Required keys: domain, on_approve, authenticate_admin_with.
+- `configuration.rb` - `Prospector.configure` DSL. Required keys: domain, on_approve, authenticate_admin_with. `default_classifier_model` defaults to nil so the classifier's own `model` applies; `resolved_classifier_model` reports the model that will actually run.
 - `engine.rb` - Rails::Engine with isolate_namespace.
 - `error.rb` - Error hierarchy: Error, ConfigurationError, AdapterError, MissingApiKeyError, ClassificationError.

data/lib/prospector/classification/runner.rb

@@ -34,25 +34,24 @@ module Prospector
 
       def classify_candidate(candidate)
         result = invoke_classifier(candidate)
+        return record_error(candidate, result.error) if result.failure?
+
         store_result(candidate, result)
         auto_reject_if_empty(candidate, result)
         @processed += 1
       rescue => e
-        candidate.update!(metadata: candidate.metadata.merge("llm_error" => e.message))
-        @errors += 1
+        record_error(candidate, e.message)
       end
 
       def invoke_classifier(candidate)
-        classifier = resolve_classifier
+        classifier = Prospector.config.classifier_class
         input = build_input(candidate)
         classifier.classify(input, model: @model)
       end
 
-      def resolve_classifier
-        raw = Prospector.config.classifier
-        raise ConfigurationError, "Prospector.config.classifier must be set" unless raw
-
-        raw.is_a?(String) ? raw.constantize : raw
+      def record_error(candidate, message)
+        candidate.update!(metadata: candidate.metadata.merge("llm_error" => message))
+        @errors += 1
       end
 
       def build_input(candidate)
@@ -66,7 +65,7 @@ module Prospector
       end
 
       def store_result(candidate, result)
-        candidate.update!(metadata: candidate.metadata.merge(
+        candidate.update!(metadata: candidate.metadata.except("llm_error").merge(
           "llm_categories" => result.categories,
           "llm_confidence" => result.confidence,
           "llm_reasoning" => result.reasoning,

data/lib/prospector/configuration.rb

@@ -12,7 +12,7 @@ module Prospector
     def initialize
       @domain = nil
       @default_source = :google_places
-      @default_classifier_model = "anthropic:claude-sonnet-4-20250514"
+      @default_classifier_model = nil
       @classifier = nil
       @turbo_enabled = defined?(Turbo::Broadcastable)
       @queue_name = :default
@@ -48,6 +48,18 @@ module Prospector
       adapter.is_a?(String) ? adapter.constantize : adapter
     end
 
+    def classifier_class
+      raise ConfigurationError, "Prospector.config.classifier must be set" unless classifier
+
+      classifier.is_a?(String) ? classifier.constantize : classifier
+    end
+
+    def resolved_classifier_model
+      require "ruby_llm"
+
+      default_classifier_model || classifier_class.model || RubyLLM.config.default_model
+    end
+
     def validate!
       raise ConfigurationError, "Prospector.config.domain must be set" if domain.blank?
       raise ConfigurationError, "Prospector.config.authenticate_admin_with must be set" unless @authenticate_admin_block

data/lib/prospector/engine.rb

@@ -2,8 +2,16 @@ module Prospector
   class Engine < ::Rails::Engine
     isolate_namespace Prospector
 
-    config.autoload_paths << File.expand_path("..", __dir__)
-    config.eager_load_paths << File.expand_path("..", __dir__)
+    initializer "prospector.autoload", before: :set_autoload_paths do |app|
+      lib_path = root.join("lib")
+      app.config.autoload_paths << lib_path
+      app.config.eager_load_paths << lib_path
+
+      Rails.autoloaders.main.ignore(
+        root.join("lib/prospector/version.rb"),
+        root.join("lib/generators")
+      )
+    end
 
     initializer "prospector.configuration" do
       Prospector.config.validate! if Rails.env.production?

data/lib/prospector/keywords/generator.rb

@@ -30,6 +30,7 @@ module Prospector
 
         chat = RubyLLM.chat(model: Prospector.config.default_classifier_model)
         response = chat.ask(prompt_for(category))
+        @generated_by = chat.model.id
         parse_keywords(response.content)
       end
 
@@ -63,7 +64,7 @@ module Prospector
         ) do |record|
           record.source = "llm"
           record.metadata = {
-            "generated_by" => Prospector.config.default_classifier_model,
+            "generated_by" => @generated_by,
             "generated_at" => Time.current.iso8601
           }
         end

data/lib/prospector/version.rb

@@ -1,3 +1,3 @@
 module Prospector
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prospector_engine
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - AxiumFoundry
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-06 00:00:00.000000000 Z
+date: 2026-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -42,16 +42,16 @@ dependencies:
   name: llm_classifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -178,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.20
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Business discovery from multiple sources with AI classification