prospector_engine 0.1.1

data/lib/prospector/pipeline/orchestrator.rb

@@ -0,0 +1,151 @@
+module Prospector
+  module Pipeline
+    class Orchestrator
+      def initialize(run)
+        @run = run
+        @adapter = Prospector.config.resolve_source(run.source_adapter.to_sym).new
+        @geography = run.geography
+        @skipped_count = 0
+        @error_count = 0
+        @errors = []
+      end
+
+      def perform
+        @run.update!(status: "running", started_at: Time.current)
+        fetch_and_process
+        complete_fetch!
+      rescue => e
+        fail_run!(e)
+        raise
+      end
+
+      private
+
+      def fetch_and_process
+        keywords = collect_keywords
+        results = @adapter.fetch(geography: @geography, keywords: keywords)
+        @run.update!(total_found: results.size)
+
+        results.each_with_index do |result, index|
+          break if cancelled?(index)
+
+          process_result(result)
+        end
+      end
+
+      def complete_fetch!
+        @run.update!(
+          status: "classifying",
+          skipped_count: @skipped_count,
+          error_count: @error_count,
+          error_messages: @errors.join("\n").presence
+        )
+        ClassifyJob.perform_later(@run.id)
+      end
+
+      def fail_run!(error)
+        @run.update!(
+          status: "failed",
+          completed_at: Time.current,
+          error_messages: "Import failed: #{error.message}"
+        )
+      end
+
+      def collect_keywords
+        categories = @run.categories.presence || [Prospector.config.domain]
+        Keywords::Generator.for(domain: Prospector.config.domain, categories: categories)
+      end
+
+      def process_result(result)
+        normalized = Normalizer.normalize(result)
+
+        if duplicate?(result.uid, normalized)
+          @skipped_count += 1
+          return
+        end
+
+        create_candidate(result, normalized)
+      rescue => e
+        @error_count += 1
+        @errors << "Error processing #{result.name}: #{e.message}"
+      end
+
+      def duplicate?(source_uid, normalized)
+        @run.candidates.exists?(source_uid: source_uid) ||
+          check_host_duplicate(source_uid, normalized)
+      end
+
+      def check_host_duplicate(source_uid, normalized)
+        checker = Prospector.config.duplicate_check
+        return false unless checker
+
+        checker.call(
+          source_uid: source_uid,
+          name: normalized["business_name"],
+          address: normalized["street_address"],
+          city: normalized["city"],
+          state: normalized["state"]
+        )
+      end
+
+      def create_candidate(result, normalized)
+        full_address = build_address(normalized)
+
+        candidate = @run.candidates.create!(
+          name: result.name,
+          address: full_address,
+          latitude: result.latitude,
+          longitude: result.longitude,
+          phone_number: result.phone_number,
+          website: result.website,
+          category: result.category,
+          source_uid: result.uid,
+          hours_of_operation: format_hours(result.hours),
+          source_data: result.raw || {},
+          metadata: build_metadata(result, normalized),
+          status: "pending"
+        )
+        @run.increment!(:fetched_count)
+        enrich_contacts(candidate)
+      end
+
+      def enrich_contacts(candidate)
+        return if candidate.website.blank?
+
+        scraper = Enrichment::ContactScraper.scrape(candidate.website)
+        apply_contacts(candidate, scraper) if scraper.success?
+      end
+
+      def apply_contacts(candidate, scraper)
+        attrs = scraper.result.slice(:email, :facebook_url, :instagram_url, :linkedin_url, :tiktok_url, :youtube_url)
+        candidate.update!(attrs)
+      rescue ActiveRecord::RecordInvalid
+        nil
+      end
+
+      def build_address(normalized)
+        [normalized["street_address"], normalized["city"], normalized["state"], normalized["zip_code"]].compact.join(", ")
+      end
+
+      def build_metadata(result, normalized)
+        { "rating" => result.rating, "rating_count" => result.rating_count, "types" => result.types, "normalized_data" => normalized }
+      end
+
+      def format_hours(hours_array)
+        return {} if hours_array.blank?
+
+        hours = {}
+        hours_array.each_with_index do |day_hours, i|
+          hours[Date::DAYNAMES[i % 7].downcase] = day_hours
+        end
+        hours
+      end
+
+      def cancelled?(index)
+        return false unless (index + 1) % 10 == 0
+
+        @run.reload.cancelled?
+      end
+    end
+  end
+end

data/lib/prospector/sources/base.rb

@@ -0,0 +1,13 @@
+module Prospector
+  module Sources
+    class Base
+      def fetch(geography:, keywords:)
+        raise NotImplementedError, "#{self.class}#fetch must return Array<Sources::Result>"
+      end
+
+      def self.adapter_key
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/prospector/sources/google_places/adapter.rb

@@ -0,0 +1,92 @@
+require "set"
+
+module Prospector
+  module Sources
+    module GooglePlaces
+      class Adapter < Sources::Base
+        def self.adapter_key = "google_places"
+
+        def initialize(api_key: nil)
+          @client = Client.new(api_key: api_key)
+          @seen_uids = Set.new
+        end
+
+        def fetch(geography:, keywords:)
+          results = []
+          keywords.each do |keyword|
+            results.concat(fetch_for_keyword(geography, keyword))
+          end
+          results
+        end
+
+        def extract_result(place)
+          uid = place["id"]&.delete_prefix("places/")
+          return if uid.blank?
+          return if @seen_uids.include?(uid)
+          return unless valid_place?(place)
+
+          @seen_uids.add(uid)
+          build_result(place, uid)
+        end
+
+        def build_result(place, uid)
+          Sources::Result.new(
+            uid: uid,
+            name: place.dig("displayName", "text"),
+            formatted_address: place["formattedAddress"],
+            latitude: place.dig("location", "latitude"),
+            longitude: place.dig("location", "longitude"),
+            phone_number: place["nationalPhoneNumber"],
+            website: place["websiteUri"],
+            description: nil,
+            category: nil,
+            hours: place.dig("regularOpeningHours", "weekdayDescriptions"),
+            rating: place["rating"],
+            rating_count: place["userRatingCount"],
+            types: place["types"] || [],
+            raw: place
+          )
+        end
+
+        private
+
+        def fetch_for_keyword(geography, keyword)
+          query_string = geography.to_query_string
+          if query_string
+            fetch_text("#{keyword} in #{query_string}")
+          else
+            coord = geography.to_coordinate_restriction
+            fetch_nearby(coord) if coord
+          end || []
+        end
+
+        def fetch_text(query)
+          response = @client.search_text(query)
+          parse_response(response)
+        end
+
+        def fetch_nearby(location_restriction)
+          response = @client.search_nearby(location_restriction)
+          parse_response(response)
+        end
+
+        def parse_response(response)
+          return [] unless response.code == 200
+
+          places = response.parsed_response["places"] || []
+          places.filter_map { |place| extract_result(place) }
+        end
+
+        def valid_place?(place)
+          place.dig("displayName", "text").present? &&
+            place["formattedAddress"].present? &&
+            place.dig("location", "latitude").present? &&
+            place.dig("location", "longitude").present? &&
+            place["id"].present? &&
+            place["businessStatus"] == "OPERATIONAL" &&
+            UsAddressValidator.us_address?(place["formattedAddress"])
+        end
+      end
+    end
+  end
+end

data/lib/prospector/sources/google_places/client.rb

@@ -0,0 +1,58 @@
+require "httparty"
+
+module Prospector
+  module Sources
+    module GooglePlaces
+      class Client
+        include HTTParty
+        base_uri "https://places.googleapis.com/v1"
+
+        FIELD_MASK = [
+          "places.id",
+          "places.displayName",
+          "places.formattedAddress",
+          "places.location",
+          "places.rating",
+          "places.userRatingCount",
+          "places.types",
+          "places.businessStatus",
+          "places.nationalPhoneNumber",
+          "places.websiteUri",
+          "places.regularOpeningHours"
+        ].join(",").freeze
+
+        def initialize(api_key: nil)
+          @api_key = api_key || ENV["GOOGLE_MAPS_API_KEY"]
+          raise MissingApiKeyError, "Google Maps API key not configured" if @api_key.blank?
+        end
+
+        def search_text(query, region_code: "US")
+          body = { textQuery: query, regionCode: region_code }
+          post("/places:searchText", body)
+        end
+
+        def search_nearby(location_restriction, region_code: "US")
+          body = { locationRestriction: location_restriction, regionCode: region_code }
+          post("/places:searchNearby", body)
+        end
+
+        private
+
+        def post(path, body)
+          self.class.post(path, build_options(body))
+        end
+
+        def build_options(body)
+          {
+            body: body.to_json,
+            headers: {
+              "Content-Type" => "application/json",
+              "X-Goog-Api-Key" => @api_key,
+              "X-Goog-FieldMask" => FIELD_MASK
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/prospector/sources/google_places/us_address_validator.rb

@@ -0,0 +1,24 @@
+module Prospector
+  module Sources
+    module GooglePlaces
+      module UsAddressValidator
+        US_STATE_CODES = %w[
+          AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO
+          MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY DC
+          PR VI GU AS MP
+        ].freeze
+
+        US_STATE_PATTERN = /,\s*[^,]+,\s*(#{US_STATE_CODES.join('|')})\s*(\d{5}(-\d{4})?)?$/i
+
+        def self.us_address?(address)
+          return false if address.blank?
+
+          normalized = address.strip.downcase
+          return true if normalized.end_with?("usa") || normalized.end_with?("united states")
+
+          address.strip.match?(US_STATE_PATTERN)
+        end
+      end
+    end
+  end
+end

data/lib/prospector/sources/result.rb

@@ -0,0 +1,21 @@
+module Prospector
+  module Sources
+    Result = Struct.new(
+      :uid,
+      :name,
+      :formatted_address,
+      :latitude,
+      :longitude,
+      :phone_number,
+      :website,
+      :description,
+      :category,
+      :hours,
+      :rating,
+      :rating_count,
+      :types,
+      :raw,
+      keyword_init: true
+    )
+  end
+end

data/lib/prospector/version.rb

@@ -0,0 +1,3 @@
+module Prospector
+  VERSION = "0.1.1"
+end

data/lib/prospector.rb

@@ -0,0 +1,20 @@
+require "prospector/version"
+require "prospector/error"
+require "prospector/configuration"
+require "prospector/engine"
+
+module Prospector
+  class << self
+    def configure
+      yield config
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def reset_config!
+      @config = Configuration.new
+    end
+  end
+end

metadata

@@ -0,0 +1,185 @@
+--- !ruby/object:Gem::Specification
+name: prospector_engine
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- AxiumFoundry
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-04-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: llm_classifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: A Rails engine for discovering businesses from Google Places and other
+  sources, with AI-powered keyword generation and classification.
+email:
+- dev@axiumfoundry.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/CLAUDE.md
+- app/assets/stylesheets/prospector/application.css
+- app/controllers/prospector/application_controller.rb
+- app/controllers/prospector/candidates_controller.rb
+- app/controllers/prospector/keyword_generations_controller.rb
+- app/controllers/prospector/keywords_controller.rb
+- app/controllers/prospector/run_bulk_approvals_controller.rb
+- app/controllers/prospector/run_cancellations_controller.rb
+- app/controllers/prospector/run_reclassifications_controller.rb
+- app/controllers/prospector/run_restarts_controller.rb
+- app/controllers/prospector/run_retries_controller.rb
+- app/controllers/prospector/runs_controller.rb
+- app/jobs/prospector/application_job.rb
+- app/jobs/prospector/bulk_approve_job.rb
+- app/jobs/prospector/classify_job.rb
+- app/jobs/prospector/fetch_job.rb
+- app/models/prospector/application_record.rb
+- app/models/prospector/candidate.rb
+- app/models/prospector/classification_run.rb
+- app/models/prospector/keyword.rb
+- app/models/prospector/run.rb
+- app/views/prospector/candidates/show.html.erb
+- app/views/prospector/keywords/index.html.erb
+- app/views/prospector/layouts/prospector.html.erb
+- app/views/prospector/runs/index.html.erb
+- app/views/prospector/runs/new.html.erb
+- app/views/prospector/runs/show.html.erb
+- config/routes.rb
+- db/prospector_schema.rb
+- lib/generators/prospector/install/install_generator.rb
+- lib/generators/prospector/install/templates/create_prospector_tables.rb
+- lib/generators/prospector/install/templates/prospector.rb
+- lib/prospector.rb
+- lib/prospector/CLAUDE.md
+- lib/prospector/classification/runner.rb
+- lib/prospector/configuration.rb
+- lib/prospector/engine.rb
+- lib/prospector/enrichment/contact_scraper.rb
+- lib/prospector/error.rb
+- lib/prospector/geography/base.rb
+- lib/prospector/geography/bounding_box.rb
+- lib/prospector/geography/city.rb
+- lib/prospector/geography/coordinates.rb
+- lib/prospector/geography/metro_area.rb
+- lib/prospector/geography/zip_code.rb
+- lib/prospector/keywords/generator.rb
+- lib/prospector/pipeline/normalizer.rb
+- lib/prospector/pipeline/orchestrator.rb
+- lib/prospector/sources/base.rb
+- lib/prospector/sources/google_places/adapter.rb
+- lib/prospector/sources/google_places/client.rb
+- lib/prospector/sources/google_places/us_address_validator.rb
+- lib/prospector/sources/result.rb
+- lib/prospector/version.rb
+homepage: https://github.com/AxiumFoundry/prospector_engine
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.20
+signing_key:
+specification_version: 4
+summary: Business discovery from multiple sources with AI classification
+test_files: []