prospector_engine 0.1.1

data/lib/prospector/enrichment/contact_scraper.rb

@@ -0,0 +1,188 @@
+require "httparty"
+require "ipaddr"
+
+module Prospector
+  module Enrichment
+    class ContactScraper
+      include HTTParty
+
+      SOCIAL_PATTERNS = {
+        facebook_url: %r{https?://(?:www\.)?facebook\.com/(?!(?:sharer|share)(?:/|\?|$))[^"'\s<>]{1,200}}i,
+        instagram_url: %r{https?://(?:www\.)?instagram\.com/[^"'\s<>]{1,200}}i,
+        linkedin_url: %r{https?://(?:www\.)?linkedin\.com/[^"'\s<>]{1,200}}i,
+        tiktok_url: %r{https?://(?:www\.)?tiktok\.com/[^"'\s<>]{1,200}}i,
+        youtube_url: %r{https?://(?:www\.)?youtube\.com/[^"'\s<>]{1,200}}i
+      }.freeze
+
+      SHARE_PATTERNS = [
+        %r{facebook\.com/sharer}i, %r{facebook\.com/share}i,
+        %r{twitter\.com/intent}i, %r{linkedin\.com/shareArticle}i
+      ].freeze
+
+      MAILTO_PATTERN = /mailto:([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})/i
+      EMAIL_PATTERN = /\b([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})\b/
+
+      FILTERED_EMAIL_PREFIXES = %w[
+        noreply no-reply donotreply do-not-reply
+        bounce mailer-daemon postmaster abuse spam
+        webmaster hostmaster admin administrator
+        unsubscribe notifications notification alerts
+        newsletter privacy security legal compliance dmca
+      ].freeze
+
+      BLOCKED_IP_RANGES = [
+        IPAddr.new("10.0.0.0/8"), IPAddr.new("172.16.0.0/12"),
+        IPAddr.new("192.168.0.0/16"), IPAddr.new("127.0.0.0/8"),
+        IPAddr.new("169.254.0.0/16"), IPAddr.new("0.0.0.0/8"),
+        IPAddr.new("::1/128"), IPAddr.new("fc00::/7"),
+        IPAddr.new("fe80::/10"), IPAddr.new("::ffff:0:0/96")
+      ].freeze
+
+      ALLOWED_SCHEMES = %w[http https].freeze
+      TIMEOUT_SECONDS = 10
+      MAX_REDIRECTS = 3
+      USER_AGENT = "Mozilla/5.0 (compatible; Prospector/1.0)"
+
+      attr_reader :result, :error
+
+      def self.scrape(url)
+        new(url).scrape
+      end
+
+      def initialize(url)
+        @url = url
+      end
+
+      def scrape
+        return self if @url.blank?
+
+        html = validate_and_fetch(normalize_url(@url))
+        @result = extract_all(html) if html
+        self
+      end
+
+      def success? = @error.nil? && @result.present?
+
+      private
+
+      def validate_and_fetch(url)
+        return set_error_nil("Invalid URL scheme") unless allowed_scheme?(url)
+        return set_error_nil("URL targets a private network") unless safe_url?(url)
+
+        fetch_html(url)
+      end
+
+      def extract_all(html)
+        extract_social_links(html).merge(extract_emails(html))
+      end
+
+      def normalize_url(url)
+        url.match?(%r{\Ahttps?://}i) ? url : "https://#{url}"
+      end
+
+      def allowed_scheme?(url)
+        ALLOWED_SCHEMES.include?(URI.parse(url).scheme&.downcase)
+      rescue URI::InvalidURIError
+        false
+      end
+
+      def safe_url?(url)
+        ip = IPSocket.getaddress(URI.parse(url).host.to_s)
+        BLOCKED_IP_RANGES.none? { |range| range.include?(ip) }
+      rescue SocketError, URI::InvalidURIError
+        false
+      end
+
+      def fetch_html(url, redirects = MAX_REDIRECTS)
+        response = make_request(url)
+        return response.body if response.code == 200
+
+        handle_non_ok(response, redirects, url)
+      rescue StandardError => e
+        set_error(e.message); nil
+      end
+
+      def make_request(url)
+        self.class.get(url, headers: { "User-Agent" => USER_AGENT }, timeout: TIMEOUT_SECONDS, follow_redirects: false)
+      end
+
+      def handle_non_ok(response, redirects, url)
+        return follow_redirect(response, redirects, url) if redirect?(response)
+
+        set_error("HTTP #{response.code}"); nil
+      end
+
+      def redirect?(response)
+        response.code.between?(300, 399) && response.headers["location"].present?
+      end
+
+      def follow_redirect(response, remaining, original_url)
+        return (set_error("Too many redirects"); nil) if remaining <= 0
+
+        location = resolve_redirect(response.headers["location"], original_url)
+        return (set_error("Invalid redirect"); nil) unless location && safe_url?(location)
+
+        fetch_html(location, remaining - 1)
+      end
+
+      def resolve_redirect(location, base)
+        URI.join(base, location).to_s
+      rescue URI::InvalidURIError, URI::BadURIError
+        nil
+      end
+
+      def extract_social_links(html)
+        return {} if html.blank?
+
+        SOCIAL_PATTERNS.each_with_object({}) do |(key, pattern), links|
+          match = first_valid_match(html, pattern)
+          links[key] = clean_url(match) if match
+        end
+      end
+
+      def first_valid_match(html, pattern)
+        html.scan(pattern).uniq.reject { |u| share_intent?(u) }.first
+      end
+
+      def share_intent?(url)
+        SHARE_PATTERNS.any? { |p| url.match?(p) }
+      end
+
+      def extract_emails(html)
+        return {} if html.blank?
+
+        email = find_contact_email(html)
+        email ? { email: email } : {}
+      end
+
+      def find_contact_email(html)
+        emails = html.scan(MAILTO_PATTERN).flatten.map(&:downcase).uniq
+        emails = html.scan(EMAIL_PATTERN).flatten.map(&:downcase).uniq if emails.empty?
+        emails.find { |e| valid_contact_email?(e) }
+      end
+
+      def valid_contact_email?(email)
+        FILTERED_EMAIL_PREFIXES.none? { |f| email.split("@").first.start_with?(f) }
+      end
+
+      def clean_url(url)
+        return nil if url.blank?
+
+        cleaned = url.gsub(/["'>].*$/, "").strip
+        URI.parse(cleaned) && cleaned
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def set_error(msg)
+        @error = msg
+        self
+      end
+
+      def set_error_nil(msg)
+        @error = msg
+        nil
+      end
+    end
+  end
+end

data/lib/prospector/error.rb

@@ -0,0 +1,8 @@
+module Prospector
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class AdapterError < Error; end
+  class AdapterNotFoundError < AdapterError; end
+  class MissingApiKeyError < AdapterError; end
+  class ClassificationError < Error; end
+end

data/lib/prospector/geography/base.rb

@@ -0,0 +1,40 @@
+module Prospector
+  module Geography
+    class Base
+      TYPES = %w[metro_area city coordinates zip_code bounding_box].freeze
+
+      def self.from_h(data)
+        type = data["type"] || data[:type]
+        klass = case type.to_s
+                when "metro_area"   then MetroArea
+                when "city"         then City
+                when "coordinates"  then Coordinates
+                when "zip_code"     then ZipCode
+                when "bounding_box" then BoundingBox
+                else raise ArgumentError, "Unknown geography type: #{type}"
+                end
+        klass.from_h(data)
+      end
+
+      def type
+        raise NotImplementedError
+      end
+
+      def to_query_string
+        raise NotImplementedError
+      end
+
+      def to_coordinate_restriction
+        nil
+      end
+
+      def label
+        to_query_string || "#{type}: #{to_h.inspect}"
+      end
+
+      def to_h
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/prospector/geography/bounding_box.rb

@@ -0,0 +1,58 @@
+module Prospector
+  module Geography
+    class BoundingBox < Base
+      attr_reader :ne_lat, :ne_lng, :sw_lat, :sw_lng
+
+      def initialize(ne_lat:, ne_lng:, sw_lat:, sw_lng:)
+        @ne_lat = ne_lat.to_f
+        @ne_lng = ne_lng.to_f
+        @sw_lat = sw_lat.to_f
+        @sw_lng = sw_lng.to_f
+      end
+
+      def self.from_h(data)
+        new(
+          ne_lat: data["ne_lat"] || data[:ne_lat],
+          ne_lng: data["ne_lng"] || data[:ne_lng],
+          sw_lat: data["sw_lat"] || data[:sw_lat],
+          sw_lng: data["sw_lng"] || data[:sw_lng]
+        )
+      end
+
+      def type = "bounding_box"
+
+      def to_query_string = nil
+
+      def to_coordinate_restriction
+        center_lat = (ne_lat + sw_lat) / 2.0
+        center_lng = (ne_lng + sw_lng) / 2.0
+        radius = haversine_distance(sw_lat, sw_lng, ne_lat, ne_lng) / 2.0
+
+        {
+          circle: {
+            center: { latitude: center_lat, longitude: center_lng },
+            radius: [radius, 50_000.0].min
+          }
+        }
+      end
+
+      def label = "Box (#{sw_lat},#{sw_lng}) to (#{ne_lat},#{ne_lng})"
+
+      def to_h
+        { "ne_lat" => ne_lat, "ne_lng" => ne_lng, "sw_lat" => sw_lat, "sw_lng" => sw_lng }
+      end
+
+      private
+
+      def haversine_distance(lat1, lng1, lat2, lng2)
+        r = 6_371_000
+        d_lat = to_rad(lat2 - lat1)
+        d_lng = to_rad(lng2 - lng1)
+        a = Math.sin(d_lat / 2)**2 + Math.cos(to_rad(lat1)) * Math.cos(to_rad(lat2)) * Math.sin(d_lng / 2)**2
+        r * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a))
+      end
+
+      def to_rad(deg) = deg * Math::PI / 180
+    end
+  end
+end

data/lib/prospector/geography/city.rb

@@ -0,0 +1,29 @@
+module Prospector
+  module Geography
+    class City < Base
+      attr_reader :city, :state
+
+      def initialize(city:, state:)
+        @city = city
+        @state = state
+      end
+
+      def self.from_h(data)
+        new(
+          city: data["city"] || data[:city],
+          state: data["state"] || data[:state]
+        )
+      end
+
+      def type = "city"
+
+      def to_query_string = "#{city}, #{state}"
+
+      def label = to_query_string
+
+      def to_h
+        { "city" => city, "state" => state }
+      end
+    end
+  end
+end

data/lib/prospector/geography/coordinates.rb

@@ -0,0 +1,43 @@
+module Prospector
+  module Geography
+    class Coordinates < Base
+      attr_reader :lat, :lng, :radius_meters
+
+      def initialize(lat:, lng:, radius_meters: 10_000)
+        @lat = lat.to_f
+        @lng = lng.to_f
+        @radius_meters = radius_meters.to_f
+      end
+
+      def self.from_h(data)
+        new(
+          lat: data["lat"] || data[:lat],
+          lng: data["lng"] || data[:lng],
+          radius_meters: data["radius_meters"] || data[:radius_meters] || 10_000
+        )
+      end
+
+      def type = "coordinates"
+
+      def to_query_string = nil
+
+      def to_coordinate_restriction
+        {
+          circle: {
+            center: { latitude: lat, longitude: lng },
+            radius: radius_meters
+          }
+        }
+      end
+
+      def label
+        radius_km = (radius_meters / 1000).round(1)
+        "#{lat}, #{lng} (#{radius_km}km radius)"
+      end
+
+      def to_h
+        { "lat" => lat, "lng" => lng, "radius_meters" => radius_meters }
+      end
+    end
+  end
+end

data/lib/prospector/geography/metro_area.rb

@@ -0,0 +1,74 @@
+module Prospector
+  module Geography
+    class MetroArea < Base
+      PRELOADED = [
+        { name: "Atlanta", primary_state: "GA" },
+        { name: "Austin", primary_state: "TX" },
+        { name: "Baltimore", primary_state: "MD" },
+        { name: "Boston", primary_state: "MA" },
+        { name: "Charlotte", primary_state: "NC" },
+        { name: "Chicago", primary_state: "IL" },
+        { name: "Cincinnati", primary_state: "OH" },
+        { name: "Cleveland", primary_state: "OH" },
+        { name: "Columbus", primary_state: "OH" },
+        { name: "Dallas-Fort Worth", primary_state: "TX" },
+        { name: "Denver", primary_state: "CO" },
+        { name: "Detroit", primary_state: "MI" },
+        { name: "Houston", primary_state: "TX" },
+        { name: "Indianapolis", primary_state: "IN" },
+        { name: "Jacksonville", primary_state: "FL" },
+        { name: "Kansas City", primary_state: "MO" },
+        { name: "Las Vegas", primary_state: "NV" },
+        { name: "Los Angeles", primary_state: "CA" },
+        { name: "Memphis", primary_state: "TN" },
+        { name: "Miami", primary_state: "FL" },
+        { name: "Milwaukee", primary_state: "WI" },
+        { name: "Minneapolis-St. Paul", primary_state: "MN" },
+        { name: "Nashville", primary_state: "TN" },
+        { name: "New Orleans", primary_state: "LA" },
+        { name: "New York", primary_state: "NY" },
+        { name: "Oklahoma City", primary_state: "OK" },
+        { name: "Orlando", primary_state: "FL" },
+        { name: "Philadelphia", primary_state: "PA" },
+        { name: "Phoenix", primary_state: "AZ" },
+        { name: "Pittsburgh", primary_state: "PA" },
+        { name: "Portland", primary_state: "OR" },
+        { name: "Raleigh-Durham", primary_state: "NC" },
+        { name: "Richmond", primary_state: "VA" },
+        { name: "Sacramento", primary_state: "CA" },
+        { name: "Salt Lake City", primary_state: "UT" },
+        { name: "San Antonio", primary_state: "TX" },
+        { name: "San Diego", primary_state: "CA" },
+        { name: "San Francisco Bay Area", primary_state: "CA" },
+        { name: "Seattle", primary_state: "WA" },
+        { name: "St. Louis", primary_state: "MO" },
+        { name: "Tampa Bay", primary_state: "FL" },
+        { name: "Washington D.C.", primary_state: "DC" }
+      ].freeze
+
+      attr_reader :name, :primary_state
+
+      def initialize(name:, primary_state:)
+        @name = name
+        @primary_state = primary_state
+      end
+
+      def self.from_h(data)
+        new(
+          name: data["name"] || data[:name],
+          primary_state: data["primary_state"] || data[:primary_state]
+        )
+      end
+
+      def type = "metro_area"
+
+      def to_query_string = "#{name}, #{primary_state}"
+
+      def label = to_query_string
+
+      def to_h
+        { "name" => name, "primary_state" => primary_state }
+      end
+    end
+  end
+end

data/lib/prospector/geography/zip_code.rb

@@ -0,0 +1,25 @@
+module Prospector
+  module Geography
+    class ZipCode < Base
+      attr_reader :zip
+
+      def initialize(zip:)
+        @zip = zip.to_s
+      end
+
+      def self.from_h(data)
+        new(zip: data["zip"] || data[:zip])
+      end
+
+      def type = "zip_code"
+
+      def to_query_string = zip
+
+      def label = "ZIP #{zip}"
+
+      def to_h
+        { "zip" => zip }
+      end
+    end
+  end
+end

data/lib/prospector/keywords/generator.rb

@@ -0,0 +1,74 @@
+module Prospector
+  module Keywords
+    class Generator
+      def self.for(domain:, categories:)
+        new(domain: domain, categories: categories).generate
+      end
+
+      def initialize(domain:, categories:)
+        @domain = domain
+        @categories = Array(categories)
+      end
+
+      def generate
+        @categories.flat_map { |category| keywords_for(category) }
+      end
+
+      private
+
+      def keywords_for(category)
+        existing = Keyword.keywords_for(domain: @domain, category: category)
+        return existing if existing.any?
+
+        generated = generate_via_llm(category)
+        persist_keywords(category, generated)
+        generated
+      end
+
+      def generate_via_llm(category)
+        require "ruby_llm"
+
+        chat = RubyLLM.chat(model: Prospector.config.default_classifier_model)
+        response = chat.ask(prompt_for(category))
+        parse_keywords(response.content)
+      end
+
+      def prompt_for(category)
+        <<~PROMPT
+          Generate 4-6 search keywords for finding businesses in the "#{category}" category
+          within the "#{@domain}" domain.
+
+          These keywords will be used as Google Places text search queries appended with a location
+          (e.g., "keyword in Dallas, TX"). Generate keywords that would find relevant businesses.
+
+          Return ONLY the keywords, one per line, no numbering or bullet points.
+        PROMPT
+      end
+
+      def parse_keywords(text)
+        text.strip.split("\n").map(&:strip).reject(&:blank?)
+      end
+
+      def persist_keywords(category, keywords)
+        keywords.each do |kw|
+          upsert_keyword(category, kw)
+        end
+      end
+
+      def upsert_keyword(category, kw)
+        Keyword.find_or_create_by!(
+          domain: @domain,
+          category: category,
+          keyword: kw
+        ) do |record|
+          record.source = "llm"
+          record.metadata = {
+            "generated_by" => Prospector.config.default_classifier_model,
+            "generated_at" => Time.current.iso8601
+          }
+        end
+      rescue ActiveRecord::RecordNotUnique
+      end
+    end
+  end
+end

data/lib/prospector/pipeline/normalizer.rb

@@ -0,0 +1,57 @@
+module Prospector
+  module Pipeline
+    class Normalizer
+      STATE_ZIP_PATTERN = /\b([A-Z]{2})\s+(\d{5})\b/
+
+      def self.normalize(result)
+        new(result).call
+      end
+
+      def initialize(result)
+        @result = result
+      end
+
+      def call
+        parts = parse_address
+        {
+          "business_name" => @result.name,
+          "street_address" => parts[:street],
+          "city" => parts[:city],
+          "state" => parts[:state],
+          "zip_code" => parts[:zip],
+          "phone_number" => @result.phone_number,
+          "website" => @result.website,
+          "latitude" => @result.latitude,
+          "longitude" => @result.longitude
+        }
+      end
+
+      private
+
+      def parse_address
+        address = @result.formatted_address
+        return {} if address.blank?
+
+        segments = address.split(",").map(&:strip)
+        state_zip_index = find_state_zip_index(segments)
+        return { street: segments[0] } unless state_zip_index
+
+        build_parts(segments, state_zip_index)
+      end
+
+      def find_state_zip_index(segments)
+        segments.rindex { |s| s.match?(STATE_ZIP_PATTERN) }
+      end
+
+      def build_parts(segments, idx)
+        match = segments[idx].match(STATE_ZIP_PATTERN)
+        {
+          street: idx >= 2 ? segments[0..(idx - 2)].join(", ") : (idx == 1 ? segments[0] : nil),
+          city: idx >= 2 ? segments[idx - 1] : nil,
+          state: match[1],
+          zip: match[2]
+        }
+      end
+    end
+  end
+end