@xapp/arachne-web-service 1.33.0 → 1.34.1

package/lib/services/ServiceExtractor.d.ts

@@ -0,0 +1,195 @@
+import { LLMProvider } from "../llm/LLMProvider";
+import { ServiceTaxonomy } from "../taxonomy/ServiceTaxonomy";
+import { ServicePageKind } from "./selectServicePages";
+/**
+ * Bumped whenever the prompt, schema or normalizer changes, so stored extractions can be compared by version.
+ */
+export declare const EXTRACTOR_VERSION = "2026-10-30.2";
+/**
+ * Default cap on the page text sent to the LLM, in tokens (estimated at four characters a token).
+ */
+export declare const DEFAULT_MAX_INPUT_TOKENS = 40000;
+export interface ServicePageText {
+    url: string;
+    kind: ServicePageKind;
+    /**
+     * The page as markdown, navigation stripped.
+     */
+    markdown: string;
+}
+/**
+ * Where a value was found: a page that was read, and text copied from it.
+ */
+export interface ServiceEvidence {
+    url: string;
+    snippet: string;
+}
+export interface EvidencedValue {
+    value: string;
+    evidence: ServiceEvidence[];
+}
+export interface ScoredValue extends EvidencedValue {
+    /**
+     * 0 to 1, as the model reported it.
+     */
+    confidence: number;
+}
+export interface ExtractedService {
+    /**
+     * Canonical taxonomy job slug, or a family slug for a service the site advertises only in general terms.
+     */
+    slug: string;
+    trade: string;
+    /**
+     * Undefined when the job sits directly under its trade.
+     */
+    family?: string;
+    confidence: number;
+    evidence: ServiceEvidence[];
+}
+export interface ExtractedFlag {
+    /**
+     * False only when the site says so; a flag with no evidence either way is absent.
+     */
+    value: boolean;
+    confidence: number;
+    evidence: ServiceEvidence[];
+}
+/**
+ * A value the normalizer dropped.
+ */
+export interface RejectedValue {
+    field: string;
+    value: string;
+    reason: "no_evidence" | "homeowner_only" | "not_in_taxonomy";
+    /**
+     * The evidence the model gave for a value dropped for `no_evidence`, for auditing.
+     */
+    evidence?: ServiceEvidence[];
+}
+/**
+ * The services, capabilities and service area a business website advertises, in Service Taxonomy terms.
+ *
+ * Every value carries evidence copied from a page that was read; values without it are dropped and listed in
+ * `rejected`.
+ */
+export interface ServiceExtraction {
+    taxonomyVersion: string;
+    extractorVersion: string;
+    /**
+     * The LLM that read the pages.
+     */
+    provider: string;
+    /**
+     * Undefined when the site is not a home-services business.
+     */
+    primaryTrade?: ScoredValue;
+    secondaryTrades: ScoredValue[];
+    services: ExtractedService[];
+    /**
+     * Services the site offers that have no job in the taxonomy, as the site words them.
+     */
+    unmappedServices: EvidencedValue[];
+    /**
+     * What the site says it does not do.
+     */
+    explicitExclusions: EvidencedValue[];
+    /**
+     * Capability flags by slug.
+     */
+    flags: Record<string, ExtractedFlag>;
+    certifications: EvidencedValue[];
+    equipmentTypes: EvidencedValue[];
+    specialtyMaterials: EvidencedValue[];
+    gear: EvidencedValue[];
+    subTrades: EvidencedValue[];
+    /**
+     * Financing or rebate programs named.
+     */
+    programs: EvidencedValue[];
+    homeWarrantyNetworks: EvidencedValue[];
+    writtenReportsCertifications: EvidencedValue[];
+    languages: EvidencedValue[];
+    brandsServiced: EvidencedValue[];
+    serviceArea: {
+        cities: EvidencedValue[];
+        counties: EvidencedValue[];
+        zips: EvidencedValue[];
+        radiusMiles: EvidencedValue[];
+    };
+    /**
+     * The pages sent to the LLM, in order.
+     */
+    pagesRead: string[];
+    rejected: RejectedValue[];
+    extractedAt: string;
+}
+export interface ExtractServicesCallOptions {
+    /**
+     * Trades the business is known to be in.  Skips the classifier call; the first is the primary trade.
+     */
+    trades?: string[];
+    /**
+     * Cap on the page text sent, in tokens.  Default `DEFAULT_MAX_INPUT_TOKENS`.
+     */
+    maxInputTokens?: number;
+}
+/**
+ * Text as a reader sees it: markdown links, emphasis, escapes and heading underlines removed, quotes and dashes
+ * straightened, case and spacing folded.
+ */
+export declare function normalizeText(text: string): string;
+/**
+ * Reads the pages of a business website and returns the services it advertises in Service Taxonomy terms.
+ *
+ * LLM calls at temperature 0: a classifier picks the trades from the start of each page (skipped when the trades
+ * are given), then the extractor chooses from those trades' advertised jobs.  When it leaves services unmapped, a
+ * third call matches them against the jobs of every trade, since a site often advertises work outside the trades
+ * it was classified in.  Each answer must match its JSON schema or it is retried once.  Values whose snippet is not
+ * on the page they cite are dropped.
+ */
+export declare class ServiceExtractor {
+    private readonly provider;
+    private readonly taxonomy;
+    constructor(props: {
+        provider: LLMProvider;
+        taxonomy?: ServiceTaxonomy;
+    });
+    /**
+     * @param pages The pages to read, most important first
+     * @returns The extraction, or undefined when no page has text or an LLM answer is unusable after a retry
+     */
+    extract(pages: ServicePageText[], options?: ExtractServicesCallOptions): Promise<ServiceExtraction | undefined>;
+    private normalize;
+    /**
+     * Asks which job of any trade each unmapped service is, and moves the ones that match into services.  A match in a
+     * trade the business was not classified in adds that trade to the secondary trades.  When the answer is unusable
+     * the services stay unmapped.
+     */
+    private matchUnmapped;
+    /**
+     * The job or family a slug names, following aliases and deprecations.
+     */
+    private lookup;
+    private addService;
+    /**
+     * Keeps every page, trimming the longest ones until the total fits the budget.
+     */
+    private fit;
+    private pagesBlock;
+    private classifyPrompt;
+    private classifySchema;
+    /**
+     * The slugs the extractor may choose for the trades: each family, for a service advertised in general terms,
+     * followed by its advertised jobs, then the jobs that sit directly under a trade.
+     */
+    private catalog;
+    private catalogText;
+    private matchPrompt;
+    private extractPrompt;
+    private extractSchema;
+    /**
+     * Calls the LLM and checks the answer's shape, retrying once.
+     */
+    private ask;
+}