@xapp/arachne-web-service 1.32.0 → 1.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +52 -3
- package/lib/ArachneWebService.js +56 -9
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/index.d.ts +2 -0
- package/lib/index.js +2 -0
- package/lib/index.js.map +1 -1
- package/lib/llm/LLMProvider.d.ts +48 -7
- package/lib/llm/LLMProvider.js +66 -12
- package/lib/llm/LLMProvider.js.map +1 -1
- package/lib/services/ServiceExtractor.d.ts +180 -0
- package/lib/services/ServiceExtractor.js +474 -0
- package/lib/services/ServiceExtractor.js.map +1 -0
- package/lib/services/readServicePages.d.ts +37 -0
- package/lib/services/readServicePages.js +58 -0
- package/lib/services/readServicePages.js.map +1 -0
- package/lib/services/selectServicePages.d.ts +34 -0
- package/lib/services/selectServicePages.js +177 -0
- package/lib/services/selectServicePages.js.map +1 -0
- package/lib/taxonomy/ServiceTaxonomy.d.ts +5 -2
- package/lib/taxonomy/ServiceTaxonomy.js +10 -7
- package/lib/taxonomy/ServiceTaxonomy.js.map +1 -1
- package/package.json +2 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMProvider.js","sourceRoot":"","sources":["../../src/llm/LLMProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"LLMProvider.js","sourceRoot":"","sources":["../../src/llm/LLMProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;AAiTA,gDAUC;AA3TD,kCAAkC;AAClC,6DAAiD;AACjD,mDAAqC;AAyCrC,MAAM,YAAY,GAAG,wBAAwB,CAAC;AAE9C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;GAEG;AACU,QAAA,oBAAoB,GAAG,kBAAkB,CAAC;AAEvD;;GAEG;AACH,SAAe,QAAQ,CACnB,QAAgB,EAChB,GAAW,EACX,OAA+B,EAC/B,IAAY,EACZ,KAAa;;QAEb,sCAAsC;QACtC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC9B,MAAM,EAAE,MAAM;gBACd,OAAO,kBAAI,cAAc,EAAE,kBAAkB,IAAK,OAAO,CAAE;gBAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC7B,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC1B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,GAAG,QAAQ,0BAA0B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBACnE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,GAAG,QAAQ,oBAAqB,CAAW,aAAX,CAAC,uBAAD,CAAC,CAAY,OAAO,EAAE,CAAC,CAAC;YACnE,OAAO,SAAS,CAAC;QACrB,CAAC;gBAAS,CAAC;YACP,sCAAsC;YACtC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;IACL,CAAC;CAAA;AAED,MAAa,cAAc;IAGvB,YAA6B,GAAW;QAAX,QAAG,GAAH,GAAG,CAAQ;QAF/B,SAAI,GAAG,QAAQ,CAAC;IAEkB,CAAC;IAEtC,QAAQ;6DAAC,EACX,MAAM,EACN,SAAS,EACT,KAAK,EACL,UAAU,EACV,WAAW,GAAG,mBAAmB,GACd;;YACnB,MAAM,MAAM,GAAG,MAAM,QAAQ,CACzB,QAAQ,EACR,4CAA4C,EAC5C,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,GAAG,EAAE,EAAE,EACvC;gBACI,KAAK,EAAE,YAAY;gBACnB,eAAe,EAAE,UAAU;oBACvB,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;oBAC5F,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE;gBAC7B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC/C,WAAW;gBACX,UAAU,EAAE,SAAS;gBACrB,KAAK,EAAE,CAAC;gBACR,iBAAiB,EAAE,CAAC;gBACpB,gBAAgB,EAAE,CAAC;aACtB,EACD,KAAK,CACR,CAAC;YAEF,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,IAAI,KAAK,EAAE,CAAC;gBACR,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CACxH,CAAC;YACN,CAAC;YAED,OAAO,MAAA,MAAA,MAAA,MAAA,MAAM,CAAC,OAAO,0CAAG,CAAC,CAAC,0CAAE,OAAO,0CAAE,OAAO,mCAAI,SAAS,CAAC;QAC9D,CAAC;KAAA;CACJ;AA5CD,wCA4CC;AAwBD,MAAa,cAAc;IAcvB;;OAEG;IACH,YAAY,IAA+B,EAAE,KAAc;QAhBlD,SAAI,GAAG,QAAQ,CAAC;QAiBrB,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,4BAAoB,CAAC;QACvE,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;QACpB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;IACL,CAAC;IAEO,GAAG;;QACP,MAAA,IAAI,CAAC,UAAU,oCAAf,IAAI,CAAC,UAAU,GAAK,IAAI,gCAAU,CAAC,EAAE,MAAM,EAAE,gDAAgD,EAAE,CAAC,EAAC;QACjG,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACW,MAAM;;;YAChB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACf,OAAO;oBACH,GAAG,EAAE,2DAA2D,IAAI,CAAC,KAAK,kBAAkB;oBAC5F,OAAO,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,GAAa,EAAE;iBACpD,CAAC;YACN,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAA,IAAI,CAAC,MAAM,CAAC,cAAc,mCAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC1F,MAAM,OAAO,GACT,IAAI,CAAC,MAAM,CAAC,OAAO;oBACnB,OAAO,CAAC,GAAG,CAAC,oBAAoB;oBAChC,CAAC,MAAM,CAAC,MAAA,IAAI,CAAC,MAAM,CAAC,YAAY,mCAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAE9E,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;oBAC/E,OAAO,SAAS,CAAC;gBACrB,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,QAAQ,CAAC;gBACvF,MAAM,IAAI,GAAG,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,GAAG,QAAQ,4BAA4B,CAAC;gBAE3G,OAAO;oBACH,GAAG,EAAE,WAAW,IAAI,gBAAgB,OAAO,cAAc,QAAQ,6BAA6B,IAAI,CAAC,KAAK,kBAAkB;oBAC1H,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE;iBAChD,CAAC;YACN,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mDAAoD,CAAW,aAAX,CAAC,uBAAD,CAAC,CAAY,OAAO,EAAE,CAAC,CAAC;gBACvF,OAAO,SAAS,CAAC;YACrB,CAAC;QACL,CAAC;KAAA;IAEK,QAAQ;6DAAC,EACX,MAAM,EACN,SAAS,EACT,KAAK,EACL,UAAU,EACV,WAAW,GAAG,mBAAmB,GACd;;YACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CACzB,QAAQ,EACR,MAAM,CAAC,GAAG,EACV,MAAM,CAAC,OAAO,EACd;gBACI,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;gBACvD,gBAAgB,8CACZ,gBAAgB,EAAE,kBAAkB,IACjC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KACzD,WAAW,EACX,eAAe,EAAE,SAAS,KAGvB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC,CACnF;aACJ,EACD,KAAK,CACR,CAAC;YAEF,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;YACnC,IAAI,KAAK,EAAE,CAAC;gBACR,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,8BAA8B,KAAK,CAAC,gBAAgB,eAAe,KAAK,CAAC,oBAAoB,UAAU,KAAK,CAAC,eAAe,EAAE,CACjI,CAAC;YACN,CAAC;YAED,MAAM,SAAS,GAAG,MAAA,MAAM,CAAC,UAAU,0CAAG,CAAC,CAAC,CAAC;YACzC,MAAM,KAAK,GAAuD,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,0CAAE,KAAK,CAAC;YAC5F,MAAM,IAAI,GAAG,KAAK,aAAL,KAAK,uBAAL,KAAK,CACZ,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,EAC/B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,IAAI,mCAAI,EAAE,CAAA,EAAA,EAC7B,IAAI,CAAC,EAAE,CAAC,CAAC;YAEd,IAAI,CAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,YAAY,MAAK,YAAY,EAAE,CAAC;gBAC3C,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gDAAgD,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,0BAA0B,CAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,YAAY,EAAC,CAAC,CAAC,mBAAmB,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,GAChG,CAAA,MAAA,MAAM,CAAC,cAAc,0CAAE,WAAW,EAAC,CAAC,CAAC,cAAc,MAAM,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAC7F,EAAE,CACL,CAAC;gBACF,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;KAAA;CACJ;AApID,wCAoIC;AAWD;;;;;;GAMG;AACH,SAAgB,kBAAkB,CAAC,OAA2B;IAC1D,IAAI,OAAO,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC5D,OAAO,IAAI,cAAc,CACrB,GAAG,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE,EAC3E,OAAO,CAAC,WAAW,CACtB,CAAC;IACN,CAAC;IAED,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACjF,CAAC"}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { LLMProvider } from "../llm/LLMProvider";
|
|
2
|
+
import { ServiceTaxonomy } from "../taxonomy/ServiceTaxonomy";
|
|
3
|
+
import { ServicePageKind } from "./selectServicePages";
|
|
4
|
+
/**
|
|
5
|
+
* Bumped whenever the prompt, schema or normalizer changes, so stored extractions can be compared by version.
|
|
6
|
+
*/
|
|
7
|
+
export declare const EXTRACTOR_VERSION = "2026-10-30.1";
|
|
8
|
+
/**
|
|
9
|
+
* Default cap on the page text sent to the LLM, in tokens (estimated at four characters a token).
|
|
10
|
+
*/
|
|
11
|
+
export declare const DEFAULT_MAX_INPUT_TOKENS = 40000;
|
|
12
|
+
export interface ServicePageText {
|
|
13
|
+
url: string;
|
|
14
|
+
kind: ServicePageKind;
|
|
15
|
+
/**
|
|
16
|
+
* The page as markdown, navigation stripped.
|
|
17
|
+
*/
|
|
18
|
+
markdown: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Where a value was found: a page that was read, and text copied from it.
|
|
22
|
+
*/
|
|
23
|
+
export interface ServiceEvidence {
|
|
24
|
+
url: string;
|
|
25
|
+
snippet: string;
|
|
26
|
+
}
|
|
27
|
+
export interface EvidencedValue {
|
|
28
|
+
value: string;
|
|
29
|
+
evidence: ServiceEvidence[];
|
|
30
|
+
}
|
|
31
|
+
export interface ScoredValue extends EvidencedValue {
|
|
32
|
+
/**
|
|
33
|
+
* 0 to 1, as the model reported it.
|
|
34
|
+
*/
|
|
35
|
+
confidence: number;
|
|
36
|
+
}
|
|
37
|
+
export interface ExtractedService {
|
|
38
|
+
/**
|
|
39
|
+
* Canonical taxonomy job slug, or a family slug for a service the site advertises only in general terms.
|
|
40
|
+
*/
|
|
41
|
+
slug: string;
|
|
42
|
+
trade: string;
|
|
43
|
+
/**
|
|
44
|
+
* Undefined when the job sits directly under its trade.
|
|
45
|
+
*/
|
|
46
|
+
family?: string;
|
|
47
|
+
confidence: number;
|
|
48
|
+
evidence: ServiceEvidence[];
|
|
49
|
+
}
|
|
50
|
+
export interface ExtractedFlag {
|
|
51
|
+
/**
|
|
52
|
+
* False only when the site says so; a flag with no evidence either way is absent.
|
|
53
|
+
*/
|
|
54
|
+
value: boolean;
|
|
55
|
+
confidence: number;
|
|
56
|
+
evidence: ServiceEvidence[];
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* A value the normalizer dropped.
|
|
60
|
+
*/
|
|
61
|
+
export interface RejectedValue {
|
|
62
|
+
field: string;
|
|
63
|
+
value: string;
|
|
64
|
+
reason: "no_evidence" | "homeowner_only" | "not_in_taxonomy";
|
|
65
|
+
/**
|
|
66
|
+
* The evidence the model gave for a value dropped for `no_evidence`, for auditing.
|
|
67
|
+
*/
|
|
68
|
+
evidence?: ServiceEvidence[];
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The services, capabilities and service area a business website advertises, in Service Taxonomy terms.
|
|
72
|
+
*
|
|
73
|
+
* Every value carries evidence copied from a page that was read; values without it are dropped and listed in
|
|
74
|
+
* `rejected`.
|
|
75
|
+
*/
|
|
76
|
+
export interface ServiceExtraction {
|
|
77
|
+
taxonomyVersion: string;
|
|
78
|
+
extractorVersion: string;
|
|
79
|
+
/**
|
|
80
|
+
* The LLM that read the pages.
|
|
81
|
+
*/
|
|
82
|
+
provider: string;
|
|
83
|
+
/**
|
|
84
|
+
* Undefined when the site is not a home-services business.
|
|
85
|
+
*/
|
|
86
|
+
primaryTrade?: ScoredValue;
|
|
87
|
+
secondaryTrades: ScoredValue[];
|
|
88
|
+
services: ExtractedService[];
|
|
89
|
+
/**
|
|
90
|
+
* Services the site offers that have no job in the taxonomy, as the site words them.
|
|
91
|
+
*/
|
|
92
|
+
unmappedServices: EvidencedValue[];
|
|
93
|
+
/**
|
|
94
|
+
* What the site says it does not do.
|
|
95
|
+
*/
|
|
96
|
+
explicitExclusions: EvidencedValue[];
|
|
97
|
+
/**
|
|
98
|
+
* Capability flags by slug.
|
|
99
|
+
*/
|
|
100
|
+
flags: Record<string, ExtractedFlag>;
|
|
101
|
+
certifications: EvidencedValue[];
|
|
102
|
+
equipmentTypes: EvidencedValue[];
|
|
103
|
+
specialtyMaterials: EvidencedValue[];
|
|
104
|
+
gear: EvidencedValue[];
|
|
105
|
+
subTrades: EvidencedValue[];
|
|
106
|
+
/**
|
|
107
|
+
* Financing or rebate programs named.
|
|
108
|
+
*/
|
|
109
|
+
programs: EvidencedValue[];
|
|
110
|
+
homeWarrantyNetworks: EvidencedValue[];
|
|
111
|
+
writtenReportsCertifications: EvidencedValue[];
|
|
112
|
+
languages: EvidencedValue[];
|
|
113
|
+
brandsServiced: EvidencedValue[];
|
|
114
|
+
serviceArea: {
|
|
115
|
+
cities: EvidencedValue[];
|
|
116
|
+
counties: EvidencedValue[];
|
|
117
|
+
zips: EvidencedValue[];
|
|
118
|
+
radiusMiles: EvidencedValue[];
|
|
119
|
+
};
|
|
120
|
+
/**
|
|
121
|
+
* The pages sent to the LLM, in order.
|
|
122
|
+
*/
|
|
123
|
+
pagesRead: string[];
|
|
124
|
+
rejected: RejectedValue[];
|
|
125
|
+
extractedAt: string;
|
|
126
|
+
}
|
|
127
|
+
export interface ExtractServicesCallOptions {
|
|
128
|
+
/**
|
|
129
|
+
* Trades the business is known to be in. Skips the classifier call; the first is the primary trade.
|
|
130
|
+
*/
|
|
131
|
+
trades?: string[];
|
|
132
|
+
/**
|
|
133
|
+
* Cap on the page text sent, in tokens. Default `DEFAULT_MAX_INPUT_TOKENS`.
|
|
134
|
+
*/
|
|
135
|
+
maxInputTokens?: number;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Text as a reader sees it: markdown links, emphasis, escapes and heading underlines removed, quotes and dashes
|
|
139
|
+
* straightened, case and spacing folded.
|
|
140
|
+
*/
|
|
141
|
+
export declare function normalizeText(text: string): string;
|
|
142
|
+
/**
|
|
143
|
+
* Reads the pages of a business website and returns the services it advertises in Service Taxonomy terms.
|
|
144
|
+
*
|
|
145
|
+
* Two LLM calls at temperature 0: a classifier picks the trades from the start of each page (skipped when the
|
|
146
|
+
* trades are given), then the extractor chooses from those trades' advertised jobs. Each answer must match its
|
|
147
|
+
* JSON schema or it is retried once. Values whose snippet is not on the page they cite are dropped.
|
|
148
|
+
*/
|
|
149
|
+
export declare class ServiceExtractor {
|
|
150
|
+
private readonly provider;
|
|
151
|
+
private readonly taxonomy;
|
|
152
|
+
constructor(props: {
|
|
153
|
+
provider: LLMProvider;
|
|
154
|
+
taxonomy?: ServiceTaxonomy;
|
|
155
|
+
});
|
|
156
|
+
/**
|
|
157
|
+
* @param pages The pages to read, most important first
|
|
158
|
+
* @returns The extraction, or undefined when no page has text or an LLM answer is unusable after a retry
|
|
159
|
+
*/
|
|
160
|
+
extract(pages: ServicePageText[], options?: ExtractServicesCallOptions): Promise<ServiceExtraction | undefined>;
|
|
161
|
+
private normalize;
|
|
162
|
+
/**
|
|
163
|
+
* Keeps every page, trimming the longest ones until the total fits the budget.
|
|
164
|
+
*/
|
|
165
|
+
private fit;
|
|
166
|
+
private pagesBlock;
|
|
167
|
+
private classifyPrompt;
|
|
168
|
+
private classifySchema;
|
|
169
|
+
/**
|
|
170
|
+
* The slugs the extractor may choose for the trades: each family, for a service advertised in general terms,
|
|
171
|
+
* followed by its advertised jobs, then the jobs that sit directly under a trade.
|
|
172
|
+
*/
|
|
173
|
+
private catalog;
|
|
174
|
+
private extractPrompt;
|
|
175
|
+
private extractSchema;
|
|
176
|
+
/**
|
|
177
|
+
* Calls the LLM and checks the answer's shape, retrying once.
|
|
178
|
+
*/
|
|
179
|
+
private ask;
|
|
180
|
+
}
|
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.ServiceExtractor = exports.DEFAULT_MAX_INPUT_TOKENS = exports.EXTRACTOR_VERSION = void 0;
|
|
13
|
+
exports.normalizeText = normalizeText;
|
|
14
|
+
/*! Copyright (c) 2026, XAPP AI */
|
|
15
|
+
const stentor_logger_1 = require("stentor-logger");
|
|
16
|
+
const ServiceTaxonomy_1 = require("../taxonomy/ServiceTaxonomy");
|
|
17
|
+
/**
|
|
18
|
+
* Bumped whenever the prompt, schema or normalizer changes, so stored extractions can be compared by version.
|
|
19
|
+
*/
|
|
20
|
+
exports.EXTRACTOR_VERSION = "2026-10-30.1";
|
|
21
|
+
/**
|
|
22
|
+
* Default cap on the page text sent to the LLM, in tokens (estimated at four characters a token).
|
|
23
|
+
*/
|
|
24
|
+
exports.DEFAULT_MAX_INPUT_TOKENS = 40000;
|
|
25
|
+
const MAX_SECONDARY_TRADES = 3;
|
|
26
|
+
const CLASSIFY_PAGE_CHARS = 3000;
|
|
27
|
+
const CLASSIFY_MAX_TOKENS = 1024;
|
|
28
|
+
const EXTRACT_MAX_TOKENS = 16384;
|
|
29
|
+
const CHARS_PER_TOKEN = 4;
|
|
30
|
+
const ATTRIBUTE_KINDS = {
|
|
31
|
+
certification: "certifications",
|
|
32
|
+
equipment_type: "equipmentTypes",
|
|
33
|
+
specialty_material: "specialtyMaterials",
|
|
34
|
+
gear: "gear",
|
|
35
|
+
sub_trade: "subTrades",
|
|
36
|
+
};
|
|
37
|
+
const LIST_KINDS = {
|
|
38
|
+
programs: "programs",
|
|
39
|
+
home_warranty_networks: "homeWarrantyNetworks",
|
|
40
|
+
written_reports_certifications: "writtenReportsCertifications",
|
|
41
|
+
languages: "languages",
|
|
42
|
+
};
|
|
43
|
+
const AREA_KINDS = { city: "cities", county: "counties", zip: "zips", radius_miles: "radiusMiles" };
|
|
44
|
+
/**
|
|
45
|
+
* Text as a reader sees it: markdown links, emphasis, escapes and heading underlines removed, quotes and dashes
|
|
46
|
+
* straightened, case and spacing folded.
|
|
47
|
+
*/
|
|
48
|
+
function normalizeText(text) {
|
|
49
|
+
return text
|
|
50
|
+
.normalize("NFKC")
|
|
51
|
+
// setext heading underlines, then the backslash escapes turndown adds, e.g. "80123\." for "80123."
|
|
52
|
+
.replace(/^\s*[=-]{3,}\s*$/gm, " ")
|
|
53
|
+
.replace(/\\([\\`*_{}[\]()#+\-.!>~|])/g, "$1")
|
|
54
|
+
.replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1")
|
|
55
|
+
.replace(/[*_#`>|]/g, " ")
|
|
56
|
+
.replace(/[‘’]/g, "'")
|
|
57
|
+
.replace(/[“”]/g, '"')
|
|
58
|
+
.replace(/[–—]/g, "-")
|
|
59
|
+
.replace(/\s+/g, " ")
|
|
60
|
+
.trim()
|
|
61
|
+
.toLowerCase();
|
|
62
|
+
}
|
|
63
|
+
function evidenceSchema(urls) {
|
|
64
|
+
return {
|
|
65
|
+
type: "array",
|
|
66
|
+
items: {
|
|
67
|
+
type: "object",
|
|
68
|
+
properties: { url: { type: "string", enum: urls }, snippet: { type: "string" } },
|
|
69
|
+
required: ["url", "snippet"],
|
|
70
|
+
additionalProperties: false,
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function objectSchema(properties) {
|
|
75
|
+
return { type: "object", properties, required: Object.keys(properties), additionalProperties: false };
|
|
76
|
+
}
|
|
77
|
+
function arrayOf(properties) {
|
|
78
|
+
return { type: "array", items: objectSchema(properties) };
|
|
79
|
+
}
|
|
80
|
+
const isString = (v) => typeof v === "string";
|
|
81
|
+
const isNumber = (v) => typeof v === "number" && Number.isFinite(v);
|
|
82
|
+
function isEvidence(v) {
|
|
83
|
+
return Array.isArray(v) && v.every((e) => e && isString(e.url) && isString(e.snippet));
|
|
84
|
+
}
|
|
85
|
+
function isArrayOf(v, check) {
|
|
86
|
+
return Array.isArray(v) && v.every((item) => !!item && typeof item === "object" && check(item));
|
|
87
|
+
}
|
|
88
|
+
function isTradeAnswer(v) {
|
|
89
|
+
return (!!v &&
|
|
90
|
+
isString(v.primary_trade) &&
|
|
91
|
+
isNumber(v.confidence) &&
|
|
92
|
+
Array.isArray(v.secondary_trades) &&
|
|
93
|
+
v.secondary_trades.every(isString) &&
|
|
94
|
+
isEvidence(v.evidence));
|
|
95
|
+
}
|
|
96
|
+
function isExtractionAnswer(v) {
|
|
97
|
+
const evidenced = (i) => isString(i.value) && isEvidence(i.evidence);
|
|
98
|
+
return (!!v &&
|
|
99
|
+
isArrayOf(v.services, (i) => isString(i.slug) && isNumber(i.confidence) && isEvidence(i.evidence)) &&
|
|
100
|
+
isArrayOf(v.unmapped_services, evidenced) &&
|
|
101
|
+
isArrayOf(v.explicit_exclusions, evidenced) &&
|
|
102
|
+
isArrayOf(v.flags, (i) => isString(i.flag) && typeof i.value === "boolean" && isNumber(i.confidence) && isEvidence(i.evidence)) &&
|
|
103
|
+
isArrayOf(v.attributes, (i) => isString(i.kind) && evidenced(i)) &&
|
|
104
|
+
isArrayOf(v.lists, (i) => isString(i.list) && evidenced(i)) &&
|
|
105
|
+
isArrayOf(v.brands_serviced, evidenced) &&
|
|
106
|
+
isArrayOf(v.service_area, (i) => isString(i.kind) && evidenced(i)));
|
|
107
|
+
}
|
|
108
|
+
const clamp = (n) => Math.min(1, Math.max(0, n));
|
|
109
|
+
/**
|
|
110
|
+
* Reads the pages of a business website and returns the services it advertises in Service Taxonomy terms.
|
|
111
|
+
*
|
|
112
|
+
* Two LLM calls at temperature 0: a classifier picks the trades from the start of each page (skipped when the
|
|
113
|
+
* trades are given), then the extractor chooses from those trades' advertised jobs. Each answer must match its
|
|
114
|
+
* JSON schema or it is retried once. Values whose snippet is not on the page they cite are dropped.
|
|
115
|
+
*/
|
|
116
|
+
class ServiceExtractor {
|
|
117
|
+
constructor(props) {
|
|
118
|
+
var _a;
|
|
119
|
+
this.provider = props.provider;
|
|
120
|
+
this.taxonomy = (_a = props.taxonomy) !== null && _a !== void 0 ? _a : new ServiceTaxonomy_1.ServiceTaxonomy();
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* @param pages The pages to read, most important first
|
|
124
|
+
* @returns The extraction, or undefined when no page has text or an LLM answer is unusable after a retry
|
|
125
|
+
*/
|
|
126
|
+
extract(pages_1) {
|
|
127
|
+
return __awaiter(this, arguments, void 0, function* (pages, options = {}) {
|
|
128
|
+
var _a, _b;
|
|
129
|
+
const fitted = this.fit(pages.filter((p) => p.markdown.trim()), ((_a = options.maxInputTokens) !== null && _a !== void 0 ? _a : exports.DEFAULT_MAX_INPUT_TOKENS) * CHARS_PER_TOKEN);
|
|
130
|
+
if (fitted.length === 0) {
|
|
131
|
+
(0, stentor_logger_1.log)().info(`No page text to extract services from.`);
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
// Spacing is ignored: the markdown can run words together that the model copies apart
|
|
135
|
+
const squash = (text) => normalizeText(text).replace(/ /g, "");
|
|
136
|
+
const texts = new Map(fitted.map((p) => [p.url, squash(p.markdown)]));
|
|
137
|
+
const rejected = [];
|
|
138
|
+
const onPage = (e) => {
|
|
139
|
+
var _a;
|
|
140
|
+
const snippet = squash(e.snippet);
|
|
141
|
+
return snippet.length >= 3 && !!((_a = texts.get(e.url)) === null || _a === void 0 ? void 0 : _a.includes(snippet));
|
|
142
|
+
};
|
|
143
|
+
const checked = (field, value, evidence) => {
|
|
144
|
+
const kept = evidence.filter(onPage);
|
|
145
|
+
if (kept.length === 0) {
|
|
146
|
+
rejected.push({ field, value, reason: "no_evidence", evidence });
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
return kept;
|
|
150
|
+
};
|
|
151
|
+
const result = {
|
|
152
|
+
taxonomyVersion: this.taxonomy.version,
|
|
153
|
+
extractorVersion: exports.EXTRACTOR_VERSION,
|
|
154
|
+
provider: this.provider.name,
|
|
155
|
+
secondaryTrades: [],
|
|
156
|
+
services: [],
|
|
157
|
+
unmappedServices: [],
|
|
158
|
+
explicitExclusions: [],
|
|
159
|
+
flags: {},
|
|
160
|
+
certifications: [],
|
|
161
|
+
equipmentTypes: [],
|
|
162
|
+
specialtyMaterials: [],
|
|
163
|
+
gear: [],
|
|
164
|
+
subTrades: [],
|
|
165
|
+
programs: [],
|
|
166
|
+
homeWarrantyNetworks: [],
|
|
167
|
+
writtenReportsCertifications: [],
|
|
168
|
+
languages: [],
|
|
169
|
+
brandsServiced: [],
|
|
170
|
+
serviceArea: { cities: [], counties: [], zips: [], radiusMiles: [] },
|
|
171
|
+
pagesRead: fitted.map((p) => p.url),
|
|
172
|
+
rejected,
|
|
173
|
+
extractedAt: new Date().toISOString(),
|
|
174
|
+
};
|
|
175
|
+
let trades = ((_b = options.trades) !== null && _b !== void 0 ? _b : []).filter((t) => this.taxonomy.isTrade(t));
|
|
176
|
+
if (trades.length > 0) {
|
|
177
|
+
result.primaryTrade = { value: trades[0], confidence: 1, evidence: [] };
|
|
178
|
+
result.secondaryTrades = trades.slice(1).map((value) => ({ value, confidence: 1, evidence: [] }));
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
const answer = yield this.ask(this.classifyPrompt(fitted), this.classifySchema(fitted), CLASSIFY_MAX_TOKENS, isTradeAnswer, "services_trades");
|
|
182
|
+
if (!answer) {
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
if (!this.taxonomy.isTrade(answer.primary_trade)) {
|
|
186
|
+
(0, stentor_logger_1.log)().info(`Not a home-services business, no trade found.`);
|
|
187
|
+
return result;
|
|
188
|
+
}
|
|
189
|
+
const evidence = answer.evidence.filter(onPage);
|
|
190
|
+
const secondary = [...new Set(answer.secondary_trades)]
|
|
191
|
+
.filter((t) => t !== answer.primary_trade && this.taxonomy.isTrade(t))
|
|
192
|
+
.slice(0, MAX_SECONDARY_TRADES);
|
|
193
|
+
result.primaryTrade = { value: answer.primary_trade, confidence: clamp(answer.confidence), evidence };
|
|
194
|
+
result.secondaryTrades = secondary.map((value) => ({
|
|
195
|
+
value,
|
|
196
|
+
confidence: clamp(answer.confidence),
|
|
197
|
+
evidence: [],
|
|
198
|
+
}));
|
|
199
|
+
trades = [answer.primary_trade, ...secondary];
|
|
200
|
+
}
|
|
201
|
+
const catalog = this.catalog(trades);
|
|
202
|
+
const answer = yield this.ask(this.extractPrompt(fitted, catalog), this.extractSchema(fitted, catalog), EXTRACT_MAX_TOKENS, isExtractionAnswer, "services_extract");
|
|
203
|
+
if (!answer) {
|
|
204
|
+
return undefined;
|
|
205
|
+
}
|
|
206
|
+
this.normalize(answer, result, checked);
|
|
207
|
+
(0, stentor_logger_1.log)().info(`Extracted ${result.services.length} services for ${trades.join(", ")} from ${fitted.length} pages, rejected ${rejected.length} values`);
|
|
208
|
+
return result;
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
normalize(answer, result, checked) {
|
|
212
|
+
const definitions = serviceLists(this.taxonomy);
|
|
213
|
+
for (const service of answer.services) {
|
|
214
|
+
const family = this.taxonomy.isFamily(service.slug) ? service.slug : undefined;
|
|
215
|
+
const slug = family !== null && family !== void 0 ? family : this.taxonomy.resolve(service.slug);
|
|
216
|
+
const job = family
|
|
217
|
+
? { slug: family, trade: family.split(".")[0], family, homeowner_only: false }
|
|
218
|
+
: slug
|
|
219
|
+
? this.taxonomy.job(slug)
|
|
220
|
+
: undefined;
|
|
221
|
+
if (!job) {
|
|
222
|
+
const evidence = checked("unmappedServices", service.slug, service.evidence);
|
|
223
|
+
if (evidence) {
|
|
224
|
+
result.unmappedServices.push({ value: service.slug, evidence });
|
|
225
|
+
}
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
if (job.homeowner_only) {
|
|
229
|
+
result.rejected.push({ field: "services", value: service.slug, reason: "homeowner_only" });
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
const evidence = checked("services", service.slug, service.evidence);
|
|
233
|
+
if (!evidence) {
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
const existing = result.services.find((s) => s.slug === job.slug);
|
|
237
|
+
if (existing) {
|
|
238
|
+
existing.confidence = Math.max(existing.confidence, clamp(service.confidence));
|
|
239
|
+
existing.evidence.push(...evidence);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
result.services.push(Object.assign(Object.assign({ slug: job.slug, trade: job.trade }, (job.family ? { family: job.family } : {})), { confidence: clamp(service.confidence), evidence }));
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
const push = (field, target, item) => {
|
|
246
|
+
const evidence = checked(field, item.value, item.evidence);
|
|
247
|
+
if (evidence) {
|
|
248
|
+
target.push({ value: item.value, evidence });
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
answer.unmapped_services.forEach((item) => push("unmappedServices", result.unmappedServices, item));
|
|
252
|
+
answer.explicit_exclusions.forEach((item) => push("explicitExclusions", result.explicitExclusions, item));
|
|
253
|
+
answer.brands_serviced.forEach((item) => push("brandsServiced", result.brandsServiced, item));
|
|
254
|
+
for (const flag of answer.flags) {
|
|
255
|
+
if (!definitions.flags.includes(flag.flag)) {
|
|
256
|
+
result.rejected.push({ field: "flags", value: flag.flag, reason: "not_in_taxonomy" });
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const evidence = checked("flags", flag.flag, flag.evidence);
|
|
260
|
+
if (evidence) {
|
|
261
|
+
result.flags[flag.flag] = { value: flag.value, confidence: clamp(flag.confidence), evidence };
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
for (const attribute of answer.attributes) {
|
|
265
|
+
const field = ATTRIBUTE_KINDS[attribute.kind];
|
|
266
|
+
if (!field ||
|
|
267
|
+
!definitions.attributes[attribute.kind].includes(attribute.value)) {
|
|
268
|
+
result.rejected.push({
|
|
269
|
+
field: field !== null && field !== void 0 ? field : "attributes",
|
|
270
|
+
value: attribute.value,
|
|
271
|
+
reason: "not_in_taxonomy",
|
|
272
|
+
});
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
push(field, result[field], attribute);
|
|
276
|
+
}
|
|
277
|
+
for (const item of answer.lists) {
|
|
278
|
+
const field = LIST_KINDS[item.list];
|
|
279
|
+
if (!field) {
|
|
280
|
+
result.rejected.push({ field: "lists", value: item.value, reason: "not_in_taxonomy" });
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
push(field, result[field], item);
|
|
284
|
+
}
|
|
285
|
+
for (const item of answer.service_area) {
|
|
286
|
+
const field = AREA_KINDS[item.kind];
|
|
287
|
+
if (!field) {
|
|
288
|
+
result.rejected.push({ field: "serviceArea", value: item.value, reason: "not_in_taxonomy" });
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
push(`serviceArea.${field}`, result.serviceArea[field], item);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Keeps every page, trimming the longest ones until the total fits the budget.
|
|
296
|
+
*/
|
|
297
|
+
fit(pages, budgetChars) {
|
|
298
|
+
const lengths = pages.map((p) => p.markdown.length).sort((a, b) => a - b);
|
|
299
|
+
if (lengths.reduce((sum, n) => sum + n, 0) <= budgetChars) {
|
|
300
|
+
return pages;
|
|
301
|
+
}
|
|
302
|
+
// Pages shorter than an even share keep their text and hand what they leave over to the longer ones
|
|
303
|
+
let remaining = budgetChars;
|
|
304
|
+
let cap = 0;
|
|
305
|
+
for (let i = 0; i < lengths.length; i++) {
|
|
306
|
+
cap = Math.floor(remaining / (lengths.length - i));
|
|
307
|
+
if (lengths[i] > cap) {
|
|
308
|
+
break;
|
|
309
|
+
}
|
|
310
|
+
remaining -= lengths[i];
|
|
311
|
+
}
|
|
312
|
+
return pages.map((p) => (p.markdown.length > cap ? Object.assign(Object.assign({}, p), { markdown: p.markdown.slice(0, cap) }) : p));
|
|
313
|
+
}
|
|
314
|
+
pagesBlock(pages, chars) {
|
|
315
|
+
return pages
|
|
316
|
+
.map((p) => `=== PAGE ${p.url} (${p.kind}) ===\n${chars ? p.markdown.slice(0, chars) : p.markdown}`)
|
|
317
|
+
.join("\n\n");
|
|
318
|
+
}
|
|
319
|
+
classifyPrompt(pages) {
|
|
320
|
+
const trades = serviceLists(this.taxonomy).trades.join(", ");
|
|
321
|
+
return `You read the start of each page of a business website and decide which home-service trades the business works in.
|
|
322
|
+
|
|
323
|
+
Trades: ${trades}
|
|
324
|
+
|
|
325
|
+
Rules:
|
|
326
|
+
- primary_trade is the trade most of its services belong to, or "none" when it is not a home-services business (e.g. a spa, a retailer, a software company).
|
|
327
|
+
- secondary_trades are other trades it clearly advertises, at most ${MAX_SECONDARY_TRADES}.
|
|
328
|
+
- evidence is one or two snippets copied exactly from the pages that show the primary trade.
|
|
329
|
+
|
|
330
|
+
${this.pagesBlock(pages, CLASSIFY_PAGE_CHARS)}`;
|
|
331
|
+
}
|
|
332
|
+
classifySchema(pages) {
|
|
333
|
+
const trades = serviceLists(this.taxonomy).trades;
|
|
334
|
+
return objectSchema({
|
|
335
|
+
primary_trade: { type: "string", enum: [...trades, "none"] },
|
|
336
|
+
confidence: { type: "number" },
|
|
337
|
+
secondary_trades: { type: "array", items: { type: "string", enum: trades } },
|
|
338
|
+
evidence: evidenceSchema(pages.map((p) => p.url)),
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* The slugs the extractor may choose for the trades: each family, for a service advertised in general terms,
|
|
343
|
+
* followed by its advertised jobs, then the jobs that sit directly under a trade.
|
|
344
|
+
*/
|
|
345
|
+
catalog(trades) {
|
|
346
|
+
var _a, _b;
|
|
347
|
+
const jobs = this.taxonomy.extractableJobs(trades);
|
|
348
|
+
const entries = [];
|
|
349
|
+
for (const trade of this.taxonomy.definition.trades.filter((t) => trades.includes(t.slug))) {
|
|
350
|
+
for (const family of (_a = trade.families) !== null && _a !== void 0 ? _a : []) {
|
|
351
|
+
entries.push({ slug: family.slug, general: true, terms: (_b = family.website_terms) !== null && _b !== void 0 ? _b : [] });
|
|
352
|
+
jobs.filter((j) => j.family === family.slug).forEach((j) => { var _a; return entries.push({ slug: j.slug, terms: (_a = j.synonyms) !== null && _a !== void 0 ? _a : [] }); });
|
|
353
|
+
}
|
|
354
|
+
jobs.filter((j) => j.trade === trade.slug && !j.family).forEach((j) => { var _a; return entries.push({ slug: j.slug, terms: (_a = j.synonyms) !== null && _a !== void 0 ? _a : [] }); });
|
|
355
|
+
}
|
|
356
|
+
return entries;
|
|
357
|
+
}
|
|
358
|
+
extractPrompt(pages, catalog) {
|
|
359
|
+
const lists = serviceLists(this.taxonomy);
|
|
360
|
+
const catalogText = catalog
|
|
361
|
+
.map((e) => `- ${e.slug}${e.general ? " (general)" : ""}${e.terms.length ? `: ${e.terms.join("; ")}` : ""}`)
|
|
362
|
+
.join("\n");
|
|
363
|
+
const flags = this.taxonomy.definition.capability_flags
|
|
364
|
+
.map((f) => `- ${f.slug}: ${f.definition}`)
|
|
365
|
+
.join("\n");
|
|
366
|
+
const listDefs = this.taxonomy.definition.capability_lists
|
|
367
|
+
.map((l) => `- ${l.slug}: ${l.definition}`)
|
|
368
|
+
.join("\n");
|
|
369
|
+
const attributes = Object.entries(lists.attributes)
|
|
370
|
+
.map(([kind, values]) => `- ${kind}: ${values.join(", ")}`)
|
|
371
|
+
.join("\n");
|
|
372
|
+
return `You read the pages of a home-services business website and list what the business advertises, using the catalog below.
|
|
373
|
+
|
|
374
|
+
Every value needs evidence: the URL of the page it is on and a snippet copied exactly, word for word, from that page (5 to 25 words). No evidence, no value. Do not infer, guess or generalise.
|
|
375
|
+
|
|
376
|
+
- services: catalog slugs for jobs the business says it performs. Pick the most specific slug the page supports; a slug marked (general) is for a service the site advertises only in general terms, e.g. "AC repair". Anything the business offers that has no catalog slug goes in unmapped_services, worded as the site words it.
|
|
377
|
+
- explicit_exclusions: anything the site says it does not do.
|
|
378
|
+
- flags: include a flag only when the pages state it. value is false only when the site says the opposite. Leave out flags the pages do not mention.
|
|
379
|
+
- attributes: certifications, equipment types, specialty materials, gear and sub-trade specialisms from the lists below that the pages name.
|
|
380
|
+
- lists: financing programs, home warranty networks, written reports and languages the pages name, as free text.
|
|
381
|
+
- brands_serviced: equipment or product brands the business installs or services.
|
|
382
|
+
- service_area: cities, counties, ZIP codes, or a radius in miles that the business says it serves.
|
|
383
|
+
|
|
384
|
+
Catalog (slug: words customers and websites use):
|
|
385
|
+
${catalogText}
|
|
386
|
+
|
|
387
|
+
Flags:
|
|
388
|
+
${flags}
|
|
389
|
+
|
|
390
|
+
Attributes:
|
|
391
|
+
${attributes}
|
|
392
|
+
|
|
393
|
+
Lists:
|
|
394
|
+
${listDefs}
|
|
395
|
+
|
|
396
|
+
${this.pagesBlock(pages)}`;
|
|
397
|
+
}
|
|
398
|
+
extractSchema(pages, catalog) {
|
|
399
|
+
const lists = serviceLists(this.taxonomy);
|
|
400
|
+
const evidence = evidenceSchema(pages.map((p) => p.url));
|
|
401
|
+
const valued = (extra = {}) => arrayOf(Object.assign(Object.assign({}, extra), { value: { type: "string" }, evidence }));
|
|
402
|
+
const attributeValues = [...new Set(Object.values(lists.attributes).flat())];
|
|
403
|
+
return objectSchema({
|
|
404
|
+
services: arrayOf({
|
|
405
|
+
slug: { type: "string", enum: catalog.map((e) => e.slug) },
|
|
406
|
+
confidence: { type: "number" },
|
|
407
|
+
evidence,
|
|
408
|
+
}),
|
|
409
|
+
unmapped_services: valued(),
|
|
410
|
+
explicit_exclusions: valued(),
|
|
411
|
+
flags: arrayOf({
|
|
412
|
+
flag: { type: "string", enum: lists.flags },
|
|
413
|
+
value: { type: "boolean" },
|
|
414
|
+
confidence: { type: "number" },
|
|
415
|
+
evidence,
|
|
416
|
+
}),
|
|
417
|
+
attributes: arrayOf({
|
|
418
|
+
kind: { type: "string", enum: Object.keys(ATTRIBUTE_KINDS) },
|
|
419
|
+
value: { type: "string", enum: attributeValues },
|
|
420
|
+
evidence,
|
|
421
|
+
}),
|
|
422
|
+
lists: valued({ list: { type: "string", enum: Object.keys(LIST_KINDS) } }),
|
|
423
|
+
brands_serviced: valued(),
|
|
424
|
+
service_area: valued({ kind: { type: "string", enum: Object.keys(AREA_KINDS) } }),
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Calls the LLM and checks the answer's shape, retrying once.
|
|
429
|
+
*/
|
|
430
|
+
ask(prompt, jsonSchema, maxTokens, check, label) {
|
|
431
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
432
|
+
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
433
|
+
const text = yield this.provider.complete({
|
|
434
|
+
prompt,
|
|
435
|
+
jsonSchema,
|
|
436
|
+
maxTokens,
|
|
437
|
+
temperature: 0,
|
|
438
|
+
label: `${label}_${attempt}`,
|
|
439
|
+
});
|
|
440
|
+
let parsed;
|
|
441
|
+
try {
|
|
442
|
+
parsed = text === undefined ? undefined : JSON.parse(text);
|
|
443
|
+
}
|
|
444
|
+
catch (_a) {
|
|
445
|
+
parsed = undefined;
|
|
446
|
+
}
|
|
447
|
+
if (check(parsed)) {
|
|
448
|
+
return parsed;
|
|
449
|
+
}
|
|
450
|
+
(0, stentor_logger_1.log)().warn(`${label}: answer ${text === undefined ? "missing" : "does not match the schema"} on attempt ${attempt}`);
|
|
451
|
+
}
|
|
452
|
+
return undefined;
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
exports.ServiceExtractor = ServiceExtractor;
|
|
457
|
+
/**
|
|
458
|
+
* The enumerations the prompts and schemas offer.
|
|
459
|
+
*/
|
|
460
|
+
function serviceLists(taxonomy) {
|
|
461
|
+
const d = taxonomy.definition;
|
|
462
|
+
return {
|
|
463
|
+
trades: d.trades.map((t) => t.slug),
|
|
464
|
+
flags: d.capability_flags.map((f) => f.slug),
|
|
465
|
+
attributes: {
|
|
466
|
+
certification: d.certifications,
|
|
467
|
+
equipment_type: d.equipment_types,
|
|
468
|
+
specialty_material: d.specialty_materials,
|
|
469
|
+
gear: d.gear,
|
|
470
|
+
sub_trade: d.sub_trades,
|
|
471
|
+
},
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
//# sourceMappingURL=ServiceExtractor.js.map
|