@webcrawl/mcp 1.4.2 → 1.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/README.md +1 -1
  2. package/dist/bin.js +5 -7
  3. package/dist/core/engine/awswaf.d.ts +59 -0
  4. package/dist/core/engine/awswaf.js +164 -0
  5. package/dist/core/engine/cache.d.ts +70 -0
  6. package/dist/core/engine/cache.js +297 -0
  7. package/dist/core/engine/chrome-cdp.d.ts +22 -0
  8. package/dist/core/engine/chrome-cdp.js +318 -0
  9. package/dist/core/engine/chrome-launch.d.ts +38 -0
  10. package/dist/core/engine/chrome-launch.js +214 -0
  11. package/dist/core/engine/chrome.d.ts +28 -0
  12. package/dist/core/engine/chrome.js +208 -0
  13. package/dist/core/engine/crawl-policy.d.ts +36 -0
  14. package/dist/core/engine/crawl-policy.js +134 -0
  15. package/dist/core/engine/crawler.d.ts +152 -0
  16. package/dist/core/engine/crawler.js +918 -0
  17. package/dist/core/engine/fetcher.d.ts +92 -0
  18. package/dist/core/engine/fetcher.js +494 -0
  19. package/dist/core/engine/index.d.ts +7 -0
  20. package/dist/core/engine/index.js +7 -0
  21. package/dist/core/engine/markdown.d.ts +33 -0
  22. package/dist/core/engine/markdown.js +334 -0
  23. package/dist/core/engine/path-scope.d.ts +21 -0
  24. package/dist/core/engine/path-scope.js +88 -0
  25. package/dist/core/engine/render.d.ts +41 -0
  26. package/dist/core/engine/render.js +138 -0
  27. package/dist/core/engine/safe-url.d.ts +29 -0
  28. package/dist/core/engine/safe-url.js +109 -0
  29. package/dist/core/engine/session.d.ts +93 -0
  30. package/dist/core/engine/session.js +250 -0
  31. package/dist/core/engine/stealth.d.ts +83 -0
  32. package/dist/core/engine/stealth.js +222 -0
  33. package/dist/core/engine.d.ts +1 -0
  34. package/dist/core/engine.js +1 -0
  35. package/dist/core/extract/business.d.ts +28 -0
  36. package/dist/core/extract/business.js +344 -0
  37. package/dist/core/extract/currency.d.ts +15 -0
  38. package/dist/core/extract/currency.js +259 -0
  39. package/dist/core/extract/cve-cache.d.ts +62 -0
  40. package/dist/core/extract/cve-cache.js +277 -0
  41. package/dist/core/extract/cve.d.ts +57 -0
  42. package/dist/core/extract/cve.js +397 -0
  43. package/dist/core/extract/dedupe-key.d.ts +12 -0
  44. package/dist/core/extract/dedupe-key.js +36 -0
  45. package/dist/core/extract/flights.d.ts +37 -0
  46. package/dist/core/extract/flights.js +373 -0
  47. package/dist/core/extract/hotels.d.ts +35 -0
  48. package/dist/core/extract/hotels.js +473 -0
  49. package/dist/core/extract/index.d.ts +9 -0
  50. package/dist/core/extract/index.js +9 -0
  51. package/dist/core/extract/real-estate.d.ts +53 -0
  52. package/dist/core/extract/real-estate.js +835 -0
  53. package/dist/core/extract/scaledown.d.ts +14 -0
  54. package/dist/core/extract/scaledown.js +484 -0
  55. package/dist/core/extract/shopping.d.ts +44 -0
  56. package/dist/core/extract/shopping.js +393 -0
  57. package/dist/core/extract/structured.d.ts +33 -0
  58. package/dist/core/extract/structured.js +316 -0
  59. package/dist/core/image-url.d.ts +5 -0
  60. package/dist/core/image-url.js +13 -0
  61. package/dist/core/interact/driver.d.ts +72 -0
  62. package/dist/core/interact/driver.js +384 -0
  63. package/dist/core/interact/remote.d.ts +41 -0
  64. package/dist/core/interact/remote.js +133 -0
  65. package/dist/core/interact/spans.d.ts +46 -0
  66. package/dist/core/interact/spans.js +32 -0
  67. package/dist/core/interact/validate.d.ts +34 -0
  68. package/dist/core/interact/validate.js +135 -0
  69. package/dist/core/interact/vm.d.ts +43 -0
  70. package/dist/core/interact/vm.js +338 -0
  71. package/dist/core/local-engine.d.ts +62 -0
  72. package/dist/core/local-engine.js +726 -0
  73. package/dist/core/parse/detect.d.ts +29 -0
  74. package/dist/core/parse/detect.js +152 -0
  75. package/dist/core/parse/fetch.d.ts +20 -0
  76. package/dist/core/parse/fetch.js +39 -0
  77. package/dist/core/parse/image.d.ts +13 -0
  78. package/dist/core/parse/image.js +100 -0
  79. package/dist/core/parse/index.d.ts +53 -0
  80. package/dist/core/parse/index.js +153 -0
  81. package/dist/core/parse/limits.d.ts +12 -0
  82. package/dist/core/parse/limits.js +29 -0
  83. package/dist/core/parse/media.d.ts +42 -0
  84. package/dist/core/parse/media.js +238 -0
  85. package/dist/core/parse/merge.d.ts +19 -0
  86. package/dist/core/parse/merge.js +48 -0
  87. package/dist/core/parse/ocr.d.ts +25 -0
  88. package/dist/core/parse/ocr.js +121 -0
  89. package/dist/core/parse/office.d.ts +28 -0
  90. package/dist/core/parse/office.js +83 -0
  91. package/dist/core/parse/pdf.d.ts +27 -0
  92. package/dist/core/parse/pdf.js +156 -0
  93. package/dist/core/parse/types.d.ts +38 -0
  94. package/dist/core/parse/types.js +1 -0
  95. package/dist/core/parse/youtube.d.ts +30 -0
  96. package/dist/core/parse/youtube.js +264 -0
  97. package/dist/core/ranker/decision.d.ts +28 -0
  98. package/dist/core/ranker/decision.js +151 -0
  99. package/dist/core/ranker/index.d.ts +5 -0
  100. package/dist/core/ranker/index.js +361 -0
  101. package/dist/core/ranker/lexical.d.ts +82 -0
  102. package/dist/core/ranker/lexical.js +643 -0
  103. package/dist/core/ranker/types.d.ts +90 -0
  104. package/dist/core/ranker/types.js +1 -0
  105. package/dist/core/ranker.d.ts +1 -0
  106. package/dist/core/ranker.js +1 -0
  107. package/dist/core/sandbox.d.ts +29 -0
  108. package/dist/core/sandbox.js +222 -0
  109. package/dist/core/search/catalog.d.ts +176 -0
  110. package/dist/core/search/catalog.js +488 -0
  111. package/dist/core/search/index.d.ts +3 -0
  112. package/dist/core/search/index.js +3 -0
  113. package/dist/core/search/pipeline.d.ts +7 -0
  114. package/dist/core/search/pipeline.js +721 -0
  115. package/dist/core/search/providers.d.ts +84 -0
  116. package/dist/core/search/providers.js +1408 -0
  117. package/dist/core/search.d.ts +1 -0
  118. package/dist/core/search.js +1 -0
  119. package/dist/core/targets/developer.d.ts +44 -0
  120. package/dist/core/targets/developer.js +338 -0
  121. package/dist/core/targets/index.d.ts +3 -0
  122. package/dist/core/targets/index.js +3 -0
  123. package/dist/core/targets/registry.d.ts +32 -0
  124. package/dist/core/targets/registry.js +411 -0
  125. package/dist/core/targets/types.d.ts +72 -0
  126. package/dist/core/targets/types.js +8 -0
  127. package/dist/core/targets.d.ts +1 -0
  128. package/dist/core/targets.js +1 -0
  129. package/dist/format.js +5 -1
  130. package/dist/index.d.ts +1 -0
  131. package/dist/index.js +1 -0
  132. package/dist/load-local-engine.d.ts +7 -0
  133. package/dist/load-local-engine.js +22 -0
  134. package/dist/server.js +6 -0
  135. package/package.json +11 -3
@@ -0,0 +1,259 @@
1
+ const VALID_ISO_CURRENCIES = new Set([
2
+ "USD",
3
+ "AUD",
4
+ "GBP",
5
+ "EUR",
6
+ "CAD",
7
+ "NZD",
8
+ "SGD",
9
+ "HKD",
10
+ "JPY",
11
+ "CNY",
12
+ "INR",
13
+ "CHF",
14
+ "SEK",
15
+ "NOK",
16
+ "DKK",
17
+ "ZAR",
18
+ "BRL",
19
+ "MXN",
20
+ "AED",
21
+ "SAR",
22
+ "THB",
23
+ "MYR",
24
+ "IDR",
25
+ "PHP",
26
+ "VND",
27
+ "KRW",
28
+ "PLN",
29
+ "TRY",
30
+ "TWD",
31
+ ]);
32
+ /**
33
+ * Derive ISO 4217 currency code from a raw price string and/or 2-letter ISO country code.
34
+ * Uses standard currency symbols and ISO country conventions without hardcoded retailer lists.
35
+ */
36
+ export function resolveCurrencyCode(rawPriceOrCurrency, countryCode) {
37
+ const s = (rawPriceOrCurrency || "").trim();
38
+ const isoMatches = s.match(/\b([A-Z]{3})\b/g);
39
+ if (isoMatches) {
40
+ for (const code of isoMatches) {
41
+ if (VALID_ISO_CURRENCIES.has(code)) {
42
+ return code;
43
+ }
44
+ }
45
+ }
46
+ if (/A\$|AU\$/i.test(s)) {
47
+ return "AUD";
48
+ }
49
+ if (/NZ\$/i.test(s)) {
50
+ return "NZD";
51
+ }
52
+ if (/CA\$|C\$/i.test(s)) {
53
+ return "CAD";
54
+ }
55
+ if (/S\$|SG\$/i.test(s)) {
56
+ return "SGD";
57
+ }
58
+ if (/HK\$/i.test(s)) {
59
+ return "HKD";
60
+ }
61
+ if (/NT\$/i.test(s)) {
62
+ return "TWD";
63
+ }
64
+ if (s.includes("£")) {
65
+ return "GBP";
66
+ }
67
+ if (s.includes("€")) {
68
+ return "EUR";
69
+ }
70
+ if (s.includes("¥")) {
71
+ return (countryCode || "").toUpperCase() === "CN" ? "CNY" : "JPY";
72
+ }
73
+ if (s.includes("₹") ||
74
+ /\bRs\.?(?:\s|\d)/i.test(s) ||
75
+ /\b(?:crores?|cr|lakhs?|lacs?)\b/i.test(s)) {
76
+ return "INR";
77
+ }
78
+ if (s.includes("₫") || /\b(?:đồng|vnd|triệu|tỷ)\b/i.test(s)) {
79
+ return "VND";
80
+ }
81
+ if (s.includes("฿")) {
82
+ return "THB";
83
+ }
84
+ if (s.includes("₱")) {
85
+ return "PHP";
86
+ }
87
+ if (s.includes("₩")) {
88
+ return "KRW";
89
+ }
90
+ if (/\bzł\b/i.test(s)) {
91
+ return "PLN";
92
+ }
93
+ if (/\bRM\s*\d/i.test(s)) {
94
+ return "MYR";
95
+ }
96
+ if (/\bRp\.?\s*\d/i.test(s)) {
97
+ return "IDR";
98
+ }
99
+ const cc = (countryCode || "").trim().toUpperCase();
100
+ // Bare "$" in non-dollar countries (e.g. EU, GB, IN, VN, JP) indicates USD pricing
101
+ if (s.includes("$") &&
102
+ !["US", "AU", "CA", "NZ", "SG", "HK", "TW", "MX", ""].includes(cc)) {
103
+ return "USD";
104
+ }
105
+ const COUNTRY_TO_CURRENCY = {
106
+ AU: "AUD",
107
+ GB: "GBP",
108
+ UK: "GBP",
109
+ CA: "CAD",
110
+ NZ: "NZD",
111
+ SG: "SGD",
112
+ IN: "INR",
113
+ VN: "VND",
114
+ JP: "JPY",
115
+ CN: "CNY",
116
+ HK: "HKD",
117
+ TW: "TWD",
118
+ KR: "KRW",
119
+ TH: "THB",
120
+ MY: "MYR",
121
+ ID: "IDR",
122
+ PH: "PHP",
123
+ CH: "CHF",
124
+ SE: "SEK",
125
+ NO: "NOK",
126
+ DK: "DKK",
127
+ PL: "PLN",
128
+ CZ: "CZK",
129
+ HU: "HUF",
130
+ RO: "RON",
131
+ TR: "TRY",
132
+ AE: "AED",
133
+ SA: "SAR",
134
+ ZA: "ZAR",
135
+ BR: "BRL",
136
+ MX: "MXN",
137
+ };
138
+ if (COUNTRY_TO_CURRENCY[cc]) {
139
+ return COUNTRY_TO_CURRENCY[cc];
140
+ }
141
+ if ([
142
+ "DE",
143
+ "FR",
144
+ "IT",
145
+ "ES",
146
+ "NL",
147
+ "IE",
148
+ "AT",
149
+ "BE",
150
+ "PT",
151
+ "FI",
152
+ "GR",
153
+ "LU",
154
+ "SK",
155
+ "SI",
156
+ "EE",
157
+ "LV",
158
+ "LT",
159
+ "HR",
160
+ "CY",
161
+ "MT",
162
+ "EU",
163
+ ].includes(cc)) {
164
+ return "EUR";
165
+ }
166
+ return "USD";
167
+ }
168
+ /**
169
+ * Parse a numeric price and currency from a human price string such as
170
+ * "$360.41", "A$399.00", "£173.49", "1.299,00 €", "₹1,19,900", "₹1.45 Crores", "S$429", "7.490.000 ₫".
171
+ */
172
+ export function parsePriceString(raw, countryCode, explicitCurrency, maxAllowedPrice = 1_000_000) {
173
+ if (typeof raw === "number") {
174
+ if (!Number.isFinite(raw) || raw <= 0) {
175
+ return null;
176
+ }
177
+ const numCurrency = resolveCurrencyCode(explicitCurrency, countryCode);
178
+ return {
179
+ price: Number(raw.toFixed(2)),
180
+ rawPrice: `${numCurrency} ${raw.toFixed(2)}`,
181
+ currency: numCurrency,
182
+ };
183
+ }
184
+ if (!raw || typeof raw !== "string") {
185
+ return null;
186
+ }
187
+ const cleaned = raw.replace(/\s+/g, " ").trim();
188
+ if (!cleaned) {
189
+ return null;
190
+ }
191
+ const currency = resolveCurrencyCode(explicitCurrency || cleaned, countryCode);
192
+ // High-denomination currencies (VND, IDR, KRW, JPY, INR) allow much higher nominal numeric caps
193
+ const highDenomMultiplier = currency === "VND" || currency === "IDR"
194
+ ? 25_000
195
+ : currency === "KRW" || currency === "JPY" || currency === "INR"
196
+ ? 500
197
+ : 1;
198
+ const effectiveMaxPrice = maxAllowedPrice * highDenomMultiplier;
199
+ // Handle Indian Crore / Lakh and Vietnamese tỷ / triệu multipliers
200
+ const unitMatch = cleaned.match(/(\d[\d,.]*)\s*(crores?|cr|lakhs?|lacs?|tỷ|triệu)\b/i);
201
+ if (unitMatch) {
202
+ const baseNum = Number.parseFloat(unitMatch[1].replace(/,/g, ""));
203
+ const unit = unitMatch[2].toLowerCase();
204
+ const mult = unit.startsWith("cr")
205
+ ? 10_000_000
206
+ : unit.startsWith("la")
207
+ ? 100_000
208
+ : unit === "tỷ"
209
+ ? 1_000_000_000
210
+ : 1_000_000;
211
+ const total = Math.round(baseNum * mult);
212
+ if (Number.isFinite(total) && total > 0 && total <= effectiveMaxPrice) {
213
+ return {
214
+ price: total,
215
+ rawPrice: cleaned,
216
+ currency,
217
+ };
218
+ }
219
+ }
220
+ const numMatch = cleaned.match(/(\d[\d,.]*)/);
221
+ if (!numMatch) {
222
+ return null;
223
+ }
224
+ let numStr = numMatch[1].replace(/[.,]$/, "");
225
+ // Handle multi-dot thousands (e.g. "7.490.000" in VND/EUR/IDR) and European decimal comma ("194,99" or "1.299,00")
226
+ if (/^\d{1,3}(?:\.\d{3}){2,}$/.test(numStr)) {
227
+ numStr = numStr.replace(/\./g, "");
228
+ }
229
+ else if (/^\d{1,3}\.\d{3}$/.test(numStr) &&
230
+ (currency === "VND" || currency === "IDR" || currency === "EUR")) {
231
+ numStr = numStr.replace(/\./g, "");
232
+ }
233
+ else if (/\.\d{3},\d{1,2}$/.test(numStr)) {
234
+ numStr = numStr.replace(/\./g, "").replace(",", ".");
235
+ }
236
+ else if (/^\d+,\d{1,2}$/.test(numStr)) {
237
+ numStr = numStr.replace(",", ".");
238
+ }
239
+ else {
240
+ numStr = numStr.replace(/,/g, "");
241
+ }
242
+ const value = Number.parseFloat(numStr);
243
+ if (!Number.isFinite(value) || value <= 0 || value > effectiveMaxPrice) {
244
+ return null;
245
+ }
246
+ return {
247
+ price: Number(value.toFixed(2)),
248
+ rawPrice: cleaned,
249
+ currency,
250
+ };
251
+ }
252
+ export function extractDomainHost(url) {
253
+ try {
254
+ return new URL(url).hostname.replace(/^www\./i, "").toLowerCase();
255
+ }
256
+ catch {
257
+ return "";
258
+ }
259
+ }
@@ -0,0 +1,62 @@
1
+ import { type CveRecord } from "./cve.js";
2
+ /** Only CVEs published (or numbered) in this year or later are stored. */
3
+ export declare const CVE_STORE_SINCE = 2020;
4
+ /** A cached keyword result is trusted for this long before NVD is asked again. */
5
+ export declare const QUERY_TTL_MS: number;
6
+ export interface CachedCves {
7
+ records: CveRecord[];
8
+ /** Ids served from cache that are due a background refresh. */
9
+ stale: string[];
10
+ }
11
+ export interface CveCacheStore {
12
+ get(ids: string[]): Promise<CachedCves>;
13
+ getQuery(keyword: string): Promise<string[] | null>;
14
+ putQuery(keyword: string, ids: string[]): Promise<void>;
15
+ search(keyword: string, limit: number): Promise<CveRecord[]>;
16
+ upsert(records: CveRecord[]): Promise<number>;
17
+ }
18
+ /** True when a record falls inside the stored window (published year, else id year). */
19
+ export declare function isCacheableCve(r: CveRecord): boolean;
20
+ export declare function isStale(fetchedAt: Date, published: string | undefined, now?: number): boolean;
21
+ export declare function cveQueryKey(keyword: string): string;
22
+ /** One row of the batched upsert; exported so the SQL shape is unit-testable. */
23
+ export declare function cveRow(r: CveRecord): unknown[];
24
+ /**
25
+ * INSERT … ON CONFLICT for `n` rows. An existing row is only replaced when the
26
+ * incoming record is at least as new (`modified`) and actually differs, so cache
27
+ * hits written back unchanged cost no row churn.
28
+ */
29
+ export declare function buildUpsertSql(n: number): string;
30
+ interface PgPoolLike {
31
+ query(sql: string, params?: unknown[]): Promise<{
32
+ rows: Array<Record<string, any>>;
33
+ rowCount?: number | null;
34
+ }>;
35
+ }
36
+ /**
37
+ * Registers an external Postgres pool & schema (e.g., Supabase Postgres in webcrawl.app cloud)
38
+ * when the engine is consumed as an installed npm package (`webcrawl/extract/cve-cache`).
39
+ */
40
+ export declare function registerExternalCveDb(mods: {
41
+ getPgPool?: () => Promise<PgPoolLike | null>;
42
+ CVE_SCHEMA_SQL?: string;
43
+ } | null): void;
44
+ /** Swap the backing store (tests). Pass null to restore Postgres. */
45
+ export declare function setCveCacheStoreForTests(store: CveCacheStore | null): void;
46
+ /** Cached records for exact ids (CVE or alias such as GHSA). Never throws. */
47
+ export declare function getCachedCves(ids: string[]): Promise<CachedCves>;
48
+ /** Ids NVD returned for this keyword within QUERY_TTL_MS, or null. Never throws. */
49
+ export declare function getCachedQuery(keyword: string): Promise<string[] | null>;
50
+ export declare function rememberQuery(keyword: string, ids: string[]): void;
51
+ /** Full-text search over cached rows; the fallback when NVD is down or rate limited. */
52
+ export declare function searchCachedCves(keyword: string, limit: number): Promise<CveRecord[]>;
53
+ /** Store records in the window; resolves to rows written (0 on any failure). */
54
+ export declare function upsertCves(records: CveRecord[]): Promise<number>;
55
+ /** Background write that outlives the response on Vercel (no-op elsewhere). */
56
+ export declare function runInBackground(p: Promise<unknown>): void;
57
+ /**
58
+ * Background refresh of stale ids: fetch live, merge over the cached copy (keeps
59
+ * OSV fixed versions the live NVD record lacks), write back.
60
+ */
61
+ export declare function refreshStaleCves(stale: string[], cached: CveRecord[], fetchLive: (ids: string[]) => Promise<CveRecord[]>): void;
62
+ export {};
@@ -0,0 +1,277 @@
1
+ // Read-through CVE cache in Postgres (Supabase in prod). Search fills it as
2
+ // requests resolve CVEs from NVD/OSV, so it grows incrementally with real traffic.
3
+ // Every function degrades to a miss / no-op when there is no DB, the table can't
4
+ // be created, or a query is slow: the cache must never fail or stall a search.
5
+ import { mergeCveRecords, normalizeCveId } from "./cve.js";
6
+ /** Only CVEs published (or numbered) in this year or later are stored. */
7
+ export const CVE_STORE_SINCE = 2020;
8
+ /** Re-fetch a cached record after this long (recent CVEs change faster). */
9
+ const STALE_MS = 7 * 24 * 3600_000;
10
+ const STALE_RECENT_MS = 24 * 3600_000;
11
+ const RECENT_MS = 60 * 24 * 3600_000;
12
+ /** A cached keyword result is trusted for this long before NVD is asked again. */
13
+ export const QUERY_TTL_MS = 24 * 3600_000;
14
+ const DB_TIMEOUT_MS = 1500;
15
+ const UPSERT_CHUNK = 200;
16
+ /** True when a record falls inside the stored window (published year, else id year). */
17
+ export function isCacheableCve(r) {
18
+ if (r.source === "page")
19
+ return false;
20
+ const year = Number((r.published ?? "").slice(0, 4)) || Number(r.id.match(/^CVE-(\d{4})-/i)?.[1]);
21
+ return Number.isFinite(year) && year >= CVE_STORE_SINCE;
22
+ }
23
+ export function isStale(fetchedAt, published, now = Date.now()) {
24
+ const age = now - fetchedAt.getTime();
25
+ const pub = published ? Date.parse(published) : NaN;
26
+ const recent = Number.isFinite(pub) && now - pub < RECENT_MS;
27
+ return age > (recent ? STALE_RECENT_MS : STALE_MS);
28
+ }
29
+ export function cveQueryKey(keyword) {
30
+ return keyword.toLowerCase().replace(/\s+/g, " ").trim().slice(0, 200);
31
+ }
32
+ /** One row of the batched upsert; exported so the SQL shape is unit-testable. */
33
+ export function cveRow(r) {
34
+ const products = (r.affected ?? [])
35
+ .map((a) => [a.vendor, a.product, a.ecosystem].filter(Boolean).join(" "))
36
+ .join(" ")
37
+ .slice(0, 2000);
38
+ return [
39
+ normalizeCveId(r.id),
40
+ (r.aliases ?? []).map(normalizeCveId),
41
+ r.title ?? null,
42
+ r.summary ?? "",
43
+ r.severity ?? null,
44
+ r.cvssScore ?? null,
45
+ products,
46
+ !!r.kev,
47
+ r.published ?? null,
48
+ r.modified ?? null,
49
+ JSON.stringify(r),
50
+ ];
51
+ }
52
+ const COLS = 11;
53
+ /**
54
+ * INSERT … ON CONFLICT for `n` rows. An existing row is only replaced when the
55
+ * incoming record is at least as new (`modified`) and actually differs, so cache
56
+ * hits written back unchanged cost no row churn.
57
+ */
58
+ export function buildUpsertSql(n) {
59
+ const values = Array.from({ length: n }, (_, i) => {
60
+ const b = i * COLS;
61
+ return `($${b + 1}, $${b + 2}::text[], $${b + 3}, $${b + 4}, $${b + 5}, $${b + 6}::real, $${b + 7}, $${b + 8}, $${b + 9}::timestamptz, $${b + 10}::timestamptz, $${b + 11}::jsonb)`;
62
+ }).join(",\n");
63
+ return `INSERT INTO cves (id, aliases, title, summary, severity, cvss_score, products, kev, published, modified, record)
64
+ VALUES ${values}
65
+ ON CONFLICT (id) DO UPDATE SET
66
+ aliases = EXCLUDED.aliases, title = EXCLUDED.title, summary = EXCLUDED.summary,
67
+ severity = EXCLUDED.severity, cvss_score = EXCLUDED.cvss_score, products = EXCLUDED.products,
68
+ kev = EXCLUDED.kev, published = EXCLUDED.published, modified = EXCLUDED.modified,
69
+ record = EXCLUDED.record, fetched_at = NOW()
70
+ WHERE (cves.modified IS NULL OR EXCLUDED.modified IS NULL OR EXCLUDED.modified >= cves.modified)
71
+ AND (cves.record IS DISTINCT FROM EXCLUDED.record OR cves.fetched_at < NOW() - INTERVAL '1 hour')`;
72
+ }
73
+ function withTimeout(p, fallback, ms = DB_TIMEOUT_MS) {
74
+ let timer;
75
+ return Promise.race([
76
+ p.catch(() => fallback),
77
+ new Promise((resolve) => {
78
+ timer = setTimeout(() => resolve(fallback), ms);
79
+ }),
80
+ ]).finally(() => clearTimeout(timer));
81
+ }
82
+ let tableState = null;
83
+ let optionalDbPromise = null;
84
+ /**
85
+ * Registers an external Postgres pool & schema (e.g., Supabase Postgres in webcrawl.app cloud)
86
+ * when the engine is consumed as an installed npm package (`webcrawl/extract/cve-cache`).
87
+ */
88
+ export function registerExternalCveDb(mods) {
89
+ optionalDbPromise = Promise.resolve(mods);
90
+ tableState = null;
91
+ }
92
+ function loadOptionalDbModules() {
93
+ if (!optionalDbPromise) {
94
+ const dbIndexPath = "../db/index.js";
95
+ const dbSchemaPath = "../db/schema.js";
96
+ optionalDbPromise = Promise.all([
97
+ import(/* @vite-ignore */ dbIndexPath).catch(() => null),
98
+ import(/* @vite-ignore */ dbSchemaPath).catch(() => null),
99
+ ]).then(([dbMod, schemaMod]) => {
100
+ if (!dbMod?.getPgPool)
101
+ return null;
102
+ return {
103
+ getPgPool: dbMod.getPgPool,
104
+ CVE_SCHEMA_SQL: schemaMod?.CVE_SCHEMA_SQL,
105
+ };
106
+ });
107
+ }
108
+ return optionalDbPromise;
109
+ }
110
+ async function cvePool() {
111
+ const mods = await loadOptionalDbModules();
112
+ if (!mods?.getPgPool)
113
+ return null;
114
+ const pool = await mods.getPgPool();
115
+ if (!pool)
116
+ return null;
117
+ tableState ??= (async () => {
118
+ try {
119
+ const r = await pool.query("SELECT to_regclass('public.cves') IS NOT NULL AND to_regclass('public.cve_queries') IS NOT NULL AS ok");
120
+ if (!r.rows[0]?.ok && mods.CVE_SCHEMA_SQL) {
121
+ await pool.query(mods.CVE_SCHEMA_SQL);
122
+ }
123
+ return true;
124
+ }
125
+ catch (err) {
126
+ console.warn("[cve-cache] disabled:", err instanceof Error ? err.message : err);
127
+ return false;
128
+ }
129
+ })();
130
+ return (await tableState) ? pool : null;
131
+ }
132
+ const pgStore = {
133
+ async get(ids) {
134
+ const pool = await cvePool();
135
+ if (!pool || ids.length === 0)
136
+ return { records: [], stale: [] };
137
+ const norm = [...new Set(ids.map(normalizeCveId))];
138
+ const res = await pool.query("SELECT id, record, fetched_at FROM cves WHERE id = ANY($1::text[]) OR aliases && $1::text[]", [norm]);
139
+ const records = [];
140
+ const stale = [];
141
+ for (const row of res.rows) {
142
+ const rec = row.record;
143
+ records.push(rec);
144
+ if (isStale(new Date(row.fetched_at), rec.published))
145
+ stale.push(String(row.id));
146
+ }
147
+ if (res.rows.length) {
148
+ runInBackground(pool.query("UPDATE cves SET hits = hits + 1 WHERE id = ANY($1::text[])", [
149
+ res.rows.map((r) => r.id),
150
+ ]));
151
+ }
152
+ return { records, stale };
153
+ },
154
+ async getQuery(keyword) {
155
+ const pool = await cvePool();
156
+ if (!pool)
157
+ return null;
158
+ const res = await pool.query("SELECT ids FROM cve_queries WHERE keyword = $1 AND fetched_at > NOW() - make_interval(secs => $2)", [cveQueryKey(keyword), QUERY_TTL_MS / 1000]);
159
+ return res.rows[0] ? res.rows[0].ids : null;
160
+ },
161
+ async putQuery(keyword, ids) {
162
+ const pool = await cvePool();
163
+ if (!pool)
164
+ return;
165
+ await pool.query(`INSERT INTO cve_queries (keyword, ids, fetched_at) VALUES ($1, $2::text[], NOW())
166
+ ON CONFLICT (keyword) DO UPDATE SET ids = EXCLUDED.ids, fetched_at = NOW()`, [cveQueryKey(keyword), ids.map(normalizeCveId)]);
167
+ },
168
+ async search(keyword, limit) {
169
+ const pool = await cvePool();
170
+ if (!pool || !keyword.trim())
171
+ return [];
172
+ const res = await pool.query(`SELECT record FROM cves, websearch_to_tsquery('simple', $1) q
173
+ WHERE search @@ q
174
+ ORDER BY ts_rank(search, q) DESC, published DESC NULLS LAST
175
+ LIMIT $2`, [keyword, limit]);
176
+ return res.rows.map((r) => r.record);
177
+ },
178
+ async upsert(records) {
179
+ const pool = await cvePool();
180
+ if (!pool)
181
+ return 0;
182
+ let written = 0;
183
+ for (let i = 0; i < records.length; i += UPSERT_CHUNK) {
184
+ const chunk = records.slice(i, i + UPSERT_CHUNK);
185
+ const res = await pool.query(buildUpsertSql(chunk.length), chunk.flatMap(cveRow));
186
+ written += res.rowCount ?? 0;
187
+ }
188
+ return written;
189
+ },
190
+ };
191
+ let override = null;
192
+ /** Swap the backing store (tests). Pass null to restore Postgres. */
193
+ export function setCveCacheStoreForTests(store) {
194
+ override = store;
195
+ tableState = null;
196
+ }
197
+ function store() {
198
+ return override ?? pgStore;
199
+ }
200
+ function debug(msg) {
201
+ if (process.env.DEBUG_SEARCH_TIMING || process.env.DEBUG_CVE_CACHE)
202
+ console.log(`[cve-cache] ${msg}`);
203
+ }
204
+ /** Cached records for exact ids (CVE or alias such as GHSA). Never throws. */
205
+ export async function getCachedCves(ids) {
206
+ if (ids.length === 0)
207
+ return { records: [], stale: [] };
208
+ const out = await withTimeout(store().get(ids), { records: [], stale: [] });
209
+ debug(`get ${ids.join(",")} -> ${out.records.length} hit, ${out.stale.length} stale`);
210
+ return out;
211
+ }
212
+ /** Ids NVD returned for this keyword within QUERY_TTL_MS, or null. Never throws. */
213
+ export async function getCachedQuery(keyword) {
214
+ const ids = await withTimeout(store().getQuery(keyword), null);
215
+ debug(`query "${keyword}" -> ${ids ? `${ids.length} ids` : "miss"}`);
216
+ return ids;
217
+ }
218
+ export function rememberQuery(keyword, ids) {
219
+ runInBackground(store().putQuery(keyword, ids));
220
+ }
221
+ /** Full-text search over cached rows; the fallback when NVD is down or rate limited. */
222
+ export async function searchCachedCves(keyword, limit) {
223
+ return withTimeout(store().search(keyword, limit), []);
224
+ }
225
+ /** Store records in the window; resolves to rows written (0 on any failure). */
226
+ export async function upsertCves(records) {
227
+ const keep = records.filter(isCacheableCve);
228
+ if (keep.length === 0)
229
+ return 0;
230
+ try {
231
+ const n = await store().upsert(keep);
232
+ debug(`upsert ${keep.length} -> ${n} written`);
233
+ return n;
234
+ }
235
+ catch (err) {
236
+ console.warn("[cve-cache] upsert failed:", err instanceof Error ? err.message : err);
237
+ return 0;
238
+ }
239
+ }
240
+ let vercelWaitUntilPromise = null;
241
+ function getOptionalWaitUntil() {
242
+ if (!vercelWaitUntilPromise) {
243
+ const pkgName = "@vercel/functions";
244
+ vercelWaitUntilPromise = import(/* @vite-ignore */ pkgName)
245
+ .then((m) => (typeof m?.waitUntil === "function" ? m.waitUntil : null))
246
+ .catch(() => null);
247
+ }
248
+ return vercelWaitUntilPromise;
249
+ }
250
+ /** Background write that outlives the response on Vercel (no-op elsewhere). */
251
+ export function runInBackground(p) {
252
+ const safe = p.catch(() => { });
253
+ void getOptionalWaitUntil().then((fn) => {
254
+ try {
255
+ fn?.(safe);
256
+ }
257
+ catch {
258
+ void safe;
259
+ }
260
+ });
261
+ }
262
+ /**
263
+ * Background refresh of stale ids: fetch live, merge over the cached copy (keeps
264
+ * OSV fixed versions the live NVD record lacks), write back.
265
+ */
266
+ export function refreshStaleCves(stale, cached, fetchLive) {
267
+ if (stale.length === 0)
268
+ return;
269
+ runInBackground((async () => {
270
+ const live = await fetchLive(stale);
271
+ if (live.length === 0)
272
+ return;
273
+ const staleSet = new Set(stale);
274
+ const merged = mergeCveRecords([...live, ...cached.filter((c) => staleSet.has(normalizeCveId(c.id)))], stale.length * 2);
275
+ await upsertCves(merged);
276
+ })());
277
+ }
@@ -0,0 +1,57 @@
1
+ export type CveSeverity = "critical" | "high" | "medium" | "low" | "none";
2
+ export interface CveAffected {
3
+ vendor?: string;
4
+ product: string;
5
+ ecosystem?: string;
6
+ /** Human range, e.g. ">=5.6.0 <5.6.2" or "5.6.0, 5.6.1" */
7
+ versions?: string;
8
+ fixed?: string;
9
+ }
10
+ export interface CveRecord {
11
+ id: string;
12
+ aliases?: string[];
13
+ title?: string;
14
+ summary: string;
15
+ severity?: CveSeverity;
16
+ /** 0-10, highest available of CVSS v4.0 / v3.1 / v3.0 / v2.0 */
17
+ cvssScore?: number;
18
+ cvssVector?: string;
19
+ cvssVersion?: "4.0" | "3.1" | "3.0" | "2.0";
20
+ cwe?: string[];
21
+ affected?: CveAffected[];
22
+ published?: string;
23
+ modified?: string;
24
+ /** Listed in CISA's Known Exploited Vulnerabilities catalog */
25
+ kev?: boolean;
26
+ references: string[];
27
+ url: string;
28
+ source: "nvd" | "osv" | "page";
29
+ }
30
+ export declare const SEVERITY_ORDER: Record<CveSeverity, number>;
31
+ /** CVSS v3.0/v3.1 base score from a vector string (FIRST spec). Undefined for v4 or malformed vectors. */
32
+ export declare function cvss3BaseScore(vector: string): number | undefined;
33
+ /** CVSS v3/v4 qualitative rating bands. */
34
+ export declare function severityFromScore(score: number | undefined): CveSeverity | undefined;
35
+ export declare function normalizeCveId(id: string): string;
36
+ /** Unique CVE ids in order of first appearance. */
37
+ export declare function extractCveIdsFromText(text: string, max?: number): string[];
38
+ export declare function extractGhsaIdsFromText(text: string, max?: number): string[];
39
+ export declare function looksLikeVulnId(query: string): boolean;
40
+ export declare const nvdUrl: (id: string) => string;
41
+ type Json = Record<string, any>;
42
+ /** One NVD `vulnerabilities[].cve` object (or the wrapper) -> CveRecord. */
43
+ export declare function parseNvdCve(input: Json): CveRecord | null;
44
+ /** NVD 2.0 response body -> records. */
45
+ export declare function parseNvdResponse(body: Json): CveRecord[];
46
+ /** One OSV vulnerability object -> CveRecord, keyed by its CVE alias when it has one. */
47
+ export declare function parseOsvVuln(v: Json): CveRecord | null;
48
+ /**
49
+ * Merge records for the same vulnerability across sources (matched on id or any alias).
50
+ * NVD wins for scores, CWE and KEV; OSV fills ecosystem packages and fixed versions.
51
+ */
52
+ export declare function mergeCveRecords(records: CveRecord[], max?: number): CveRecord[];
53
+ /** Exact-id hits first, then KEV, then severity, then newest. */
54
+ export declare function sortCveRecords(records: CveRecord[], pinnedIds?: string[]): CveRecord[];
55
+ /** "9.8 Critical" / "High" for cards and MCP text. */
56
+ export declare function cveSeverityLabel(r: Pick<CveRecord, "cvssScore" | "severity">): string | undefined;
57
+ export {};