@thanh01.pmt/research-kit 1.4.0 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/academic/index.cjs +17 -16
  2. package/dist/academic/index.cjs.map +1 -1
  3. package/dist/academic/index.d.cts +1 -1
  4. package/dist/academic/index.d.ts +1 -1
  5. package/dist/academic/index.js +3 -3
  6. package/dist/agent/index.cjs +287 -126
  7. package/dist/agent/index.cjs.map +1 -1
  8. package/dist/agent/index.d.cts +4 -4
  9. package/dist/agent/index.d.ts +4 -4
  10. package/dist/agent/index.js +7 -7
  11. package/dist/{catalogBuilder-D-UiM37y.d.ts → catalogBuilder-BN3VwEsE.d.ts} +15 -2
  12. package/dist/{catalogBuilder-BBCa9jqI.d.cts → catalogBuilder-mba5BIHD.d.cts} +15 -2
  13. package/dist/{chunk-RYLZYZRF.js → chunk-5HSLUAM6.js} +11 -3
  14. package/dist/chunk-5HSLUAM6.js.map +1 -0
  15. package/dist/{chunk-RAMDHDRJ.js → chunk-6QW5TFDJ.js} +70 -342
  16. package/dist/chunk-6QW5TFDJ.js.map +1 -0
  17. package/dist/chunk-BPZLZKVJ.js +1012 -0
  18. package/dist/chunk-BPZLZKVJ.js.map +1 -0
  19. package/dist/{chunk-AKEI7H2C.js → chunk-EWIAA2CC.js} +4 -4
  20. package/dist/{chunk-AKEI7H2C.js.map → chunk-EWIAA2CC.js.map} +1 -1
  21. package/dist/{chunk-QQZQNBJC.js → chunk-JV6Z6WUQ.js} +4 -4
  22. package/dist/{chunk-QQZQNBJC.js.map → chunk-JV6Z6WUQ.js.map} +1 -1
  23. package/dist/{chunk-4O3FXMXO.js → chunk-LTXXARCA.js} +5 -5
  24. package/dist/{chunk-4O3FXMXO.js.map → chunk-LTXXARCA.js.map} +1 -1
  25. package/dist/{chunk-UHKDJZNL.js → chunk-PFECHAAR.js} +3 -3
  26. package/dist/{chunk-UHKDJZNL.js.map → chunk-PFECHAAR.js.map} +1 -1
  27. package/dist/{chunk-MFRQ2VJZ.js → chunk-QLBHTCV6.js} +98 -38
  28. package/dist/chunk-QLBHTCV6.js.map +1 -0
  29. package/dist/{chunk-SDDZFFL6.js → chunk-SB7WYK7A.js} +15 -14
  30. package/dist/chunk-SB7WYK7A.js.map +1 -0
  31. package/dist/{chunk-SHBDLP64.js → chunk-SEHHDO7X.js} +5 -5
  32. package/dist/{chunk-SHBDLP64.js.map → chunk-SEHHDO7X.js.map} +1 -1
  33. package/dist/evidence/index.cjs.map +1 -1
  34. package/dist/evidence/index.js +2 -2
  35. package/dist/harvester/index.cjs +225 -92
  36. package/dist/harvester/index.cjs.map +1 -1
  37. package/dist/harvester/index.d.cts +2 -2
  38. package/dist/harvester/index.d.ts +2 -2
  39. package/dist/harvester/index.js +5 -5
  40. package/dist/{index-CGa7lf3L.d.ts → index-9Z-uMwdI.d.ts} +13 -5
  41. package/dist/{index-DrjfrbGw.d.cts → index-eTcF4XVo.d.cts} +13 -5
  42. package/dist/index.cjs +2578 -2348
  43. package/dist/index.cjs.map +1 -1
  44. package/dist/index.d.cts +5 -5
  45. package/dist/index.d.ts +5 -5
  46. package/dist/index.js +19 -19
  47. package/dist/{literatureReview-Bvlqk4DB.d.cts → literatureReview-BUtPL7bj.d.cts} +1 -0
  48. package/dist/{literatureReview-Bn2iEGRO.d.ts → literatureReview-CHz-JfSR.d.ts} +1 -0
  49. package/dist/pipeline/index.cjs +326 -132
  50. package/dist/pipeline/index.cjs.map +1 -1
  51. package/dist/pipeline/index.d.cts +15 -3
  52. package/dist/pipeline/index.d.ts +15 -3
  53. package/dist/pipeline/index.js +7 -7
  54. package/dist/{socialPulseHarvester-CN4MwrpY.d.cts → socialPulseHarvester-BCFxGwx2.d.cts} +4 -1
  55. package/dist/{socialPulseHarvester-IbdT3zD1.d.ts → socialPulseHarvester-DjMvBufT.d.ts} +4 -1
  56. package/package.json +1 -1
  57. package/dist/chunk-2RMBLKP5.js +0 -581
  58. package/dist/chunk-2RMBLKP5.js.map +0 -1
  59. package/dist/chunk-MFRQ2VJZ.js.map +0 -1
  60. package/dist/chunk-RAMDHDRJ.js.map +0 -1
  61. package/dist/chunk-RYLZYZRF.js.map +0 -1
  62. package/dist/chunk-SDDZFFL6.js.map +0 -1
@@ -0,0 +1,1012 @@
1
+ import { AcademicSearchEngine } from './chunk-EWIAA2CC.js';
2
+
3
+ // src/harvester/domainClassifier.ts
4
+ var HARDWARE_PATTERNS = [
5
+ /\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
6
+ /\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
7
+ /(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
8
+ ];
9
+ var PLATFORM_PATTERNS = [
10
+ /\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
11
+ /(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
12
+ ];
13
+ var ACADEMIC_PATTERNS = [
14
+ /\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
15
+ /(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
16
+ ];
17
+ function inferPrimaryCodeLanguage(topic, context, domain) {
18
+ const combined = [
19
+ topic,
20
+ context?.courseTitle,
21
+ context?.hardwarePlatform,
22
+ context?.softwareEnv,
23
+ ...context?.inScopeTopics || []
24
+ ].filter(Boolean).join(" ").toLowerCase();
25
+ if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
26
+ if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
27
+ if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
28
+ if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
29
+ if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
30
+ if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
31
+ if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
32
+ if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
33
+ if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
34
+ if (/\b(rust|cargo)\b/.test(combined)) return "rust";
35
+ if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
36
+ if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
37
+ if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
38
+ if (domain === "hardware") return "cpp";
39
+ return "text";
40
+ }
41
+ function classifyTechnicalDomain(topic, explicitDomain, context) {
42
+ if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
43
+ const domain = explicitDomain.toLowerCase();
44
+ return {
45
+ domain,
46
+ confidence: 1,
47
+ rationale: "Explicitly configured by user flag.",
48
+ recommendedExtractors: getExtractorsForDomain(domain),
49
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
50
+ };
51
+ }
52
+ const combined = [
53
+ topic,
54
+ context?.courseTitle,
55
+ context?.positioning,
56
+ context?.hardwarePlatform,
57
+ context?.softwareEnv,
58
+ ...context?.inScopeTopics || []
59
+ ].filter(Boolean).join(" ");
60
+ if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
61
+ return {
62
+ domain: "hardware",
63
+ confidence: 0.95,
64
+ rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
65
+ recommendedExtractors: getExtractorsForDomain("hardware"),
66
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
67
+ };
68
+ }
69
+ if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
70
+ return {
71
+ domain: "platform",
72
+ confidence: 0.9,
73
+ rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
74
+ recommendedExtractors: getExtractorsForDomain("platform"),
75
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
76
+ };
77
+ }
78
+ if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
79
+ return {
80
+ domain: "academic",
81
+ confidence: 0.9,
82
+ rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
83
+ recommendedExtractors: getExtractorsForDomain("academic"),
84
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
85
+ };
86
+ }
87
+ return {
88
+ domain: "software",
89
+ confidence: 0.85,
90
+ rationale: "Topic is a programming language, game engine, client/server library, or software application.",
91
+ recommendedExtractors: getExtractorsForDomain("software"),
92
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
93
+ };
94
+ }
95
+ function getExtractorsForDomain(domain) {
96
+ switch (domain) {
97
+ case "hardware":
98
+ return ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"];
99
+ case "platform":
100
+ return ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"];
101
+ case "academic":
102
+ return ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"];
103
+ case "software":
104
+ default:
105
+ return ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"];
106
+ }
107
+ }
108
+
109
+ // src/harvester/multiSourceSearch.ts
110
+ function isUrlAllowedByDomainFilter(url, domainFilter) {
111
+ if (!domainFilter || domainFilter.length === 0) return true;
112
+ try {
113
+ const hostname = new URL(url).hostname.toLowerCase();
114
+ return domainFilter.some((d) => {
115
+ const cleanD = d.toLowerCase().trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
116
+ return hostname === cleanD || hostname.endsWith(`.${cleanD}`);
117
+ });
118
+ } catch {
119
+ return false;
120
+ }
121
+ }
122
+ var MultiSourceSearchEngine = class {
123
+ tavilyKey;
124
+ exaKey;
125
+ serperKey;
126
+ brightDataKey;
127
+ brightDataZone;
128
+ searxngHost;
129
+ searxngHeaders;
130
+ allowEnvFallback;
131
+ customAdapter;
132
+ academicEngine;
133
+ lastErrors = [];
134
+ getLastErrors() {
135
+ return [...this.lastErrors];
136
+ }
137
+ recordError(engine, error, status, traceErrors) {
138
+ const errObj = {
139
+ engine,
140
+ error,
141
+ status,
142
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
143
+ };
144
+ this.lastErrors.push(errObj);
145
+ traceErrors?.push(errObj);
146
+ }
147
+ constructor(options = {}) {
148
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
149
+ this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
150
+ this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
151
+ this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
152
+ this.brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || (this.allowEnvFallback ? process.env.BRIGHTDATA_API_KEY || process.env.BRIGHT_DATA_KEY : void 0);
153
+ this.brightDataZone = options.brightDataZone || (this.allowEnvFallback ? process.env.BRIGHTDATA_SERP_ZONE || process.env.BRIGHTDATA_UNLOCKER_ZONE : void 0) || "cli_unlocker";
154
+ this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
155
+ const cfId = process.env.CF_ACCESS_CLIENT_ID;
156
+ const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
157
+ this.searxngHeaders = {
158
+ ...options.searxngHeaders || {},
159
+ ...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
160
+ };
161
+ this.customAdapter = options.customAdapter;
162
+ this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
163
+ }
164
+ setCustomAdapter(adapter) {
165
+ this.customAdapter = adapter;
166
+ }
167
+ setApiKeys(keys) {
168
+ if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
169
+ if (keys.exa !== void 0) this.exaKey = keys.exa;
170
+ if (keys.serper !== void 0) this.serperKey = keys.serper;
171
+ if (keys.brightdata !== void 0) this.brightDataKey = keys.brightdata;
172
+ if (keys.brightData !== void 0) this.brightDataKey = keys.brightData;
173
+ }
174
+ setBrightData(key, zone) {
175
+ if (key !== void 0) this.brightDataKey = key;
176
+ if (zone !== void 0) this.brightDataZone = zone;
177
+ }
178
+ setSearxng(host, headers) {
179
+ this.searxngHost = host;
180
+ if (headers) {
181
+ this.searxngHeaders = { ...this.searxngHeaders, ...headers };
182
+ }
183
+ }
184
+ async search(query, options = {}) {
185
+ const limit = options.limit ?? 5;
186
+ const adapter = options.customAdapter || this.customAdapter;
187
+ const preferred = options.preferredEngine;
188
+ if (adapter && (!preferred || preferred === "custom")) {
189
+ try {
190
+ const customResults = await adapter.search(query, { limit, domainFilter: options.domainFilter });
191
+ if (Array.isArray(customResults)) {
192
+ return customResults.map((r) => ({
193
+ title: r.title,
194
+ url: r.url,
195
+ content: r.content,
196
+ score: r.score,
197
+ engine: "custom"
198
+ }));
199
+ }
200
+ } catch (e) {
201
+ console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
202
+ }
203
+ }
204
+ if (preferred === "academic") {
205
+ try {
206
+ const papers = await this.academicEngine.searchPapers(query, { limit });
207
+ if (papers.length > 0) {
208
+ return papers.map((p) => ({
209
+ title: p.title,
210
+ url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
211
+ content: p.tldr || p.abstract || "",
212
+ score: p.citationCount,
213
+ engine: "academic"
214
+ }));
215
+ }
216
+ } catch (e) {
217
+ console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
218
+ }
219
+ }
220
+ const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
221
+ const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
222
+ const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
223
+ const brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || this.brightDataKey;
224
+ const brightDataZone = options.brightDataZone || this.brightDataZone;
225
+ const searxngHost = options.searxngHost || this.searxngHost;
226
+ if (preferred === "searxng" && searxngHost) {
227
+ const searxHits = await this.querySearxng(
228
+ query,
229
+ searxngHost,
230
+ limit,
231
+ options.searxngHeaders,
232
+ options.timeRange,
233
+ options.domainFilter
234
+ );
235
+ if (searxHits.length > 0) return searxHits;
236
+ }
237
+ if (preferred === "brightdata" && brightDataKey) {
238
+ const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
239
+ zone: brightDataZone,
240
+ engine: options.brightDataEngine,
241
+ domainFilter: options.domainFilter
242
+ });
243
+ if (bdHits.length > 0) return bdHits;
244
+ }
245
+ if (preferred === "exa" && exaKey) {
246
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
247
+ if (exaHits.length > 0) return exaHits;
248
+ }
249
+ if (preferred === "serper" && serperKey) {
250
+ const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
251
+ if (serperHits.length > 0) return serperHits;
252
+ }
253
+ if (preferred === "tavily" && tavilyKey) {
254
+ const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
255
+ if (tavilyHits.length > 0) return tavilyHits;
256
+ }
257
+ if (preferred === "duckduckgo") {
258
+ const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
259
+ if (ddgHits.length > 0) return ddgHits;
260
+ }
261
+ if (tavilyKey && preferred !== "tavily") {
262
+ const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
263
+ if (tavilyHits.length > 0) return tavilyHits;
264
+ }
265
+ if (exaKey && preferred !== "exa") {
266
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
267
+ if (exaHits.length > 0) return exaHits;
268
+ }
269
+ if (serperKey && preferred !== "serper") {
270
+ const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
271
+ if (serperHits.length > 0) return serperHits;
272
+ }
273
+ if (brightDataKey && preferred !== "brightdata") {
274
+ const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
275
+ zone: brightDataZone,
276
+ engine: options.brightDataEngine,
277
+ domainFilter: options.domainFilter
278
+ });
279
+ if (bdHits.length > 0) return bdHits;
280
+ }
281
+ if (searxngHost && preferred !== "searxng") {
282
+ const searxHits = await this.querySearxng(
283
+ query,
284
+ searxngHost,
285
+ limit,
286
+ options.searxngHeaders,
287
+ options.timeRange,
288
+ options.domainFilter
289
+ );
290
+ if (searxHits.length > 0) return searxHits;
291
+ }
292
+ if (preferred !== "duckduckgo") {
293
+ const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
294
+ if (ddgHits.length > 0) return ddgHits;
295
+ }
296
+ return [];
297
+ }
298
+ async queryTavily(query, tavilyKey, limit, timeRange, domainFilter) {
299
+ try {
300
+ const body = {
301
+ api_key: tavilyKey,
302
+ query,
303
+ search_depth: "advanced",
304
+ include_answer: true,
305
+ max_results: limit
306
+ };
307
+ if (timeRange) {
308
+ body.time_range = timeRange;
309
+ }
310
+ if (domainFilter && domainFilter.length > 0) {
311
+ body.include_domains = domainFilter;
312
+ }
313
+ const res = await fetch("https://api.tavily.com/search", {
314
+ method: "POST",
315
+ headers: { "Content-Type": "application/json" },
316
+ body: JSON.stringify(body),
317
+ signal: AbortSignal.timeout(12e3)
318
+ });
319
+ if (res.ok) {
320
+ const data = await res.json();
321
+ if (Array.isArray(data?.results) && data.results.length > 0) {
322
+ return data.results.map((r) => ({
323
+ title: r.title || "Untitled",
324
+ url: r.url,
325
+ content: r.content || "",
326
+ score: r.score,
327
+ engine: "tavily"
328
+ }));
329
+ }
330
+ }
331
+ } catch (e) {
332
+ console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
333
+ }
334
+ return [];
335
+ }
336
+ async queryBrightData(query, key, limit = 5, options = {}) {
337
+ try {
338
+ const zone = options.zone || this.brightDataZone || "cli_unlocker";
339
+ const engine = options.engine || "bing";
340
+ let effectiveQuery = query;
341
+ if (options.domainFilter && options.domainFilter.length === 1) {
342
+ effectiveQuery = `${query} site:${options.domainFilter[0]}`;
343
+ }
344
+ let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
345
+ if (engine === "google") {
346
+ searchUrl = `https://www.google.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1&hl=en&gl=us`;
347
+ } else if (engine === "yandex") {
348
+ searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
349
+ }
350
+ const res = await fetch("https://api.brightdata.com/request", {
351
+ method: "POST",
352
+ headers: {
353
+ Authorization: `Bearer ${key}`,
354
+ "Content-Type": "application/json"
355
+ },
356
+ body: JSON.stringify({
357
+ zone,
358
+ url: searchUrl,
359
+ format: "raw"
360
+ }),
361
+ signal: AbortSignal.timeout(15e3)
362
+ });
363
+ if (res.ok) {
364
+ let data;
365
+ try {
366
+ if (typeof res.json === "function") {
367
+ data = await res.json();
368
+ } else if (typeof res.text === "function") {
369
+ const rawText = await res.text();
370
+ data = JSON.parse(rawText);
371
+ }
372
+ } catch {
373
+ data = null;
374
+ }
375
+ const hits = [];
376
+ if (data && Array.isArray(data?.organic)) {
377
+ for (let idx = 0; idx < data.organic.length; idx++) {
378
+ const item = data.organic[idx];
379
+ const hitUrl = item.link || item.url;
380
+ if (!hitUrl || isUrlAllowedByDomainFilter(hitUrl, options.domainFilter)) {
381
+ hits.push({
382
+ title: item.title || "Untitled",
383
+ url: hitUrl,
384
+ content: item.description || item.snippet || "",
385
+ score: item.rank ? 1 / item.rank : 1 / (idx + 1),
386
+ engine: "brightdata"
387
+ });
388
+ }
389
+ }
390
+ }
391
+ if (hits.length > 0) return hits.slice(0, limit);
392
+ }
393
+ } catch (e) {
394
+ console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
395
+ }
396
+ return [];
397
+ }
398
+ async querySearxng(query, searxngHost, limit, customHeaders, timeRange, domainFilter) {
399
+ try {
400
+ const cleanHost = searxngHost.replace(/\/$/, "");
401
+ let effectiveQuery = query;
402
+ if (domainFilter && domainFilter.length === 1) {
403
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
404
+ }
405
+ let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(effectiveQuery)}&format=json`;
406
+ if (timeRange) {
407
+ searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
408
+ }
409
+ const headers = {
410
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
411
+ ...this.searxngHeaders || {},
412
+ ...customHeaders || {}
413
+ };
414
+ const res = await fetch(searchUrl, {
415
+ headers,
416
+ signal: AbortSignal.timeout(1e4)
417
+ });
418
+ if (res.ok) {
419
+ const data = await res.json();
420
+ if (Array.isArray(data?.results) && data.results.length > 0) {
421
+ return data.results.filter((r) => !r.url || isUrlAllowedByDomainFilter(r.url, domainFilter)).slice(0, limit).map((r) => ({
422
+ title: r.title || "Untitled",
423
+ url: r.url,
424
+ content: r.content || r.snippet || "",
425
+ score: r.score,
426
+ engine: "searxng"
427
+ }));
428
+ }
429
+ }
430
+ } catch (e) {
431
+ console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
432
+ }
433
+ return [];
434
+ }
435
+ async queryExa(query, exaKey, limit, domainFilter, timeRange) {
436
+ try {
437
+ const body = {
438
+ query,
439
+ numResults: limit,
440
+ useAutoprompt: true
441
+ };
442
+ if (domainFilter && domainFilter.length > 0) {
443
+ body.includeDomains = domainFilter;
444
+ }
445
+ if (timeRange) {
446
+ const now = /* @__PURE__ */ new Date();
447
+ if (timeRange === "day") now.setDate(now.getDate() - 1);
448
+ else if (timeRange === "week") now.setDate(now.getDate() - 7);
449
+ else if (timeRange === "month") now.setMonth(now.getMonth() - 1);
450
+ else if (timeRange === "year") now.setFullYear(now.getFullYear() - 1);
451
+ body.startPublishedDate = now.toISOString();
452
+ }
453
+ const res = await fetch("https://api.exa.ai/search", {
454
+ method: "POST",
455
+ headers: {
456
+ "x-api-key": exaKey,
457
+ "Content-Type": "application/json"
458
+ },
459
+ body: JSON.stringify(body),
460
+ signal: AbortSignal.timeout(1e4)
461
+ });
462
+ if (res.ok) {
463
+ const data = await res.json();
464
+ if (Array.isArray(data?.results) && data.results.length > 0) {
465
+ return data.results.map((r) => ({
466
+ title: r.title || "Untitled",
467
+ url: r.url,
468
+ content: r.text || r.snippet || "",
469
+ score: r.score,
470
+ engine: "exa"
471
+ }));
472
+ }
473
+ }
474
+ } catch (e) {
475
+ console.warn("[MultiSourceSearchEngine] Exa attempt failed:", e?.message || e);
476
+ }
477
+ return [];
478
+ }
479
+ async querySerper(query, serperKey, limit, timeRange, domainFilter) {
480
+ try {
481
+ let effectiveQuery = query;
482
+ if (domainFilter && domainFilter.length === 1) {
483
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
484
+ }
485
+ const body = { q: effectiveQuery, num: limit };
486
+ if (timeRange) {
487
+ const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
488
+ if (tbsMap[timeRange]) {
489
+ body.tbs = tbsMap[timeRange];
490
+ }
491
+ }
492
+ const res = await fetch("https://google.serper.dev/search", {
493
+ method: "POST",
494
+ headers: {
495
+ "X-API-KEY": serperKey,
496
+ "Content-Type": "application/json"
497
+ },
498
+ body: JSON.stringify(body),
499
+ signal: AbortSignal.timeout(1e4)
500
+ });
501
+ if (res.ok) {
502
+ const data = await res.json();
503
+ if (Array.isArray(data?.organic) && data.organic.length > 0) {
504
+ return data.organic.filter((r) => !r.link || isUrlAllowedByDomainFilter(r.link, domainFilter)).slice(0, limit).map((r) => ({
505
+ title: r.title || "Untitled",
506
+ url: r.link,
507
+ content: r.snippet || "",
508
+ engine: "serper"
509
+ }));
510
+ }
511
+ }
512
+ } catch (e) {
513
+ console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
514
+ }
515
+ return [];
516
+ }
517
+ async queryDuckDuckGo(query, limit, domainFilter) {
518
+ try {
519
+ let effectiveQuery = query;
520
+ if (domainFilter && domainFilter.length === 1) {
521
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
522
+ }
523
+ const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(effectiveQuery)}`;
524
+ const res = await fetch(ddgUrl, {
525
+ headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
526
+ signal: AbortSignal.timeout(8e3)
527
+ });
528
+ if (res.ok) {
529
+ const html = await res.text();
530
+ const hits = [];
531
+ const resultBlocks = html.split(/class="[^"]*result__body/i);
532
+ for (const block of resultBlocks.slice(1)) {
533
+ if (hits.length >= limit) break;
534
+ const titleMatch = block.match(/<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
535
+ if (!titleMatch) continue;
536
+ const rawHref = titleMatch[1]?.trim() || "";
537
+ const cleanUrl = rawHref.startsWith("//duckduckgo.com/l/?uddg=") || rawHref.includes("uddg=") ? decodeURIComponent(rawHref.split("uddg=")[1]?.split("&")[0] || "") : rawHref;
538
+ const rawTitle = titleMatch[2]?.replace(/<[^>]+>/g, "").trim() || query;
539
+ const snippetMatch = block.match(/<(?:a|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div)>/i);
540
+ const snippet = snippetMatch ? snippetMatch[1]?.replace(/<[^>]+>/g, "").trim() || "" : "";
541
+ if (cleanUrl && !cleanUrl.includes("duckduckgo.com") && cleanUrl.startsWith("http")) {
542
+ if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
543
+ hits.push({
544
+ title: rawTitle,
545
+ url: cleanUrl,
546
+ content: snippet,
547
+ engine: "duckduckgo"
548
+ });
549
+ }
550
+ }
551
+ }
552
+ if (hits.length === 0) {
553
+ const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
554
+ let match;
555
+ while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
556
+ const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
557
+ const rawUrl = match[2]?.trim();
558
+ const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
559
+ const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
560
+ if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
561
+ if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
562
+ hits.push({
563
+ title: rawTitle,
564
+ url: cleanUrl,
565
+ content: snippet,
566
+ engine: "duckduckgo"
567
+ });
568
+ }
569
+ }
570
+ }
571
+ }
572
+ if (hits.length > 0) return hits;
573
+ }
574
+ } catch (e) {
575
+ console.warn("[MultiSourceSearchEngine] DuckDuckGo attempt failed:", e?.message || e);
576
+ }
577
+ return [];
578
+ }
579
+ };
580
+
581
+ // src/harvester/noiseFilter.ts
582
+ var ECOMMERCE_DOMAINS = /* @__PURE__ */ new Set([
583
+ "shopee.vn",
584
+ "shopee.com",
585
+ "lazada.vn",
586
+ "lazada.com",
587
+ "tiki.vn",
588
+ "sendo.vn",
589
+ "aliexpress.com",
590
+ "taobao.com",
591
+ "amazon.com",
592
+ "ebay.com",
593
+ "banggood.com"
594
+ ]);
595
+ var ECOMMERCE_AND_COMMERCIAL_PATTERNS = [
596
+ /\b(shopee|lazada|tiki|sendo|taobao|aliexpress)\b/i,
597
+ /\b(mua ngay|giá tốt|khuyến mãi|freeship|bán chạy|nhập khẩu chính hãng|cửa hàng|bảng giá|đặt hàng)\b/i,
598
+ /\b(buy now|add to cart|free shipping|discount|best price|on sale|shop now|in stock|order now)\b/i,
599
+ /\b(\d+[kKđ]|₫|\$\d+|\d+\s*(?:VNĐ|VND|usd))\b/i
600
+ ];
601
+ function isCommercialOrNoise(input) {
602
+ if (input.url) {
603
+ try {
604
+ const hostname = new URL(input.url).hostname.toLowerCase().replace(/^www\./, "");
605
+ if (ECOMMERCE_DOMAINS.has(hostname) || [...ECOMMERCE_DOMAINS].some((d) => hostname.endsWith(`.${d}`))) {
606
+ return true;
607
+ }
608
+ } catch {
609
+ }
610
+ }
611
+ const combined = `${input.title || ""} ${input.snippet || ""}`;
612
+ if (!combined.trim()) return false;
613
+ for (const pattern of ECOMMERCE_AND_COMMERCIAL_PATTERNS) {
614
+ if (pattern.test(combined)) {
615
+ return true;
616
+ }
617
+ }
618
+ return false;
619
+ }
620
+ function evaluateTechnicalDensity(markdown) {
621
+ if (!markdown || markdown.trim().length < 80) {
622
+ return {
623
+ score: 1,
624
+ isSufficientQuality: false,
625
+ codeDensity: 0,
626
+ technicalTermDensity: 0,
627
+ hasStructure: false,
628
+ reasons: ["Content is too short or empty."]
629
+ };
630
+ }
631
+ const reasons = [];
632
+ const totalLength = markdown.length;
633
+ const codeBlocks = markdown.match(/```[\s\S]*?```/g) || [];
634
+ const inlineCode = markdown.match(/`[^`\n]+`/g) || [];
635
+ const totalCodeChars = codeBlocks.reduce((sum, b) => sum + b.length, 0) + inlineCode.reduce((sum, c) => sum + c.length, 0);
636
+ const codeDensity = Math.min(1, totalCodeChars / Math.max(1, totalLength));
637
+ const headings = markdown.match(/^#{1,4}\s+.+$/gm) || [];
638
+ const hasStructure = headings.length >= 2;
639
+ const techKeywords = markdown.match(
640
+ /\b(function|class|method|struct|interface|api|hardware|sensor|pin|circuit|logic|loop|state|algorithm|test|debug|schema|protocol|variable)\b/gi
641
+ ) || [];
642
+ const technicalTermDensity = Math.min(1, techKeywords.length * 10 / Math.max(100, markdown.split(/\s+/).length));
643
+ let score = 5;
644
+ if (codeDensity > 0.15) {
645
+ score += 2;
646
+ } else if (codeDensity > 0.05) {
647
+ score += 1;
648
+ } else {
649
+ reasons.push("Low code or API snippet density.");
650
+ }
651
+ if (hasStructure) {
652
+ score += 1.5;
653
+ } else {
654
+ reasons.push("Lacks clear hierarchical markdown headings.");
655
+ }
656
+ if (technicalTermDensity > 0.08) {
657
+ score += 1.5;
658
+ }
659
+ const finalScore = Math.min(10, Math.max(1, Math.round(score * 10) / 10));
660
+ return {
661
+ score: finalScore,
662
+ isSufficientQuality: finalScore >= 6,
663
+ codeDensity,
664
+ technicalTermDensity,
665
+ hasStructure,
666
+ reasons
667
+ };
668
+ }
669
+
670
+ // src/agent/evidenceLedger.ts
671
+ function normalizeUrlKey(rawUrl) {
672
+ try {
673
+ const u = new URL(rawUrl.trim());
674
+ u.hash = "";
675
+ for (const param of ["utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "ref"]) {
676
+ u.searchParams.delete(param);
677
+ }
678
+ return u.toString().replace(/\/$/, "").toLowerCase();
679
+ } catch {
680
+ return rawUrl.trim().replace(/\/$/, "").toLowerCase();
681
+ }
682
+ }
683
+ function computeCanonicalId(url) {
684
+ try {
685
+ let hash = 0;
686
+ const clean = url.trim().replace(/^https?:\/\//i, "").replace(/[\/?#].*$/, "").toLowerCase();
687
+ for (let i = 0; i < clean.length; i++) {
688
+ hash = hash * 31 + clean.charCodeAt(i) >>> 0;
689
+ }
690
+ return `SRC-${hash.toString(16).padStart(8, "0")}`;
691
+ } catch {
692
+ return `SRC-${Math.random().toString(36).slice(2, 10)}`;
693
+ }
694
+ }
695
+ var EvidenceLedger = class {
696
+ items = [];
697
+ urlIndex = /* @__PURE__ */ new Map();
698
+ idIndex = /* @__PURE__ */ new Map();
699
+ allocateId() {
700
+ const nextNum = this.items.length + 1;
701
+ return `RES-${String(nextNum).padStart(2, "0")}`;
702
+ }
703
+ /**
704
+ * Records or upgrades a ledger entry from a full `ScrapedDocument`.
705
+ */
706
+ addScrapedDocument(doc, relevanceScore, extraTakeaways = []) {
707
+ const key = normalizeUrlKey(doc.url);
708
+ const existing = this.urlIndex.get(key);
709
+ const mergedExcerpts = Array.from(
710
+ new Set([
711
+ ...extraTakeaways,
712
+ ...doc.keyExcerpts || [],
713
+ doc.markdownContent.replace(/```[\s\S]*?```/g, " ").replace(/\s+/g, " ").trim().slice(0, 450)
714
+ ].filter((e) => e && e.trim().length > 20))
715
+ ).slice(0, 6);
716
+ const sourceType = doc.fetchProvider === "youtube" ? "youtube" : "web";
717
+ if (existing) {
718
+ existing.title = doc.title || existing.title;
719
+ existing.relevanceScore = Math.max(existing.relevanceScore, relevanceScore);
720
+ existing.headings = doc.extractedHeadings?.slice(0, 8) || existing.headings;
721
+ existing.keyExcerpts = mergedExcerpts.length > 0 ? mergedExcerpts : existing.keyExcerpts;
722
+ existing.codeBlocks = (doc.extractedCodeBlocks || []).slice(0, 4);
723
+ existing.tables = (doc.extractedTables || []).slice(0, 4);
724
+ existing.rawByteSize = Math.max(existing.rawByteSize, doc.byteSize || 0);
725
+ return this.toCompactReceipt(existing);
726
+ }
727
+ const item = {
728
+ id: this.allocateId(),
729
+ canonicalId: computeCanonicalId(doc.url),
730
+ sourceType,
731
+ title: doc.title || "Untitled Document",
732
+ url: doc.url,
733
+ relevanceScore: Math.max(1, Math.min(10, Math.round(relevanceScore))),
734
+ headings: (doc.extractedHeadings || []).slice(0, 8),
735
+ keyExcerpts: mergedExcerpts,
736
+ codeBlocks: (doc.extractedCodeBlocks || []).slice(0, 4),
737
+ tables: (doc.extractedTables || []).slice(0, 4),
738
+ rawByteSize: doc.byteSize || Buffer.byteLength(doc.markdownContent || "", "utf-8"),
739
+ accessedAt: (/* @__PURE__ */ new Date()).toISOString()
740
+ };
741
+ this.items.push(item);
742
+ this.urlIndex.set(key, item);
743
+ this.idIndex.set(item.id, item);
744
+ return this.toCompactReceipt(item);
745
+ }
746
+ /**
747
+ * Records a lightweight search hit when full scraping is skipped or pending.
748
+ */
749
+ addSearchHit(hit, relevanceScore = 6) {
750
+ const key = normalizeUrlKey(hit.url);
751
+ const existing = this.urlIndex.get(key);
752
+ if (existing) {
753
+ return this.toCompactReceipt(existing);
754
+ }
755
+ const snippet = (hit.content || "").replace(/\s+/g, " ").trim();
756
+ const item = {
757
+ id: this.allocateId(),
758
+ canonicalId: computeCanonicalId(hit.url),
759
+ sourceType: hit.engine === "academic" ? "academic" : "web",
760
+ title: hit.title || "Search Result",
761
+ url: hit.url,
762
+ relevanceScore: Math.max(1, Math.min(10, Math.round(relevanceScore))),
763
+ headings: [],
764
+ keyExcerpts: snippet ? [snippet.slice(0, 450)] : [],
765
+ codeBlocks: [],
766
+ tables: [],
767
+ rawByteSize: Buffer.byteLength(snippet, "utf-8"),
768
+ accessedAt: (/* @__PURE__ */ new Date()).toISOString()
769
+ };
770
+ this.items.push(item);
771
+ this.urlIndex.set(key, item);
772
+ this.idIndex.set(item.id, item);
773
+ return this.toCompactReceipt(item);
774
+ }
775
+ /**
776
+ * Records a peer-reviewed or preprint paper from Semantic Scholar, OpenAlex, Crossref, or arXiv.
777
+ */
778
+ addAcademicPaper(paper) {
779
+ const url = paper.url || paper.pdfUrl || (paper.doi ? `https://doi.org/${paper.doi}` : `https://openalex.org/${paper.id}`);
780
+ const key = normalizeUrlKey(url);
781
+ const existing = this.urlIndex.get(key);
782
+ if (existing) {
783
+ return this.toCompactReceipt(existing);
784
+ }
785
+ const abstractOrTldr = [paper.tldr, paper.abstract].filter(Boolean).join(" \u2014 ").trim();
786
+ const citationBonus = paper.citationCount >= 50 ? 9 : paper.citationCount >= 10 ? 8 : 7;
787
+ const item = {
788
+ id: this.allocateId(),
789
+ canonicalId: computeCanonicalId(url),
790
+ sourceType: "academic",
791
+ title: paper.title,
792
+ url,
793
+ authorOrVenue: [paper.authors?.slice(0, 3).join(", "), paper.venue, paper.year].filter(Boolean).join(" \xB7 "),
794
+ publishedDate: paper.year ? String(paper.year) : void 0,
795
+ relevanceScore: citationBonus,
796
+ headings: paper.fieldsOfStudy || [],
797
+ keyExcerpts: abstractOrTldr ? [abstractOrTldr.slice(0, 600)] : [`Academic paper: ${paper.title}`],
798
+ codeBlocks: [],
799
+ tables: [],
800
+ rawByteSize: Buffer.byteLength(abstractOrTldr || paper.title, "utf-8"),
801
+ accessedAt: (/* @__PURE__ */ new Date()).toISOString()
802
+ };
803
+ this.items.push(item);
804
+ this.urlIndex.set(key, item);
805
+ this.idIndex.set(item.id, item);
806
+ return this.toCompactReceipt(item);
807
+ }
808
+ /**
809
+ * Records a 30-day social/developer pulse signal (HackerNews, GitHub, Reddit).
810
+ */
811
+ addSocialPulseItem(pulse) {
812
+ const key = normalizeUrlKey(pulse.url);
813
+ const existing = this.urlIndex.get(key);
814
+ if (existing) {
815
+ return this.toCompactReceipt(existing);
816
+ }
817
+ const item = {
818
+ id: this.allocateId(),
819
+ canonicalId: computeCanonicalId(pulse.url),
820
+ sourceType: "social_pulse",
821
+ title: pulse.title,
822
+ url: pulse.url,
823
+ authorOrVenue: `${pulse.platform.toUpperCase()}${pulse.author ? ` (@${pulse.author})` : ""}`,
824
+ publishedDate: pulse.createdAt,
825
+ relevanceScore: Math.max(1, Math.min(10, pulse.engagementScore)),
826
+ headings: [pulse.platform.toUpperCase()],
827
+ keyExcerpts: [pulse.snippet.slice(0, 450)],
828
+ codeBlocks: [],
829
+ tables: [],
830
+ rawByteSize: Buffer.byteLength(pulse.snippet, "utf-8"),
831
+ accessedAt: (/* @__PURE__ */ new Date()).toISOString()
832
+ };
833
+ this.items.push(item);
834
+ this.urlIndex.set(key, item);
835
+ this.idIndex.set(item.id, item);
836
+ return this.toCompactReceipt(item);
837
+ }
838
+ /**
839
+ * Records a verified STEM OER simulation/resource (PhET, OpenStax, NASA, CS Unplugged).
840
+ */
841
+ addStemResource(resource) {
842
+ const key = normalizeUrlKey(resource.url);
843
+ const existing = this.urlIndex.get(key);
844
+ if (existing) {
845
+ return this.toCompactReceipt(existing);
846
+ }
847
+ const excerpt = `${resource.description} [Provider: ${resource.provider}${resource.license ? `, License: ${resource.license}` : ""}]`;
848
+ const item = {
849
+ id: this.allocateId(),
850
+ canonicalId: computeCanonicalId(resource.url),
851
+ sourceType: "stem_oer",
852
+ title: resource.title,
853
+ url: resource.url,
854
+ authorOrVenue: resource.provider,
855
+ relevanceScore: 8,
856
+ headings: [resource.provider],
857
+ keyExcerpts: [excerpt.slice(0, 450)],
858
+ codeBlocks: [],
859
+ tables: [],
860
+ rawByteSize: Buffer.byteLength(excerpt, "utf-8"),
861
+ accessedAt: (/* @__PURE__ */ new Date()).toISOString()
862
+ };
863
+ this.items.push(item);
864
+ this.urlIndex.set(key, item);
865
+ this.idIndex.set(item.id, item);
866
+ return this.toCompactReceipt(item);
867
+ }
868
+ /**
869
+ * Projects a full `LedgerItem` into a compact receipt (<= 150 tokens) for Stage A planning & gap analysis.
870
+ */
871
+ toCompactReceipt(item) {
872
+ const rawExcerpt = item.keyExcerpts[0] || item.title;
873
+ return {
874
+ id: item.id,
875
+ canonicalId: item.canonicalId || computeCanonicalId(item.url),
876
+ sourceType: item.sourceType,
877
+ title: item.title.slice(0, 140),
878
+ url: item.url,
879
+ relevanceScore: item.relevanceScore,
880
+ headings: item.headings.slice(0, 5),
881
+ codeBlockCount: item.codeBlocks.length,
882
+ tableCount: item.tables.length,
883
+ briefExcerpt: rawExcerpt.slice(0, 450)
884
+ };
885
+ }
886
+ getItems() {
887
+ return [...this.items];
888
+ }
889
+ getApprovedItems(minScore = 6) {
890
+ return this.items.filter((i) => i.relevanceScore >= minScore);
891
+ }
892
+ getReceipts() {
893
+ return this.items.map((i) => this.toCompactReceipt(i));
894
+ }
895
+ getById(id) {
896
+ return this.idIndex.get(id.toUpperCase());
897
+ }
898
+ getTotalCodeBlocks() {
899
+ return this.items.reduce((sum, i) => sum + i.codeBlocks.length, 0);
900
+ }
901
+ getTotalTables() {
902
+ return this.items.reduce((sum, i) => sum + i.tables.length, 0);
903
+ }
904
+ /**
905
+ * Formats the top verified items with `[RES-xx]` tags, substantive excerpts, code blocks,
906
+ * and tables for Stage B Contract Synthesis.
907
+ */
908
+ buildSynthesisCorpus(maxItems = 12) {
909
+ const sorted = [...this.items].sort((a, b) => b.relevanceScore - a.relevanceScore).slice(0, maxItems);
910
+ if (sorted.length === 0) {
911
+ return "No verified sources recorded in EvidenceLedger.";
912
+ }
913
+ return sorted.map((item) => {
914
+ const parts = [
915
+ `[${item.id}] (${item.sourceType.toUpperCase()} \xB7 Score ${item.relevanceScore}/10) Title: ${item.title}`,
916
+ `URL: ${item.url}`
917
+ ];
918
+ if (item.authorOrVenue) {
919
+ parts.push(`Source/Author: ${item.authorOrVenue}`);
920
+ }
921
+ if (item.headings.length > 0) {
922
+ parts.push(`Sections: ${item.headings.join(" | ")}`);
923
+ }
924
+ if (item.keyExcerpts.length > 0) {
925
+ parts.push(`Verified Excerpts:
926
+ ${item.keyExcerpts.map((e) => `- ${e}`).join("\n")}`);
927
+ }
928
+ if (item.codeBlocks.length > 0) {
929
+ parts.push(
930
+ `Extracted Code Blocks:
931
+ ` + item.codeBlocks.slice(0, 2).map((cb) => `\`\`\`${cb.language || "text"}
932
+ ${cb.code.slice(0, 600)}
933
+ \`\`\``).join("\n")
934
+ );
935
+ }
936
+ if (item.tables.length > 0) {
937
+ parts.push(`Extracted Tables:
938
+ ${item.tables.slice(0, 2).join("\n\n")}`);
939
+ }
940
+ return parts.join("\n");
941
+ }).join("\n\n---\n\n");
942
+ }
943
+ /**
944
+ * Returns concatenated plain text of all ledger items for deterministic/LLM `ClaimVerifier` checks.
945
+ */
946
+ buildFullRawCorpus() {
947
+ return this.items.map(
948
+ (i) => `${i.title}
949
+ ${i.keyExcerpts.join("\n")}
950
+ ${i.codeBlocks.map((c) => c.code).join("\n")}
951
+ ${i.tables.join("\n")}`
952
+ ).join("\n\n");
953
+ }
954
+ /**
955
+ * Scans any synthesized report or JSON string for `[RES-xx]` citation markers
956
+ * and verifies every marker against the ledger.
957
+ */
958
+ verifyCitationTags(synthesizedText) {
959
+ const matches = [...synthesizedText.matchAll(/\[?(RES-\d{2,3})\]?/gi)];
960
+ const validSet = /* @__PURE__ */ new Set();
961
+ const invalidSet = /* @__PURE__ */ new Set();
962
+ for (const m of matches) {
963
+ const tag = m[1].toUpperCase();
964
+ if (this.idIndex.has(tag)) {
965
+ validSet.add(tag);
966
+ } else {
967
+ invalidSet.add(tag);
968
+ }
969
+ }
970
+ return {
971
+ validCitationIds: Array.from(validSet),
972
+ invalidCitationIds: Array.from(invalidSet)
973
+ };
974
+ }
975
+ /**
976
+ * Maps ledger items into `ProvenanceRecord[]` for `ProvenanceTracker`.
977
+ */
978
+ toProvenanceRecords(defaultDomain = "general") {
979
+ return this.items.map((item) => ({
980
+ resourceId: item.id,
981
+ sourceUrl: item.url,
982
+ accessedAt: item.accessedAt,
983
+ domain: item.sourceType || defaultDomain,
984
+ relevanceScore: item.relevanceScore,
985
+ byteSize: item.rawByteSize
986
+ }));
987
+ }
988
+ };
989
+
990
+ // src/harvester/healthAuditor.ts
991
+ var HealthAuditor = class {
992
+ auditDocument(doc) {
993
+ const issues = [];
994
+ const warnings = [];
995
+ if (!doc.markdownContent || doc.markdownContent.length < 80) {
996
+ issues.push("Document content is too short or empty (< 80 characters).");
997
+ }
998
+ if (doc.markdownContent.includes("403 Forbidden") || doc.markdownContent.includes("Cloudflare Ray ID") || doc.markdownContent.includes("Access Denied")) {
999
+ issues.push("Document blocked by firewall / Cloudflare / WAF.");
1000
+ }
1001
+ if (doc.extractedCodeSnippets.length === 0 && !doc.markdownContent.includes("|")) {
1002
+ warnings.push("Document does not contain any code snippets or reference tables.");
1003
+ }
1004
+ const passed = issues.length === 0;
1005
+ const score = passed ? warnings.length > 0 ? 80 : 100 : 20;
1006
+ return { passed, score, issues, warnings };
1007
+ }
1008
+ };
1009
+
1010
+ export { ECOMMERCE_AND_COMMERCIAL_PATTERNS, ECOMMERCE_DOMAINS, EvidenceLedger, HealthAuditor, MultiSourceSearchEngine, classifyTechnicalDomain, computeCanonicalId, evaluateTechnicalDensity, inferPrimaryCodeLanguage, isCommercialOrNoise };
1011
+ //# sourceMappingURL=chunk-BPZLZKVJ.js.map
1012
+ //# sourceMappingURL=chunk-BPZLZKVJ.js.map