@thanh01.pmt/research-kit 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +213 -0
  3. package/dist/academic/index.cjs +332 -127
  4. package/dist/academic/index.cjs.map +1 -1
  5. package/dist/academic/index.d.cts +3 -3
  6. package/dist/academic/index.d.ts +3 -3
  7. package/dist/academic/index.js +2 -2
  8. package/dist/{catalogBuilder-8A-Dg_A-.d.ts → catalogBuilder-CmmQzUnf.d.ts} +11 -3
  9. package/dist/{catalogBuilder-feU79wQQ.d.cts → catalogBuilder-DgXYxL2F.d.cts} +11 -3
  10. package/dist/chunk-2JEVBCIF.js +3 -0
  11. package/dist/{chunk-KGCDYFFD.js.map → chunk-2JEVBCIF.js.map} +1 -1
  12. package/dist/chunk-2XIEWCUE.js +244 -0
  13. package/dist/chunk-2XIEWCUE.js.map +1 -0
  14. package/dist/chunk-AUGVVZTL.js +514 -0
  15. package/dist/chunk-AUGVVZTL.js.map +1 -0
  16. package/dist/chunk-DOXSPRSP.js +559 -0
  17. package/dist/chunk-DOXSPRSP.js.map +1 -0
  18. package/dist/chunk-GKF3NDEX.js +162 -0
  19. package/dist/chunk-GKF3NDEX.js.map +1 -0
  20. package/dist/chunk-J3VCGH7Z.js +3 -0
  21. package/dist/chunk-J3VCGH7Z.js.map +1 -0
  22. package/dist/{chunk-67V2GJGK.js → chunk-J7XKDMGL.js} +7 -5
  23. package/dist/chunk-J7XKDMGL.js.map +1 -0
  24. package/dist/chunk-KAFQVI2R.js +333 -0
  25. package/dist/chunk-KAFQVI2R.js.map +1 -0
  26. package/dist/chunk-XIF5YNI7.js +799 -0
  27. package/dist/chunk-XIF5YNI7.js.map +1 -0
  28. package/dist/config/index.cjs +5 -3
  29. package/dist/config/index.cjs.map +1 -1
  30. package/dist/config/index.d.cts +1 -1
  31. package/dist/config/index.d.ts +1 -1
  32. package/dist/config/index.js +1 -1
  33. package/dist/evidence/index.cjs +337 -21
  34. package/dist/evidence/index.cjs.map +1 -1
  35. package/dist/evidence/index.d.cts +44 -7
  36. package/dist/evidence/index.d.ts +44 -7
  37. package/dist/evidence/index.js +3 -2
  38. package/dist/harvester/index.cjs +805 -64
  39. package/dist/harvester/index.cjs.map +1 -1
  40. package/dist/harvester/index.d.cts +79 -8
  41. package/dist/harvester/index.d.ts +79 -8
  42. package/dist/harvester/index.js +3 -2
  43. package/dist/{index-BU5n1ip_.d.ts → index-t8pahlJ4.d.cts} +104 -15
  44. package/dist/{index-BU5n1ip_.d.cts → index-t8pahlJ4.d.ts} +104 -15
  45. package/dist/index.cjs +2001 -252
  46. package/dist/index.cjs.map +1 -1
  47. package/dist/index.d.cts +156 -8
  48. package/dist/index.d.ts +156 -8
  49. package/dist/index.js +237 -8
  50. package/dist/index.js.map +1 -1
  51. package/dist/literatureReview-BTPFe7eb.d.cts +64 -0
  52. package/dist/literatureReview-hGbfrBmw.d.ts +64 -0
  53. package/dist/pipeline/index.cjs +1570 -241
  54. package/dist/pipeline/index.cjs.map +1 -1
  55. package/dist/pipeline/index.d.cts +49 -9
  56. package/dist/pipeline/index.d.ts +49 -9
  57. package/dist/pipeline/index.js +5 -4
  58. package/dist/webScraper-BhiQ8s9x.d.ts +68 -0
  59. package/dist/webScraper-D9Ldus9p.d.cts +68 -0
  60. package/package.json +8 -9
  61. package/dist/chunk-47L7ZHMY.js +0 -316
  62. package/dist/chunk-47L7ZHMY.js.map +0 -1
  63. package/dist/chunk-63X2HPCI.js +0 -47
  64. package/dist/chunk-63X2HPCI.js.map +0 -1
  65. package/dist/chunk-67V2GJGK.js.map +0 -1
  66. package/dist/chunk-6Q2XCUG7.js +0 -40
  67. package/dist/chunk-6Q2XCUG7.js.map +0 -1
  68. package/dist/chunk-KGCDYFFD.js +0 -3
  69. package/dist/chunk-MHAYD555.js +0 -239
  70. package/dist/chunk-MHAYD555.js.map +0 -1
  71. package/dist/chunk-MPLDSHFM.js +0 -396
  72. package/dist/chunk-MPLDSHFM.js.map +0 -1
  73. package/dist/chunk-QZNLB5X4.js +0 -65
  74. package/dist/chunk-QZNLB5X4.js.map +0 -1
  75. package/dist/literatureReview-C4AgVljZ.d.ts +0 -43
  76. package/dist/literatureReview-Dlods4Aw.d.cts +0 -43
  77. package/dist/webScraper-pKzdXhfI.d.ts +0 -46
  78. package/dist/webScraper-yBcYOHoL.d.cts +0 -46
package/dist/index.cjs CHANGED
@@ -5,11 +5,13 @@ var zod = require('zod');
5
5
  // src/config/researchConfig.ts
6
6
  var ResearchConfigSchema = zod.z.object({
7
7
  search: zod.z.object({
8
- preferredProvider: zod.z.enum(["tavily", "exa", "searxng", "duckduckgo", "semantic_scholar", "openalex", "arxiv", "custom"]).default("tavily"),
9
- fallbackChain: zod.z.array(zod.z.string()).default(["tavily", "exa", "duckduckgo"]),
8
+ preferredProvider: zod.z.enum(["tavily", "exa", "serper", "searxng", "duckduckgo", "semantic_scholar", "openalex", "arxiv", "custom"]).default("tavily"),
9
+ fallbackChain: zod.z.array(zod.z.string()).default(["tavily", "exa", "serper", "duckduckgo"]),
10
10
  apiKeys: zod.z.object({
11
11
  tavily: zod.z.string().optional(),
12
12
  exa: zod.z.string().optional(),
13
+ jina: zod.z.string().optional(),
14
+ serper: zod.z.string().optional(),
13
15
  semanticScholar: zod.z.string().optional()
14
16
  }).default({}),
15
17
  searxngHost: zod.z.string().optional(),
@@ -17,7 +19,7 @@ var ResearchConfigSchema = zod.z.object({
17
19
  maxResultsPerQuery: zod.z.number().default(5)
18
20
  }).default({}),
19
21
  crawler: zod.z.object({
20
- preferredProvider: zod.z.enum(["jina", "crawl4ai", "defuddle", "direct", "custom"]).default("jina"),
22
+ preferredProvider: zod.z.enum(["jina", "crawl4ai", "youtube", "defuddle", "direct", "custom"]).default("jina"),
21
23
  fallbackChain: zod.z.array(zod.z.string()).default(["jina", "crawl4ai", "direct"]),
22
24
  crawl4aiHost: zod.z.string().default("http://localhost:11235"),
23
25
  timeoutMs: zod.z.number().default(15e3),
@@ -30,6 +32,52 @@ var ResearchConfigSchema = zod.z.object({
30
32
  }).default({})
31
33
  });
32
34
  var DEFAULT_RESEARCH_CONFIG = ResearchConfigSchema.parse({});
35
+
36
+ // src/academic/arxivClient.ts
37
+ var ArXivSearchEngine = class {
38
+ /**
39
+ * Search ArXiv repository using the public atom XML API
40
+ */
41
+ async search(query, maxResults = 5) {
42
+ const papers = [];
43
+ try {
44
+ const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
45
+ const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
46
+ if (res.ok) {
47
+ const xmlText = await res.text();
48
+ const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
49
+ for (const entry of entryMatches) {
50
+ const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
51
+ const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
52
+ const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
53
+ const publishedMatch = entry.match(/<published>(\d{4})/);
54
+ const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
55
+ const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
56
+ const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
57
+ if (titleMatch) {
58
+ papers.push({
59
+ id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
60
+ title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
61
+ abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
62
+ authors: authorMatches,
63
+ year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
64
+ citationCount: 0,
65
+ venue: "arXiv preprint",
66
+ url: arxivUrl,
67
+ pdfUrl,
68
+ fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
69
+ });
70
+ }
71
+ }
72
+ }
73
+ } catch (e) {
74
+ console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
75
+ }
76
+ return papers;
77
+ }
78
+ };
79
+
80
+ // src/academic/semanticScholar.ts
33
81
  var AcademicPaperSchema = zod.z.object({
34
82
  id: zod.z.string(),
35
83
  title: zod.z.string(),
@@ -44,22 +92,56 @@ var AcademicPaperSchema = zod.z.object({
44
92
  tldr: zod.z.string().optional(),
45
93
  fieldsOfStudy: zod.z.array(zod.z.string()).default([])
46
94
  });
95
+ function reconstructOpenAlexAbstract(invertedIndex) {
96
+ if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
97
+ const entries = Object.entries(invertedIndex);
98
+ if (entries.length === 0) return void 0;
99
+ let maxIndex = 0;
100
+ for (const [, positions] of entries) {
101
+ if (Array.isArray(positions)) {
102
+ for (const p of positions) {
103
+ if (typeof p === "number" && p > maxIndex) maxIndex = p;
104
+ }
105
+ }
106
+ }
107
+ if (maxIndex > 5e3) maxIndex = 5e3;
108
+ const words = new Array(maxIndex + 1);
109
+ for (const [word, positions] of entries) {
110
+ if (Array.isArray(positions)) {
111
+ for (const p of positions) {
112
+ if (typeof p === "number" && p >= 0 && p <= maxIndex) {
113
+ words[p] = word;
114
+ }
115
+ }
116
+ }
117
+ }
118
+ const reconstructed = words.filter(Boolean).join(" ").trim();
119
+ return reconstructed.length > 0 ? reconstructed : void 0;
120
+ }
47
121
  var AcademicSearchEngine = class {
48
122
  apiKey;
123
+ allowEnvFallback;
49
124
  customAdapter;
125
+ arxivEngine;
50
126
  constructor(options = {}) {
127
+ this.arxivEngine = new ArXivSearchEngine();
51
128
  if (typeof options === "string") {
52
129
  this.apiKey = options;
130
+ this.allowEnvFallback = true;
53
131
  } else {
54
- this.apiKey = options.apiKey || process.env.SEMANTIC_SCHOLAR_API_KEY;
132
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
133
+ this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
55
134
  this.customAdapter = options.customAdapter;
56
135
  }
57
136
  }
58
137
  setCustomAdapter(adapter) {
59
138
  this.customAdapter = adapter;
60
139
  }
140
+ setApiKey(apiKey) {
141
+ this.apiKey = apiKey;
142
+ }
61
143
  /**
62
- * Search academic papers across Semantic Scholar with OpenAlex fallback or Custom Adapter
144
+ * Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
63
145
  */
64
146
  async searchPapers(query, options = {}) {
65
147
  const adapter = options.customAdapter || this.customAdapter;
@@ -75,6 +157,13 @@ var AcademicSearchEngine = class {
75
157
  }
76
158
  const limit = options.limit ?? 5;
77
159
  const papers = [];
160
+ const seenTitles = /* @__PURE__ */ new Set();
161
+ const pushUnique = (p) => {
162
+ const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
163
+ if (!normTitle || seenTitles.has(normTitle)) return;
164
+ seenTitles.add(normTitle);
165
+ papers.push(p);
166
+ };
78
167
  try {
79
168
  const apiKey = options.apiKey || this.apiKey;
80
169
  const headers = { "Content-Type": "application/json" };
@@ -97,104 +186,111 @@ var AcademicSearchEngine = class {
97
186
  const data = await res.json();
98
187
  if (Array.isArray(data?.data)) {
99
188
  for (const item of data.data) {
100
- papers.push({
189
+ pushUnique({
101
190
  id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
102
191
  title: item.title || "Untitled",
103
- abstract: item.abstract,
104
- authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name) : [],
192
+ abstract: item.abstract || void 0,
193
+ authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
105
194
  year: item.year,
106
195
  citationCount: item.citationCount || 0,
107
- venue: item.venue,
196
+ venue: item.venue || void 0,
108
197
  doi: item.externalIds?.DOI,
109
- url: item.url,
198
+ url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
110
199
  pdfUrl: item.openAccessPdf?.url,
111
200
  tldr: item.tldr?.text,
112
- fieldsOfStudy: item.fieldsOfStudy || ["Computer Science"]
201
+ fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
113
202
  });
114
203
  }
115
- if (papers.length > 0) return papers;
204
+ if (papers.length >= limit) return papers.slice(0, limit);
116
205
  }
117
206
  }
118
207
  } catch (e) {
119
208
  console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
120
209
  }
121
- try {
122
- const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
123
- headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
124
- signal: AbortSignal.timeout(6e3)
125
- });
126
- if (res.ok) {
127
- const data = await res.json();
128
- if (Array.isArray(data?.results)) {
129
- for (const item of data.results) {
130
- papers.push({
131
- id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
132
- title: item.title || "Untitled",
133
- abstract: item.abstract_inverted_index ? "Abstract indexed" : void 0,
134
- authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
135
- year: item.publication_year,
136
- citationCount: item.cited_by_count || 0,
137
- venue: item.primary_location?.source?.display_name,
138
- doi: item.doi,
139
- url: item.doi || item.id,
140
- pdfUrl: item.open_access?.oa_url,
141
- fieldsOfStudy: ["Computer Science"]
142
- });
210
+ if (papers.length < limit) {
211
+ try {
212
+ const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
213
+ headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
214
+ signal: AbortSignal.timeout(6e3)
215
+ });
216
+ if (res.ok) {
217
+ const data = await res.json();
218
+ if (Array.isArray(data?.results)) {
219
+ for (const item of data.results) {
220
+ const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
221
+ const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
222
+ pushUnique({
223
+ id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
224
+ title: item.title || "Untitled",
225
+ abstract: abstractText,
226
+ authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
227
+ year: item.publication_year,
228
+ citationCount: item.cited_by_count || 0,
229
+ venue: item.primary_location?.source?.display_name,
230
+ doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
231
+ url: item.doi || item.id,
232
+ pdfUrl: item.open_access?.oa_url,
233
+ fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
234
+ });
235
+ }
143
236
  }
144
237
  }
238
+ } catch (e) {
239
+ console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
145
240
  }
146
- } catch (e) {
147
- console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
148
241
  }
149
- return papers;
150
- }
151
- };
152
-
153
- // src/academic/arxivClient.ts
154
- var ArXivSearchEngine = class {
155
- /**
156
- * Search ArXiv repository using the public atom XML API
157
- */
158
- async search(query, maxResults = 5) {
159
- const papers = [];
160
- try {
161
- const url = `http://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
162
- const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
163
- if (res.ok) {
164
- const xmlText = await res.text();
165
- const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
166
- for (const entry of entryMatches) {
167
- const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
168
- const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
169
- const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
170
- const publishedMatch = entry.match(/<published>(\d{4})/);
171
- const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
172
- const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
173
- const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
174
- if (titleMatch) {
175
- papers.push({
176
- id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
177
- title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
178
- abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
179
- authors: authorMatches,
180
- year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
181
- citationCount: 0,
182
- venue: "arXiv preprint",
183
- url: arxivUrl,
184
- pdfUrl,
185
- fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
186
- });
187
- }
242
+ if (papers.length < limit && options.includeArxivFallback !== false) {
243
+ try {
244
+ const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
245
+ for (const p of arxivPapers) {
246
+ pushUnique(p);
188
247
  }
248
+ } catch (e) {
249
+ console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
189
250
  }
190
- } catch (e) {
191
- console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
192
251
  }
193
- return papers;
252
+ return papers.slice(0, limit);
194
253
  }
195
254
  };
196
255
 
197
256
  // src/academic/cs2023Taxonomy.ts
257
+ var CSTA_K12_STRANDS = {
258
+ CS: {
259
+ code: "CS",
260
+ name: "Computing Systems",
261
+ nameVi: "H\u1EC7 th\u1ED1ng M\xE1y t\xEDnh & Ph\u1EA7n c\u1EE9ng",
262
+ description: "Devices, hardware and software integration, sensors, actuators, and troubleshooting.",
263
+ subconcepts: ["Devices", "Hardware & Software", "Troubleshooting"]
264
+ },
265
+ NI: {
266
+ code: "NI",
267
+ name: "Networks and the Internet",
268
+ nameVi: "M\u1EA1ng m\xE1y t\xEDnh & An to\xE0n th\xF4ng tin",
269
+ description: "Network communication, protocols, IoT connectivity, and cybersecurity.",
270
+ subconcepts: ["Network Communication & Organization", "Cybersecurity"]
271
+ },
272
+ DA: {
273
+ code: "DA",
274
+ name: "Data and Analysis",
275
+ nameVi: "D\u1EEF li\u1EC7u & Ph\xE2n t\xEDch",
276
+ description: "Storage, collection, visualization, and transformation of sensor or application data.",
277
+ subconcepts: ["Storage", "Collection, Visualization & Transformation", "Inference & Models"]
278
+ },
279
+ AP: {
280
+ code: "AP",
281
+ name: "Algorithms and Programming",
282
+ nameVi: "Thu\u1EADt to\xE1n & L\u1EADp tr\xECnh",
283
+ description: "Algorithms, variables, control structures, modularity, and iterative program development.",
284
+ subconcepts: ["Algorithms", "Variables", "Control", "Modularity", "Program Development"]
285
+ },
286
+ IC: {
287
+ code: "IC",
288
+ name: "Impacts of Computing",
289
+ nameVi: "T\xE1c \u0111\u1ED9ng c\u1EE7a Tin h\u1ECDc & AI",
290
+ description: "Culture, social interactions, safety, law, and ethics of computing and AI systems.",
291
+ subconcepts: ["Culture", "Social Interactions", "Safety, Law, & Ethics"]
292
+ }
293
+ };
198
294
  var CS2023_KNOWLEDGE_AREAS = {
199
295
  AL: {
200
296
  code: "AL",
@@ -205,8 +301,8 @@ var CS2023_KNOWLEDGE_AREAS = {
205
301
  AR: {
206
302
  code: "AR",
207
303
  name: "Architecture and Organization",
208
- description: "Hardware architecture, assembly, memory hierarchy, pipelining, and system bus.",
209
- coreTopics: ["Digital Logic", "Memory Hierarchy & Cache", "Instruction Set Architecture (ISA)", "Interfacing & I/O"]
304
+ description: "Hardware architecture, microcontrollers, sensors, memory hierarchy, and interfacing.",
305
+ coreTopics: ["Digital Logic", "Memory Hierarchy & Cache", "Microcontrollers & Embedded I/O", "Sensor & Actuator Interfacing"]
210
306
  },
211
307
  AI: {
212
308
  code: "AI",
@@ -214,6 +310,30 @@ var CS2023_KNOWLEDGE_AREAS = {
214
310
  description: "Neural networks, search algorithms, knowledge representation, reinforcement learning, and LLMs.",
215
311
  coreTopics: ["Supervised Learning", "Deep Learning & Transformers", "Search & Optimization", "AI Ethics & Alignment"]
216
312
  },
313
+ DM: {
314
+ code: "DM",
315
+ name: "Data Management",
316
+ description: "Relational databases, NoSQL, query optimization, data modeling, and transactions.",
317
+ coreTopics: ["Relational & SQL Modeling", "Indexing & Query Optimization", "Transactions & ACID", "Distributed Data Stores"]
318
+ },
319
+ FPL: {
320
+ code: "FPL",
321
+ name: "Foundations of Programming Languages",
322
+ description: "Language paradigms, type systems, event-driven programming, memory safety, and compilers.",
323
+ coreTopics: ["Event-Driven & Reactive Programming", "Functional & OOP Paradigms", "Static & Dynamic Typing", "Memory Safety & Ownership"]
324
+ },
325
+ GIT: {
326
+ code: "GIT",
327
+ name: "Graphics and Interactive Techniques",
328
+ description: "Game engines, 2D/3D rendering, physics simulation, collision detection, and interactive media.",
329
+ coreTopics: ["Game Loop & Frame Timing", "2D/3D Coordinate Systems & Physics", "Collision Detection & State Machines", "Shader & Rendering Pipelines"]
330
+ },
331
+ HCI: {
332
+ code: "HCI",
333
+ name: "Human-Computer Interaction",
334
+ description: "User interface design, usability, accessibility, state-driven UI frameworks, and interaction loops.",
335
+ coreTopics: ["Declarative & State-Driven UI", "Usability & Cognitive Ergonomics", "Accessibility Standards", "Interactive Prototyping"]
336
+ },
217
337
  OS: {
218
338
  code: "OS",
219
339
  name: "Operating Systems",
@@ -232,6 +352,12 @@ var CS2023_KNOWLEDGE_AREAS = {
232
352
  description: "Language paradigms, type systems, memory safety, compilers, and runtime environments.",
233
353
  coreTopics: ["Functional & OOP Paradigms", "Static & Dynamic Typing", "Memory Safety & Ownership", "Language Runtimes"]
234
354
  },
355
+ SDF: {
356
+ code: "SDF",
357
+ name: "Software Development Fundamentals",
358
+ description: "Core programming constructs, variables, control flow, data structures, debugging, and modularization.",
359
+ coreTopics: ["Variables, Loops & Control Flow", "Data Structures (Arrays, Lists, Maps)", "Systematic Debugging & Testing", "Modular & Procedural Design"]
360
+ },
235
361
  SE: {
236
362
  code: "SE",
237
363
  name: "Software Engineering",
@@ -244,37 +370,53 @@ var CS2023_KNOWLEDGE_AREAS = {
244
370
  description: "Authentication, cryptography, secure coding, threat modeling, and network defense.",
245
371
  coreTopics: ["Symmetric & Asymmetric Encryption", "OAuth & JWT Auth", "Secure Coding & Vulnerabilities", "Identity & Access Management"]
246
372
  },
247
- SDF: {
248
- code: "SDF",
249
- name: "Software Development Fundamentals",
250
- description: "Core programming constructs, data structures, debugging, and modularization.",
251
- coreTopics: ["Variables, Loops & Control Flow", "Data Structures (Arrays, Lists, Trees)", "Debugging & Version Control", "OOP & Modular Design"]
373
+ SPD: {
374
+ code: "SPD",
375
+ name: "Specialized Platform Development",
376
+ description: "Web, mobile, cloud, IoT, and robotics platform engineering.",
377
+ coreTopics: ["Mobile & Desktop App Lifecycle", "Web Full-Stack Architecture", "IoT & Edge Device Connectivity", "Platform SDK Integration"]
252
378
  }
253
379
  };
254
- function mapTopicToCS2023(topic) {
255
- const t = topic.toLowerCase();
256
- if (t.includes("concurren") || t.includes("async") || t.includes("actor") || t.includes("thread") || t.includes("distributed")) {
257
- return { area: CS2023_KNOWLEDGE_AREAS.PDC, matchConfidence: 0.95 };
380
+ function mapTopicToCS2023(topic, context) {
381
+ const combined = [
382
+ topic,
383
+ context?.courseTitle,
384
+ context?.positioning,
385
+ context?.hardwarePlatform,
386
+ context?.softwareEnv,
387
+ ...context?.inScopeTopics || []
388
+ ].filter(Boolean).join(" ").toLowerCase();
389
+ if (/\b(concurren|async|actor|thread|distributed|parallel|đồng thời|bất đồng bộ|đa luồng)\b/.test(combined)) {
390
+ return { area: CS2023_KNOWLEDGE_AREAS.PDC, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.95 };
391
+ }
392
+ if (/\b(ai|machine learning|deep learning|neural|llm|computer vision|trí tuệ nhân tạo|học máy|mạng nơ-ron)\b/.test(combined)) {
393
+ return { area: CS2023_KNOWLEDGE_AREAS.AI, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.95 };
258
394
  }
259
- if (t.includes("ai") || t.includes("machine learning") || t.includes("deep learning") || t.includes("robot") || t.includes("neural")) {
260
- return { area: CS2023_KNOWLEDGE_AREAS.AI, matchConfidence: 0.95 };
395
+ if (/\b(arduino|esp32|raspberry|micro:?bit|mblock|makecode|lego|sensor|circuit|gpio|iot|hardware|robotics|cảm biến|vi điều khiển|mạch điện|điện tử)\b/.test(combined)) {
396
+ return { area: CS2023_KNOWLEDGE_AREAS.AR, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.92 };
261
397
  }
262
- if (t.includes("os") || t.includes("operating system") || t.includes("kernel") || t.includes("memory") || t.includes("linux")) {
263
- return { area: CS2023_KNOWLEDGE_AREAS.OS, matchConfidence: 0.9 };
398
+ if (/\b(roblox|unity|godot|pygame|game|2d|3d|sprite|collision|physics|shader|trò chơi|làm game)\b/.test(combined)) {
399
+ return { area: CS2023_KNOWLEDGE_AREAS.GIT, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.92 };
264
400
  }
265
- if (t.includes("security") || t.includes("auth") || t.includes("crypto") || t.includes("cyber")) {
266
- return { area: CS2023_KNOWLEDGE_AREAS.SEC, matchConfidence: 0.9 };
401
+ if (/\b(swiftui|ui|ux|interface|frontend|accessibility|figma|giao diện|tương tác người-máy)\b/.test(combined)) {
402
+ return { area: CS2023_KNOWLEDGE_AREAS.HCI, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
267
403
  }
268
- if (t.includes("architecture") || t.includes("pattern") || t.includes("software engineer") || t.includes("clean code")) {
269
- return { area: CS2023_KNOWLEDGE_AREAS.SE, matchConfidence: 0.85 };
404
+ if (/\b(os|operating system|kernel|virtual memory|linux|hệ điều hành)\b/.test(combined)) {
405
+ return { area: CS2023_KNOWLEDGE_AREAS.OS, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.9 };
270
406
  }
271
- if (t.includes("algorithm") || t.includes("data structure") || t.includes("tree") || t.includes("graph")) {
272
- return { area: CS2023_KNOWLEDGE_AREAS.AL, matchConfidence: 0.9 };
407
+ if (/\b(security|auth|crypto|cyber|jwt|oauth|bảo mật|an toàn thông tin|mã hóa)\b/.test(combined)) {
408
+ return { area: CS2023_KNOWLEDGE_AREAS.SEC, k12Strand: CSTA_K12_STRANDS.NI, matchConfidence: 0.9 };
273
409
  }
274
- if (t.includes("arduino") || t.includes("esp32") || t.includes("raspberry") || t.includes("hardware") || t.includes("circuit")) {
275
- return { area: CS2023_KNOWLEDGE_AREAS.AR, matchConfidence: 0.9 };
410
+ if (/\b(sql|database|postgres|supabase|mongodb|data model|cơ sở dữ liệu|truy vấn)\b/.test(combined)) {
411
+ return { area: CS2023_KNOWLEDGE_AREAS.DM, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.9 };
276
412
  }
277
- return { area: CS2023_KNOWLEDGE_AREAS.SDF, matchConfidence: 0.7 };
413
+ if (/\b(architecture|design pattern|software engineer|clean code|testing|tdd|ci\/cd|kiến trúc phần mềm|kiểm thử)\b/.test(combined)) {
414
+ return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88 };
415
+ }
416
+ if (/\b(algorithm|data structure|tree|graph|dynamic programming|sorting|thuật toán|cấu trúc dữ liệu|đồ thị|quy hoạch động)\b/.test(combined)) {
417
+ return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
418
+ }
419
+ return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.8 };
278
420
  }
279
421
 
280
422
  // src/academic/citationManager.ts
@@ -326,105 +468,217 @@ var CitationManager = class {
326
468
 
327
469
  // src/academic/literatureReview.ts
328
470
  var LiteratureReviewSynthesizer = class {
329
- synthesize(topic, papers, cs2023Area) {
471
+ synthesize(topic, papers, cs2023Area, k12Strand) {
330
472
  const citationManager = new CitationManager();
331
473
  const sotaMatrix = [];
474
+ const foundations = [];
332
475
  for (const paper of papers) {
333
476
  citationManager.addPaper(paper);
477
+ const summaryText = paper.tldr?.trim() || (paper.abstract ? paper.abstract.trim().split(/(?<=[.!?])\s+/)[0] || paper.abstract.slice(0, 160) : "");
334
478
  sotaMatrix.push({
335
- approach: paper.title.split(":")[0] || paper.title.slice(0, 30),
336
- sourcePaper: `${paper.authors[0] || "Unknown"} (${paper.year || "n.d."})`,
337
- keyStrengths: paper.tldr || (paper.abstract ? paper.abstract.slice(0, 120) + "..." : "Foundational theoretical concept"),
338
- limitations: "Context-specific or hardware-constrained",
339
- relevanceToCourse: `Informs pedagogical sequencing and theoretical grounding for ${topic}.`
479
+ approach: paper.title.split(":")[0]?.trim() || paper.title.slice(0, 45),
480
+ sourcePaper: `${paper.authors[0] || "Unknown"} (${paper.year || "n.d."})${paper.venue ? ` \u2014 ${paper.venue}` : ""}`,
481
+ keyStrengths: summaryText ? summaryText.slice(0, 180) : `Peer-reviewed study (${paper.citationCount} citations)`,
482
+ limitations: paper.year && paper.year < 2020 ? `Published in ${paper.year}; verify against modern tooling` : "Requires pedagogical scaffolding for target learners",
483
+ relevanceToCourse: `Grounds "${topic}" in peer-reviewed evidence (${paper.fieldsOfStudy.slice(0, 2).join(", ") || "CS"}).`
340
484
  });
485
+ if (summaryText && foundations.length < 4) {
486
+ foundations.push(`[${paper.authors[0] || "Ref"} ${paper.year || ""}] ${summaryText}`);
487
+ }
341
488
  }
489
+ if (cs2023Area) {
490
+ foundations.push(
491
+ `IEEE/ACM CS2023 (${cs2023Area.code} - ${cs2023Area.name}): ${cs2023Area.coreTopics.join(", ")}`
492
+ );
493
+ }
494
+ if (k12Strand) {
495
+ foundations.push(
496
+ `CSTA K-12 (${k12Strand.code} - ${k12Strand.name} / ${k12Strand.nameVi}): ${k12Strand.subconcepts.join(", ")}`
497
+ );
498
+ }
499
+ const openQuestions = [
500
+ `How to scaffold core concepts of "${topic}" (${cs2023Area?.code || "CS"}) so learners master mental models before syntax memorization?`,
501
+ `What observable debugging and verification tasks best diagnose learner misconceptions in "${topic}"?`
502
+ ];
342
503
  const mdLines = [
343
504
  `# \u{1F393} Academic Literature Review & Theoretical Ground Truth`,
344
505
  `> **Topic:** ${topic}`,
345
- cs2023Area ? `> **IEEE/ACM CS2023 Standard Area:** \`${cs2023Area.code}\` \u2014 *${cs2023Area.name}*
506
+ cs2023Area ? `> **IEEE/ACM CS2023 Standard Area:** \`${cs2023Area.code}\` \u2014 *${cs2023Area.name}*` : "",
507
+ k12Strand ? `> **CSTA K-12 Strand:** \`${k12Strand.code}\` \u2014 *${k12Strand.name} (${k12Strand.nameVi})*
346
508
  ` : "",
347
509
  `## 1. Executive Summary`,
348
- `Synthesized from **${papers.length} peer-reviewed works** and international curricular standards directly relevant to \`${topic}\`.`,
510
+ papers.length > 0 ? `Synthesized from **${papers.length} peer-reviewed works** and curricular standards directly relevant to \`${topic}\`.` : `No peer-reviewed papers matched directly; aligned to curricular taxonomy standards for \`${topic}\`.`,
349
511
  `
350
512
  ## 2. State-of-the-Art (SOTA) Comparison Matrix`,
351
- `| Paradigm / Approach | Source Reference | Core Strengths | Constraints & Limitations | Pedagogical Application |`,
352
- `|---|---|---|---|---|`,
353
- ...sotaMatrix.map((r) => `| **${r.approach}** | ${r.sourcePaper} | ${r.keyStrengths} | ${r.limitations} | ${r.relevanceToCourse} |`),
513
+ ...sotaMatrix.length > 0 ? [
514
+ `| Paradigm / Approach | Source Reference | Core Findings / Abstract | Pedagogical Note | Course Application |`,
515
+ `|---|---|---|---|---|`,
516
+ ...sotaMatrix.map(
517
+ (r) => `| **${r.approach.replace(/\|/g, "/")}** | ${r.sourcePaper.replace(/\|/g, "/")} | ${r.keyStrengths.replace(/\|/g, "/")} | ${r.limitations.replace(/\|/g, "/")} | ${r.relevanceToCourse.replace(/\|/g, "/")} |`
518
+ )
519
+ ] : ["*No peer-reviewed papers returned from academic engines for this query.*"],
354
520
  `
355
- ## 3. Core Theoretical Foundations`,
356
- `- **Academic Rigor:** Curriculum models strictly adhere to peer-reviewed principles.`,
357
- `- **CS2023 Competency Alignment:** Maps directly to IEEE/ACM recommended graduate competencies.`,
358
- `- **Theory-to-Practice Bridge:** Direct translational pipeline from formal papers to executable lab operations.`,
521
+ ## 3. Core Theoretical & Curricular Foundations`,
522
+ ...foundations.map((f) => `- ${f}`),
359
523
  `
360
- ## 4. Open Pedagogical & Research Questions`,
361
- `- How to distill complex formal mechanisms for high-school / undergraduate learners without diluting accuracy?`,
362
- `- What automated verification setups validate theoretical guarantees in real-world student code?`,
524
+ ## 4. Open Pedagogical & Curriculum Design Questions`,
525
+ ...openQuestions.map((q) => `- ${q}`),
363
526
  `
364
527
  ${citationManager.getMarkdownFootnotes()}`
365
- ];
528
+ ].filter((line) => line !== "");
366
529
  return {
367
530
  topic,
368
531
  cs2023Area,
532
+ k12Strand,
369
533
  totalPapersFound: papers.length,
370
534
  papers,
371
535
  sotaMatrix,
372
- keyTheoreticalFoundations: [
373
- "Immutable state guarantees and memory safety semantics",
374
- "Dataflow and message-passing distributed concurrency models",
375
- "Empirical latency and throughput benchmark validation"
376
- ],
377
- openResearchQuestions: [
378
- "How to distill complex formal mechanisms for high-school / undergraduate learners without diluting accuracy?",
379
- "What automated verification setups validate theoretical guarantees in real-world student code?"
380
- ],
536
+ keyTheoreticalFoundations: foundations,
537
+ openResearchQuestions: openQuestions,
381
538
  markdownReport: mdLines.join("\n"),
382
539
  bibtex: citationManager.getBibTeX()
383
540
  };
384
541
  }
542
+ /**
543
+ * Optional LLM semantic synthesis of dynamic open research questions and pedagogical takeaways
544
+ * based on the retrieved academic literature.
545
+ */
546
+ async synthesizeWithLlm(topic, papers, cs2023Area, k12Strand, options = {}) {
547
+ const base = this.synthesize(topic, papers, cs2023Area, k12Strand);
548
+ if (!options.llm || papers.length === 0) {
549
+ return base;
550
+ }
551
+ try {
552
+ const abstractsSummary = papers.map((p, i) => `[Paper ${i + 1}] ${p.title} (${p.authors[0]} ${p.year}): ${p.tldr || p.abstract?.slice(0, 300) || "No abstract"}`).join("\n\n");
553
+ const prompt = [
554
+ `You are a Professor of Computing Education. Review these academic research papers retrieved for the course topic "${topic}".`,
555
+ cs2023Area ? `IEEE/ACM CS2023 Area: ${cs2023Area.code} - ${cs2023Area.name}` : "",
556
+ k12Strand ? `CSTA K-12 Strand: ${k12Strand.code} - ${k12Strand.name}` : "",
557
+ options.context?.ageBand ? `Target Age: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
558
+ `
559
+ Retrieved Academic Papers:
560
+ ${abstractsSummary}`,
561
+ `
562
+ Generate 2-3 deep, actionable pedagogical research & curriculum design questions directly inspired by these papers.`,
563
+ `Return ONLY a JSON array of strings: ["Question 1", "Question 2"]`
564
+ ].filter(Boolean).join("\n");
565
+ const raw = await options.llm(prompt, {
566
+ system: "You are an educational research synthesis specialist. Output strictly a JSON array of strings.",
567
+ temperature: 0.2,
568
+ jsonMode: true
569
+ });
570
+ const match = raw.match(/\[[\s\S]*\]/);
571
+ if (match) {
572
+ const parsed = JSON.parse(match[0]);
573
+ if (Array.isArray(parsed) && parsed.length > 0) {
574
+ const dynamicQuestions = parsed.map(String).filter((q) => q.length > 10);
575
+ if (dynamicQuestions.length > 0) {
576
+ base.openResearchQuestions = dynamicQuestions;
577
+ base.markdownReport = base.markdownReport.replace(
578
+ /## 4\. Open Pedagogical & Curriculum Design Questions[\s\S]*?(?=\n\n\[\^|\n\n$|$)/,
579
+ `## 4. Open Pedagogical & Curriculum Design Questions
580
+ ${dynamicQuestions.map((q) => `- ${q}`).join("\n")}`
581
+ );
582
+ }
583
+ }
584
+ }
585
+ } catch (e) {
586
+ console.warn("[LiteratureReviewSynthesizer] LLM question synthesis failed, using baseline:", e?.message || e);
587
+ }
588
+ return base;
589
+ }
385
590
  };
386
591
 
387
592
  // src/harvester/domainClassifier.ts
388
- function classifyTechnicalDomain(topic, explicitDomain) {
593
+ var HARDWARE_PATTERNS = [
594
+ /\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
595
+ /\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
596
+ /(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
597
+ ];
598
+ var PLATFORM_PATTERNS = [
599
+ /\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
600
+ /(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
601
+ ];
602
+ var ACADEMIC_PATTERNS = [
603
+ /\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
604
+ /(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
605
+ ];
606
+ function inferPrimaryCodeLanguage(topic, context, domain) {
607
+ const combined = [
608
+ topic,
609
+ context?.courseTitle,
610
+ context?.hardwarePlatform,
611
+ context?.softwareEnv,
612
+ ...context?.inScopeTopics || []
613
+ ].filter(Boolean).join(" ").toLowerCase();
614
+ if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
615
+ if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
616
+ if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
617
+ if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
618
+ if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
619
+ if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
620
+ if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
621
+ if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
622
+ if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
623
+ if (/\b(rust|cargo)\b/.test(combined)) return "rust";
624
+ if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
625
+ if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
626
+ if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
627
+ if (domain === "hardware") return "cpp";
628
+ return "text";
629
+ }
630
+ function classifyTechnicalDomain(topic, explicitDomain, context) {
389
631
  if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
390
632
  const domain = explicitDomain.toLowerCase();
391
633
  return {
392
634
  domain,
393
635
  confidence: 1,
394
636
  rationale: "Explicitly configured by user flag.",
395
- recommendedExtractors: getExtractorsForDomain(domain)
637
+ recommendedExtractors: getExtractorsForDomain(domain),
638
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
396
639
  };
397
640
  }
398
- const t = topic.toLowerCase();
399
- if (t.includes("esp32") || t.includes("arduino") || t.includes("raspberry") || t.includes("stm32") || t.includes("sensor") || t.includes("circuit") || t.includes("gpio") || t.includes("iot hardware")) {
641
+ const combined = [
642
+ topic,
643
+ context?.courseTitle,
644
+ context?.positioning,
645
+ context?.hardwarePlatform,
646
+ context?.softwareEnv,
647
+ ...context?.inScopeTopics || []
648
+ ].filter(Boolean).join(" ");
649
+ if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
400
650
  return {
401
651
  domain: "hardware",
402
652
  confidence: 0.95,
403
- rationale: "Topic contains microcontrollers, sensors, or physical electronics keywords.",
404
- recommendedExtractors: ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"]
653
+ rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
654
+ recommendedExtractors: getExtractorsForDomain("hardware"),
655
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
405
656
  };
406
657
  }
407
- if (t.includes("supabase") || t.includes("firebase") || t.includes("aws") || t.includes("cloud") || t.includes("vercel") || t.includes("backend-as-a-service") || t.includes("docker") || t.includes("k8s")) {
658
+ if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
408
659
  return {
409
660
  domain: "platform",
410
661
  confidence: 0.9,
411
- rationale: "Topic represents a cloud infrastructure, hosting, or BaaS platform.",
412
- recommendedExtractors: ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"]
662
+ rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
663
+ recommendedExtractors: getExtractorsForDomain("platform"),
664
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
413
665
  };
414
666
  }
415
- if (t.includes("theorem") || t.includes("complexity") || t.includes("pedagogy") || t.includes("bloom") || t.includes("constructivism") || t.includes("formal verification") || t.includes("computational theory")) {
667
+ if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
416
668
  return {
417
669
  domain: "academic",
418
670
  confidence: 0.9,
419
671
  rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
420
- recommendedExtractors: ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"]
672
+ recommendedExtractors: getExtractorsForDomain("academic"),
673
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
421
674
  };
422
675
  }
423
676
  return {
424
677
  domain: "software",
425
678
  confidence: 0.85,
426
- rationale: "Topic is a programming language, client/server library, or software framework.",
427
- recommendedExtractors: ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"]
679
+ rationale: "Topic is a programming language, game engine, client/server library, or software application.",
680
+ recommendedExtractors: getExtractorsForDomain("software"),
681
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
428
682
  };
429
683
  }
430
684
  function getExtractorsForDomain(domain) {
@@ -445,15 +699,24 @@ function getExtractorsForDomain(domain) {
445
699
  var MultiSourceSearchEngine = class {
446
700
  tavilyKey;
447
701
  exaKey;
702
+ serperKey;
703
+ allowEnvFallback;
448
704
  customAdapter;
449
705
  constructor(options = {}) {
450
- this.tavilyKey = options.tavilyKey || process.env.TAVILY_API_KEY;
451
- this.exaKey = options.exaKey || process.env.EXA_API_KEY;
706
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
707
+ this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
708
+ this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
709
+ this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
452
710
  this.customAdapter = options.customAdapter;
453
711
  }
454
712
  setCustomAdapter(adapter) {
455
713
  this.customAdapter = adapter;
456
714
  }
715
+ setApiKeys(keys) {
716
+ if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
717
+ if (keys.exa !== void 0) this.exaKey = keys.exa;
718
+ if (keys.serper !== void 0) this.serperKey = keys.serper;
719
+ }
457
720
  async search(query, options = {}) {
458
721
  const limit = options.limit ?? 5;
459
722
  const adapter = options.customAdapter || this.customAdapter;
@@ -474,28 +737,37 @@ var MultiSourceSearchEngine = class {
474
737
  console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
475
738
  }
476
739
  }
477
- const tavilyKey = options.tavilyKey || this.tavilyKey;
478
- const exaKey = options.exaKey || this.exaKey;
740
+ const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
741
+ const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
742
+ const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
479
743
  if (preferred === "exa" && exaKey) {
480
- const exaHits = await this.queryExa(query, exaKey, limit);
744
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
481
745
  if (exaHits.length > 0) return exaHits;
482
746
  }
747
+ if (preferred === "serper" && serperKey) {
748
+ const serperHits = await this.querySerper(query, serperKey, limit);
749
+ if (serperHits.length > 0) return serperHits;
750
+ }
483
751
  if (preferred === "duckduckgo") {
484
752
  const ddgHits2 = await this.queryDuckDuckGo(query, limit);
485
753
  if (ddgHits2.length > 0) return ddgHits2;
486
754
  }
487
755
  if (tavilyKey && (!preferred || preferred === "tavily")) {
488
756
  try {
757
+ const body = {
758
+ api_key: tavilyKey,
759
+ query,
760
+ search_depth: "advanced",
761
+ include_answer: true,
762
+ max_results: limit
763
+ };
764
+ if (options.domainFilter && options.domainFilter.length > 0) {
765
+ body.include_domains = options.domainFilter;
766
+ }
489
767
  const res = await fetch("https://api.tavily.com/search", {
490
768
  method: "POST",
491
769
  headers: { "Content-Type": "application/json" },
492
- body: JSON.stringify({
493
- api_key: tavilyKey,
494
- query,
495
- search_depth: "advanced",
496
- include_answer: true,
497
- max_results: limit
498
- }),
770
+ body: JSON.stringify(body),
499
771
  signal: AbortSignal.timeout(12e3)
500
772
  });
501
773
  if (res.ok) {
@@ -515,26 +787,34 @@ var MultiSourceSearchEngine = class {
515
787
  }
516
788
  }
517
789
  if (exaKey) {
518
- const exaHits = await this.queryExa(query, exaKey, limit);
790
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
519
791
  if (exaHits.length > 0) return exaHits;
520
792
  }
793
+ if (serperKey) {
794
+ const serperHits = await this.querySerper(query, serperKey, limit);
795
+ if (serperHits.length > 0) return serperHits;
796
+ }
521
797
  const ddgHits = await this.queryDuckDuckGo(query, limit);
522
798
  if (ddgHits.length > 0) return ddgHits;
523
799
  return [];
524
800
  }
525
- async queryExa(query, exaKey, limit) {
801
+ async queryExa(query, exaKey, limit, domainFilter) {
526
802
  try {
803
+ const body = {
804
+ query,
805
+ numResults: limit,
806
+ useAutoprompt: true
807
+ };
808
+ if (domainFilter && domainFilter.length > 0) {
809
+ body.includeDomains = domainFilter;
810
+ }
527
811
  const res = await fetch("https://api.exa.ai/search", {
528
812
  method: "POST",
529
813
  headers: {
530
814
  "x-api-key": exaKey,
531
815
  "Content-Type": "application/json"
532
816
  },
533
- body: JSON.stringify({
534
- query,
535
- numResults: limit,
536
- useAutoprompt: true
537
- }),
817
+ body: JSON.stringify(body),
538
818
  signal: AbortSignal.timeout(1e4)
539
819
  });
540
820
  if (res.ok) {
@@ -554,6 +834,33 @@ var MultiSourceSearchEngine = class {
554
834
  }
555
835
  return [];
556
836
  }
837
+ async querySerper(query, serperKey, limit) {
838
+ try {
839
+ const res = await fetch("https://google.serper.dev/search", {
840
+ method: "POST",
841
+ headers: {
842
+ "X-API-KEY": serperKey,
843
+ "Content-Type": "application/json"
844
+ },
845
+ body: JSON.stringify({ q: query, num: limit }),
846
+ signal: AbortSignal.timeout(1e4)
847
+ });
848
+ if (res.ok) {
849
+ const data = await res.json();
850
+ if (Array.isArray(data?.organic) && data.organic.length > 0) {
851
+ return data.organic.slice(0, limit).map((r) => ({
852
+ title: r.title || "Untitled",
853
+ url: r.link,
854
+ content: r.snippet || "",
855
+ engine: "serper"
856
+ }));
857
+ }
858
+ }
859
+ } catch (e) {
860
+ console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
861
+ }
862
+ return [];
863
+ }
557
864
  async queryDuckDuckGo(query, limit) {
558
865
  try {
559
866
  const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
@@ -564,15 +871,16 @@ var MultiSourceSearchEngine = class {
564
871
  if (res.ok) {
565
872
  const html = await res.text();
566
873
  const hits = [];
567
- const resultRegex = /<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
874
+ const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
568
875
  let match;
569
876
  while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
570
- const rawUrl = match[1]?.trim();
877
+ const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
878
+ const rawUrl = match[2]?.trim();
571
879
  const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
572
- const snippet = match[2]?.replace(/<[^>]+>/g, "").trim() || "";
880
+ const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
573
881
  if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
574
882
  hits.push({
575
- title: query,
883
+ title: rawTitle,
576
884
  url: cleanUrl,
577
885
  content: snippet,
578
886
  engine: "duckduckgo"
@@ -588,15 +896,210 @@ var MultiSourceSearchEngine = class {
588
896
  }
589
897
  };
590
898
 
899
+ // src/harvester/youtubeTranscript.ts
900
+ var SPONSOR_NOISE_PATTERNS = [
901
+ /\b(sponsored by|this video is brought to you by|use (?:promo )?code\b|don't forget to like and subscribe|hit the bell icon|link in the description)\b/i,
902
+ /(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\s+này)?\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i
903
+ ];
904
+ function extractYouTubeVideoId(urlOrId) {
905
+ const trimmed = urlOrId.trim();
906
+ if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;
907
+ try {
908
+ const u = new URL(trimmed);
909
+ const host = u.hostname.replace(/^www\./, "").toLowerCase();
910
+ if (host === "youtu.be") {
911
+ const id = u.pathname.split("/").filter(Boolean)[0];
912
+ return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;
913
+ }
914
+ if (host === "youtube.com" || host === "m.youtube.com" || host === "youtube-nocookie.com") {
915
+ const v = u.searchParams.get("v");
916
+ if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;
917
+ const parts = u.pathname.split("/").filter(Boolean);
918
+ if ((parts[0] === "embed" || parts[0] === "shorts" || parts[0] === "v") && parts[1]) {
919
+ return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;
920
+ }
921
+ }
922
+ } catch {
923
+ }
924
+ return null;
925
+ }
926
+ function isYouTubeUrl(url) {
927
+ return extractYouTubeVideoId(url) !== null;
928
+ }
929
+ function formatTimestamp(seconds) {
930
+ const total = Math.max(0, Math.floor(seconds));
931
+ const hrs = Math.floor(total / 3600);
932
+ const mins = Math.floor(total % 3600 / 60);
933
+ const secs = total % 60;
934
+ if (hrs > 0) {
935
+ return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
936
+ }
937
+ return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
938
+ }
939
+ function decodeHtmlEntities(str) {
940
+ return str.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&#x27;/g, "'").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
941
+ }
942
+ function stitchCaptionSegments(segments) {
943
+ const cleaned = segments.map((s) => ({
944
+ startSec: s.startSec,
945
+ durationSec: s.durationSec,
946
+ text: decodeHtmlEntities(s.text)
947
+ })).filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));
948
+ if (cleaned.length === 0) return [];
949
+ const stitched = [];
950
+ let bucketStart = cleaned[0].startSec;
951
+ let bucketEnd = cleaned[0].startSec + cleaned[0].durationSec;
952
+ let bucketTexts = [];
953
+ for (const seg of cleaned) {
954
+ bucketTexts.push(seg.text);
955
+ bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);
956
+ const joined = bucketTexts.join(" ");
957
+ const elapsed = seg.startSec - bucketStart;
958
+ const endsSentence = /[.!?…]$/.test(seg.text);
959
+ if (elapsed >= 30 && endsSentence || elapsed >= 50 || joined.length >= 360) {
960
+ stitched.push({
961
+ startSec: bucketStart,
962
+ durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
963
+ timestamp: formatTimestamp(bucketStart),
964
+ text: joined
965
+ });
966
+ bucketTexts = [];
967
+ bucketStart = seg.startSec + seg.durationSec;
968
+ }
969
+ }
970
+ if (bucketTexts.length > 0) {
971
+ stitched.push({
972
+ startSec: bucketStart,
973
+ durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
974
+ timestamp: formatTimestamp(bucketStart),
975
+ text: bucketTexts.join(" ")
976
+ });
977
+ }
978
+ return stitched;
979
+ }
980
+ var YouTubeIngestionService = class {
981
+ async fetchTranscript(urlOrId, preferredLanguages = ["vi", "en"]) {
982
+ const videoId = extractYouTubeVideoId(urlOrId);
983
+ if (!videoId) {
984
+ throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);
985
+ }
986
+ const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;
987
+ const res = await fetch(canonicalUrl, {
988
+ headers: {
989
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
990
+ "Accept-Language": "vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7"
991
+ },
992
+ signal: AbortSignal.timeout(12e3)
993
+ });
994
+ if (!res.ok) {
995
+ throw new Error(`Failed to fetch YouTube watch page (${res.status})`);
996
+ }
997
+ const html = await res.text();
998
+ const playerMatch = html.match(/ytInitialPlayerResponse\s*=\s*(\{[\s\S]*?\});\s*(?:var\s|window\.|<\/script>)/);
999
+ let title = `YouTube Video (${videoId})`;
1000
+ let author;
1001
+ let shortDescription = "";
1002
+ let rawSegments = [];
1003
+ let selectedLang = "en";
1004
+ let isAutoGenerated = false;
1005
+ if (playerMatch?.[1]) {
1006
+ try {
1007
+ const playerData = JSON.parse(playerMatch[1]);
1008
+ title = playerData?.videoDetails?.title || title;
1009
+ author = playerData?.videoDetails?.author;
1010
+ shortDescription = playerData?.videoDetails?.shortDescription || "";
1011
+ const tracks = playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];
1012
+ if (tracks.length > 0) {
1013
+ let chosenTrack = tracks.find(
1014
+ (t) => preferredLanguages.includes(t.languageCode) && t.kind !== "asr"
1015
+ ) || tracks.find((t) => preferredLanguages.includes(t.languageCode)) || tracks[0];
1016
+ if (chosenTrack?.baseUrl) {
1017
+ selectedLang = chosenTrack.languageCode || "en";
1018
+ isAutoGenerated = chosenTrack.kind === "asr";
1019
+ const trackRes = await fetch(chosenTrack.baseUrl, {
1020
+ signal: AbortSignal.timeout(1e4)
1021
+ });
1022
+ if (trackRes.ok) {
1023
+ const xml = await trackRes.text();
1024
+ const textMatches = [
1025
+ ...xml.matchAll(/<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g)
1026
+ ];
1027
+ for (const m of textMatches) {
1028
+ rawSegments.push({
1029
+ startSec: parseFloat(m[1] || "0"),
1030
+ durationSec: parseFloat(m[2] || "2"),
1031
+ text: (m[3] || "").replace(/<[^>]+>/g, " ")
1032
+ });
1033
+ }
1034
+ }
1035
+ }
1036
+ }
1037
+ } catch {
1038
+ }
1039
+ }
1040
+ if (title === `YouTube Video (${videoId})`) {
1041
+ const titleMeta = html.match(/<title>([\s\S]*?)<\/title>/i);
1042
+ if (titleMeta?.[1]) {
1043
+ title = decodeHtmlEntities(titleMeta[1].replace(/\s*-\s*YouTube$/i, ""));
1044
+ }
1045
+ }
1046
+ const stitched = stitchCaptionSegments(rawSegments);
1047
+ const mdLines = [
1048
+ `# ${title}`,
1049
+ `> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ""} | **Language:** \`${selectedLang}\`${isAutoGenerated ? " (Auto-caption)" : ""}`
1050
+ ];
1051
+ if (shortDescription.trim()) {
1052
+ mdLines.push(`
1053
+ ## Video Overview & Description
1054
+ ${shortDescription.trim().slice(0, 1500)}`);
1055
+ }
1056
+ if (stitched.length > 0) {
1057
+ mdLines.push(`
1058
+ ## Timestamped Transcript`);
1059
+ for (const seg of stitched) {
1060
+ mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);
1061
+ }
1062
+ }
1063
+ return {
1064
+ videoId,
1065
+ url: canonicalUrl,
1066
+ title,
1067
+ author,
1068
+ language: selectedLang,
1069
+ isAutoGenerated,
1070
+ segments: stitched,
1071
+ stitchedMarkdown: mdLines.join("\n")
1072
+ };
1073
+ }
1074
+ };
1075
+
591
1076
  // src/harvester/webScraper.ts
1077
+ function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
1078
+ const withoutCode = markdown.replace(/```[\s\S]*?```/g, "\n");
1079
+ const blocks = withoutCode.split(/\n{2,}/).map((b) => b.replace(/\s+/g, " ").trim()).filter((b) => {
1080
+ if (b.length < 60 || b.length > 900) return false;
1081
+ if (b.startsWith("#") || b.startsWith("|")) return false;
1082
+ const linkRatio = (b.match(/\[[^\]]+\]\([^)]+\)/g) || []).join("").length / b.length;
1083
+ if (linkRatio > 0.45) return false;
1084
+ if (/\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\b/i.test(b)) {
1085
+ return false;
1086
+ }
1087
+ return true;
1088
+ });
1089
+ return blocks.slice(0, maxExcerpts);
1090
+ }
592
1091
  var WebScraper = class {
593
1092
  crawl4aiHost;
594
1093
  jinaKey;
1094
+ allowEnvFallback;
595
1095
  customAdapter;
1096
+ youtubeService;
596
1097
  constructor(options = {}) {
597
- this.crawl4aiHost = options.crawl4aiHost || process.env.CRAWL4AI_HOST || "http://localhost:11235";
598
- this.jinaKey = options.jinaKey || process.env.JINA_API_KEY;
1098
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
1099
+ this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
1100
+ this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
599
1101
  this.customAdapter = options.customAdapter;
1102
+ this.youtubeService = new YouTubeIngestionService();
600
1103
  }
601
1104
  setCustomAdapter(adapter) {
602
1105
  this.customAdapter = adapter;
@@ -610,12 +1113,22 @@ var WebScraper = class {
610
1113
  try {
611
1114
  const result = await adapter.crawl(targetUrl);
612
1115
  if (result && result.markdownContent) {
613
- return this.packageDocument(targetUrl, result.markdownContent, "custom");
1116
+ return this.packageDocument(targetUrl, result.markdownContent, "custom", result.title);
614
1117
  }
615
1118
  } catch (e) {
616
1119
  console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);
617
1120
  }
618
1121
  }
1122
+ if (isYouTubeUrl(targetUrl)) {
1123
+ try {
1124
+ const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);
1125
+ if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {
1126
+ return this.packageDocument(yt.url, yt.stitchedMarkdown, "youtube", yt.title);
1127
+ }
1128
+ } catch (e) {
1129
+ console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
1130
+ }
1131
+ }
619
1132
  try {
620
1133
  const jinaUrl = `https://r.jina.ai/${targetUrl}`;
621
1134
  const jinaKey = options.jinaKey || this.jinaKey;
@@ -666,35 +1179,435 @@ var WebScraper = class {
666
1179
  signal: AbortSignal.timeout(1e4)
667
1180
  });
668
1181
  if (res.ok) {
669
- const rawText = await res.text();
670
- const clean = rawText.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
671
- return this.packageDocument(targetUrl, clean, "direct");
1182
+ const rawHtml = await res.text();
1183
+ const titleTag = rawHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
1184
+ const clean = rawHtml.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "").replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, "").replace(/<h([1-3])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, lvl, txt) => `
1185
+ ${"#".repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, "").trim()}
1186
+ `).replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => `
1187
+ \`\`\`
1188
+ ${code.replace(/<[^>]+>/g, "").trim()}
1189
+ \`\`\`
1190
+ `).replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, p) => `
1191
+ ${p.replace(/<[^>]+>/g, " ").trim()}
1192
+ `).replace(/<[^>]+>/g, " ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
1193
+ return this.packageDocument(targetUrl, clean, "direct", titleTag);
672
1194
  }
673
1195
  } catch (e) {
674
1196
  console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);
675
1197
  }
676
1198
  throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
677
1199
  }
678
- packageDocument(url, markdown, provider) {
679
- const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/Title:\s*(.+)/i);
680
- const title = titleMatch ? titleMatch[1].trim() : url.split("/").pop() || "Document";
681
- const codeMatches = [...markdown.matchAll(/```(?:[a-z0-9_-]+)?\n([\s\S]*?)```/gi)].map((m) => m[1]?.trim()).filter(Boolean);
682
- const tableMatches = markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
1200
+ packageDocument(url, markdown, provider, explicitTitle) {
1201
+ const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
1202
+ const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
1203
+ const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```/g)];
1204
+ const extractedCodeBlocks = codeBlockMatches.map((m) => ({
1205
+ language: (m[1] || "").trim().toLowerCase(),
1206
+ code: (m[2] || "").trim()
1207
+ })).filter((b) => b.code.length > 0);
1208
+ const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);
1209
+ const tableBlocks = markdown.match(/(?:^\|.+\|\s*\n)+^\|[-:| ]+\|\s*\n(?:^\|.+\|\s*\n?)+/gm) || markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
1210
+ const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);
1211
+ const extractedHeadings = [...markdown.matchAll(/^#{1,3}\s+(.+)$/gm)].map((m) => m[1].trim()).filter((h) => h.length > 2 && h.length < 120);
1212
+ const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);
683
1213
  return {
684
1214
  url,
685
1215
  title,
686
1216
  markdownContent: markdown,
687
1217
  byteSize: Buffer.byteLength(markdown, "utf-8"),
688
- extractedCodeSnippets: codeMatches,
689
- tablesCount: tableMatches.length,
1218
+ extractedCodeSnippets,
1219
+ extractedCodeBlocks,
1220
+ extractedTables,
1221
+ extractedHeadings,
1222
+ keyExcerpts,
1223
+ tablesCount: extractedTables.length,
690
1224
  fetchProvider: provider
691
1225
  };
692
1226
  }
693
1227
  };
694
1228
 
1229
+ // src/harvester/termExtractor.ts
1230
+ var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
1231
+ "l\xE0",
1232
+ "c\u1EE7a",
1233
+ "trong",
1234
+ "\u0111\u01B0\u1EE3c",
1235
+ "nh\u1EEFng",
1236
+ "c\xE1c",
1237
+ "cho",
1238
+ "v\u1EDBi",
1239
+ "khi",
1240
+ "n\u1EBFu",
1241
+ "th\xEC",
1242
+ "v\xE0",
1243
+ "ho\u1EB7c",
1244
+ "\u0111\u1EC3",
1245
+ "t\u1EEB",
1246
+ "\u0111\u1EBFn",
1247
+ "v\u1EC1",
1248
+ "n\xE0y",
1249
+ "\u0111\xF3",
1250
+ "m\u1ED9t",
1251
+ "nh\u01B0",
1252
+ "s\u1EBD",
1253
+ "\u0111ang",
1254
+ "\u0111\xE3",
1255
+ "kh\xF4ng",
1256
+ "c\xF3",
1257
+ "ra",
1258
+ "v\xE0o",
1259
+ "b\u1EB1ng",
1260
+ "theo",
1261
+ "tr\xEAn",
1262
+ "d\u01B0\u1EDBi",
1263
+ "qua",
1264
+ "nh\u1EDD",
1265
+ "v\xEC",
1266
+ "do",
1267
+ "t\u1EA1i",
1268
+ "b\u1EDFi",
1269
+ "nh\u01B0ng",
1270
+ "m\xE0",
1271
+ "c\u0169ng",
1272
+ "v\u1EABn",
1273
+ "r\u1EA5t",
1274
+ "qu\xE1",
1275
+ "h\u01A1n",
1276
+ "nh\u1EA5t",
1277
+ "c\u1EA7n",
1278
+ "ph\u1EA3i",
1279
+ "n\xEAn",
1280
+ "th\u1EC3",
1281
+ "vi\u1EC7c",
1282
+ "c\xE1ch",
1283
+ "ph\u1EA7n",
1284
+ "b\xE0i",
1285
+ "bu\u1ED5i",
1286
+ "h\u1ECDc",
1287
+ "sinh",
1288
+ "gi\xE1o",
1289
+ "vi\xEAn",
1290
+ "ng\u01B0\u1EDDi",
1291
+ "d\xF9ng",
1292
+ "ch\xFAng",
1293
+ "ta",
1294
+ "b\u1EA1n",
1295
+ "em",
1296
+ "gi\xFAp",
1297
+ "l\xE0m",
1298
+ "t\u1EA1o",
1299
+ "s\u1EED",
1300
+ "d\u1EE5ng",
1301
+ "th\u1EF1c",
1302
+ "hi\u1EC7n",
1303
+ "hi\u1EC3u",
1304
+ "bi\u1EBFt",
1305
+ "xem",
1306
+ "th\xEAm",
1307
+ "sau",
1308
+ "tr\u01B0\u1EDBc"
1309
+ ]);
1310
+ var ENGLISH_STOPWORDS = /* @__PURE__ */ new Set([
1311
+ "the",
1312
+ "and",
1313
+ "for",
1314
+ "with",
1315
+ "that",
1316
+ "this",
1317
+ "from",
1318
+ "into",
1319
+ "your",
1320
+ "you",
1321
+ "are",
1322
+ "was",
1323
+ "were",
1324
+ "have",
1325
+ "has",
1326
+ "had",
1327
+ "will",
1328
+ "would",
1329
+ "can",
1330
+ "could",
1331
+ "should",
1332
+ "about",
1333
+ "more",
1334
+ "other",
1335
+ "such",
1336
+ "than",
1337
+ "then",
1338
+ "them",
1339
+ "these",
1340
+ "those",
1341
+ "their",
1342
+ "what",
1343
+ "when",
1344
+ "where",
1345
+ "which",
1346
+ "while",
1347
+ "who",
1348
+ "how",
1349
+ "why",
1350
+ "each",
1351
+ "some",
1352
+ "many",
1353
+ "most",
1354
+ "also",
1355
+ "only",
1356
+ "just",
1357
+ "very",
1358
+ "using",
1359
+ "used",
1360
+ "use",
1361
+ "overview",
1362
+ "introduction",
1363
+ "getting",
1364
+ "started",
1365
+ "guide",
1366
+ "tutorial",
1367
+ "example",
1368
+ "examples",
1369
+ "documentation"
1370
+ ]);
1371
+ var VIETNAMESE_TECHNICAL_HEADS = /^(cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|chuỗi|đối tượng|lớp|phương thức|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|mạng|giao thức|khái niệm|mô hình|kiến trúc|kiểm thử|gỡ lỗi)\b/i;
1372
+ var SYNTAX_OR_API_PATTERN = /(?:`[^`]+`|\b[a-z]+[A-Z][a-zA-Z0-9]*\b|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b|@[A-Za-z]+|\b[a-z_]+\(\)|\b(?:pinMode|digitalWrite|digitalRead|analogRead|analogWrite|async|await|actor|struct|class|func|def|let|const|var)\b)/;
1373
+ var HARDWARE_COMPONENT_PATTERN = /\b(esp32|arduino|micro:?bit|raspberry|sensor|led|servo|ultrasonic|ldr|dht11|dht22|breadboard|gpio|pwm|i2c|spi|uart|cảm biến|động cơ|mạch|đèn led|vi điều khiển)\b/i;
1374
+ function normalizeTermKey(raw) {
1375
+ return raw.normalize("NFC").toLowerCase().replace(/[`*_~#"']/g, "").replace(/\s+/g, " ").trim();
1376
+ }
1377
+ function cleanDisplayTerm(raw) {
1378
+ return raw.normalize("NFC").replace(/^[-*•#\d.)\s]+/, "").replace(/[`*_~]+/g, "").replace(/\s+/g, " ").trim();
1379
+ }
1380
+ function isNoiseTerm(term) {
1381
+ const clean = normalizeTermKey(term);
1382
+ if (clean.length < 2 || clean.length > 65) return true;
1383
+ if (/^\d+$/.test(clean)) return true;
1384
+ if (/^https?:\/\//i.test(clean)) return true;
1385
+ if (ENGLISH_STOPWORDS.has(clean) || VIETNAMESE_STOPWORDS.has(clean)) return true;
1386
+ const words = clean.split(" ");
1387
+ if (words.length > 6) return true;
1388
+ if (words.length === 1 && clean.length < 3 && !/^(ai|ui|ux|os|io|db|id|ip)$/i.test(clean)) return true;
1389
+ return false;
1390
+ }
1391
+ function classifyMarrAndCategory(term, domain) {
1392
+ if (HARDWARE_COMPONENT_PATTERN.test(term)) {
1393
+ return { category: "hardware_component", marrLayer: "CIO", bloomLevel: "apply" };
1394
+ }
1395
+ if (SYNTAX_OR_API_PATTERN.test(term)) {
1396
+ return { category: "api_syntax", marrLayer: "SIO", bloomLevel: "apply" };
1397
+ }
1398
+ if (/\b(architecture|pattern|model|design|debugging|verification|mô hình|kiến trúc|tư duy|quy trình)\b/i.test(term)) {
1399
+ return { category: "pedagogical_pattern", marrLayer: "ULO", bloomLevel: "analyze" };
1400
+ }
1401
+ return {
1402
+ category: "core_concept",
1403
+ marrLayer: domain === "academic" ? "ULO" : "CIO",
1404
+ bloomLevel: "understand"
1405
+ };
1406
+ }
1407
+ function findSentenceDefinition(term, markdown) {
1408
+ const clean = cleanDisplayTerm(term);
1409
+ const escaped = clean.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1410
+ const sentences = markdown.replace(/```[\s\S]*?```/g, " ").split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").trim()).filter((s) => s.length >= 25 && s.length <= 280);
1411
+ const re = new RegExp(escaped, "i");
1412
+ const match = sentences.find((s) => re.test(s));
1413
+ return match || `Core concept in ${clean} extracted from primary technical reference.`;
1414
+ }
1415
+ var PedagogicalTermExtractor = class {
1416
+ extractDeterministic(topic, domain, docs, maxTerms = 15) {
1417
+ const candidates = /* @__PURE__ */ new Map();
1418
+ let orderCounter = 0;
1419
+ const upsertCandidate = (rawTerm, weight, resourceId, markdown, pair) => {
1420
+ const display = cleanDisplayTerm(rawTerm);
1421
+ if (isNoiseTerm(display)) return;
1422
+ const key = normalizeTermKey(display);
1423
+ const existing = candidates.get(key);
1424
+ if (existing) {
1425
+ existing.score += weight;
1426
+ existing.sourceIds.add(resourceId);
1427
+ if (pair?.termVi && !existing.termVi) existing.termVi = pair.termVi;
1428
+ if (pair?.termEn && !existing.termEn) existing.termEn = pair.termEn;
1429
+ } else {
1430
+ candidates.set(key, {
1431
+ display,
1432
+ termVi: pair?.termVi,
1433
+ termEn: pair?.termEn,
1434
+ score: weight,
1435
+ firstSeenOrder: orderCounter++,
1436
+ sourceIds: /* @__PURE__ */ new Set([resourceId]),
1437
+ definition: findSentenceDefinition(display, markdown)
1438
+ });
1439
+ }
1440
+ };
1441
+ for (const { resourceId, doc } of docs) {
1442
+ const md = doc.markdownContent || "";
1443
+ const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
1444
+ const bilingualMatches = mdWithoutCode.matchAll(
1445
+ /([A-ZÀÁẠẢÃÂẦẤẬẨẪĂẰẮẶẲẴÈÉẸẺẼÊỀẾỆỂỄÌÍỊỈĨÒÓỌỎÕÔỒỐỘỔỖƠỜỚỢỞỠÙÚỤỦŨƯỪỨỰỬỮỲÝỴỶỸĐ][a-zA-Z0-9\u00C0-\u1EF9\s/-]{2,40}?)\s*\(([A-Za-z0-9\u00C0-\u1EF9\s/_:-]{2,40})\)/g
1446
+ );
1447
+ for (const m of bilingualMatches) {
1448
+ const left = cleanDisplayTerm(m[1] || "");
1449
+ const right = cleanDisplayTerm(m[2] || "");
1450
+ if (!isNoiseTerm(left) && !isNoiseTerm(right)) {
1451
+ const leftHasVi = /[\u00C0-\u1EF9]/.test(left);
1452
+ const rightHasVi = /[\u00C0-\u1EF9]/.test(right);
1453
+ if (leftHasVi && !rightHasVi) {
1454
+ upsertCandidate(`${left} (${right})`, 6, resourceId, mdWithoutCode, { termVi: left, termEn: right });
1455
+ } else if (!leftHasVi && rightHasVi) {
1456
+ upsertCandidate(`${right} (${left})`, 6, resourceId, mdWithoutCode, { termVi: right, termEn: left });
1457
+ } else {
1458
+ upsertCandidate(left, 4, resourceId, mdWithoutCode);
1459
+ }
1460
+ }
1461
+ }
1462
+ for (const heading of doc.extractedHeadings || []) {
1463
+ const cleaned = cleanDisplayTerm(heading.replace(/^\d+(\.\d+)*\s*/, ""));
1464
+ if (!isNoiseTerm(cleaned)) {
1465
+ upsertCandidate(cleaned, 4, resourceId, mdWithoutCode);
1466
+ }
1467
+ }
1468
+ for (const m of mdWithoutCode.matchAll(/\*\*([^*\n]{3,45})\*\*/g)) {
1469
+ upsertCandidate(m[1] || "", 3.5, resourceId, mdWithoutCode);
1470
+ }
1471
+ for (const m of mdWithoutCode.matchAll(/`([a-zA-Z_@][a-zA-Z0-9_.:()-]{2,35})`/g)) {
1472
+ upsertCandidate(m[1] || "", 3, resourceId, mdWithoutCode);
1473
+ }
1474
+ for (const line of mdWithoutCode.split("\n")) {
1475
+ const clauses = line.split(/[.,;:!?()[\]{}"“”‘’/\\|]+/);
1476
+ for (const clause of clauses) {
1477
+ const trimmed = clause.trim();
1478
+ const viMatch = trimmed.match(
1479
+ /\b((?:cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|đối tượng|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|giao thức|mô hình|kiến trúc|kiểm thử|gỡ lỗi)(?:\s+[a-zA-Z0-9\u00C0-\u1EF9_-]+){1,3})\b/i
1480
+ );
1481
+ if (viMatch?.[1]) {
1482
+ const tokens = viMatch[1].trim().split(/\s+/);
1483
+ while (tokens.length > 1 && VIETNAMESE_STOPWORDS.has(tokens[tokens.length - 1].toLowerCase())) {
1484
+ tokens.pop();
1485
+ }
1486
+ const phrase = tokens.join(" ");
1487
+ if (tokens.length >= 2 && VIETNAMESE_TECHNICAL_HEADS.test(phrase)) {
1488
+ upsertCandidate(phrase, 3.5, resourceId, mdWithoutCode, { termVi: phrase });
1489
+ }
1490
+ }
1491
+ }
1492
+ }
1493
+ }
1494
+ const sorted = [...candidates.values()].sort((a, b) => b.score - a.score || a.firstSeenOrder - b.firstSeenOrder).slice(0, maxTerms);
1495
+ const orderedByAppearance = [...sorted].sort((a, b) => a.firstSeenOrder - b.firstSeenOrder);
1496
+ const maxScore = Math.max(1, sorted[0]?.score || 1);
1497
+ const concepts = orderedByAppearance.map((item, idx) => {
1498
+ const id = `C_${String(idx + 1).padStart(2, "0")}`;
1499
+ const { category, marrLayer, bloomLevel } = classifyMarrAndCategory(item.display, domain);
1500
+ const prerequisites = [];
1501
+ return {
1502
+ id,
1503
+ term: item.display,
1504
+ termVi: item.termVi,
1505
+ termEn: item.termEn,
1506
+ category,
1507
+ marrLayer,
1508
+ bloomLevel,
1509
+ salienceScore: Number(Math.min(10, Math.max(1, item.score / maxScore * 10)).toFixed(1)),
1510
+ definition: item.definition,
1511
+ prerequisites,
1512
+ sourceIds: [...item.sourceIds]
1513
+ };
1514
+ });
1515
+ return {
1516
+ concepts,
1517
+ conceptDagTsv: this.formatConceptDagTsv(concepts),
1518
+ isLlmRefined: false,
1519
+ unrefinedCandidatesCount: candidates.size
1520
+ };
1521
+ }
1522
+ async extractWithOptionalLlm(topic, domain, docs, options = {}) {
1523
+ const detBundle = this.extractDeterministic(topic, domain, docs, options.maxTerms ?? 15);
1524
+ if (!options.llm || docs.length === 0) {
1525
+ return detBundle;
1526
+ }
1527
+ try {
1528
+ const corpusPreview = docs.slice(0, 6).map(
1529
+ (d) => `[${d.resourceId}] ${d.doc.title} (${d.doc.url}):
1530
+ ${(d.doc.keyExcerpts || []).slice(0, 3).join("\n")}`
1531
+ ).join("\n\n");
1532
+ const candidateSeed = detBundle.concepts.slice(0, 12).map((c) => `${c.term} (${c.sourceIds.join(",")})`).join("; ");
1533
+ const prompt = [
1534
+ `Analyze the harvested technical & pedagogical corpus for topic "${topic}" (Domain: ${domain}).`,
1535
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]} years old.` : "",
1536
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}.` : "",
1537
+ options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}.` : "",
1538
+ `Deterministic Candidate Terms: ${candidateSeed}`,
1539
+ `
1540
+ Harvested Source Excerpts:
1541
+ ${corpusPreview}`,
1542
+ `
1543
+ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
1544
+ `IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
1545
+ `Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
1546
+ ].filter(Boolean).join("\n");
1547
+ const rawResponse = await options.llm(prompt, {
1548
+ system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
1549
+ temperature: 0.2,
1550
+ jsonMode: true
1551
+ });
1552
+ const jsonMatch = rawResponse.match(/\[[\s\S]*\]/);
1553
+ if (jsonMatch) {
1554
+ const parsed = JSON.parse(jsonMatch[0]);
1555
+ if (Array.isArray(parsed) && parsed.length > 0) {
1556
+ const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
1557
+ const concepts = parsed.map((item, idx) => {
1558
+ const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
1559
+ const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
1560
+ return {
1561
+ id,
1562
+ term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
1563
+ termVi: item.termVi ? String(item.termVi) : void 0,
1564
+ termEn: item.termEn ? String(item.termEn) : void 0,
1565
+ category: ["core_concept", "api_syntax", "hardware_component", "pedagogical_pattern"].includes(item.category) ? item.category : "core_concept",
1566
+ marrLayer: ["ULO", "CIO", "SIO"].includes(item.marrLayer) ? item.marrLayer : "CIO",
1567
+ bloomLevel: ["remember", "understand", "apply", "analyze", "evaluate", "create"].includes(item.bloomLevel) ? item.bloomLevel : "understand",
1568
+ salienceScore: Number(item.salienceScore) || 8,
1569
+ definition: String(item.definition || ""),
1570
+ prerequisites: prereqs,
1571
+ sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
1572
+ };
1573
+ });
1574
+ return {
1575
+ concepts,
1576
+ conceptDagTsv: this.formatConceptDagTsv(concepts),
1577
+ isLlmRefined: true,
1578
+ unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
1579
+ };
1580
+ }
1581
+ }
1582
+ } catch (e) {
1583
+ console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
1584
+ }
1585
+ return detBundle;
1586
+ }
1587
+ formatConceptDagTsv(concepts) {
1588
+ const header = "id term term_vi term_en marr_layer bloom_level salience prerequisites sources definition";
1589
+ const rows = concepts.map(
1590
+ (c) => [
1591
+ c.id,
1592
+ c.term.replace(/\t|\n/g, " "),
1593
+ (c.termVi || "").replace(/\t|\n/g, " "),
1594
+ (c.termEn || "").replace(/\t|\n/g, " "),
1595
+ c.marrLayer,
1596
+ c.bloomLevel,
1597
+ c.salienceScore,
1598
+ c.prerequisites.join(","),
1599
+ c.sourceIds.join(","),
1600
+ c.definition.replace(/\t|\n/g, " ")
1601
+ ].join(" ")
1602
+ );
1603
+ return [header, ...rows].join("\n");
1604
+ }
1605
+ };
1606
+
695
1607
  // src/harvester/catalogBuilder.ts
696
1608
  var CatalogBuilder = class {
697
- buildBundle(topic, domain, items) {
1609
+ buildBundle(topic, domain, items, options = {}) {
1610
+ const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
698
1611
  const indexList = items.map((item, idx) => ({
699
1612
  id: `RES-${String(idx + 1).padStart(2, "0")}`,
700
1613
  title: item.doc.title,
@@ -704,7 +1617,8 @@ var CatalogBuilder = class {
704
1617
  byteSize: item.doc.byteSize,
705
1618
  tablesCount: item.doc.tablesCount,
706
1619
  codeSnippetsCount: item.doc.extractedCodeSnippets.length,
707
- summary: item.summary
1620
+ summary: item.summary,
1621
+ keyTakeaways: item.keyTakeaways || item.doc.keyExcerpts?.slice(0, 3) || []
708
1622
  }));
709
1623
  const resourceIndexJson = JSON.stringify(
710
1624
  {
@@ -723,36 +1637,60 @@ var CatalogBuilder = class {
723
1637
  `
724
1638
  | ID | Resource Title | Relevance (1-10) | Code Snippets | Tables | Source Link |`,
725
1639
  `|---|---|---|---|---|---|`,
726
- ...indexList.map((r) => `| **${r.id}** | ${r.title} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`),
1640
+ ...indexList.map(
1641
+ (r) => `| **${r.id}** | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`
1642
+ ),
727
1643
  `
728
1644
  ## Resource Summaries & Technical Ground Truth
729
1645
  `,
730
- ...indexList.map((r) => `### ${r.id}: ${r.title}
1646
+ ...indexList.map((r) => {
1647
+ const takeaways = r.keyTakeaways && r.keyTakeaways.length > 0 ? r.keyTakeaways.map((t) => ` - ${t}`).join("\n") : ` - ${r.summary}`;
1648
+ return `### ${r.id}: ${r.title}
731
1649
  - **URL:** ${r.url}
732
- - **Summary:** ${r.summary}
733
1650
  - **Relevance:** ${r.relevanceScore}/10
734
- `)
1651
+ - **Key Excerpts:**
1652
+ ${takeaways}
1653
+ `;
1654
+ })
735
1655
  ];
1656
+ const codeSections = items.flatMap((item) => {
1657
+ if (item.doc.extractedCodeBlocks && item.doc.extractedCodeBlocks.length > 0) {
1658
+ return item.doc.extractedCodeBlocks.slice(0, 2).map((b) => {
1659
+ const lang = b.language || defaultLang;
1660
+ return `\`\`\`${lang}
1661
+ ${b.code}
1662
+ \`\`\``;
1663
+ });
1664
+ }
1665
+ return item.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
1666
+ ${c}
1667
+ \`\`\``);
1668
+ });
1669
+ const conceptLines = options.concepts && options.concepts.length > 0 ? [
1670
+ `| ID | Concept / Term | Vietnamese | English | Marr | Bloom | Prereqs | Sources |`,
1671
+ `|---|---|---|---|---|---|---|---|`,
1672
+ ...options.concepts.map(
1673
+ (c) => `| \`${c.id}\` | **${c.term.replace(/\|/g, "/")}** | ${c.termVi || "\u2014"} | ${c.termEn || "\u2014"} | \`${c.marrLayer}\` | \`${c.bloomLevel}\` | ${c.prerequisites.join(", ") || "\u2014"} | ${c.sourceIds.join(", ")} |`
1674
+ )
1675
+ ] : [`- Canonical technical breakdown for **${topic}** derived from ${items.length} verified primary sources.`];
736
1676
  const kgLines = [
737
1677
  `# \u{1F9EC} Distilled Knowledge Graph: ${topic}`,
738
1678
  `> **Domain:** \`${domain.toUpperCase()}\` | **Ground Truth Verification:** PASSED`,
739
1679
  `
740
- ## 1. Architectural Blueprint & Component Decomposition`,
741
- `- Canonical technical breakdown for **${topic}** derived from ${items.length} peer-reviewed and verified primary sources.`,
1680
+ ## 1. Architectural Blueprint & Concept Hierarchy (Marr & Bloom)`,
1681
+ ...conceptLines,
742
1682
  `
743
1683
  ## 2. Core APIs & Code Primitives`,
744
- ...items.flatMap((i) => i.doc.extractedCodeSnippets.slice(0, 2)).map((c) => `\`\`\`${domain === "hardware" ? "cpp" : "swift"}
745
- ${c}
746
- \`\`\``),
1684
+ ...codeSections.length > 0 ? codeSections : ["*No explicit code snippets captured in harvested sources.*"],
747
1685
  `
748
- ## 3. Technical Constraints & Common Pitfalls`,
749
- `- Strict compliance with ${domain.toUpperCase()} safety and concurrency contracts.`,
750
- `- Proactively prevent documented anti-patterns and performance regressions.`
1686
+ ## 3. Extracted Reference Tables`,
1687
+ ...items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)).length > 0 ? items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)) : ["*No markdown reference tables extracted.*"]
751
1688
  ];
752
1689
  return {
753
1690
  resourceIndexJson,
754
1691
  resourceCatalogMd: catalogLines.join("\n"),
755
- knowledgeGraphMd: kgLines.join("\n")
1692
+ knowledgeGraphMd: kgLines.join("\n"),
1693
+ conceptDagTsv: options.conceptDagTsv
756
1694
  };
757
1695
  }
758
1696
  };
@@ -778,73 +1716,372 @@ var HealthAuditor = class {
778
1716
  };
779
1717
 
780
1718
  // src/evidence/evidenceScorer.ts
1719
+ function tokenizeBilingualKeywords(text) {
1720
+ return text.normalize("NFC").toLowerCase().replace(/[.,;:!?()[\]{}"“”‘’/\\|`*_~#-]+/g, " ").split(/\s+/).filter((w) => w.length >= 2 && !/^(and|the|for|with|from|into|của|trong|được|những|các|cho|với|khi|và|hoặc|để|từ|đến|về)$/.test(w));
1721
+ }
1722
+ function extractTopSentences(markdown, targetTopic, maxSentences = 3) {
1723
+ const keywords = tokenizeBilingualKeywords(targetTopic);
1724
+ const withoutCode = markdown.replace(/```[\s\S]*?```/g, " ");
1725
+ const rawSentences = withoutCode.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => {
1726
+ if (s.length < 45 || s.length > 320) return false;
1727
+ if (s.startsWith("|") || s.startsWith("http")) return false;
1728
+ if (/\b(cookie|privacy policy|sign in|subscribe|all rights reserved)\b/i.test(s)) return false;
1729
+ return true;
1730
+ });
1731
+ if (rawSentences.length === 0) return [];
1732
+ const scored = rawSentences.map((sentence, idx) => {
1733
+ const lower = sentence.normalize("NFC").toLowerCase();
1734
+ const matchedCount = keywords.filter((k) => lower.includes(k)).length;
1735
+ const positionBonus = idx < 8 ? 1.5 : idx < 20 ? 0.5 : 0;
1736
+ const technicalBonus = /(`[^`]+`|\b(?:api|function|class|actor|async|sensor|pin|loop|state|biến|hàm|cảm biến|vòng lặp|mạch)\b)/i.test(
1737
+ sentence
1738
+ ) ? 1.5 : 0;
1739
+ return {
1740
+ sentence,
1741
+ idx,
1742
+ score: matchedCount * 2 + positionBonus + technicalBonus
1743
+ };
1744
+ });
1745
+ return scored.sort((a, b) => b.score - a.score || a.idx - b.idx).slice(0, maxSentences).sort((a, b) => a.idx - b.idx).map((s) => s.sentence);
1746
+ }
781
1747
  var EvidenceScorer = class {
782
1748
  scoreDocument(doc, domain, targetTopic) {
783
1749
  let depthScore = 5;
784
- let codeScore = doc.extractedCodeSnippets.length > 0 ? 8 : 4;
1750
+ const hasCode = doc.extractedCodeSnippets.length > 0;
1751
+ let codeScore = hasCode ? Math.min(10, 7 + Math.min(3, doc.extractedCodeSnippets.length)) : 5;
785
1752
  let noiseScore = 2;
786
- const lowerContent = doc.markdownContent.toLowerCase();
787
- const targetKeywords = targetTopic.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
1753
+ const lowerContent = doc.markdownContent.normalize("NFC").toLowerCase();
1754
+ const targetKeywords = tokenizeBilingualKeywords(targetTopic);
788
1755
  const keywordMatches = targetKeywords.filter((k) => lowerContent.includes(k)).length;
789
- if (keywordMatches >= targetKeywords.length) depthScore += 3;
1756
+ const matchRatio = targetKeywords.length > 0 ? keywordMatches / targetKeywords.length : 1;
1757
+ if (matchRatio >= 0.75) depthScore += 3;
1758
+ else if (matchRatio >= 0.4) depthScore += 2;
1759
+ else if (keywordMatches >= 1) depthScore += 1;
790
1760
  if (doc.tablesCount > 0) depthScore += 1;
791
- if (doc.byteSize > 2e3) depthScore += 1;
1761
+ if (doc.byteSize > 1500) depthScore += 1;
1762
+ if ((doc.extractedHeadings?.length || 0) >= 2) depthScore += 1;
1763
+ depthScore = Math.min(10, depthScore);
1764
+ if (!hasCode && doc.byteSize > 1200 && matchRatio >= 0.5) {
1765
+ codeScore = 6;
1766
+ }
792
1767
  if (lowerContent.includes("subscribe") && lowerContent.includes("advertisement")) {
793
1768
  noiseScore += 3;
794
1769
  }
795
- const rawTotal = depthScore * 0.5 + codeScore * 0.4 - noiseScore * 0.1;
1770
+ const rawTotal = depthScore * 0.55 + codeScore * 0.35 - noiseScore * 0.1 + 0.5;
796
1771
  const score = Math.max(1, Math.min(10, Math.round(rawTotal)));
797
1772
  const isApproved = score >= 6;
798
- const summary = `Evidence scored ${score}/10 (${isApproved ? "APPROVED" : "REJECTED"}). Contains ${doc.extractedCodeSnippets.length} code snippets and ${doc.tablesCount} reference tables for domain ${domain}.`;
1773
+ const topSentences = extractTopSentences(doc.markdownContent, targetTopic, 3);
1774
+ const excerpts = doc.keyExcerpts?.length ? doc.keyExcerpts : extractSubstantiveExcerpts(doc.markdownContent, 3);
1775
+ const keyTakeaways = topSentences.length > 0 ? topSentences : excerpts.slice(0, 2);
1776
+ const summary = keyTakeaways.length > 0 ? `${keyTakeaways[0]} [Evidence ${score}/10 \xB7 ${doc.extractedCodeSnippets.length} code blocks \xB7 ${doc.tablesCount} tables]` : `Evidence scored ${score}/10 (${isApproved ? "APPROVED" : "REJECTED"}) for ${domain} (${doc.byteSize} bytes).`;
799
1777
  return {
800
1778
  doc,
801
1779
  score,
802
1780
  isApproved,
1781
+ isLlmEvaluated: false,
803
1782
  breakdown: {
804
1783
  technicalDepth: depthScore,
1784
+ structuralRichness: depthScore,
805
1785
  codeSnippetQuality: codeScore,
806
1786
  noiseFactor: noiseScore
807
1787
  },
808
- summary
1788
+ summary,
1789
+ keyTakeaways
809
1790
  };
810
1791
  }
1792
+ /**
1793
+ * Evaluates evidence with host LLM for genuine pedagogical relevance, conceptual depth,
1794
+ * and accuracy for the target learner age band.
1795
+ */
1796
+ async scoreDocumentWithLlm(doc, domain, targetTopic, options = {}) {
1797
+ const baseline = this.scoreDocument(doc, domain, targetTopic);
1798
+ if (!options.llm) {
1799
+ return baseline;
1800
+ }
1801
+ try {
1802
+ const prompt = [
1803
+ `You are a Senior Curriculum Researcher. Evaluate this harvested web document for the course topic "${targetTopic}" (Domain: ${domain}).`,
1804
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
1805
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : "",
1806
+ options.context?.softwareEnv ? `Software / Tools: ${options.context.softwareEnv}` : "",
1807
+ `
1808
+ Document Title: ${doc.title}`,
1809
+ `Document URL: ${doc.url}`,
1810
+ `Sample Excerpts:
1811
+ ${(doc.keyExcerpts || []).slice(0, 3).join("\n")}`,
1812
+ `Code Blocks Found: ${doc.extractedCodeSnippets.length}`,
1813
+ `
1814
+ Return ONLY a JSON object:`,
1815
+ `{"score": 8, "isApproved": true, "pedagogicalRelevance": 9, "summary": "1-sentence evaluative summary", "keyTakeaways": ["Takeaway 1 grounded in text", "Takeaway 2"]}`
1816
+ ].filter(Boolean).join("\n");
1817
+ const raw = await options.llm(prompt, {
1818
+ system: "You are an educational curriculum auditor. Output strictly valid JSON.",
1819
+ temperature: 0.2,
1820
+ jsonMode: true
1821
+ });
1822
+ const match = raw.match(/\{[\s\S]*\}/);
1823
+ if (match) {
1824
+ const parsed = JSON.parse(match[0]);
1825
+ const score = Math.max(1, Math.min(10, Number(parsed.score) || baseline.score));
1826
+ return {
1827
+ doc,
1828
+ score,
1829
+ isApproved: score >= 6,
1830
+ isLlmEvaluated: true,
1831
+ breakdown: {
1832
+ ...baseline.breakdown,
1833
+ pedagogicalRelevance: Number(parsed.pedagogicalRelevance) || score
1834
+ },
1835
+ summary: String(parsed.summary || baseline.summary),
1836
+ keyTakeaways: Array.isArray(parsed.keyTakeaways) && parsed.keyTakeaways.length > 0 ? parsed.keyTakeaways.map(String) : baseline.keyTakeaways
1837
+ };
1838
+ }
1839
+ } catch (e) {
1840
+ console.warn("[EvidenceScorer] LLM evidence evaluation failed, using structural baseline:", e?.message || e);
1841
+ }
1842
+ return baseline;
1843
+ }
811
1844
  };
812
1845
 
813
1846
  // src/evidence/claimVerifier.ts
1847
+ var CLAIM_STOPWORDS = /* @__PURE__ */ new Set([
1848
+ "the",
1849
+ "and",
1850
+ "for",
1851
+ "with",
1852
+ "that",
1853
+ "this",
1854
+ "from",
1855
+ "into",
1856
+ "are",
1857
+ "was",
1858
+ "were",
1859
+ "have",
1860
+ "has",
1861
+ "will",
1862
+ "would",
1863
+ "through",
1864
+ "their",
1865
+ "these",
1866
+ "those",
1867
+ "which",
1868
+ "while",
1869
+ "c\u1EE7a",
1870
+ "trong",
1871
+ "\u0111\u01B0\u1EE3c",
1872
+ "nh\u1EEFng",
1873
+ "c\xE1c",
1874
+ "cho",
1875
+ "v\u1EDBi",
1876
+ "v\xE0",
1877
+ "ho\u1EB7c",
1878
+ "\u0111\u1EC3",
1879
+ "t\u1EEB",
1880
+ "\u0111\u1EBFn",
1881
+ "v\u1EC1",
1882
+ "n\xE0y",
1883
+ "\u0111\xF3",
1884
+ "m\u1ED9t",
1885
+ "nh\u01B0",
1886
+ "s\u1EBD",
1887
+ "\u0111ang",
1888
+ "\u0111\xE3",
1889
+ "ra",
1890
+ "v\xE0o",
1891
+ "l\xE0"
1892
+ ]);
1893
+ function extractSignificantTokens(text) {
1894
+ return text.normalize("NFC").toLowerCase().replace(/[.,;:!?()[\]{}"“”‘’/\\|`*_~#-]+/g, " ").split(/\s+/).filter((w) => w.length >= 3 && !CLAIM_STOPWORDS.has(w));
1895
+ }
1896
+ function tokenMatchesCorpus(token, lowerCorpus) {
1897
+ if (lowerCorpus.includes(token)) return true;
1898
+ if (token.length >= 6) {
1899
+ const stem = token.replace(/(?:ing|tion|ions|ment|ments|ed|es|s)$/, "");
1900
+ if (stem.length >= 4 && lowerCorpus.includes(stem)) return true;
1901
+ }
1902
+ return false;
1903
+ }
814
1904
  var ClaimVerifier = class {
815
- verifyClaims(claims, sourceCorpus) {
1905
+ /**
1906
+ * Deterministic lexical match baseline.
1907
+ * Explicitly marks verdict as 'lexical_match_only' to prevent false sense of semantic verification.
1908
+ */
1909
+ verifyClaimsDeterministic(claims, sourceCorpus) {
816
1910
  const verifiedList = [];
817
- const lowerCorpus = sourceCorpus.toLowerCase();
1911
+ const normCorpus = sourceCorpus.normalize("NFC");
1912
+ const lowerCorpus = normCorpus.toLowerCase();
1913
+ const corpusSentences = normCorpus.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => {
1914
+ if (s.length < 25) return false;
1915
+ if (/^(title:|url source:|final url:|http|!\[)/i.test(s)) return false;
1916
+ return true;
1917
+ });
818
1918
  for (const claim of claims) {
819
- const claimKeywords = claim.toLowerCase().split(/\s+/).filter((w) => w.length > 4);
820
- const matchCount = claimKeywords.filter((k) => lowerCorpus.includes(k)).length;
821
- const matchRatio = claimKeywords.length > 0 ? matchCount / claimKeywords.length : 0;
822
- const isSupported = matchRatio >= 0.6;
1919
+ const claimTokens = extractSignificantTokens(claim);
1920
+ const matchCount = claimTokens.filter((t) => tokenMatchesCorpus(t, lowerCorpus)).length;
1921
+ const matchRatio = claimTokens.length > 0 ? matchCount / claimTokens.length : 0;
1922
+ let bestQuote;
1923
+ let bestSentenceRatio = 0;
1924
+ for (const sent of corpusSentences) {
1925
+ const lowerSent = sent.toLowerCase();
1926
+ const sentMatches = claimTokens.filter((t) => tokenMatchesCorpus(t, lowerSent)).length;
1927
+ const sentRatio = claimTokens.length > 0 ? sentMatches / claimTokens.length : 0;
1928
+ if (sentRatio > bestSentenceRatio) {
1929
+ bestSentenceRatio = sentRatio;
1930
+ bestQuote = sent.slice(0, 240);
1931
+ }
1932
+ }
1933
+ const effectiveRatio = Math.max(matchRatio, bestSentenceRatio);
1934
+ const hasLexicalEvidence = effectiveRatio >= 0.55;
823
1935
  verifiedList.push({
824
1936
  claimText: claim,
825
- isSupported,
826
- confidenceScore: Math.round(matchRatio * 100)
1937
+ verdict: "lexical_match_only",
1938
+ isSupported: hasLexicalEvidence,
1939
+ isLlmVerified: false,
1940
+ confidenceScore: Math.round(effectiveRatio * 100),
1941
+ extractedQuote: hasLexicalEvidence ? bestQuote : void 0,
1942
+ explanation: hasLexicalEvidence ? `Lexical token match found (${Math.round(effectiveRatio * 100)}% token overlap). Semantic entailment requires LLM verification.` : `Insufficient lexical overlap in harvest corpus (${Math.round(effectiveRatio * 100)}%).`
827
1943
  });
828
1944
  }
829
1945
  const supportedCount = verifiedList.filter((c) => c.isSupported).length;
830
1946
  const supportPct = claims.length > 0 ? supportedCount / claims.length * 100 : 100;
831
1947
  const hallucinationRisk = supportPct >= 80 ? "low" : supportPct >= 50 ? "medium" : "high";
832
1948
  const mdLines = [
833
- `# \u{1F50D} Claim-Support Sweep & Verification Report`,
834
- `> **Verification Rate:** \`${supportPct.toFixed(1)}%\` (${supportedCount}/${claims.length} claims supported) | **Hallucination Risk:** \`${hallucinationRisk.toUpperCase()}\``,
1949
+ `# \u{1F50D} Claim-Support Sweep & Verification Report (Lexical Scan Mode)`,
1950
+ `> **Notice:** *Scored via deterministic lexical token matching. True semantic entailment requires an LLM call.*`,
1951
+ `> **Lexical Coverage:** \`${supportPct.toFixed(1)}%\` (${supportedCount}/${claims.length} claims with token matches) | **Hallucination Risk (Lexical):** \`${hallucinationRisk.toUpperCase()}\``,
835
1952
  `
836
- | Pedagogical / Technical Claim | Evidence Status | Confidence Match |`,
837
- `|---|---|---|`,
838
- ...verifiedList.map((c) => `| ${c.claimText} | ${c.isSupported ? "\u2705 **SUPPORTED**" : "\u26A0\uFE0F *UNVERIFIED / GAP*"} | \`${c.confidenceScore}%\` |`)
1953
+ | Pedagogical / Technical Claim | Evidence Status | Token Match | Supporting Quote | Semantic Status |`,
1954
+ `|---|---|---|---|---|`,
1955
+ ...verifiedList.map(
1956
+ (c) => `| ${c.claimText.replace(/\|/g, "/")} | ${c.isSupported ? "\u{1F7E1} **LEXICAL MATCH**" : "\u26A0\uFE0F *UNVERIFIED / GAP*"} | \`${c.confidenceScore}%\` | ${(c.extractedQuote || "\u2014").replace(/\|/g, "/")} | *Awaiting LLM Entailment* |`
1957
+ )
839
1958
  ];
840
1959
  return {
841
1960
  totalClaimsChecked: claims.length,
842
1961
  supportedClaimsCount: supportedCount,
1962
+ refutedClaimsCount: 0,
1963
+ unverifiedClaimsCount: claims.length - supportedCount,
843
1964
  hallucinationRisk,
1965
+ isLlmVerified: false,
844
1966
  claims: verifiedList,
845
1967
  reportMd: mdLines.join("\n")
846
1968
  };
847
1969
  }
1970
+ /**
1971
+ * Primary verification interface.
1972
+ * If an LLM callback is provided, performs genuine Semantic Entailment (NLI).
1973
+ * Otherwise falls back to deterministic lexical retrieval with honest labeling.
1974
+ */
1975
+ async verifyClaims(claims, sourceCorpus, options = {}) {
1976
+ const detReport = this.verifyClaimsDeterministic(claims, sourceCorpus);
1977
+ if (!options.llm || claims.length === 0) {
1978
+ return detReport;
1979
+ }
1980
+ try {
1981
+ const normCorpus = sourceCorpus.normalize("NFC");
1982
+ const cleanSubstantiveText = normCorpus.replace(/```[\s\S]*?```/g, "\n").split("\n").filter((l) => !/^(title:|url source:|final url:|http|!\[|#)/i.test(l.trim())).join("\n").trim();
1983
+ const corpusSentences = cleanSubstantiveText.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => s.length >= 25 && !/^(cookie|privacy|sign in|all rights reserved)/i.test(s));
1984
+ const fallbackExcerpts = corpusSentences.slice(0, 8).join("\n");
1985
+ const claimContexts = claims.map((claim, idx) => {
1986
+ const tokens = extractSignificantTokens(claim);
1987
+ const topSentences = corpusSentences.map((sent) => ({
1988
+ sent,
1989
+ overlap: tokens.filter((t) => tokenMatchesCorpus(t, sent.toLowerCase())).length
1990
+ })).filter((item) => item.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, 5).map((item) => item.sent);
1991
+ const candidates = topSentences.length > 0 ? `${topSentences.join("\n")}
1992
+
1993
+ [General Reference Excerpts]:
1994
+ ${fallbackExcerpts}` : fallbackExcerpts || "No substantive text found.";
1995
+ return {
1996
+ idx,
1997
+ claim,
1998
+ candidatePassages: candidates
1999
+ };
2000
+ });
2001
+ const prompt = [
2002
+ `You are a Rigorous Scientific & Pedagogical Fact-Checking System (Natural Language Inference).`,
2003
+ `Evaluate whether each technical/pedagogical claim is strictly SUPPORTED, directly REFUTED, or has NOT_ENOUGH_EVIDENCE in the harvested technical reference text.`,
2004
+ `Pay careful attention to NEGATION words ("kh\xF4ng", "never", "tr\xE1nh", "do not"), causal relationships, hardware pin constraints, and edge cases. Do not assume truth without textual evidence.`,
2005
+ `
2006
+ Harvested Reference Excerpts for Claims:
2007
+ `,
2008
+ claimContexts.map((c) => `[Claim ${c.idx + 1}]: "${c.claim}"
2009
+ [Passage Candidates]: ${c.candidatePassages}`).join("\n\n"),
2010
+ `
2011
+ Return ONLY a valid JSON array of objects with the exact structure:`,
2012
+ `[{"claimIndex": 1, "verdict": "supported"|"refuted"|"not_enough_evidence", "confidenceScore": 95, "extractedQuote": "exact quote from passage", "explanation": "1-sentence semantic reason"}]`
2013
+ ].join("\n");
2014
+ const raw = await options.llm(prompt, {
2015
+ system: "You are an exacting academic fact-checker. Output strictly valid JSON array without commentary.",
2016
+ temperature: 0.1,
2017
+ jsonMode: true
2018
+ });
2019
+ let parsed = [];
2020
+ const arrayMatch = raw.match(/\[[\s\S]*\]/);
2021
+ if (arrayMatch) {
2022
+ try {
2023
+ const res = JSON.parse(arrayMatch[0]);
2024
+ if (Array.isArray(res)) parsed = res;
2025
+ } catch {
2026
+ }
2027
+ }
2028
+ if (parsed.length === 0) {
2029
+ const objMatch = raw.match(/\{[\s\S]*\}/);
2030
+ if (objMatch) {
2031
+ try {
2032
+ const obj = JSON.parse(objMatch[0]);
2033
+ const candidateList = obj.claims || obj.results || obj.verifications || Object.values(obj).find(Array.isArray);
2034
+ if (Array.isArray(candidateList)) parsed = candidateList;
2035
+ } catch {
2036
+ }
2037
+ }
2038
+ }
2039
+ if (parsed.length > 0) {
2040
+ const verifiedList = claims.map((claim, idx) => {
2041
+ const item = parsed.find((p) => p.claimIndex === idx + 1) || parsed[idx];
2042
+ const verdict = item && ["supported", "refuted", "not_enough_evidence"].includes(item.verdict) ? item.verdict : detReport.claims[idx]?.isSupported ? "supported" : "not_enough_evidence";
2043
+ const isSupported = verdict === "supported";
2044
+ return {
2045
+ claimText: claim,
2046
+ verdict,
2047
+ isSupported,
2048
+ isLlmVerified: true,
2049
+ confidenceScore: Number(item?.confidenceScore) || (isSupported ? 90 : 30),
2050
+ extractedQuote: item?.extractedQuote || detReport.claims[idx]?.extractedQuote,
2051
+ explanation: item?.explanation || (isSupported ? "Semantically entailed by reference corpus." : "Not verified.")
2052
+ };
2053
+ });
2054
+ const supportedCount = verifiedList.filter((c) => c.verdict === "supported").length;
2055
+ const refutedCount = verifiedList.filter((c) => c.verdict === "refuted").length;
2056
+ const unverifiedCount = claims.length - supportedCount;
2057
+ const supportPct = claims.length > 0 ? supportedCount / claims.length * 100 : 100;
2058
+ const hallucinationRisk = refutedCount > 0 ? "high" : supportPct >= 75 ? "low" : "medium";
2059
+ const mdLines = [
2060
+ `# \u{1F50D} Claim-Support Sweep & Semantic Verification Report (LLM NLI Evaluated)`,
2061
+ `> **Semantic Verification Rate:** \`${supportPct.toFixed(1)}%\` (${supportedCount}/${claims.length} claims supported) | **Refuted / Contradictions:** \`${refutedCount}\` | **Hallucination Risk:** \`${hallucinationRisk.toUpperCase()}\``,
2062
+ `
2063
+ | Pedagogical / Technical Claim | Semantic Verdict | Confidence | Supporting Quote / Evidence | Explanation |`,
2064
+ `|---|---|---|---|---|`,
2065
+ ...verifiedList.map(
2066
+ (c) => `| ${c.claimText.replace(/\|/g, "/")} | ${c.verdict === "supported" ? "\u2705 **SUPPORTED**" : c.verdict === "refuted" ? "\u274C **REFUTED**" : "\u26A0\uFE0F *NOT ENOUGH EVIDENCE*"} | \`${c.confidenceScore}%\` | ${(c.extractedQuote || "\u2014").replace(/\|/g, "/")} | ${(c.explanation || "").replace(/\|/g, "/")} |`
2067
+ )
2068
+ ];
2069
+ return {
2070
+ totalClaimsChecked: claims.length,
2071
+ supportedClaimsCount: supportedCount,
2072
+ refutedClaimsCount: refutedCount,
2073
+ unverifiedClaimsCount: unverifiedCount,
2074
+ hallucinationRisk,
2075
+ isLlmVerified: true,
2076
+ claims: verifiedList,
2077
+ reportMd: mdLines.join("\n")
2078
+ };
2079
+ }
2080
+ } catch (e) {
2081
+ console.warn("[ClaimVerifier] LLM semantic verification failed, falling back to deterministic lexical report:", e?.message || e);
2082
+ }
2083
+ return detReport;
2084
+ }
848
2085
  };
849
2086
 
850
2087
  // src/evidence/provenanceTracker.ts
@@ -879,16 +2116,33 @@ var ProvenanceTracker = class {
879
2116
  // src/pipeline/campaignPlanner.ts
880
2117
  var CampaignPlanner = class {
881
2118
  createPlan(topic, domain, options = {}) {
882
- const slug = topic.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40);
2119
+ const slug = topic.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40) || "research-topic";
883
2120
  const scaleMode = options.isDeep ? "deep" : domain === "academic" ? "academic_survey" : "direct";
884
2121
  const maxRounds = options.maxRounds ?? (scaleMode === "deep" ? 3 : 1);
885
- const plannedQueries = [
886
- `${topic} official documentation getting started`,
887
- `${topic} architecture design best practices`,
888
- `${topic} practical code examples and pitfalls`
889
- ];
2122
+ const ctx = options.context;
2123
+ const toolAnchor = [ctx?.hardwarePlatform, ctx?.softwareEnv].filter(Boolean).join(" ").trim();
2124
+ const coreSubject = toolAnchor ? `${toolAnchor} ${topic}` : topic;
2125
+ const plannedQueries = [];
2126
+ if (ctx && (ctx.hardwarePlatform || ctx.softwareEnv || ctx.inScopeTopics && ctx.inScopeTopics.length > 0)) {
2127
+ plannedQueries.push(
2128
+ `${toolAnchor || topic} official documentation getting started guide ${ctx.inScopeTopics?.slice(0, 2).join(" ") || ""}`.trim()
2129
+ );
2130
+ const ageHint = ctx.ageBand ? `ages ${ctx.ageBand[0]}-${ctx.ageBand[1]}` : "beginner project";
2131
+ plannedQueries.push(
2132
+ `${toolAnchor || topic} hands-on project tutorial curriculum ${ageHint}`.trim()
2133
+ );
2134
+ plannedQueries.push(
2135
+ `${toolAnchor || topic} common mistakes pitfalls debugging troubleshooting`.trim()
2136
+ );
2137
+ } else {
2138
+ plannedQueries.push(
2139
+ `${coreSubject} official documentation getting started`,
2140
+ `${coreSubject} architecture design best practices`,
2141
+ `${coreSubject} practical code examples and pitfalls`
2142
+ );
2143
+ }
890
2144
  if (domain === "hardware") {
891
- plannedQueries.push(`${topic} pinout schematic datasheet specifications`);
2145
+ plannedQueries.push(`${ctx?.hardwarePlatform || topic} pinout schematic sensor wiring specifications`);
892
2146
  } else if (domain === "academic") {
893
2147
  plannedQueries.push(`${topic} state of the art benchmarks survey paper`);
894
2148
  }
@@ -897,14 +2151,14 @@ var CampaignPlanner = class {
897
2151
  `> **Slug:** \`${slug}\` | **Domain:** \`${domain.toUpperCase()}\` | **Scale Mode:** \`${scaleMode.toUpperCase()}\` | **Max Rounds:** \`${maxRounds}\``,
898
2152
  `
899
2153
  ## 1. Campaign Objective`,
900
- `- Exhaustively harvest and verify Ground Truth for **${topic}** to anchor all downstream SOT artifacts in Curriculum OS.`,
2154
+ `- Exhaustively harvest and verify Technical & Pedagogical Ground Truth for **${topic}** to anchor all downstream SOT artifacts in Curriculum OS.`,
901
2155
  `
902
2156
  ## 2. Planned Multi-Source Queries`,
903
2157
  ...plannedQueries.map((q, i) => `${i + 1}. \`${q}\``),
904
2158
  `
905
2159
  ## 3. Acceptance Criteria & Quality Gates`,
906
2160
  `- Minimum 3 primary technical resources with Relevancy Score >= 6/10.`,
907
- `- Extract 100% executable syntax-verified code snippets.`,
2161
+ `- Extract verified code blocks, reference tables, and bilingual domain concepts.`,
908
2162
  `- Produce the complete 3-layer catalog: \`RESOURCE_INDEX.json\`, \`RESOURCE_CATALOG.md\`, and \`KNOWLEDGE_GRAPH.md\`.`
909
2163
  ];
910
2164
  return {
@@ -917,33 +2171,75 @@ var CampaignPlanner = class {
917
2171
  planMd: mdLines.join("\n")
918
2172
  };
919
2173
  }
2174
+ /**
2175
+ * Optional LLM-augmented query planner when the host app provides an `llm` callback.
2176
+ */
2177
+ async createPlanWithLlm(topic, domain, options = {}) {
2178
+ const basePlan = this.createPlan(topic, domain, options);
2179
+ if (!options.llm) return basePlan;
2180
+ try {
2181
+ const ctx = options.context;
2182
+ const prompt = [
2183
+ `Generate 4 high-precision search engine queries (in English or bilingual as best suited for technical documentation and curriculum discovery) for researching this course:`,
2184
+ `- Topic / Title: ${topic}`,
2185
+ ctx?.positioning ? `- Positioning: ${ctx.positioning}` : "",
2186
+ ctx?.hardwarePlatform ? `- Hardware Platform: ${ctx.hardwarePlatform}` : "",
2187
+ ctx?.softwareEnv ? `- Software / Tools: ${ctx.softwareEnv}` : "",
2188
+ ctx?.ageBand ? `- Learner Age Band: ${ctx.ageBand[0]}-${ctx.ageBand[1]}` : "",
2189
+ ctx?.inScopeTopics?.length ? `- In-Scope Topics: ${ctx.inScopeTopics.join(", ")}` : "",
2190
+ `The 4 queries must cover: (1) Official technical/API/hardware documentation, (2) Real-world project/course exemplars for this age group, (3) Common beginner misconceptions & debugging pitfalls, (4) Best pedagogical scaffolding practices.`,
2191
+ `Return ONLY a JSON array of 4 query strings: ["query 1", "query 2", "query 3", "query 4"]`
2192
+ ].filter(Boolean).join("\n");
2193
+ const raw = await options.llm(prompt, {
2194
+ system: "You are a technical research librarian. Return only a valid JSON array of 4 search query strings.",
2195
+ temperature: 0.2,
2196
+ jsonMode: true
2197
+ });
2198
+ const match = raw.match(/\[[\s\S]*\]/);
2199
+ if (match) {
2200
+ const parsed = JSON.parse(match[0]);
2201
+ if (Array.isArray(parsed) && parsed.length >= 2) {
2202
+ const queries = parsed.map((q) => String(q).trim()).filter((q) => q.length > 4).slice(0, 5);
2203
+ if (queries.length >= 2) {
2204
+ basePlan.plannedQueries = queries;
2205
+ }
2206
+ }
2207
+ }
2208
+ } catch (e) {
2209
+ console.warn("[CampaignPlanner] LLM query expansion failed, using deterministic queries:", e?.message || e);
2210
+ }
2211
+ return basePlan;
2212
+ }
920
2213
  };
921
2214
 
922
2215
  // src/pipeline/referencePackBuilder.ts
923
2216
  var ReferencePackBuilder = class {
924
2217
  buildReferencePack(topic, domain, options = {}) {
925
2218
  const registry = [];
2219
+ const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
926
2220
  if (options.academicPapers) {
927
2221
  options.academicPapers.forEach((p, idx) => {
2222
+ const takeaway = p.tldr?.trim() || (p.abstract ? p.abstract.slice(0, 220).trim() : `Peer-reviewed publication (${p.year || "n.d."}, ${p.citationCount} citations).`);
928
2223
  registry.push({
929
2224
  id: `REF-ACAD-${String(idx + 1).padStart(2, "0")}`,
930
2225
  title: p.title,
931
2226
  sourceType: "academic_paper",
932
- url: p.url || `https://doi.org/${p.doi || ""}`,
2227
+ url: p.url || (p.doi ? `https://doi.org/${p.doi}` : ""),
933
2228
  relevanceScore: 9,
934
- extractedKeyTakeaways: [p.tldr || p.abstract?.slice(0, 100) || "Peer-reviewed theoretical foundation."]
2229
+ extractedKeyTakeaways: [takeaway]
935
2230
  });
936
2231
  });
937
2232
  }
938
2233
  if (options.scrapedDocs) {
939
2234
  options.scrapedDocs.forEach((d, idx) => {
2235
+ const takeaways = d.keyTakeaways && d.keyTakeaways.length > 0 ? d.keyTakeaways : d.doc.keyExcerpts && d.doc.keyExcerpts.length > 0 ? d.doc.keyExcerpts.slice(0, 3) : [d.summary];
940
2236
  registry.push({
941
2237
  id: `REF-DOC-${String(idx + 1).padStart(2, "0")}`,
942
2238
  title: d.doc.title,
943
- sourceType: "official_docs",
2239
+ sourceType: d.doc.fetchProvider === "youtube" ? "video_transcript" : "official_docs",
944
2240
  url: d.doc.url,
945
2241
  relevanceScore: d.score,
946
- extractedKeyTakeaways: [d.summary]
2242
+ extractedKeyTakeaways: takeaways
947
2243
  });
948
2244
  });
949
2245
  }
@@ -953,33 +2249,83 @@ var ReferencePackBuilder = class {
953
2249
  domain,
954
2250
  generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
955
2251
  totalReferences: registry.length,
956
- cs2023StandardArea: options.cs2023Area?.code || "PDC",
2252
+ cs2023StandardArea: options.cs2023Area?.code || "SDF",
2253
+ cstaK12Strand: options.k12Strand?.code || "AP",
2254
+ synthesizedInsights: options.synthesizedInsights,
2255
+ concepts: options.concepts || [],
957
2256
  registry
958
2257
  },
959
2258
  null,
960
2259
  2
961
2260
  );
2261
+ const codeBlocks = options.scrapedDocs?.flatMap((d) => {
2262
+ if (d.doc.extractedCodeBlocks && d.doc.extractedCodeBlocks.length > 0) {
2263
+ return d.doc.extractedCodeBlocks.slice(0, 2).map((b) => `\`\`\`${b.language || defaultLang}
2264
+ ${b.code}
2265
+ \`\`\``);
2266
+ }
2267
+ return d.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
2268
+ ${c}
2269
+ \`\`\``);
2270
+ }) || [];
2271
+ const insightsSection = [];
2272
+ if (options.synthesizedInsights) {
2273
+ if (options.synthesizedInsights.executiveSummary) {
2274
+ insightsSection.push(`
2275
+ ### Synthesized Executive Summary
2276
+ ${options.synthesizedInsights.executiveSummary}`);
2277
+ }
2278
+ if (options.synthesizedInsights.keyTakeaways?.length > 0) {
2279
+ insightsSection.push(
2280
+ `
2281
+ ### Key Technical & Pedagogical Takeaways
2282
+ ` + options.synthesizedInsights.keyTakeaways.map((t) => `- ${t}`).join("\n")
2283
+ );
2284
+ }
2285
+ if (options.synthesizedInsights.commonMisconceptionsAndPitfalls?.length > 0) {
2286
+ insightsSection.push(
2287
+ `
2288
+ ### Documented Learner Misconceptions & Debugging Pitfalls
2289
+ ` + options.synthesizedInsights.commonMisconceptionsAndPitfalls.map((m) => `- ${m}`).join("\n")
2290
+ );
2291
+ }
2292
+ if (options.synthesizedInsights.recommendedExemplars?.length > 0) {
2293
+ insightsSection.push(
2294
+ `
2295
+ ### Reference Project & Curriculum Exemplars
2296
+ ` + options.synthesizedInsights.recommendedExemplars.map((e) => `- ${e}`).join("\n")
2297
+ );
2298
+ }
2299
+ } else if (registry.length > 0) {
2300
+ const topTakeaways = registry.flatMap((r) => r.extractedKeyTakeaways.slice(0, 2).map((t) => `- **[${r.id}]** ${t}`));
2301
+ if (topTakeaways.length > 0) {
2302
+ insightsSection.push(`
2303
+ ### Extracted Source Takeaways
2304
+ ` + topTakeaways.join("\n"));
2305
+ }
2306
+ }
962
2307
  const mdLines = [
963
2308
  `# \u{1F4DA} REFERENCE_PACK.md: ${topic}`,
964
- `> **Domain:** \`${domain.toUpperCase()}\` | **CS2023 Area:** \`${options.cs2023Area?.code || "SE"}\` \u2014 *${options.cs2023Area?.name || "Software Engineering"}*`,
2309
+ `> **Domain:** \`${domain.toUpperCase()}\` | **CS2023 Area:** \`${options.cs2023Area?.code || "SE"}\` \u2014 *${options.cs2023Area?.name || "Software Engineering"}*${options.k12Strand ? ` | **CSTA K-12:** \`${options.k12Strand.code}\` (${options.k12Strand.name})` : ""}`,
965
2310
  `> **Curriculum OS Status:** \`<!-- [SME_APPROVED] -->\``,
966
2311
  `
967
2312
  ## 1. Executive Summary & Ground Truth Mandate`,
968
2313
  `This reference pack serves as the immutable ground truth for all downstream lesson plans, code labs, and rubrics for **${topic}**.`,
2314
+ ...insightsSection,
969
2315
  `
970
2316
  ## 2. Technical Ground Truth Registry`,
971
2317
  `| Reference ID | Type | Title | Relevance | Source Link |`,
972
2318
  `|---|---|---|---|---|`,
973
- ...registry.map((r) => `| **${r.id}** | \`${r.sourceType}\` | ${r.title} | \u2B50 \`${r.relevanceScore}/10\` | [Access](${r.url}) |`),
2319
+ ...registry.map(
2320
+ (r) => `| **${r.id}** | \`${r.sourceType}\` | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | [Access](${r.url}) |`
2321
+ ),
974
2322
  `
975
2323
  ## 3. Extracted Code Signatures & Primitives`,
976
- ...options.scrapedDocs?.flatMap((d) => d.doc.extractedCodeSnippets.slice(0, 2)).map((c) => `\`\`\`${domain === "hardware" ? "cpp" : "swift"}
977
- ${c}
978
- \`\`\``) || ["*No explicit code signatures captured.*"],
2324
+ ...codeBlocks.length > 0 ? codeBlocks : ["*No explicit code signatures captured.*"],
979
2325
  `
980
2326
  ## 4. Academic Alignment & Standard Citations`,
981
- `- Aligns directly with IEEE/ACM Computer Science Curricula standards.`,
982
- `- All code examples guaranteed 100% syntactically valid and executable.`
2327
+ `- Aligns directly with IEEE/ACM Computer Science Curricula (${options.cs2023Area?.code || "SDF"}) and CSTA K-12 (${options.k12Strand?.code || "AP"}) standards.`,
2328
+ `- All claims and code examples are anchored to physical primary sources listed in the registry above.`
983
2329
  ];
984
2330
  return {
985
2331
  referenceRegistryJson,
@@ -989,6 +2335,22 @@ ${c}
989
2335
  };
990
2336
 
991
2337
  // src/pipeline/harvestPipeline.ts
2338
+ function normalizeUrlForDedup(rawUrl) {
2339
+ try {
2340
+ const u = new URL(rawUrl.trim());
2341
+ u.hash = "";
2342
+ for (const key of [...u.searchParams.keys()]) {
2343
+ if (/^(utm_|fbclid$|gclid$|ref$|source$)/i.test(key)) {
2344
+ u.searchParams.delete(key);
2345
+ }
2346
+ }
2347
+ const path = u.pathname.replace(/\/+$/, "") || "/";
2348
+ const query = u.searchParams.toString();
2349
+ return `${u.protocol}//${u.hostname.toLowerCase()}${path}${query ? `?${query}` : ""}`;
2350
+ } catch {
2351
+ return rawUrl.trim().toLowerCase().replace(/\/+$/, "");
2352
+ }
2353
+ }
992
2354
  var HarvestPipeline = class {
993
2355
  searchEngine;
994
2356
  scraper;
@@ -1000,14 +2362,27 @@ var HarvestPipeline = class {
1000
2362
  litReviewSynthesizer;
1001
2363
  planner;
1002
2364
  refPackBuilder;
2365
+ termExtractor;
2366
+ customSynthesizer;
2367
+ defaultLlm;
1003
2368
  constructor(options = {}) {
2369
+ const allowEnv = options.allowEnvFallback ?? true;
2370
+ const keys = {
2371
+ tavily: options.apiKeys?.tavily || options.tavilyKey,
2372
+ exa: options.apiKeys?.exa || options.exaKey,
2373
+ jina: options.apiKeys?.jina || options.jinaKey,
2374
+ serper: options.apiKeys?.serper || options.serperKey,
2375
+ semanticScholar: options.apiKeys?.semanticScholar || options.semanticScholarKey
2376
+ };
1004
2377
  this.searchEngine = new MultiSourceSearchEngine({
1005
- tavilyKey: options.tavilyKey,
1006
- exaKey: options.exaKey,
2378
+ apiKeys: keys,
2379
+ allowEnvFallback: allowEnv,
1007
2380
  customAdapter: options.customSearchAdapter
1008
2381
  });
1009
2382
  this.scraper = new WebScraper({
1010
2383
  crawl4aiHost: options.crawl4aiHost,
2384
+ jinaKey: keys.jina,
2385
+ allowEnvFallback: allowEnv,
1011
2386
  customAdapter: options.customCrawlerAdapter
1012
2387
  });
1013
2388
  this.auditor = new HealthAuditor();
@@ -1015,15 +2390,29 @@ var HarvestPipeline = class {
1015
2390
  this.catalogBuilder = new CatalogBuilder();
1016
2391
  this.provenanceTracker = new ProvenanceTracker();
1017
2392
  this.academicSearch = new AcademicSearchEngine({
2393
+ apiKey: keys.semanticScholar,
2394
+ allowEnvFallback: allowEnv,
1018
2395
  customAdapter: options.customAcademicSearchAdapter
1019
2396
  });
1020
2397
  this.litReviewSynthesizer = new LiteratureReviewSynthesizer();
1021
2398
  this.planner = new CampaignPlanner();
1022
2399
  this.refPackBuilder = new ReferencePackBuilder();
2400
+ this.termExtractor = new PedagogicalTermExtractor();
2401
+ this.customSynthesizer = options.customSynthesizerAdapter;
2402
+ this.defaultLlm = options.llm;
1023
2403
  }
1024
2404
  async run(topic, options = {}) {
1025
2405
  const notify = options.onProgress || (() => {
1026
2406
  });
2407
+ const llm = options.llm || this.defaultLlm;
2408
+ const synthesizer = options.customSynthesizerAdapter || this.customSynthesizer;
2409
+ const mergedKeys = {
2410
+ tavily: options.apiKeys?.tavily || options.tavilyKey,
2411
+ exa: options.apiKeys?.exa || options.exaKey,
2412
+ jina: options.apiKeys?.jina || options.jinaKey,
2413
+ serper: options.apiKeys?.serper || options.serperKey,
2414
+ semanticScholar: options.apiKeys?.semanticScholar || options.semanticScholarKey
2415
+ };
1027
2416
  if (options.customSearchAdapter) {
1028
2417
  this.searchEngine.setCustomAdapter(options.customSearchAdapter);
1029
2418
  }
@@ -1034,32 +2423,59 @@ var HarvestPipeline = class {
1034
2423
  this.academicSearch.setCustomAdapter(options.customAcademicSearchAdapter);
1035
2424
  }
1036
2425
  notify("PLANNING", `Classifying domain and creating Master Research Plan for "${topic}"...`);
1037
- const domainResult = classifyTechnicalDomain(topic, options.domain);
2426
+ const domainResult = classifyTechnicalDomain(topic, options.domain, options.context);
1038
2427
  const domain = domainResult.domain;
1039
- const plan = this.planner.createPlan(topic, domain, { maxRounds: options.maxRounds, isDeep: options.isDeep });
1040
- notify("ACADEMIC", `Querying academic literature & mapping to IEEE/ACM CS2023 standards...`);
1041
- const cs2023Match = mapTopicToCS2023(topic);
2428
+ const plan = llm ? await this.planner.createPlanWithLlm(topic, domain, {
2429
+ maxRounds: options.maxRounds,
2430
+ isDeep: options.isDeep,
2431
+ context: options.context,
2432
+ llm
2433
+ }) : this.planner.createPlan(topic, domain, {
2434
+ maxRounds: options.maxRounds,
2435
+ isDeep: options.isDeep,
2436
+ context: options.context
2437
+ });
2438
+ notify("ACADEMIC", `Querying academic literature & mapping to IEEE/ACM CS2023 and CSTA K-12 standards...`);
2439
+ const taxonomyMatch = mapTopicToCS2023(topic, options.context);
1042
2440
  const papers = await this.academicSearch.searchPapers(topic, {
1043
2441
  limit: 4,
1044
- apiKey: options.semanticScholarKey,
2442
+ apiKey: mergedKeys.semanticScholar,
1045
2443
  customAdapter: options.customAcademicSearchAdapter
1046
2444
  });
1047
- const litReview = this.litReviewSynthesizer.synthesize(topic, papers, cs2023Match.area);
2445
+ const litReview = llm ? await this.litReviewSynthesizer.synthesizeWithLlm(
2446
+ topic,
2447
+ papers,
2448
+ taxonomyMatch.area,
2449
+ taxonomyMatch.k12Strand,
2450
+ { llm, context: options.context }
2451
+ ) : this.litReviewSynthesizer.synthesize(
2452
+ topic,
2453
+ papers,
2454
+ taxonomyMatch.area,
2455
+ taxonomyMatch.k12Strand
2456
+ );
1048
2457
  notify("DISCOVERY", `Executing multi-source search queries across configured adapters...`);
1049
2458
  const scrapedDocs = [];
1050
2459
  const provenanceRecords = [];
2460
+ const visitedUrls = /* @__PURE__ */ new Set();
1051
2461
  for (const query of plan.plannedQueries) {
1052
2462
  notify("SEARCHING", `Searching: ${query}`);
1053
2463
  const hits = await this.searchEngine.search(query, {
1054
2464
  limit: 2,
1055
- tavilyKey: options.tavilyKey,
1056
- exaKey: options.exaKey,
2465
+ apiKeys: mergedKeys,
1057
2466
  customAdapter: options.customSearchAdapter
1058
2467
  });
1059
2468
  for (const hit of hits) {
2469
+ if (!hit?.url) continue;
2470
+ const normUrl = normalizeUrlForDedup(hit.url);
2471
+ if (visitedUrls.has(normUrl)) continue;
2472
+ visitedUrls.add(normUrl);
1060
2473
  try {
1061
2474
  notify("SCRAPING", `Extracting clean markdown from: ${hit.url}`);
1062
- const doc = await this.scraper.scrapeUrl(hit.url, { customAdapter: options.customCrawlerAdapter });
2475
+ const doc = await this.scraper.scrapeUrl(hit.url, {
2476
+ customAdapter: options.customCrawlerAdapter,
2477
+ jinaKey: mergedKeys.jina
2478
+ });
1063
2479
  const audit = this.auditor.auditDocument(doc);
1064
2480
  if (audit.passed) {
1065
2481
  const scored = this.scorer.scoreDocument(doc, domain, topic);
@@ -1067,7 +2483,8 @@ var HarvestPipeline = class {
1067
2483
  scrapedDocs.push({
1068
2484
  doc,
1069
2485
  score: scored.score,
1070
- summary: scored.summary
2486
+ summary: scored.summary,
2487
+ keyTakeaways: scored.keyTakeaways
1071
2488
  });
1072
2489
  provenanceRecords.push({
1073
2490
  resourceId: `RES-${String(scrapedDocs.length).padStart(2, "0")}`,
@@ -1084,13 +2501,36 @@ var HarvestPipeline = class {
1084
2501
  }
1085
2502
  }
1086
2503
  }
2504
+ notify("TERMS", `Extracting bilingual technical concepts and prerequisite DAG...`);
2505
+ const docsWithIds = scrapedDocs.map((item, idx) => ({
2506
+ resourceId: `RES-${String(idx + 1).padStart(2, "0")}`,
2507
+ doc: item.doc
2508
+ }));
2509
+ const termBundle = await this.termExtractor.extractWithOptionalLlm(topic, domain, docsWithIds, {
2510
+ context: options.context,
2511
+ llm
2512
+ });
2513
+ notify("SYNTHESIS", `Synthesizing pedagogical takeaways, misconceptions, and reference pack...`);
2514
+ const synthesizedInsights = await this.synthesizeInsights(topic, domain, scrapedDocs, {
2515
+ context: options.context,
2516
+ llm,
2517
+ synthesizer,
2518
+ concepts: termBundle.concepts
2519
+ });
1087
2520
  notify("CATALOGING", `Compiling Resource Index, Catalog, and Distilled Knowledge Graph...`);
1088
- const catalog = this.catalogBuilder.buildBundle(topic, domain, scrapedDocs);
1089
- notify("SYNTHESIS", `Building canonical REFERENCE_PACK.md & REFERENCE_REGISTRY.json...`);
2521
+ const catalog = this.catalogBuilder.buildBundle(topic, domain, scrapedDocs, {
2522
+ context: options.context,
2523
+ concepts: termBundle.concepts,
2524
+ conceptDagTsv: termBundle.conceptDagTsv
2525
+ });
1090
2526
  const referencePack = this.refPackBuilder.buildReferencePack(topic, domain, {
1091
- cs2023Area: cs2023Match.area,
2527
+ cs2023Area: taxonomyMatch.area,
2528
+ k12Strand: taxonomyMatch.k12Strand,
2529
+ context: options.context,
1092
2530
  scrapedDocs,
1093
- academicPapers: papers
2531
+ academicPapers: papers,
2532
+ concepts: termBundle.concepts,
2533
+ synthesizedInsights
1094
2534
  });
1095
2535
  notify("PROVENANCE", `Generating audit trail and provenance sidecar...`);
1096
2536
  const provenance = this.provenanceTracker.generateSidecar(topic, provenanceRecords);
@@ -1103,30 +2543,339 @@ var HarvestPipeline = class {
1103
2543
  literatureReview: litReview,
1104
2544
  referencePack,
1105
2545
  provenance,
2546
+ concepts: termBundle.concepts,
2547
+ conceptDagTsv: termBundle.conceptDagTsv,
2548
+ synthesizedInsights,
1106
2549
  totalDocumentsScraped: scrapedDocs.length
1107
2550
  };
1108
2551
  }
2552
+ async synthesizeInsights(topic, domain, scrapedDocs, options) {
2553
+ const excerpts = scrapedDocs.map((d, idx) => ({
2554
+ resourceId: `RES-${String(idx + 1).padStart(2, "0")}`,
2555
+ title: d.doc.title,
2556
+ url: d.doc.url,
2557
+ excerpt: d.keyTakeaways.join(" ") || d.summary,
2558
+ codeSnippets: d.doc.extractedCodeSnippets.slice(0, 2)
2559
+ }));
2560
+ if (options.synthesizer) {
2561
+ try {
2562
+ return await options.synthesizer.synthesize({
2563
+ topic,
2564
+ domain,
2565
+ context: options.context,
2566
+ excerpts
2567
+ });
2568
+ } catch (e) {
2569
+ console.warn("[HarvestPipeline] CustomSynthesizerAdapter failed, falling back:", e?.message || e);
2570
+ }
2571
+ }
2572
+ if (options.llm && excerpts.length > 0) {
2573
+ try {
2574
+ const prompt = [
2575
+ `Synthesize pedagogical and technical ground truth for the course topic "${topic}" (Domain: ${domain}).`,
2576
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
2577
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : "",
2578
+ options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}` : "",
2579
+ `
2580
+ Verified Harvested Sources:
2581
+ ` + excerpts.map((e) => `[${e.resourceId}] ${e.title} (${e.url}): ${e.excerpt}`).join("\n\n"),
2582
+ `
2583
+ Return ONLY a valid JSON object with this structure:`,
2584
+ `{"executiveSummary": "2-3 sentence synthesis anchored to sources", "keyTakeaways": ["[RES-01] ...", "[RES-02] ..."], "commonMisconceptionsAndPitfalls": ["Specific beginner bug or misconception 1", "Misconception 2"], "recommendedExemplars": ["Concrete project/activity exemplar 1", "Exemplar 2"]}`
2585
+ ].filter(Boolean).join("\n");
2586
+ const raw = await options.llm(prompt, {
2587
+ system: "You are a Principal Curriculum Researcher. Output strictly valid JSON grounded in the provided sources.",
2588
+ temperature: 0.2,
2589
+ jsonMode: true
2590
+ });
2591
+ const match = raw.match(/\{[\s\S]*\}/);
2592
+ if (match) {
2593
+ const parsed = JSON.parse(match[0]);
2594
+ return {
2595
+ executiveSummary: String(parsed.executiveSummary || ""),
2596
+ keyTakeaways: Array.isArray(parsed.keyTakeaways) ? parsed.keyTakeaways.map(String) : [],
2597
+ commonMisconceptionsAndPitfalls: Array.isArray(parsed.commonMisconceptionsAndPitfalls) ? parsed.commonMisconceptionsAndPitfalls.map(String) : [],
2598
+ recommendedExemplars: Array.isArray(parsed.recommendedExemplars) ? parsed.recommendedExemplars.map(String) : [],
2599
+ extractedConcepts: options.concepts
2600
+ };
2601
+ }
2602
+ } catch (e) {
2603
+ console.warn("[HarvestPipeline] Host LLM synthesis failed, using deterministic synthesis:", e?.message || e);
2604
+ }
2605
+ }
2606
+ const keyTakeaways = excerpts.filter((e) => e.excerpt).slice(0, 6).map((e) => `[${e.resourceId}] ${e.title}: ${e.excerpt}`);
2607
+ const pitfalls = scrapedDocs.flatMap(
2608
+ (d) => d.doc.markdownContent.split(/(?<=[.!?。\n])\s+/).map((s) => s.trim()).filter(
2609
+ (s) => s.length >= 35 && s.length <= 240 && /\b(pitfall|mistake|error|bug|warning|caution|avoid|race condition|deadlock|lỗi|nhầm lẫn|lưu ý|cảnh báo|tránh)\b/i.test(s)
2610
+ )
2611
+ ).slice(0, 4);
2612
+ return {
2613
+ executiveSummary: scrapedDocs.length > 0 ? `Synthesized from ${scrapedDocs.length} verified primary sources for "${topic}" (${domain.toUpperCase()}).` : `No external web documents passed quality threshold for "${topic}".`,
2614
+ keyTakeaways,
2615
+ commonMisconceptionsAndPitfalls: pitfalls,
2616
+ recommendedExemplars: scrapedDocs.slice(0, 3).map((d) => `${d.doc.title} (${d.doc.url})`),
2617
+ extractedConcepts: options.concepts
2618
+ };
2619
+ }
2620
+ };
2621
+
2622
+ // src/services/researchKitClient.ts
2623
+ var KnowledgeDiscoveryService = class {
2624
+ constructor(config) {
2625
+ this.config = config;
2626
+ this.searchEngine = new MultiSourceSearchEngine({
2627
+ apiKeys: config.apiKeys,
2628
+ allowEnvFallback: config.allowEnvFallback,
2629
+ customAdapter: config.customSearchAdapter
2630
+ });
2631
+ this.webScraper = new WebScraper({
2632
+ crawl4aiHost: config.crawl4aiHost,
2633
+ jinaKey: config.apiKeys?.jina,
2634
+ allowEnvFallback: config.allowEnvFallback,
2635
+ customAdapter: config.customCrawlerAdapter
2636
+ });
2637
+ this.youtubeService = new YouTubeIngestionService();
2638
+ }
2639
+ searchEngine;
2640
+ webScraper;
2641
+ youtubeService;
2642
+ async search(query, limit = 5) {
2643
+ return this.searchEngine.search(query, {
2644
+ limit,
2645
+ apiKeys: this.config.apiKeys,
2646
+ customAdapter: this.config.customSearchAdapter
2647
+ });
2648
+ }
2649
+ async scrapeUrl(url) {
2650
+ return this.webScraper.scrapeUrl(url, {
2651
+ customAdapter: this.config.customCrawlerAdapter,
2652
+ jinaKey: this.config.apiKeys?.jina
2653
+ });
2654
+ }
2655
+ async ingestYouTube(urlOrVideoId, preferredLanguages = ["vi", "en"]) {
2656
+ return this.youtubeService.fetchTranscript(urlOrVideoId, preferredLanguages);
2657
+ }
2658
+ };
2659
+ var TerminologyGraphService = class {
2660
+ constructor(config) {
2661
+ this.config = config;
2662
+ }
2663
+ extractor = new PedagogicalTermExtractor();
2664
+ extractConceptsDeterministic(topic, docs, domain) {
2665
+ const resolvedDomain = domain ?? classifyTechnicalDomain(topic, void 0, this.config.defaultCourseContext).domain;
2666
+ const normalizedDocs = typeof docs === "string" ? [
2667
+ {
2668
+ resourceId: "RES-01",
2669
+ doc: {
2670
+ url: "inline://corpus",
2671
+ title: topic,
2672
+ markdownContent: docs,
2673
+ byteSize: Buffer.byteLength(docs, "utf-8"),
2674
+ extractedCodeSnippets: [],
2675
+ tablesCount: 0,
2676
+ fetchProvider: "direct"
2677
+ }
2678
+ }
2679
+ ] : docs;
2680
+ return this.extractor.extractDeterministic(topic, resolvedDomain, normalizedDocs);
2681
+ }
2682
+ async extractConcepts(topic, docs, options) {
2683
+ const resolvedDomain = options?.domain ?? classifyTechnicalDomain(topic, void 0, options?.context ?? this.config.defaultCourseContext).domain;
2684
+ const normalizedDocs = typeof docs === "string" ? [
2685
+ {
2686
+ resourceId: "RES-01",
2687
+ doc: {
2688
+ url: "inline://corpus",
2689
+ title: topic,
2690
+ markdownContent: docs,
2691
+ byteSize: Buffer.byteLength(docs, "utf-8"),
2692
+ extractedCodeSnippets: [],
2693
+ keyExcerpts: [docs.slice(0, 600)],
2694
+ tablesCount: 0,
2695
+ fetchProvider: "direct"
2696
+ }
2697
+ }
2698
+ ] : docs;
2699
+ return this.extractor.extractWithOptionalLlm(topic, resolvedDomain, normalizedDocs, {
2700
+ context: options?.context ?? this.config.defaultCourseContext,
2701
+ llm: options?.llm ?? this.config.llm,
2702
+ maxTerms: options?.maxTerms
2703
+ });
2704
+ }
2705
+ formatConceptDagTsv(concepts) {
2706
+ return this.extractor.formatConceptDagTsv(concepts);
2707
+ }
2708
+ };
2709
+ var AcademicStandardsService = class {
2710
+ constructor(config) {
2711
+ this.config = config;
2712
+ this.academicSearch = new AcademicSearchEngine({
2713
+ apiKey: config.apiKeys?.semanticScholar,
2714
+ allowEnvFallback: config.allowEnvFallback,
2715
+ customAdapter: config.customAcademicSearchAdapter
2716
+ });
2717
+ }
2718
+ academicSearch;
2719
+ litSynthesizer = new LiteratureReviewSynthesizer();
2720
+ alignStandards(topic, context) {
2721
+ return mapTopicToCS2023(topic, context ?? this.config.defaultCourseContext);
2722
+ }
2723
+ async searchPapers(query, limit = 5) {
2724
+ return this.academicSearch.searchPapers(query, {
2725
+ limit,
2726
+ apiKey: this.config.apiKeys?.semanticScholar,
2727
+ customAdapter: this.config.customAcademicSearchAdapter
2728
+ });
2729
+ }
2730
+ synthesizeLiteratureReview(topic, papers, context) {
2731
+ const alignment = this.alignStandards(topic, context);
2732
+ return this.litSynthesizer.synthesize(topic, papers, alignment.area, alignment.k12Strand);
2733
+ }
2734
+ async synthesizeLiteratureReviewWithLlm(topic, papers, context, options) {
2735
+ const alignment = this.alignStandards(topic, context);
2736
+ return this.litSynthesizer.synthesizeWithLlm(
2737
+ topic,
2738
+ papers,
2739
+ alignment.area,
2740
+ alignment.k12Strand,
2741
+ {
2742
+ llm: options?.llm ?? this.config.llm,
2743
+ context: context ?? this.config.defaultCourseContext
2744
+ }
2745
+ );
2746
+ }
2747
+ };
2748
+ var EvidenceVerificationService = class {
2749
+ constructor(config = {}) {
2750
+ this.config = config;
2751
+ }
2752
+ healthAuditor = new HealthAuditor();
2753
+ evidenceScorer = new EvidenceScorer();
2754
+ claimVerifier = new ClaimVerifier();
2755
+ provenanceTracker = new ProvenanceTracker();
2756
+ auditDocument(doc) {
2757
+ return this.healthAuditor.auditDocument(doc);
2758
+ }
2759
+ scoreDocument(doc, domain, topic) {
2760
+ return this.evidenceScorer.scoreDocument(doc, domain, topic);
2761
+ }
2762
+ async scoreDocumentWithLlm(doc, domain, topic, options) {
2763
+ return this.evidenceScorer.scoreDocumentWithLlm(doc, domain, topic, {
2764
+ llm: options?.llm ?? this.config.llm,
2765
+ context: options?.context ?? this.config.defaultCourseContext
2766
+ });
2767
+ }
2768
+ verifyClaimsDeterministic(claimsToVerify, sourceCorpusText) {
2769
+ return this.claimVerifier.verifyClaimsDeterministic(claimsToVerify, sourceCorpusText);
2770
+ }
2771
+ async verifyClaims(claimsToVerify, sourceCorpusText, options) {
2772
+ return this.claimVerifier.verifyClaims(claimsToVerify, sourceCorpusText, {
2773
+ llm: options?.llm ?? this.config.llm
2774
+ });
2775
+ }
2776
+ buildProvenance(topic, records) {
2777
+ return this.provenanceTracker.generateSidecar(topic, records);
2778
+ }
2779
+ };
2780
+ var PedagogicalSynthesisService = class {
2781
+ constructor(config) {
2782
+ this.config = config;
2783
+ this.pipeline = new HarvestPipeline({
2784
+ apiKeys: config.apiKeys,
2785
+ allowEnvFallback: config.allowEnvFallback,
2786
+ crawl4aiHost: config.crawl4aiHost,
2787
+ llm: config.llm,
2788
+ customSearchAdapter: config.customSearchAdapter,
2789
+ customCrawlerAdapter: config.customCrawlerAdapter,
2790
+ customAcademicSearchAdapter: config.customAcademicSearchAdapter,
2791
+ customSynthesizerAdapter: config.customSynthesizerAdapter
2792
+ });
2793
+ }
2794
+ planner = new CampaignPlanner();
2795
+ pipeline;
2796
+ async planCampaign(topic, options) {
2797
+ const ctx = options?.context ?? this.config.defaultCourseContext;
2798
+ const domain = options?.domain ?? classifyTechnicalDomain(topic, void 0, ctx).domain;
2799
+ return this.planner.createPlanWithLlm(topic, domain, {
2800
+ isDeep: options?.isDeep,
2801
+ maxRounds: options?.maxRounds,
2802
+ context: ctx,
2803
+ llm: this.config.llm
2804
+ });
2805
+ }
2806
+ async runFullHarvest(topic, overrides) {
2807
+ return this.pipeline.run(topic, {
2808
+ apiKeys: this.config.apiKeys,
2809
+ allowEnvFallback: this.config.allowEnvFallback,
2810
+ crawl4aiHost: this.config.crawl4aiHost,
2811
+ llm: this.config.llm,
2812
+ context: this.config.defaultCourseContext,
2813
+ customSearchAdapter: this.config.customSearchAdapter,
2814
+ customCrawlerAdapter: this.config.customCrawlerAdapter,
2815
+ customAcademicSearchAdapter: this.config.customAcademicSearchAdapter,
2816
+ customSynthesizerAdapter: this.config.customSynthesizerAdapter,
2817
+ ...overrides
2818
+ });
2819
+ }
2820
+ };
2821
+ var ResearchKitClient = class {
2822
+ constructor(config = {}) {
2823
+ this.config = config;
2824
+ this.discovery = new KnowledgeDiscoveryService(config);
2825
+ this.terminology = new TerminologyGraphService(config);
2826
+ this.academic = new AcademicStandardsService(config);
2827
+ this.verification = new EvidenceVerificationService(config);
2828
+ this.orchestrator = new PedagogicalSynthesisService(config);
2829
+ }
2830
+ /** Service 1: Multi-Engine Web Search, Scraper & YouTube Transcript Ingestion */
2831
+ discovery;
2832
+ /** Service 2: Bilingual Pedagogical Term Extraction & Prerequisite Concept Graph (DAG) */
2833
+ terminology;
2834
+ /** Service 3: Academic Literature Search, BibTeX Citations & CS2023/CSTA K-12 Alignment */
2835
+ academic;
2836
+ /** Service 4: Health Audit, Evidence Scoring, Claim Verification & Provenance Tracking */
2837
+ verification;
2838
+ /** Service 5: Campaign Planning, Semantic Synthesis & Canonical Reference Pack Orchestration */
2839
+ orchestrator;
1109
2840
  };
1110
2841
 
1111
2842
  exports.AcademicPaperSchema = AcademicPaperSchema;
1112
2843
  exports.AcademicSearchEngine = AcademicSearchEngine;
2844
+ exports.AcademicStandardsService = AcademicStandardsService;
1113
2845
  exports.ArXivSearchEngine = ArXivSearchEngine;
1114
2846
  exports.CS2023_KNOWLEDGE_AREAS = CS2023_KNOWLEDGE_AREAS;
2847
+ exports.CSTA_K12_STRANDS = CSTA_K12_STRANDS;
1115
2848
  exports.CampaignPlanner = CampaignPlanner;
1116
2849
  exports.CatalogBuilder = CatalogBuilder;
1117
2850
  exports.CitationManager = CitationManager;
1118
2851
  exports.ClaimVerifier = ClaimVerifier;
1119
2852
  exports.DEFAULT_RESEARCH_CONFIG = DEFAULT_RESEARCH_CONFIG;
1120
2853
  exports.EvidenceScorer = EvidenceScorer;
2854
+ exports.EvidenceVerificationService = EvidenceVerificationService;
1121
2855
  exports.HarvestPipeline = HarvestPipeline;
1122
2856
  exports.HealthAuditor = HealthAuditor;
2857
+ exports.KnowledgeDiscoveryService = KnowledgeDiscoveryService;
1123
2858
  exports.LiteratureReviewSynthesizer = LiteratureReviewSynthesizer;
1124
2859
  exports.MultiSourceSearchEngine = MultiSourceSearchEngine;
2860
+ exports.PedagogicalSynthesisService = PedagogicalSynthesisService;
2861
+ exports.PedagogicalTermExtractor = PedagogicalTermExtractor;
1125
2862
  exports.ProvenanceTracker = ProvenanceTracker;
1126
2863
  exports.ReferencePackBuilder = ReferencePackBuilder;
1127
2864
  exports.ResearchConfigSchema = ResearchConfigSchema;
2865
+ exports.ResearchKitClient = ResearchKitClient;
2866
+ exports.TerminologyGraphService = TerminologyGraphService;
1128
2867
  exports.WebScraper = WebScraper;
2868
+ exports.YouTubeIngestionService = YouTubeIngestionService;
1129
2869
  exports.classifyTechnicalDomain = classifyTechnicalDomain;
2870
+ exports.extractSubstantiveExcerpts = extractSubstantiveExcerpts;
2871
+ exports.extractTopSentences = extractTopSentences;
2872
+ exports.extractYouTubeVideoId = extractYouTubeVideoId;
2873
+ exports.formatTimestamp = formatTimestamp;
2874
+ exports.inferPrimaryCodeLanguage = inferPrimaryCodeLanguage;
2875
+ exports.isYouTubeUrl = isYouTubeUrl;
1130
2876
  exports.mapTopicToCS2023 = mapTopicToCS2023;
2877
+ exports.normalizeUrlForDedup = normalizeUrlForDedup;
2878
+ exports.reconstructOpenAlexAbstract = reconstructOpenAlexAbstract;
2879
+ exports.stitchCaptionSegments = stitchCaptionSegments;
1131
2880
  //# sourceMappingURL=index.cjs.map
1132
2881
  //# sourceMappingURL=index.cjs.map