@thanh01.pmt/research-kit 1.0.0 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +213 -0
  3. package/dist/academic/index.cjs +332 -127
  4. package/dist/academic/index.cjs.map +1 -1
  5. package/dist/academic/index.d.cts +3 -3
  6. package/dist/academic/index.d.ts +3 -3
  7. package/dist/academic/index.js +2 -2
  8. package/dist/{catalogBuilder-8A-Dg_A-.d.ts → catalogBuilder-CmmQzUnf.d.ts} +11 -3
  9. package/dist/{catalogBuilder-feU79wQQ.d.cts → catalogBuilder-DgXYxL2F.d.cts} +11 -3
  10. package/dist/chunk-2JEVBCIF.js +3 -0
  11. package/dist/{chunk-KGCDYFFD.js.map → chunk-2JEVBCIF.js.map} +1 -1
  12. package/dist/chunk-2XIEWCUE.js +244 -0
  13. package/dist/chunk-2XIEWCUE.js.map +1 -0
  14. package/dist/chunk-AUGVVZTL.js +514 -0
  15. package/dist/chunk-AUGVVZTL.js.map +1 -0
  16. package/dist/chunk-DOXSPRSP.js +559 -0
  17. package/dist/chunk-DOXSPRSP.js.map +1 -0
  18. package/dist/chunk-GKF3NDEX.js +162 -0
  19. package/dist/chunk-GKF3NDEX.js.map +1 -0
  20. package/dist/chunk-J3VCGH7Z.js +3 -0
  21. package/dist/chunk-J3VCGH7Z.js.map +1 -0
  22. package/dist/{chunk-67V2GJGK.js → chunk-J7XKDMGL.js} +7 -5
  23. package/dist/chunk-J7XKDMGL.js.map +1 -0
  24. package/dist/chunk-KAFQVI2R.js +333 -0
  25. package/dist/chunk-KAFQVI2R.js.map +1 -0
  26. package/dist/chunk-XIF5YNI7.js +799 -0
  27. package/dist/chunk-XIF5YNI7.js.map +1 -0
  28. package/dist/config/index.cjs +5 -3
  29. package/dist/config/index.cjs.map +1 -1
  30. package/dist/config/index.d.cts +1 -1
  31. package/dist/config/index.d.ts +1 -1
  32. package/dist/config/index.js +1 -1
  33. package/dist/evidence/index.cjs +337 -21
  34. package/dist/evidence/index.cjs.map +1 -1
  35. package/dist/evidence/index.d.cts +44 -7
  36. package/dist/evidence/index.d.ts +44 -7
  37. package/dist/evidence/index.js +3 -2
  38. package/dist/harvester/index.cjs +805 -64
  39. package/dist/harvester/index.cjs.map +1 -1
  40. package/dist/harvester/index.d.cts +79 -8
  41. package/dist/harvester/index.d.ts +79 -8
  42. package/dist/harvester/index.js +3 -2
  43. package/dist/{index-BU5n1ip_.d.ts → index-t8pahlJ4.d.cts} +104 -15
  44. package/dist/{index-BU5n1ip_.d.cts → index-t8pahlJ4.d.ts} +104 -15
  45. package/dist/index.cjs +2001 -252
  46. package/dist/index.cjs.map +1 -1
  47. package/dist/index.d.cts +156 -8
  48. package/dist/index.d.ts +156 -8
  49. package/dist/index.js +237 -8
  50. package/dist/index.js.map +1 -1
  51. package/dist/literatureReview-BTPFe7eb.d.cts +64 -0
  52. package/dist/literatureReview-hGbfrBmw.d.ts +64 -0
  53. package/dist/pipeline/index.cjs +1570 -241
  54. package/dist/pipeline/index.cjs.map +1 -1
  55. package/dist/pipeline/index.d.cts +49 -9
  56. package/dist/pipeline/index.d.ts +49 -9
  57. package/dist/pipeline/index.js +5 -4
  58. package/dist/webScraper-BhiQ8s9x.d.ts +68 -0
  59. package/dist/webScraper-D9Ldus9p.d.cts +68 -0
  60. package/package.json +8 -9
  61. package/dist/chunk-47L7ZHMY.js +0 -316
  62. package/dist/chunk-47L7ZHMY.js.map +0 -1
  63. package/dist/chunk-63X2HPCI.js +0 -47
  64. package/dist/chunk-63X2HPCI.js.map +0 -1
  65. package/dist/chunk-67V2GJGK.js.map +0 -1
  66. package/dist/chunk-6Q2XCUG7.js +0 -40
  67. package/dist/chunk-6Q2XCUG7.js.map +0 -1
  68. package/dist/chunk-KGCDYFFD.js +0 -3
  69. package/dist/chunk-MHAYD555.js +0 -239
  70. package/dist/chunk-MHAYD555.js.map +0 -1
  71. package/dist/chunk-MPLDSHFM.js +0 -396
  72. package/dist/chunk-MPLDSHFM.js.map +0 -1
  73. package/dist/chunk-QZNLB5X4.js +0 -65
  74. package/dist/chunk-QZNLB5X4.js.map +0 -1
  75. package/dist/literatureReview-C4AgVljZ.d.ts +0 -43
  76. package/dist/literatureReview-Dlods4Aw.d.cts +0 -43
  77. package/dist/webScraper-pKzdXhfI.d.ts +0 -46
  78. package/dist/webScraper-yBcYOHoL.d.cts +0 -46
@@ -5,16 +5,33 @@ var zod = require('zod');
5
5
  // src/pipeline/campaignPlanner.ts
6
6
  var CampaignPlanner = class {
7
7
  createPlan(topic, domain, options = {}) {
8
- const slug = topic.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40);
8
+ const slug = topic.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40) || "research-topic";
9
9
  const scaleMode = options.isDeep ? "deep" : domain === "academic" ? "academic_survey" : "direct";
10
10
  const maxRounds = options.maxRounds ?? (scaleMode === "deep" ? 3 : 1);
11
- const plannedQueries = [
12
- `${topic} official documentation getting started`,
13
- `${topic} architecture design best practices`,
14
- `${topic} practical code examples and pitfalls`
15
- ];
11
+ const ctx = options.context;
12
+ const toolAnchor = [ctx?.hardwarePlatform, ctx?.softwareEnv].filter(Boolean).join(" ").trim();
13
+ const coreSubject = toolAnchor ? `${toolAnchor} ${topic}` : topic;
14
+ const plannedQueries = [];
15
+ if (ctx && (ctx.hardwarePlatform || ctx.softwareEnv || ctx.inScopeTopics && ctx.inScopeTopics.length > 0)) {
16
+ plannedQueries.push(
17
+ `${toolAnchor || topic} official documentation getting started guide ${ctx.inScopeTopics?.slice(0, 2).join(" ") || ""}`.trim()
18
+ );
19
+ const ageHint = ctx.ageBand ? `ages ${ctx.ageBand[0]}-${ctx.ageBand[1]}` : "beginner project";
20
+ plannedQueries.push(
21
+ `${toolAnchor || topic} hands-on project tutorial curriculum ${ageHint}`.trim()
22
+ );
23
+ plannedQueries.push(
24
+ `${toolAnchor || topic} common mistakes pitfalls debugging troubleshooting`.trim()
25
+ );
26
+ } else {
27
+ plannedQueries.push(
28
+ `${coreSubject} official documentation getting started`,
29
+ `${coreSubject} architecture design best practices`,
30
+ `${coreSubject} practical code examples and pitfalls`
31
+ );
32
+ }
16
33
  if (domain === "hardware") {
17
- plannedQueries.push(`${topic} pinout schematic datasheet specifications`);
34
+ plannedQueries.push(`${ctx?.hardwarePlatform || topic} pinout schematic sensor wiring specifications`);
18
35
  } else if (domain === "academic") {
19
36
  plannedQueries.push(`${topic} state of the art benchmarks survey paper`);
20
37
  }
@@ -23,14 +40,14 @@ var CampaignPlanner = class {
23
40
  `> **Slug:** \`${slug}\` | **Domain:** \`${domain.toUpperCase()}\` | **Scale Mode:** \`${scaleMode.toUpperCase()}\` | **Max Rounds:** \`${maxRounds}\``,
24
41
  `
25
42
  ## 1. Campaign Objective`,
26
- `- Exhaustively harvest and verify Ground Truth for **${topic}** to anchor all downstream SOT artifacts in Curriculum OS.`,
43
+ `- Exhaustively harvest and verify Technical & Pedagogical Ground Truth for **${topic}** to anchor all downstream SOT artifacts in Curriculum OS.`,
27
44
  `
28
45
  ## 2. Planned Multi-Source Queries`,
29
46
  ...plannedQueries.map((q, i) => `${i + 1}. \`${q}\``),
30
47
  `
31
48
  ## 3. Acceptance Criteria & Quality Gates`,
32
49
  `- Minimum 3 primary technical resources with Relevancy Score >= 6/10.`,
33
- `- Extract 100% executable syntax-verified code snippets.`,
50
+ `- Extract verified code blocks, reference tables, and bilingual domain concepts.`,
34
51
  `- Produce the complete 3-layer catalog: \`RESOURCE_INDEX.json\`, \`RESOURCE_CATALOG.md\`, and \`KNOWLEDGE_GRAPH.md\`.`
35
52
  ];
36
53
  return {
@@ -43,33 +60,181 @@ var CampaignPlanner = class {
43
60
  planMd: mdLines.join("\n")
44
61
  };
45
62
  }
63
+ /**
64
+ * Optional LLM-augmented query planner when the host app provides an `llm` callback.
65
+ */
66
+ async createPlanWithLlm(topic, domain, options = {}) {
67
+ const basePlan = this.createPlan(topic, domain, options);
68
+ if (!options.llm) return basePlan;
69
+ try {
70
+ const ctx = options.context;
71
+ const prompt = [
72
+ `Generate 4 high-precision search engine queries (in English or bilingual as best suited for technical documentation and curriculum discovery) for researching this course:`,
73
+ `- Topic / Title: ${topic}`,
74
+ ctx?.positioning ? `- Positioning: ${ctx.positioning}` : "",
75
+ ctx?.hardwarePlatform ? `- Hardware Platform: ${ctx.hardwarePlatform}` : "",
76
+ ctx?.softwareEnv ? `- Software / Tools: ${ctx.softwareEnv}` : "",
77
+ ctx?.ageBand ? `- Learner Age Band: ${ctx.ageBand[0]}-${ctx.ageBand[1]}` : "",
78
+ ctx?.inScopeTopics?.length ? `- In-Scope Topics: ${ctx.inScopeTopics.join(", ")}` : "",
79
+ `The 4 queries must cover: (1) Official technical/API/hardware documentation, (2) Real-world project/course exemplars for this age group, (3) Common beginner misconceptions & debugging pitfalls, (4) Best pedagogical scaffolding practices.`,
80
+ `Return ONLY a JSON array of 4 query strings: ["query 1", "query 2", "query 3", "query 4"]`
81
+ ].filter(Boolean).join("\n");
82
+ const raw = await options.llm(prompt, {
83
+ system: "You are a technical research librarian. Return only a valid JSON array of 4 search query strings.",
84
+ temperature: 0.2,
85
+ jsonMode: true
86
+ });
87
+ const match = raw.match(/\[[\s\S]*\]/);
88
+ if (match) {
89
+ const parsed = JSON.parse(match[0]);
90
+ if (Array.isArray(parsed) && parsed.length >= 2) {
91
+ const queries = parsed.map((q) => String(q).trim()).filter((q) => q.length > 4).slice(0, 5);
92
+ if (queries.length >= 2) {
93
+ basePlan.plannedQueries = queries;
94
+ }
95
+ }
96
+ }
97
+ } catch (e) {
98
+ console.warn("[CampaignPlanner] LLM query expansion failed, using deterministic queries:", e?.message || e);
99
+ }
100
+ return basePlan;
101
+ }
46
102
  };
47
103
 
104
+ // src/harvester/domainClassifier.ts
105
+ var HARDWARE_PATTERNS = [
106
+ /\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
107
+ /\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
108
+ /(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
109
+ ];
110
+ var PLATFORM_PATTERNS = [
111
+ /\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
112
+ /(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
113
+ ];
114
+ var ACADEMIC_PATTERNS = [
115
+ /\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
116
+ /(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
117
+ ];
118
+ function inferPrimaryCodeLanguage(topic, context, domain) {
119
+ const combined = [
120
+ topic,
121
+ context?.courseTitle,
122
+ context?.hardwarePlatform,
123
+ context?.softwareEnv,
124
+ ...context?.inScopeTopics || []
125
+ ].filter(Boolean).join(" ").toLowerCase();
126
+ if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
127
+ if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
128
+ if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
129
+ if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
130
+ if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
131
+ if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
132
+ if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
133
+ if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
134
+ if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
135
+ if (/\b(rust|cargo)\b/.test(combined)) return "rust";
136
+ if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
137
+ if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
138
+ if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
139
+ if (domain === "hardware") return "cpp";
140
+ return "text";
141
+ }
142
+ function classifyTechnicalDomain(topic, explicitDomain, context) {
143
+ if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
144
+ const domain = explicitDomain.toLowerCase();
145
+ return {
146
+ domain,
147
+ confidence: 1,
148
+ rationale: "Explicitly configured by user flag.",
149
+ recommendedExtractors: getExtractorsForDomain(domain),
150
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
151
+ };
152
+ }
153
+ const combined = [
154
+ topic,
155
+ context?.courseTitle,
156
+ context?.positioning,
157
+ context?.hardwarePlatform,
158
+ context?.softwareEnv,
159
+ ...context?.inScopeTopics || []
160
+ ].filter(Boolean).join(" ");
161
+ if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
162
+ return {
163
+ domain: "hardware",
164
+ confidence: 0.95,
165
+ rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
166
+ recommendedExtractors: getExtractorsForDomain("hardware"),
167
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
168
+ };
169
+ }
170
+ if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
171
+ return {
172
+ domain: "platform",
173
+ confidence: 0.9,
174
+ rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
175
+ recommendedExtractors: getExtractorsForDomain("platform"),
176
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
177
+ };
178
+ }
179
+ if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
180
+ return {
181
+ domain: "academic",
182
+ confidence: 0.9,
183
+ rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
184
+ recommendedExtractors: getExtractorsForDomain("academic"),
185
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
186
+ };
187
+ }
188
+ return {
189
+ domain: "software",
190
+ confidence: 0.85,
191
+ rationale: "Topic is a programming language, game engine, client/server library, or software application.",
192
+ recommendedExtractors: getExtractorsForDomain("software"),
193
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
194
+ };
195
+ }
196
+ function getExtractorsForDomain(domain) {
197
+ switch (domain) {
198
+ case "hardware":
199
+ return ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"];
200
+ case "platform":
201
+ return ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"];
202
+ case "academic":
203
+ return ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"];
204
+ case "software":
205
+ default:
206
+ return ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"];
207
+ }
208
+ }
209
+
48
210
  // src/pipeline/referencePackBuilder.ts
49
211
  var ReferencePackBuilder = class {
50
212
  buildReferencePack(topic, domain, options = {}) {
51
213
  const registry = [];
214
+ const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
52
215
  if (options.academicPapers) {
53
216
  options.academicPapers.forEach((p, idx) => {
217
+ const takeaway = p.tldr?.trim() || (p.abstract ? p.abstract.slice(0, 220).trim() : `Peer-reviewed publication (${p.year || "n.d."}, ${p.citationCount} citations).`);
54
218
  registry.push({
55
219
  id: `REF-ACAD-${String(idx + 1).padStart(2, "0")}`,
56
220
  title: p.title,
57
221
  sourceType: "academic_paper",
58
- url: p.url || `https://doi.org/${p.doi || ""}`,
222
+ url: p.url || (p.doi ? `https://doi.org/${p.doi}` : ""),
59
223
  relevanceScore: 9,
60
- extractedKeyTakeaways: [p.tldr || p.abstract?.slice(0, 100) || "Peer-reviewed theoretical foundation."]
224
+ extractedKeyTakeaways: [takeaway]
61
225
  });
62
226
  });
63
227
  }
64
228
  if (options.scrapedDocs) {
65
229
  options.scrapedDocs.forEach((d, idx) => {
230
+ const takeaways = d.keyTakeaways && d.keyTakeaways.length > 0 ? d.keyTakeaways : d.doc.keyExcerpts && d.doc.keyExcerpts.length > 0 ? d.doc.keyExcerpts.slice(0, 3) : [d.summary];
66
231
  registry.push({
67
232
  id: `REF-DOC-${String(idx + 1).padStart(2, "0")}`,
68
233
  title: d.doc.title,
69
- sourceType: "official_docs",
234
+ sourceType: d.doc.fetchProvider === "youtube" ? "video_transcript" : "official_docs",
70
235
  url: d.doc.url,
71
236
  relevanceScore: d.score,
72
- extractedKeyTakeaways: [d.summary]
237
+ extractedKeyTakeaways: takeaways
73
238
  });
74
239
  });
75
240
  }
@@ -79,33 +244,83 @@ var ReferencePackBuilder = class {
79
244
  domain,
80
245
  generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
81
246
  totalReferences: registry.length,
82
- cs2023StandardArea: options.cs2023Area?.code || "PDC",
247
+ cs2023StandardArea: options.cs2023Area?.code || "SDF",
248
+ cstaK12Strand: options.k12Strand?.code || "AP",
249
+ synthesizedInsights: options.synthesizedInsights,
250
+ concepts: options.concepts || [],
83
251
  registry
84
252
  },
85
253
  null,
86
254
  2
87
255
  );
256
+ const codeBlocks = options.scrapedDocs?.flatMap((d) => {
257
+ if (d.doc.extractedCodeBlocks && d.doc.extractedCodeBlocks.length > 0) {
258
+ return d.doc.extractedCodeBlocks.slice(0, 2).map((b) => `\`\`\`${b.language || defaultLang}
259
+ ${b.code}
260
+ \`\`\``);
261
+ }
262
+ return d.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
263
+ ${c}
264
+ \`\`\``);
265
+ }) || [];
266
+ const insightsSection = [];
267
+ if (options.synthesizedInsights) {
268
+ if (options.synthesizedInsights.executiveSummary) {
269
+ insightsSection.push(`
270
+ ### Synthesized Executive Summary
271
+ ${options.synthesizedInsights.executiveSummary}`);
272
+ }
273
+ if (options.synthesizedInsights.keyTakeaways?.length > 0) {
274
+ insightsSection.push(
275
+ `
276
+ ### Key Technical & Pedagogical Takeaways
277
+ ` + options.synthesizedInsights.keyTakeaways.map((t) => `- ${t}`).join("\n")
278
+ );
279
+ }
280
+ if (options.synthesizedInsights.commonMisconceptionsAndPitfalls?.length > 0) {
281
+ insightsSection.push(
282
+ `
283
+ ### Documented Learner Misconceptions & Debugging Pitfalls
284
+ ` + options.synthesizedInsights.commonMisconceptionsAndPitfalls.map((m) => `- ${m}`).join("\n")
285
+ );
286
+ }
287
+ if (options.synthesizedInsights.recommendedExemplars?.length > 0) {
288
+ insightsSection.push(
289
+ `
290
+ ### Reference Project & Curriculum Exemplars
291
+ ` + options.synthesizedInsights.recommendedExemplars.map((e) => `- ${e}`).join("\n")
292
+ );
293
+ }
294
+ } else if (registry.length > 0) {
295
+ const topTakeaways = registry.flatMap((r) => r.extractedKeyTakeaways.slice(0, 2).map((t) => `- **[${r.id}]** ${t}`));
296
+ if (topTakeaways.length > 0) {
297
+ insightsSection.push(`
298
+ ### Extracted Source Takeaways
299
+ ` + topTakeaways.join("\n"));
300
+ }
301
+ }
88
302
  const mdLines = [
89
303
  `# \u{1F4DA} REFERENCE_PACK.md: ${topic}`,
90
- `> **Domain:** \`${domain.toUpperCase()}\` | **CS2023 Area:** \`${options.cs2023Area?.code || "SE"}\` \u2014 *${options.cs2023Area?.name || "Software Engineering"}*`,
304
+ `> **Domain:** \`${domain.toUpperCase()}\` | **CS2023 Area:** \`${options.cs2023Area?.code || "SE"}\` \u2014 *${options.cs2023Area?.name || "Software Engineering"}*${options.k12Strand ? ` | **CSTA K-12:** \`${options.k12Strand.code}\` (${options.k12Strand.name})` : ""}`,
91
305
  `> **Curriculum OS Status:** \`<!-- [SME_APPROVED] -->\``,
92
306
  `
93
307
  ## 1. Executive Summary & Ground Truth Mandate`,
94
308
  `This reference pack serves as the immutable ground truth for all downstream lesson plans, code labs, and rubrics for **${topic}**.`,
309
+ ...insightsSection,
95
310
  `
96
311
  ## 2. Technical Ground Truth Registry`,
97
312
  `| Reference ID | Type | Title | Relevance | Source Link |`,
98
313
  `|---|---|---|---|---|`,
99
- ...registry.map((r) => `| **${r.id}** | \`${r.sourceType}\` | ${r.title} | \u2B50 \`${r.relevanceScore}/10\` | [Access](${r.url}) |`),
314
+ ...registry.map(
315
+ (r) => `| **${r.id}** | \`${r.sourceType}\` | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | [Access](${r.url}) |`
316
+ ),
100
317
  `
101
318
  ## 3. Extracted Code Signatures & Primitives`,
102
- ...options.scrapedDocs?.flatMap((d) => d.doc.extractedCodeSnippets.slice(0, 2)).map((c) => `\`\`\`${domain === "hardware" ? "cpp" : "swift"}
103
- ${c}
104
- \`\`\``) || ["*No explicit code signatures captured.*"],
319
+ ...codeBlocks.length > 0 ? codeBlocks : ["*No explicit code signatures captured.*"],
105
320
  `
106
321
  ## 4. Academic Alignment & Standard Citations`,
107
- `- Aligns directly with IEEE/ACM Computer Science Curricula standards.`,
108
- `- All code examples guaranteed 100% syntactically valid and executable.`
322
+ `- Aligns directly with IEEE/ACM Computer Science Curricula (${options.cs2023Area?.code || "SDF"}) and CSTA K-12 (${options.k12Strand?.code || "AP"}) standards.`,
323
+ `- All claims and code examples are anchored to physical primary sources listed in the registry above.`
109
324
  ];
110
325
  return {
111
326
  referenceRegistryJson,
@@ -114,76 +329,28 @@ ${c}
114
329
  }
115
330
  };
116
331
 
117
- // src/harvester/domainClassifier.ts
118
- function classifyTechnicalDomain(topic, explicitDomain) {
119
- if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
120
- const domain = explicitDomain.toLowerCase();
121
- return {
122
- domain,
123
- confidence: 1,
124
- rationale: "Explicitly configured by user flag.",
125
- recommendedExtractors: getExtractorsForDomain(domain)
126
- };
127
- }
128
- const t = topic.toLowerCase();
129
- if (t.includes("esp32") || t.includes("arduino") || t.includes("raspberry") || t.includes("stm32") || t.includes("sensor") || t.includes("circuit") || t.includes("gpio") || t.includes("iot hardware")) {
130
- return {
131
- domain: "hardware",
132
- confidence: 0.95,
133
- rationale: "Topic contains microcontrollers, sensors, or physical electronics keywords.",
134
- recommendedExtractors: ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"]
135
- };
136
- }
137
- if (t.includes("supabase") || t.includes("firebase") || t.includes("aws") || t.includes("cloud") || t.includes("vercel") || t.includes("backend-as-a-service") || t.includes("docker") || t.includes("k8s")) {
138
- return {
139
- domain: "platform",
140
- confidence: 0.9,
141
- rationale: "Topic represents a cloud infrastructure, hosting, or BaaS platform.",
142
- recommendedExtractors: ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"]
143
- };
144
- }
145
- if (t.includes("theorem") || t.includes("complexity") || t.includes("pedagogy") || t.includes("bloom") || t.includes("constructivism") || t.includes("formal verification") || t.includes("computational theory")) {
146
- return {
147
- domain: "academic",
148
- confidence: 0.9,
149
- rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
150
- recommendedExtractors: ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"]
151
- };
152
- }
153
- return {
154
- domain: "software",
155
- confidence: 0.85,
156
- rationale: "Topic is a programming language, client/server library, or software framework.",
157
- recommendedExtractors: ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"]
158
- };
159
- }
160
- function getExtractorsForDomain(domain) {
161
- switch (domain) {
162
- case "hardware":
163
- return ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"];
164
- case "platform":
165
- return ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"];
166
- case "academic":
167
- return ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"];
168
- case "software":
169
- default:
170
- return ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"];
171
- }
172
- }
173
-
174
332
  // src/harvester/multiSourceSearch.ts
175
333
  var MultiSourceSearchEngine = class {
176
334
  tavilyKey;
177
335
  exaKey;
336
+ serperKey;
337
+ allowEnvFallback;
178
338
  customAdapter;
179
339
  constructor(options = {}) {
180
- this.tavilyKey = options.tavilyKey || process.env.TAVILY_API_KEY;
181
- this.exaKey = options.exaKey || process.env.EXA_API_KEY;
340
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
341
+ this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
342
+ this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
343
+ this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
182
344
  this.customAdapter = options.customAdapter;
183
345
  }
184
346
  setCustomAdapter(adapter) {
185
347
  this.customAdapter = adapter;
186
348
  }
349
+ setApiKeys(keys) {
350
+ if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
351
+ if (keys.exa !== void 0) this.exaKey = keys.exa;
352
+ if (keys.serper !== void 0) this.serperKey = keys.serper;
353
+ }
187
354
  async search(query, options = {}) {
188
355
  const limit = options.limit ?? 5;
189
356
  const adapter = options.customAdapter || this.customAdapter;
@@ -204,28 +371,37 @@ var MultiSourceSearchEngine = class {
204
371
  console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
205
372
  }
206
373
  }
207
- const tavilyKey = options.tavilyKey || this.tavilyKey;
208
- const exaKey = options.exaKey || this.exaKey;
374
+ const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
375
+ const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
376
+ const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
209
377
  if (preferred === "exa" && exaKey) {
210
- const exaHits = await this.queryExa(query, exaKey, limit);
378
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
211
379
  if (exaHits.length > 0) return exaHits;
212
380
  }
381
+ if (preferred === "serper" && serperKey) {
382
+ const serperHits = await this.querySerper(query, serperKey, limit);
383
+ if (serperHits.length > 0) return serperHits;
384
+ }
213
385
  if (preferred === "duckduckgo") {
214
386
  const ddgHits2 = await this.queryDuckDuckGo(query, limit);
215
387
  if (ddgHits2.length > 0) return ddgHits2;
216
388
  }
217
389
  if (tavilyKey && (!preferred || preferred === "tavily")) {
218
390
  try {
391
+ const body = {
392
+ api_key: tavilyKey,
393
+ query,
394
+ search_depth: "advanced",
395
+ include_answer: true,
396
+ max_results: limit
397
+ };
398
+ if (options.domainFilter && options.domainFilter.length > 0) {
399
+ body.include_domains = options.domainFilter;
400
+ }
219
401
  const res = await fetch("https://api.tavily.com/search", {
220
402
  method: "POST",
221
403
  headers: { "Content-Type": "application/json" },
222
- body: JSON.stringify({
223
- api_key: tavilyKey,
224
- query,
225
- search_depth: "advanced",
226
- include_answer: true,
227
- max_results: limit
228
- }),
404
+ body: JSON.stringify(body),
229
405
  signal: AbortSignal.timeout(12e3)
230
406
  });
231
407
  if (res.ok) {
@@ -245,26 +421,34 @@ var MultiSourceSearchEngine = class {
245
421
  }
246
422
  }
247
423
  if (exaKey) {
248
- const exaHits = await this.queryExa(query, exaKey, limit);
424
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
249
425
  if (exaHits.length > 0) return exaHits;
250
426
  }
427
+ if (serperKey) {
428
+ const serperHits = await this.querySerper(query, serperKey, limit);
429
+ if (serperHits.length > 0) return serperHits;
430
+ }
251
431
  const ddgHits = await this.queryDuckDuckGo(query, limit);
252
432
  if (ddgHits.length > 0) return ddgHits;
253
433
  return [];
254
434
  }
255
- async queryExa(query, exaKey, limit) {
435
+ async queryExa(query, exaKey, limit, domainFilter) {
256
436
  try {
437
+ const body = {
438
+ query,
439
+ numResults: limit,
440
+ useAutoprompt: true
441
+ };
442
+ if (domainFilter && domainFilter.length > 0) {
443
+ body.includeDomains = domainFilter;
444
+ }
257
445
  const res = await fetch("https://api.exa.ai/search", {
258
446
  method: "POST",
259
447
  headers: {
260
448
  "x-api-key": exaKey,
261
449
  "Content-Type": "application/json"
262
450
  },
263
- body: JSON.stringify({
264
- query,
265
- numResults: limit,
266
- useAutoprompt: true
267
- }),
451
+ body: JSON.stringify(body),
268
452
  signal: AbortSignal.timeout(1e4)
269
453
  });
270
454
  if (res.ok) {
@@ -284,6 +468,33 @@ var MultiSourceSearchEngine = class {
284
468
  }
285
469
  return [];
286
470
  }
471
+ async querySerper(query, serperKey, limit) {
472
+ try {
473
+ const res = await fetch("https://google.serper.dev/search", {
474
+ method: "POST",
475
+ headers: {
476
+ "X-API-KEY": serperKey,
477
+ "Content-Type": "application/json"
478
+ },
479
+ body: JSON.stringify({ q: query, num: limit }),
480
+ signal: AbortSignal.timeout(1e4)
481
+ });
482
+ if (res.ok) {
483
+ const data = await res.json();
484
+ if (Array.isArray(data?.organic) && data.organic.length > 0) {
485
+ return data.organic.slice(0, limit).map((r) => ({
486
+ title: r.title || "Untitled",
487
+ url: r.link,
488
+ content: r.snippet || "",
489
+ engine: "serper"
490
+ }));
491
+ }
492
+ }
493
+ } catch (e) {
494
+ console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
495
+ }
496
+ return [];
497
+ }
287
498
  async queryDuckDuckGo(query, limit) {
288
499
  try {
289
500
  const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
@@ -294,15 +505,16 @@ var MultiSourceSearchEngine = class {
294
505
  if (res.ok) {
295
506
  const html = await res.text();
296
507
  const hits = [];
297
- const resultRegex = /<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
508
+ const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
298
509
  let match;
299
510
  while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
300
- const rawUrl = match[1]?.trim();
511
+ const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
512
+ const rawUrl = match[2]?.trim();
301
513
  const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
302
- const snippet = match[2]?.replace(/<[^>]+>/g, "").trim() || "";
514
+ const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
303
515
  if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
304
516
  hits.push({
305
- title: query,
517
+ title: rawTitle,
306
518
  url: cleanUrl,
307
519
  content: snippet,
308
520
  engine: "duckduckgo"
@@ -318,15 +530,210 @@ var MultiSourceSearchEngine = class {
318
530
  }
319
531
  };
320
532
 
533
+ // src/harvester/youtubeTranscript.ts
534
+ var SPONSOR_NOISE_PATTERNS = [
535
+ /\b(sponsored by|this video is brought to you by|use (?:promo )?code\b|don't forget to like and subscribe|hit the bell icon|link in the description)\b/i,
536
+ /(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\s+này)?\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i
537
+ ];
538
+ function extractYouTubeVideoId(urlOrId) {
539
+ const trimmed = urlOrId.trim();
540
+ if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;
541
+ try {
542
+ const u = new URL(trimmed);
543
+ const host = u.hostname.replace(/^www\./, "").toLowerCase();
544
+ if (host === "youtu.be") {
545
+ const id = u.pathname.split("/").filter(Boolean)[0];
546
+ return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;
547
+ }
548
+ if (host === "youtube.com" || host === "m.youtube.com" || host === "youtube-nocookie.com") {
549
+ const v = u.searchParams.get("v");
550
+ if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;
551
+ const parts = u.pathname.split("/").filter(Boolean);
552
+ if ((parts[0] === "embed" || parts[0] === "shorts" || parts[0] === "v") && parts[1]) {
553
+ return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;
554
+ }
555
+ }
556
+ } catch {
557
+ }
558
+ return null;
559
+ }
560
+ function isYouTubeUrl(url) {
561
+ return extractYouTubeVideoId(url) !== null;
562
+ }
563
+ function formatTimestamp(seconds) {
564
+ const total = Math.max(0, Math.floor(seconds));
565
+ const hrs = Math.floor(total / 3600);
566
+ const mins = Math.floor(total % 3600 / 60);
567
+ const secs = total % 60;
568
+ if (hrs > 0) {
569
+ return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
570
+ }
571
+ return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
572
+ }
573
+ function decodeHtmlEntities(str) {
574
+ return str.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&#x27;/g, "'").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
575
+ }
576
+ function stitchCaptionSegments(segments) {
577
+ const cleaned = segments.map((s) => ({
578
+ startSec: s.startSec,
579
+ durationSec: s.durationSec,
580
+ text: decodeHtmlEntities(s.text)
581
+ })).filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));
582
+ if (cleaned.length === 0) return [];
583
+ const stitched = [];
584
+ let bucketStart = cleaned[0].startSec;
585
+ let bucketEnd = cleaned[0].startSec + cleaned[0].durationSec;
586
+ let bucketTexts = [];
587
+ for (const seg of cleaned) {
588
+ bucketTexts.push(seg.text);
589
+ bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);
590
+ const joined = bucketTexts.join(" ");
591
+ const elapsed = seg.startSec - bucketStart;
592
+ const endsSentence = /[.!?…]$/.test(seg.text);
593
+ if (elapsed >= 30 && endsSentence || elapsed >= 50 || joined.length >= 360) {
594
+ stitched.push({
595
+ startSec: bucketStart,
596
+ durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
597
+ timestamp: formatTimestamp(bucketStart),
598
+ text: joined
599
+ });
600
+ bucketTexts = [];
601
+ bucketStart = seg.startSec + seg.durationSec;
602
+ }
603
+ }
604
+ if (bucketTexts.length > 0) {
605
+ stitched.push({
606
+ startSec: bucketStart,
607
+ durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
608
+ timestamp: formatTimestamp(bucketStart),
609
+ text: bucketTexts.join(" ")
610
+ });
611
+ }
612
+ return stitched;
613
+ }
614
+ var YouTubeIngestionService = class {
615
+ async fetchTranscript(urlOrId, preferredLanguages = ["vi", "en"]) {
616
+ const videoId = extractYouTubeVideoId(urlOrId);
617
+ if (!videoId) {
618
+ throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);
619
+ }
620
+ const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;
621
+ const res = await fetch(canonicalUrl, {
622
+ headers: {
623
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
624
+ "Accept-Language": "vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7"
625
+ },
626
+ signal: AbortSignal.timeout(12e3)
627
+ });
628
+ if (!res.ok) {
629
+ throw new Error(`Failed to fetch YouTube watch page (${res.status})`);
630
+ }
631
+ const html = await res.text();
632
+ const playerMatch = html.match(/ytInitialPlayerResponse\s*=\s*(\{[\s\S]*?\});\s*(?:var\s|window\.|<\/script>)/);
633
+ let title = `YouTube Video (${videoId})`;
634
+ let author;
635
+ let shortDescription = "";
636
+ let rawSegments = [];
637
+ let selectedLang = "en";
638
+ let isAutoGenerated = false;
639
+ if (playerMatch?.[1]) {
640
+ try {
641
+ const playerData = JSON.parse(playerMatch[1]);
642
+ title = playerData?.videoDetails?.title || title;
643
+ author = playerData?.videoDetails?.author;
644
+ shortDescription = playerData?.videoDetails?.shortDescription || "";
645
+ const tracks = playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];
646
+ if (tracks.length > 0) {
647
+ let chosenTrack = tracks.find(
648
+ (t) => preferredLanguages.includes(t.languageCode) && t.kind !== "asr"
649
+ ) || tracks.find((t) => preferredLanguages.includes(t.languageCode)) || tracks[0];
650
+ if (chosenTrack?.baseUrl) {
651
+ selectedLang = chosenTrack.languageCode || "en";
652
+ isAutoGenerated = chosenTrack.kind === "asr";
653
+ const trackRes = await fetch(chosenTrack.baseUrl, {
654
+ signal: AbortSignal.timeout(1e4)
655
+ });
656
+ if (trackRes.ok) {
657
+ const xml = await trackRes.text();
658
+ const textMatches = [
659
+ ...xml.matchAll(/<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g)
660
+ ];
661
+ for (const m of textMatches) {
662
+ rawSegments.push({
663
+ startSec: parseFloat(m[1] || "0"),
664
+ durationSec: parseFloat(m[2] || "2"),
665
+ text: (m[3] || "").replace(/<[^>]+>/g, " ")
666
+ });
667
+ }
668
+ }
669
+ }
670
+ }
671
+ } catch {
672
+ }
673
+ }
674
+ if (title === `YouTube Video (${videoId})`) {
675
+ const titleMeta = html.match(/<title>([\s\S]*?)<\/title>/i);
676
+ if (titleMeta?.[1]) {
677
+ title = decodeHtmlEntities(titleMeta[1].replace(/\s*-\s*YouTube$/i, ""));
678
+ }
679
+ }
680
+ const stitched = stitchCaptionSegments(rawSegments);
681
+ const mdLines = [
682
+ `# ${title}`,
683
+ `> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ""} | **Language:** \`${selectedLang}\`${isAutoGenerated ? " (Auto-caption)" : ""}`
684
+ ];
685
+ if (shortDescription.trim()) {
686
+ mdLines.push(`
687
+ ## Video Overview & Description
688
+ ${shortDescription.trim().slice(0, 1500)}`);
689
+ }
690
+ if (stitched.length > 0) {
691
+ mdLines.push(`
692
+ ## Timestamped Transcript`);
693
+ for (const seg of stitched) {
694
+ mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);
695
+ }
696
+ }
697
+ return {
698
+ videoId,
699
+ url: canonicalUrl,
700
+ title,
701
+ author,
702
+ language: selectedLang,
703
+ isAutoGenerated,
704
+ segments: stitched,
705
+ stitchedMarkdown: mdLines.join("\n")
706
+ };
707
+ }
708
+ };
709
+
321
710
  // src/harvester/webScraper.ts
711
+ function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
712
+ const withoutCode = markdown.replace(/```[\s\S]*?```/g, "\n");
713
+ const blocks = withoutCode.split(/\n{2,}/).map((b) => b.replace(/\s+/g, " ").trim()).filter((b) => {
714
+ if (b.length < 60 || b.length > 900) return false;
715
+ if (b.startsWith("#") || b.startsWith("|")) return false;
716
+ const linkRatio = (b.match(/\[[^\]]+\]\([^)]+\)/g) || []).join("").length / b.length;
717
+ if (linkRatio > 0.45) return false;
718
+ if (/\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\b/i.test(b)) {
719
+ return false;
720
+ }
721
+ return true;
722
+ });
723
+ return blocks.slice(0, maxExcerpts);
724
+ }
322
725
  var WebScraper = class {
323
726
  crawl4aiHost;
324
727
  jinaKey;
728
+ allowEnvFallback;
325
729
  customAdapter;
730
+ youtubeService;
326
731
  constructor(options = {}) {
327
- this.crawl4aiHost = options.crawl4aiHost || process.env.CRAWL4AI_HOST || "http://localhost:11235";
328
- this.jinaKey = options.jinaKey || process.env.JINA_API_KEY;
732
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
733
+ this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
734
+ this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
329
735
  this.customAdapter = options.customAdapter;
736
+ this.youtubeService = new YouTubeIngestionService();
330
737
  }
331
738
  setCustomAdapter(adapter) {
332
739
  this.customAdapter = adapter;
@@ -340,12 +747,22 @@ var WebScraper = class {
340
747
  try {
341
748
  const result = await adapter.crawl(targetUrl);
342
749
  if (result && result.markdownContent) {
343
- return this.packageDocument(targetUrl, result.markdownContent, "custom");
750
+ return this.packageDocument(targetUrl, result.markdownContent, "custom", result.title);
344
751
  }
345
752
  } catch (e) {
346
753
  console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);
347
754
  }
348
755
  }
756
+ if (isYouTubeUrl(targetUrl)) {
757
+ try {
758
+ const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);
759
+ if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {
760
+ return this.packageDocument(yt.url, yt.stitchedMarkdown, "youtube", yt.title);
761
+ }
762
+ } catch (e) {
763
+ console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
764
+ }
765
+ }
349
766
  try {
350
767
  const jinaUrl = `https://r.jina.ai/${targetUrl}`;
351
768
  const jinaKey = options.jinaKey || this.jinaKey;
@@ -396,27 +813,48 @@ var WebScraper = class {
396
813
  signal: AbortSignal.timeout(1e4)
397
814
  });
398
815
  if (res.ok) {
399
- const rawText = await res.text();
400
- const clean = rawText.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
401
- return this.packageDocument(targetUrl, clean, "direct");
816
+ const rawHtml = await res.text();
817
+ const titleTag = rawHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
818
+ const clean = rawHtml.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "").replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, "").replace(/<h([1-3])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, lvl, txt) => `
819
+ ${"#".repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, "").trim()}
820
+ `).replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => `
821
+ \`\`\`
822
+ ${code.replace(/<[^>]+>/g, "").trim()}
823
+ \`\`\`
824
+ `).replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, p) => `
825
+ ${p.replace(/<[^>]+>/g, " ").trim()}
826
+ `).replace(/<[^>]+>/g, " ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
827
+ return this.packageDocument(targetUrl, clean, "direct", titleTag);
402
828
  }
403
829
  } catch (e) {
404
830
  console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);
405
831
  }
406
832
  throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
407
833
  }
408
- packageDocument(url, markdown, provider) {
409
- const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/Title:\s*(.+)/i);
410
- const title = titleMatch ? titleMatch[1].trim() : url.split("/").pop() || "Document";
411
- const codeMatches = [...markdown.matchAll(/```(?:[a-z0-9_-]+)?\n([\s\S]*?)```/gi)].map((m) => m[1]?.trim()).filter(Boolean);
412
- const tableMatches = markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
834
+ packageDocument(url, markdown, provider, explicitTitle) {
835
+ const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
836
+ const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
837
+ const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```/g)];
838
+ const extractedCodeBlocks = codeBlockMatches.map((m) => ({
839
+ language: (m[1] || "").trim().toLowerCase(),
840
+ code: (m[2] || "").trim()
841
+ })).filter((b) => b.code.length > 0);
842
+ const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);
843
+ const tableBlocks = markdown.match(/(?:^\|.+\|\s*\n)+^\|[-:| ]+\|\s*\n(?:^\|.+\|\s*\n?)+/gm) || markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
844
+ const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);
845
+ const extractedHeadings = [...markdown.matchAll(/^#{1,3}\s+(.+)$/gm)].map((m) => m[1].trim()).filter((h) => h.length > 2 && h.length < 120);
846
+ const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);
413
847
  return {
414
848
  url,
415
849
  title,
416
850
  markdownContent: markdown,
417
851
  byteSize: Buffer.byteLength(markdown, "utf-8"),
418
- extractedCodeSnippets: codeMatches,
419
- tablesCount: tableMatches.length,
852
+ extractedCodeSnippets,
853
+ extractedCodeBlocks,
854
+ extractedTables,
855
+ extractedHeadings,
856
+ keyExcerpts,
857
+ tablesCount: extractedTables.length,
420
858
  fetchProvider: provider
421
859
  };
422
860
  }
@@ -424,7 +862,8 @@ var WebScraper = class {
424
862
 
425
863
  // src/harvester/catalogBuilder.ts
426
864
  var CatalogBuilder = class {
427
- buildBundle(topic, domain, items) {
865
+ buildBundle(topic, domain, items, options = {}) {
866
+ const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
428
867
  const indexList = items.map((item, idx) => ({
429
868
  id: `RES-${String(idx + 1).padStart(2, "0")}`,
430
869
  title: item.doc.title,
@@ -434,7 +873,8 @@ var CatalogBuilder = class {
434
873
  byteSize: item.doc.byteSize,
435
874
  tablesCount: item.doc.tablesCount,
436
875
  codeSnippetsCount: item.doc.extractedCodeSnippets.length,
437
- summary: item.summary
876
+ summary: item.summary,
877
+ keyTakeaways: item.keyTakeaways || item.doc.keyExcerpts?.slice(0, 3) || []
438
878
  }));
439
879
  const resourceIndexJson = JSON.stringify(
440
880
  {
@@ -453,36 +893,60 @@ var CatalogBuilder = class {
453
893
  `
454
894
  | ID | Resource Title | Relevance (1-10) | Code Snippets | Tables | Source Link |`,
455
895
  `|---|---|---|---|---|---|`,
456
- ...indexList.map((r) => `| **${r.id}** | ${r.title} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`),
896
+ ...indexList.map(
897
+ (r) => `| **${r.id}** | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`
898
+ ),
457
899
  `
458
900
  ## Resource Summaries & Technical Ground Truth
459
901
  `,
460
- ...indexList.map((r) => `### ${r.id}: ${r.title}
902
+ ...indexList.map((r) => {
903
+ const takeaways = r.keyTakeaways && r.keyTakeaways.length > 0 ? r.keyTakeaways.map((t) => ` - ${t}`).join("\n") : ` - ${r.summary}`;
904
+ return `### ${r.id}: ${r.title}
461
905
  - **URL:** ${r.url}
462
- - **Summary:** ${r.summary}
463
906
  - **Relevance:** ${r.relevanceScore}/10
464
- `)
907
+ - **Key Excerpts:**
908
+ ${takeaways}
909
+ `;
910
+ })
465
911
  ];
912
+ const codeSections = items.flatMap((item) => {
913
+ if (item.doc.extractedCodeBlocks && item.doc.extractedCodeBlocks.length > 0) {
914
+ return item.doc.extractedCodeBlocks.slice(0, 2).map((b) => {
915
+ const lang = b.language || defaultLang;
916
+ return `\`\`\`${lang}
917
+ ${b.code}
918
+ \`\`\``;
919
+ });
920
+ }
921
+ return item.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
922
+ ${c}
923
+ \`\`\``);
924
+ });
925
+ const conceptLines = options.concepts && options.concepts.length > 0 ? [
926
+ `| ID | Concept / Term | Vietnamese | English | Marr | Bloom | Prereqs | Sources |`,
927
+ `|---|---|---|---|---|---|---|---|`,
928
+ ...options.concepts.map(
929
+ (c) => `| \`${c.id}\` | **${c.term.replace(/\|/g, "/")}** | ${c.termVi || "\u2014"} | ${c.termEn || "\u2014"} | \`${c.marrLayer}\` | \`${c.bloomLevel}\` | ${c.prerequisites.join(", ") || "\u2014"} | ${c.sourceIds.join(", ")} |`
930
+ )
931
+ ] : [`- Canonical technical breakdown for **${topic}** derived from ${items.length} verified primary sources.`];
466
932
  const kgLines = [
467
933
  `# \u{1F9EC} Distilled Knowledge Graph: ${topic}`,
468
934
  `> **Domain:** \`${domain.toUpperCase()}\` | **Ground Truth Verification:** PASSED`,
469
935
  `
470
- ## 1. Architectural Blueprint & Component Decomposition`,
471
- `- Canonical technical breakdown for **${topic}** derived from ${items.length} peer-reviewed and verified primary sources.`,
936
+ ## 1. Architectural Blueprint & Concept Hierarchy (Marr & Bloom)`,
937
+ ...conceptLines,
472
938
  `
473
939
  ## 2. Core APIs & Code Primitives`,
474
- ...items.flatMap((i) => i.doc.extractedCodeSnippets.slice(0, 2)).map((c) => `\`\`\`${domain === "hardware" ? "cpp" : "swift"}
475
- ${c}
476
- \`\`\``),
940
+ ...codeSections.length > 0 ? codeSections : ["*No explicit code snippets captured in harvested sources.*"],
477
941
  `
478
- ## 3. Technical Constraints & Common Pitfalls`,
479
- `- Strict compliance with ${domain.toUpperCase()} safety and concurrency contracts.`,
480
- `- Proactively prevent documented anti-patterns and performance regressions.`
942
+ ## 3. Extracted Reference Tables`,
943
+ ...items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)).length > 0 ? items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)) : ["*No markdown reference tables extracted.*"]
481
944
  ];
482
945
  return {
483
946
  resourceIndexJson,
484
947
  resourceCatalogMd: catalogLines.join("\n"),
485
- knowledgeGraphMd: kgLines.join("\n")
948
+ knowledgeGraphMd: kgLines.join("\n"),
949
+ conceptDagTsv: options.conceptDagTsv
486
950
  };
487
951
  }
488
952
  };
@@ -507,37 +971,510 @@ var HealthAuditor = class {
507
971
  }
508
972
  };
509
973
 
974
+ // src/harvester/termExtractor.ts
975
+ var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
976
+ "l\xE0",
977
+ "c\u1EE7a",
978
+ "trong",
979
+ "\u0111\u01B0\u1EE3c",
980
+ "nh\u1EEFng",
981
+ "c\xE1c",
982
+ "cho",
983
+ "v\u1EDBi",
984
+ "khi",
985
+ "n\u1EBFu",
986
+ "th\xEC",
987
+ "v\xE0",
988
+ "ho\u1EB7c",
989
+ "\u0111\u1EC3",
990
+ "t\u1EEB",
991
+ "\u0111\u1EBFn",
992
+ "v\u1EC1",
993
+ "n\xE0y",
994
+ "\u0111\xF3",
995
+ "m\u1ED9t",
996
+ "nh\u01B0",
997
+ "s\u1EBD",
998
+ "\u0111ang",
999
+ "\u0111\xE3",
1000
+ "kh\xF4ng",
1001
+ "c\xF3",
1002
+ "ra",
1003
+ "v\xE0o",
1004
+ "b\u1EB1ng",
1005
+ "theo",
1006
+ "tr\xEAn",
1007
+ "d\u01B0\u1EDBi",
1008
+ "qua",
1009
+ "nh\u1EDD",
1010
+ "v\xEC",
1011
+ "do",
1012
+ "t\u1EA1i",
1013
+ "b\u1EDFi",
1014
+ "nh\u01B0ng",
1015
+ "m\xE0",
1016
+ "c\u0169ng",
1017
+ "v\u1EABn",
1018
+ "r\u1EA5t",
1019
+ "qu\xE1",
1020
+ "h\u01A1n",
1021
+ "nh\u1EA5t",
1022
+ "c\u1EA7n",
1023
+ "ph\u1EA3i",
1024
+ "n\xEAn",
1025
+ "th\u1EC3",
1026
+ "vi\u1EC7c",
1027
+ "c\xE1ch",
1028
+ "ph\u1EA7n",
1029
+ "b\xE0i",
1030
+ "bu\u1ED5i",
1031
+ "h\u1ECDc",
1032
+ "sinh",
1033
+ "gi\xE1o",
1034
+ "vi\xEAn",
1035
+ "ng\u01B0\u1EDDi",
1036
+ "d\xF9ng",
1037
+ "ch\xFAng",
1038
+ "ta",
1039
+ "b\u1EA1n",
1040
+ "em",
1041
+ "gi\xFAp",
1042
+ "l\xE0m",
1043
+ "t\u1EA1o",
1044
+ "s\u1EED",
1045
+ "d\u1EE5ng",
1046
+ "th\u1EF1c",
1047
+ "hi\u1EC7n",
1048
+ "hi\u1EC3u",
1049
+ "bi\u1EBFt",
1050
+ "xem",
1051
+ "th\xEAm",
1052
+ "sau",
1053
+ "tr\u01B0\u1EDBc"
1054
+ ]);
1055
+ var ENGLISH_STOPWORDS = /* @__PURE__ */ new Set([
1056
+ "the",
1057
+ "and",
1058
+ "for",
1059
+ "with",
1060
+ "that",
1061
+ "this",
1062
+ "from",
1063
+ "into",
1064
+ "your",
1065
+ "you",
1066
+ "are",
1067
+ "was",
1068
+ "were",
1069
+ "have",
1070
+ "has",
1071
+ "had",
1072
+ "will",
1073
+ "would",
1074
+ "can",
1075
+ "could",
1076
+ "should",
1077
+ "about",
1078
+ "more",
1079
+ "other",
1080
+ "such",
1081
+ "than",
1082
+ "then",
1083
+ "them",
1084
+ "these",
1085
+ "those",
1086
+ "their",
1087
+ "what",
1088
+ "when",
1089
+ "where",
1090
+ "which",
1091
+ "while",
1092
+ "who",
1093
+ "how",
1094
+ "why",
1095
+ "each",
1096
+ "some",
1097
+ "many",
1098
+ "most",
1099
+ "also",
1100
+ "only",
1101
+ "just",
1102
+ "very",
1103
+ "using",
1104
+ "used",
1105
+ "use",
1106
+ "overview",
1107
+ "introduction",
1108
+ "getting",
1109
+ "started",
1110
+ "guide",
1111
+ "tutorial",
1112
+ "example",
1113
+ "examples",
1114
+ "documentation"
1115
+ ]);
1116
+ var VIETNAMESE_TECHNICAL_HEADS = /^(cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|chuỗi|đối tượng|lớp|phương thức|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|mạng|giao thức|khái niệm|mô hình|kiến trúc|kiểm thử|gỡ lỗi)\b/i;
1117
+ var SYNTAX_OR_API_PATTERN = /(?:`[^`]+`|\b[a-z]+[A-Z][a-zA-Z0-9]*\b|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b|@[A-Za-z]+|\b[a-z_]+\(\)|\b(?:pinMode|digitalWrite|digitalRead|analogRead|analogWrite|async|await|actor|struct|class|func|def|let|const|var)\b)/;
1118
+ var HARDWARE_COMPONENT_PATTERN = /\b(esp32|arduino|micro:?bit|raspberry|sensor|led|servo|ultrasonic|ldr|dht11|dht22|breadboard|gpio|pwm|i2c|spi|uart|cảm biến|động cơ|mạch|đèn led|vi điều khiển)\b/i;
1119
+ function normalizeTermKey(raw) {
1120
+ return raw.normalize("NFC").toLowerCase().replace(/[`*_~#"']/g, "").replace(/\s+/g, " ").trim();
1121
+ }
1122
+ function cleanDisplayTerm(raw) {
1123
+ return raw.normalize("NFC").replace(/^[-*•#\d.)\s]+/, "").replace(/[`*_~]+/g, "").replace(/\s+/g, " ").trim();
1124
+ }
1125
+ function isNoiseTerm(term) {
1126
+ const clean = normalizeTermKey(term);
1127
+ if (clean.length < 2 || clean.length > 65) return true;
1128
+ if (/^\d+$/.test(clean)) return true;
1129
+ if (/^https?:\/\//i.test(clean)) return true;
1130
+ if (ENGLISH_STOPWORDS.has(clean) || VIETNAMESE_STOPWORDS.has(clean)) return true;
1131
+ const words = clean.split(" ");
1132
+ if (words.length > 6) return true;
1133
+ if (words.length === 1 && clean.length < 3 && !/^(ai|ui|ux|os|io|db|id|ip)$/i.test(clean)) return true;
1134
+ return false;
1135
+ }
1136
+ function classifyMarrAndCategory(term, domain) {
1137
+ if (HARDWARE_COMPONENT_PATTERN.test(term)) {
1138
+ return { category: "hardware_component", marrLayer: "CIO", bloomLevel: "apply" };
1139
+ }
1140
+ if (SYNTAX_OR_API_PATTERN.test(term)) {
1141
+ return { category: "api_syntax", marrLayer: "SIO", bloomLevel: "apply" };
1142
+ }
1143
+ if (/\b(architecture|pattern|model|design|debugging|verification|mô hình|kiến trúc|tư duy|quy trình)\b/i.test(term)) {
1144
+ return { category: "pedagogical_pattern", marrLayer: "ULO", bloomLevel: "analyze" };
1145
+ }
1146
+ return {
1147
+ category: "core_concept",
1148
+ marrLayer: domain === "academic" ? "ULO" : "CIO",
1149
+ bloomLevel: "understand"
1150
+ };
1151
+ }
1152
+ function findSentenceDefinition(term, markdown) {
1153
+ const clean = cleanDisplayTerm(term);
1154
+ const escaped = clean.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1155
+ const sentences = markdown.replace(/```[\s\S]*?```/g, " ").split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").trim()).filter((s) => s.length >= 25 && s.length <= 280);
1156
+ const re = new RegExp(escaped, "i");
1157
+ const match = sentences.find((s) => re.test(s));
1158
+ return match || `Core concept in ${clean} extracted from primary technical reference.`;
1159
+ }
1160
+ var PedagogicalTermExtractor = class {
1161
+ extractDeterministic(topic, domain, docs, maxTerms = 15) {
1162
+ const candidates = /* @__PURE__ */ new Map();
1163
+ let orderCounter = 0;
1164
+ const upsertCandidate = (rawTerm, weight, resourceId, markdown, pair) => {
1165
+ const display = cleanDisplayTerm(rawTerm);
1166
+ if (isNoiseTerm(display)) return;
1167
+ const key = normalizeTermKey(display);
1168
+ const existing = candidates.get(key);
1169
+ if (existing) {
1170
+ existing.score += weight;
1171
+ existing.sourceIds.add(resourceId);
1172
+ if (pair?.termVi && !existing.termVi) existing.termVi = pair.termVi;
1173
+ if (pair?.termEn && !existing.termEn) existing.termEn = pair.termEn;
1174
+ } else {
1175
+ candidates.set(key, {
1176
+ display,
1177
+ termVi: pair?.termVi,
1178
+ termEn: pair?.termEn,
1179
+ score: weight,
1180
+ firstSeenOrder: orderCounter++,
1181
+ sourceIds: /* @__PURE__ */ new Set([resourceId]),
1182
+ definition: findSentenceDefinition(display, markdown)
1183
+ });
1184
+ }
1185
+ };
1186
+ for (const { resourceId, doc } of docs) {
1187
+ const md = doc.markdownContent || "";
1188
+ const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
1189
+ const bilingualMatches = mdWithoutCode.matchAll(
1190
+ /([A-ZÀÁẠẢÃÂẦẤẬẨẪĂẰẮẶẲẴÈÉẸẺẼÊỀẾỆỂỄÌÍỊỈĨÒÓỌỎÕÔỒỐỘỔỖƠỜỚỢỞỠÙÚỤỦŨƯỪỨỰỬỮỲÝỴỶỸĐ][a-zA-Z0-9\u00C0-\u1EF9\s/-]{2,40}?)\s*\(([A-Za-z0-9\u00C0-\u1EF9\s/_:-]{2,40})\)/g
1191
+ );
1192
+ for (const m of bilingualMatches) {
1193
+ const left = cleanDisplayTerm(m[1] || "");
1194
+ const right = cleanDisplayTerm(m[2] || "");
1195
+ if (!isNoiseTerm(left) && !isNoiseTerm(right)) {
1196
+ const leftHasVi = /[\u00C0-\u1EF9]/.test(left);
1197
+ const rightHasVi = /[\u00C0-\u1EF9]/.test(right);
1198
+ if (leftHasVi && !rightHasVi) {
1199
+ upsertCandidate(`${left} (${right})`, 6, resourceId, mdWithoutCode, { termVi: left, termEn: right });
1200
+ } else if (!leftHasVi && rightHasVi) {
1201
+ upsertCandidate(`${right} (${left})`, 6, resourceId, mdWithoutCode, { termVi: right, termEn: left });
1202
+ } else {
1203
+ upsertCandidate(left, 4, resourceId, mdWithoutCode);
1204
+ }
1205
+ }
1206
+ }
1207
+ for (const heading of doc.extractedHeadings || []) {
1208
+ const cleaned = cleanDisplayTerm(heading.replace(/^\d+(\.\d+)*\s*/, ""));
1209
+ if (!isNoiseTerm(cleaned)) {
1210
+ upsertCandidate(cleaned, 4, resourceId, mdWithoutCode);
1211
+ }
1212
+ }
1213
+ for (const m of mdWithoutCode.matchAll(/\*\*([^*\n]{3,45})\*\*/g)) {
1214
+ upsertCandidate(m[1] || "", 3.5, resourceId, mdWithoutCode);
1215
+ }
1216
+ for (const m of mdWithoutCode.matchAll(/`([a-zA-Z_@][a-zA-Z0-9_.:()-]{2,35})`/g)) {
1217
+ upsertCandidate(m[1] || "", 3, resourceId, mdWithoutCode);
1218
+ }
1219
+ for (const line of mdWithoutCode.split("\n")) {
1220
+ const clauses = line.split(/[.,;:!?()[\]{}"“”‘’/\\|]+/);
1221
+ for (const clause of clauses) {
1222
+ const trimmed = clause.trim();
1223
+ const viMatch = trimmed.match(
1224
+ /\b((?:cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|đối tượng|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|giao thức|mô hình|kiến trúc|kiểm thử|gỡ lỗi)(?:\s+[a-zA-Z0-9\u00C0-\u1EF9_-]+){1,3})\b/i
1225
+ );
1226
+ if (viMatch?.[1]) {
1227
+ const tokens = viMatch[1].trim().split(/\s+/);
1228
+ while (tokens.length > 1 && VIETNAMESE_STOPWORDS.has(tokens[tokens.length - 1].toLowerCase())) {
1229
+ tokens.pop();
1230
+ }
1231
+ const phrase = tokens.join(" ");
1232
+ if (tokens.length >= 2 && VIETNAMESE_TECHNICAL_HEADS.test(phrase)) {
1233
+ upsertCandidate(phrase, 3.5, resourceId, mdWithoutCode, { termVi: phrase });
1234
+ }
1235
+ }
1236
+ }
1237
+ }
1238
+ }
1239
+ const sorted = [...candidates.values()].sort((a, b) => b.score - a.score || a.firstSeenOrder - b.firstSeenOrder).slice(0, maxTerms);
1240
+ const orderedByAppearance = [...sorted].sort((a, b) => a.firstSeenOrder - b.firstSeenOrder);
1241
+ const maxScore = Math.max(1, sorted[0]?.score || 1);
1242
+ const concepts = orderedByAppearance.map((item, idx) => {
1243
+ const id = `C_${String(idx + 1).padStart(2, "0")}`;
1244
+ const { category, marrLayer, bloomLevel } = classifyMarrAndCategory(item.display, domain);
1245
+ const prerequisites = [];
1246
+ return {
1247
+ id,
1248
+ term: item.display,
1249
+ termVi: item.termVi,
1250
+ termEn: item.termEn,
1251
+ category,
1252
+ marrLayer,
1253
+ bloomLevel,
1254
+ salienceScore: Number(Math.min(10, Math.max(1, item.score / maxScore * 10)).toFixed(1)),
1255
+ definition: item.definition,
1256
+ prerequisites,
1257
+ sourceIds: [...item.sourceIds]
1258
+ };
1259
+ });
1260
+ return {
1261
+ concepts,
1262
+ conceptDagTsv: this.formatConceptDagTsv(concepts),
1263
+ isLlmRefined: false,
1264
+ unrefinedCandidatesCount: candidates.size
1265
+ };
1266
+ }
1267
+ async extractWithOptionalLlm(topic, domain, docs, options = {}) {
1268
+ const detBundle = this.extractDeterministic(topic, domain, docs, options.maxTerms ?? 15);
1269
+ if (!options.llm || docs.length === 0) {
1270
+ return detBundle;
1271
+ }
1272
+ try {
1273
+ const corpusPreview = docs.slice(0, 6).map(
1274
+ (d) => `[${d.resourceId}] ${d.doc.title} (${d.doc.url}):
1275
+ ${(d.doc.keyExcerpts || []).slice(0, 3).join("\n")}`
1276
+ ).join("\n\n");
1277
+ const candidateSeed = detBundle.concepts.slice(0, 12).map((c) => `${c.term} (${c.sourceIds.join(",")})`).join("; ");
1278
+ const prompt = [
1279
+ `Analyze the harvested technical & pedagogical corpus for topic "${topic}" (Domain: ${domain}).`,
1280
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]} years old.` : "",
1281
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}.` : "",
1282
+ options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}.` : "",
1283
+ `Deterministic Candidate Terms: ${candidateSeed}`,
1284
+ `
1285
+ Harvested Source Excerpts:
1286
+ ${corpusPreview}`,
1287
+ `
1288
+ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
1289
+ `IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
1290
+ `Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
1291
+ ].filter(Boolean).join("\n");
1292
+ const rawResponse = await options.llm(prompt, {
1293
+ system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
1294
+ temperature: 0.2,
1295
+ jsonMode: true
1296
+ });
1297
+ const jsonMatch = rawResponse.match(/\[[\s\S]*\]/);
1298
+ if (jsonMatch) {
1299
+ const parsed = JSON.parse(jsonMatch[0]);
1300
+ if (Array.isArray(parsed) && parsed.length > 0) {
1301
+ const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
1302
+ const concepts = parsed.map((item, idx) => {
1303
+ const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
1304
+ const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
1305
+ return {
1306
+ id,
1307
+ term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
1308
+ termVi: item.termVi ? String(item.termVi) : void 0,
1309
+ termEn: item.termEn ? String(item.termEn) : void 0,
1310
+ category: ["core_concept", "api_syntax", "hardware_component", "pedagogical_pattern"].includes(item.category) ? item.category : "core_concept",
1311
+ marrLayer: ["ULO", "CIO", "SIO"].includes(item.marrLayer) ? item.marrLayer : "CIO",
1312
+ bloomLevel: ["remember", "understand", "apply", "analyze", "evaluate", "create"].includes(item.bloomLevel) ? item.bloomLevel : "understand",
1313
+ salienceScore: Number(item.salienceScore) || 8,
1314
+ definition: String(item.definition || ""),
1315
+ prerequisites: prereqs,
1316
+ sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
1317
+ };
1318
+ });
1319
+ return {
1320
+ concepts,
1321
+ conceptDagTsv: this.formatConceptDagTsv(concepts),
1322
+ isLlmRefined: true,
1323
+ unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
1324
+ };
1325
+ }
1326
+ }
1327
+ } catch (e) {
1328
+ console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
1329
+ }
1330
+ return detBundle;
1331
+ }
1332
+ formatConceptDagTsv(concepts) {
1333
+ const header = "id term term_vi term_en marr_layer bloom_level salience prerequisites sources definition";
1334
+ const rows = concepts.map(
1335
+ (c) => [
1336
+ c.id,
1337
+ c.term.replace(/\t|\n/g, " "),
1338
+ (c.termVi || "").replace(/\t|\n/g, " "),
1339
+ (c.termEn || "").replace(/\t|\n/g, " "),
1340
+ c.marrLayer,
1341
+ c.bloomLevel,
1342
+ c.salienceScore,
1343
+ c.prerequisites.join(","),
1344
+ c.sourceIds.join(","),
1345
+ c.definition.replace(/\t|\n/g, " ")
1346
+ ].join(" ")
1347
+ );
1348
+ return [header, ...rows].join("\n");
1349
+ }
1350
+ };
1351
+
510
1352
  // src/evidence/evidenceScorer.ts
1353
+ function tokenizeBilingualKeywords(text) {
1354
+ return text.normalize("NFC").toLowerCase().replace(/[.,;:!?()[\]{}"“”‘’/\\|`*_~#-]+/g, " ").split(/\s+/).filter((w) => w.length >= 2 && !/^(and|the|for|with|from|into|của|trong|được|những|các|cho|với|khi|và|hoặc|để|từ|đến|về)$/.test(w));
1355
+ }
1356
+ function extractTopSentences(markdown, targetTopic, maxSentences = 3) {
1357
+ const keywords = tokenizeBilingualKeywords(targetTopic);
1358
+ const withoutCode = markdown.replace(/```[\s\S]*?```/g, " ");
1359
+ const rawSentences = withoutCode.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => {
1360
+ if (s.length < 45 || s.length > 320) return false;
1361
+ if (s.startsWith("|") || s.startsWith("http")) return false;
1362
+ if (/\b(cookie|privacy policy|sign in|subscribe|all rights reserved)\b/i.test(s)) return false;
1363
+ return true;
1364
+ });
1365
+ if (rawSentences.length === 0) return [];
1366
+ const scored = rawSentences.map((sentence, idx) => {
1367
+ const lower = sentence.normalize("NFC").toLowerCase();
1368
+ const matchedCount = keywords.filter((k) => lower.includes(k)).length;
1369
+ const positionBonus = idx < 8 ? 1.5 : idx < 20 ? 0.5 : 0;
1370
+ const technicalBonus = /(`[^`]+`|\b(?:api|function|class|actor|async|sensor|pin|loop|state|biến|hàm|cảm biến|vòng lặp|mạch)\b)/i.test(
1371
+ sentence
1372
+ ) ? 1.5 : 0;
1373
+ return {
1374
+ sentence,
1375
+ idx,
1376
+ score: matchedCount * 2 + positionBonus + technicalBonus
1377
+ };
1378
+ });
1379
+ return scored.sort((a, b) => b.score - a.score || a.idx - b.idx).slice(0, maxSentences).sort((a, b) => a.idx - b.idx).map((s) => s.sentence);
1380
+ }
511
1381
  var EvidenceScorer = class {
512
1382
  scoreDocument(doc, domain, targetTopic) {
513
1383
  let depthScore = 5;
514
- let codeScore = doc.extractedCodeSnippets.length > 0 ? 8 : 4;
1384
+ const hasCode = doc.extractedCodeSnippets.length > 0;
1385
+ let codeScore = hasCode ? Math.min(10, 7 + Math.min(3, doc.extractedCodeSnippets.length)) : 5;
515
1386
  let noiseScore = 2;
516
- const lowerContent = doc.markdownContent.toLowerCase();
517
- const targetKeywords = targetTopic.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
1387
+ const lowerContent = doc.markdownContent.normalize("NFC").toLowerCase();
1388
+ const targetKeywords = tokenizeBilingualKeywords(targetTopic);
518
1389
  const keywordMatches = targetKeywords.filter((k) => lowerContent.includes(k)).length;
519
- if (keywordMatches >= targetKeywords.length) depthScore += 3;
1390
+ const matchRatio = targetKeywords.length > 0 ? keywordMatches / targetKeywords.length : 1;
1391
+ if (matchRatio >= 0.75) depthScore += 3;
1392
+ else if (matchRatio >= 0.4) depthScore += 2;
1393
+ else if (keywordMatches >= 1) depthScore += 1;
520
1394
  if (doc.tablesCount > 0) depthScore += 1;
521
- if (doc.byteSize > 2e3) depthScore += 1;
1395
+ if (doc.byteSize > 1500) depthScore += 1;
1396
+ if ((doc.extractedHeadings?.length || 0) >= 2) depthScore += 1;
1397
+ depthScore = Math.min(10, depthScore);
1398
+ if (!hasCode && doc.byteSize > 1200 && matchRatio >= 0.5) {
1399
+ codeScore = 6;
1400
+ }
522
1401
  if (lowerContent.includes("subscribe") && lowerContent.includes("advertisement")) {
523
1402
  noiseScore += 3;
524
1403
  }
525
- const rawTotal = depthScore * 0.5 + codeScore * 0.4 - noiseScore * 0.1;
1404
+ const rawTotal = depthScore * 0.55 + codeScore * 0.35 - noiseScore * 0.1 + 0.5;
526
1405
  const score = Math.max(1, Math.min(10, Math.round(rawTotal)));
527
1406
  const isApproved = score >= 6;
528
- const summary = `Evidence scored ${score}/10 (${isApproved ? "APPROVED" : "REJECTED"}). Contains ${doc.extractedCodeSnippets.length} code snippets and ${doc.tablesCount} reference tables for domain ${domain}.`;
1407
+ const topSentences = extractTopSentences(doc.markdownContent, targetTopic, 3);
1408
+ const excerpts = doc.keyExcerpts?.length ? doc.keyExcerpts : extractSubstantiveExcerpts(doc.markdownContent, 3);
1409
+ const keyTakeaways = topSentences.length > 0 ? topSentences : excerpts.slice(0, 2);
1410
+ const summary = keyTakeaways.length > 0 ? `${keyTakeaways[0]} [Evidence ${score}/10 \xB7 ${doc.extractedCodeSnippets.length} code blocks \xB7 ${doc.tablesCount} tables]` : `Evidence scored ${score}/10 (${isApproved ? "APPROVED" : "REJECTED"}) for ${domain} (${doc.byteSize} bytes).`;
529
1411
  return {
530
1412
  doc,
531
1413
  score,
532
1414
  isApproved,
1415
+ isLlmEvaluated: false,
533
1416
  breakdown: {
534
1417
  technicalDepth: depthScore,
1418
+ structuralRichness: depthScore,
535
1419
  codeSnippetQuality: codeScore,
536
1420
  noiseFactor: noiseScore
537
1421
  },
538
- summary
1422
+ summary,
1423
+ keyTakeaways
539
1424
  };
540
1425
  }
1426
+ /**
1427
+ * Evaluates evidence with host LLM for genuine pedagogical relevance, conceptual depth,
1428
+ * and accuracy for the target learner age band.
1429
+ */
1430
+ async scoreDocumentWithLlm(doc, domain, targetTopic, options = {}) {
1431
+ const baseline = this.scoreDocument(doc, domain, targetTopic);
1432
+ if (!options.llm) {
1433
+ return baseline;
1434
+ }
1435
+ try {
1436
+ const prompt = [
1437
+ `You are a Senior Curriculum Researcher. Evaluate this harvested web document for the course topic "${targetTopic}" (Domain: ${domain}).`,
1438
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
1439
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : "",
1440
+ options.context?.softwareEnv ? `Software / Tools: ${options.context.softwareEnv}` : "",
1441
+ `
1442
+ Document Title: ${doc.title}`,
1443
+ `Document URL: ${doc.url}`,
1444
+ `Sample Excerpts:
1445
+ ${(doc.keyExcerpts || []).slice(0, 3).join("\n")}`,
1446
+ `Code Blocks Found: ${doc.extractedCodeSnippets.length}`,
1447
+ `
1448
+ Return ONLY a JSON object:`,
1449
+ `{"score": 8, "isApproved": true, "pedagogicalRelevance": 9, "summary": "1-sentence evaluative summary", "keyTakeaways": ["Takeaway 1 grounded in text", "Takeaway 2"]}`
1450
+ ].filter(Boolean).join("\n");
1451
+ const raw = await options.llm(prompt, {
1452
+ system: "You are an educational curriculum auditor. Output strictly valid JSON.",
1453
+ temperature: 0.2,
1454
+ jsonMode: true
1455
+ });
1456
+ const match = raw.match(/\{[\s\S]*\}/);
1457
+ if (match) {
1458
+ const parsed = JSON.parse(match[0]);
1459
+ const score = Math.max(1, Math.min(10, Number(parsed.score) || baseline.score));
1460
+ return {
1461
+ doc,
1462
+ score,
1463
+ isApproved: score >= 6,
1464
+ isLlmEvaluated: true,
1465
+ breakdown: {
1466
+ ...baseline.breakdown,
1467
+ pedagogicalRelevance: Number(parsed.pedagogicalRelevance) || score
1468
+ },
1469
+ summary: String(parsed.summary || baseline.summary),
1470
+ keyTakeaways: Array.isArray(parsed.keyTakeaways) && parsed.keyTakeaways.length > 0 ? parsed.keyTakeaways.map(String) : baseline.keyTakeaways
1471
+ };
1472
+ }
1473
+ } catch (e) {
1474
+ console.warn("[EvidenceScorer] LLM evidence evaluation failed, using structural baseline:", e?.message || e);
1475
+ }
1476
+ return baseline;
1477
+ }
541
1478
  };
542
1479
 
543
1480
  // src/evidence/provenanceTracker.ts
@@ -568,6 +1505,52 @@ var ProvenanceTracker = class {
568
1505
  };
569
1506
  }
570
1507
  };
1508
+
1509
+ // src/academic/arxivClient.ts
1510
+ var ArXivSearchEngine = class {
1511
+ /**
1512
+ * Search ArXiv repository using the public atom XML API
1513
+ */
1514
+ async search(query, maxResults = 5) {
1515
+ const papers = [];
1516
+ try {
1517
+ const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
1518
+ const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
1519
+ if (res.ok) {
1520
+ const xmlText = await res.text();
1521
+ const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
1522
+ for (const entry of entryMatches) {
1523
+ const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
1524
+ const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
1525
+ const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
1526
+ const publishedMatch = entry.match(/<published>(\d{4})/);
1527
+ const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
1528
+ const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
1529
+ const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
1530
+ if (titleMatch) {
1531
+ papers.push({
1532
+ id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
1533
+ title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
1534
+ abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
1535
+ authors: authorMatches,
1536
+ year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
1537
+ citationCount: 0,
1538
+ venue: "arXiv preprint",
1539
+ url: arxivUrl,
1540
+ pdfUrl,
1541
+ fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
1542
+ });
1543
+ }
1544
+ }
1545
+ }
1546
+ } catch (e) {
1547
+ console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
1548
+ }
1549
+ return papers;
1550
+ }
1551
+ };
1552
+
1553
+ // src/academic/semanticScholar.ts
571
1554
  zod.z.object({
572
1555
  id: zod.z.string(),
573
1556
  title: zod.z.string(),
@@ -582,22 +1565,56 @@ zod.z.object({
582
1565
  tldr: zod.z.string().optional(),
583
1566
  fieldsOfStudy: zod.z.array(zod.z.string()).default([])
584
1567
  });
1568
+ function reconstructOpenAlexAbstract(invertedIndex) {
1569
+ if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
1570
+ const entries = Object.entries(invertedIndex);
1571
+ if (entries.length === 0) return void 0;
1572
+ let maxIndex = 0;
1573
+ for (const [, positions] of entries) {
1574
+ if (Array.isArray(positions)) {
1575
+ for (const p of positions) {
1576
+ if (typeof p === "number" && p > maxIndex) maxIndex = p;
1577
+ }
1578
+ }
1579
+ }
1580
+ if (maxIndex > 5e3) maxIndex = 5e3;
1581
+ const words = new Array(maxIndex + 1);
1582
+ for (const [word, positions] of entries) {
1583
+ if (Array.isArray(positions)) {
1584
+ for (const p of positions) {
1585
+ if (typeof p === "number" && p >= 0 && p <= maxIndex) {
1586
+ words[p] = word;
1587
+ }
1588
+ }
1589
+ }
1590
+ }
1591
+ const reconstructed = words.filter(Boolean).join(" ").trim();
1592
+ return reconstructed.length > 0 ? reconstructed : void 0;
1593
+ }
585
1594
  var AcademicSearchEngine = class {
586
1595
  apiKey;
1596
+ allowEnvFallback;
587
1597
  customAdapter;
1598
+ arxivEngine;
588
1599
  constructor(options = {}) {
1600
+ this.arxivEngine = new ArXivSearchEngine();
589
1601
  if (typeof options === "string") {
590
1602
  this.apiKey = options;
1603
+ this.allowEnvFallback = true;
591
1604
  } else {
592
- this.apiKey = options.apiKey || process.env.SEMANTIC_SCHOLAR_API_KEY;
1605
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
1606
+ this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
593
1607
  this.customAdapter = options.customAdapter;
594
1608
  }
595
1609
  }
596
1610
  setCustomAdapter(adapter) {
597
1611
  this.customAdapter = adapter;
598
1612
  }
1613
+ setApiKey(apiKey) {
1614
+ this.apiKey = apiKey;
1615
+ }
599
1616
  /**
600
- * Search academic papers across Semantic Scholar with OpenAlex fallback or Custom Adapter
1617
+ * Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
601
1618
  */
602
1619
  async searchPapers(query, options = {}) {
603
1620
  const adapter = options.customAdapter || this.customAdapter;
@@ -613,6 +1630,13 @@ var AcademicSearchEngine = class {
613
1630
  }
614
1631
  const limit = options.limit ?? 5;
615
1632
  const papers = [];
1633
+ const seenTitles = /* @__PURE__ */ new Set();
1634
+ const pushUnique = (p) => {
1635
+ const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
1636
+ if (!normTitle || seenTitles.has(normTitle)) return;
1637
+ seenTitles.add(normTitle);
1638
+ papers.push(p);
1639
+ };
616
1640
  try {
617
1641
  const apiKey = options.apiKey || this.apiKey;
618
1642
  const headers = { "Content-Type": "application/json" };
@@ -635,60 +1659,103 @@ var AcademicSearchEngine = class {
635
1659
  const data = await res.json();
636
1660
  if (Array.isArray(data?.data)) {
637
1661
  for (const item of data.data) {
638
- papers.push({
1662
+ pushUnique({
639
1663
  id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
640
1664
  title: item.title || "Untitled",
641
- abstract: item.abstract,
642
- authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name) : [],
1665
+ abstract: item.abstract || void 0,
1666
+ authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
643
1667
  year: item.year,
644
1668
  citationCount: item.citationCount || 0,
645
- venue: item.venue,
1669
+ venue: item.venue || void 0,
646
1670
  doi: item.externalIds?.DOI,
647
- url: item.url,
1671
+ url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
648
1672
  pdfUrl: item.openAccessPdf?.url,
649
1673
  tldr: item.tldr?.text,
650
- fieldsOfStudy: item.fieldsOfStudy || ["Computer Science"]
1674
+ fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
651
1675
  });
652
1676
  }
653
- if (papers.length > 0) return papers;
1677
+ if (papers.length >= limit) return papers.slice(0, limit);
654
1678
  }
655
1679
  }
656
1680
  } catch (e) {
657
1681
  console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
658
1682
  }
659
- try {
660
- const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
661
- headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
662
- signal: AbortSignal.timeout(6e3)
663
- });
664
- if (res.ok) {
665
- const data = await res.json();
666
- if (Array.isArray(data?.results)) {
667
- for (const item of data.results) {
668
- papers.push({
669
- id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
670
- title: item.title || "Untitled",
671
- abstract: item.abstract_inverted_index ? "Abstract indexed" : void 0,
672
- authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
673
- year: item.publication_year,
674
- citationCount: item.cited_by_count || 0,
675
- venue: item.primary_location?.source?.display_name,
676
- doi: item.doi,
677
- url: item.doi || item.id,
678
- pdfUrl: item.open_access?.oa_url,
679
- fieldsOfStudy: ["Computer Science"]
680
- });
1683
+ if (papers.length < limit) {
1684
+ try {
1685
+ const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
1686
+ headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
1687
+ signal: AbortSignal.timeout(6e3)
1688
+ });
1689
+ if (res.ok) {
1690
+ const data = await res.json();
1691
+ if (Array.isArray(data?.results)) {
1692
+ for (const item of data.results) {
1693
+ const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
1694
+ const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
1695
+ pushUnique({
1696
+ id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
1697
+ title: item.title || "Untitled",
1698
+ abstract: abstractText,
1699
+ authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
1700
+ year: item.publication_year,
1701
+ citationCount: item.cited_by_count || 0,
1702
+ venue: item.primary_location?.source?.display_name,
1703
+ doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
1704
+ url: item.doi || item.id,
1705
+ pdfUrl: item.open_access?.oa_url,
1706
+ fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
1707
+ });
1708
+ }
681
1709
  }
682
1710
  }
1711
+ } catch (e) {
1712
+ console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
683
1713
  }
684
- } catch (e) {
685
- console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
686
1714
  }
687
- return papers;
1715
+ if (papers.length < limit && options.includeArxivFallback !== false) {
1716
+ try {
1717
+ const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
1718
+ for (const p of arxivPapers) {
1719
+ pushUnique(p);
1720
+ }
1721
+ } catch (e) {
1722
+ console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
1723
+ }
1724
+ }
1725
+ return papers.slice(0, limit);
688
1726
  }
689
1727
  };
690
1728
 
691
1729
  // src/academic/cs2023Taxonomy.ts
1730
+ var CSTA_K12_STRANDS = {
1731
+ CS: {
1732
+ code: "CS",
1733
+ name: "Computing Systems",
1734
+ nameVi: "H\u1EC7 th\u1ED1ng M\xE1y t\xEDnh & Ph\u1EA7n c\u1EE9ng",
1735
+ description: "Devices, hardware and software integration, sensors, actuators, and troubleshooting.",
1736
+ subconcepts: ["Devices", "Hardware & Software", "Troubleshooting"]
1737
+ },
1738
+ NI: {
1739
+ code: "NI",
1740
+ name: "Networks and the Internet",
1741
+ nameVi: "M\u1EA1ng m\xE1y t\xEDnh & An to\xE0n th\xF4ng tin",
1742
+ description: "Network communication, protocols, IoT connectivity, and cybersecurity.",
1743
+ subconcepts: ["Network Communication & Organization", "Cybersecurity"]
1744
+ },
1745
+ DA: {
1746
+ code: "DA",
1747
+ name: "Data and Analysis",
1748
+ nameVi: "D\u1EEF li\u1EC7u & Ph\xE2n t\xEDch",
1749
+ description: "Storage, collection, visualization, and transformation of sensor or application data.",
1750
+ subconcepts: ["Storage", "Collection, Visualization & Transformation", "Inference & Models"]
1751
+ },
1752
+ AP: {
1753
+ code: "AP",
1754
+ name: "Algorithms and Programming",
1755
+ nameVi: "Thu\u1EADt to\xE1n & L\u1EADp tr\xECnh",
1756
+ description: "Algorithms, variables, control structures, modularity, and iterative program development.",
1757
+ subconcepts: ["Algorithms", "Variables", "Control", "Modularity", "Program Development"]
1758
+ }};
692
1759
  var CS2023_KNOWLEDGE_AREAS = {
693
1760
  AL: {
694
1761
  code: "AL",
@@ -699,8 +1766,8 @@ var CS2023_KNOWLEDGE_AREAS = {
699
1766
  AR: {
700
1767
  code: "AR",
701
1768
  name: "Architecture and Organization",
702
- description: "Hardware architecture, assembly, memory hierarchy, pipelining, and system bus.",
703
- coreTopics: ["Digital Logic", "Memory Hierarchy & Cache", "Instruction Set Architecture (ISA)", "Interfacing & I/O"]
1769
+ description: "Hardware architecture, microcontrollers, sensors, memory hierarchy, and interfacing.",
1770
+ coreTopics: ["Digital Logic", "Memory Hierarchy & Cache", "Microcontrollers & Embedded I/O", "Sensor & Actuator Interfacing"]
704
1771
  },
705
1772
  AI: {
706
1773
  code: "AI",
@@ -708,6 +1775,24 @@ var CS2023_KNOWLEDGE_AREAS = {
708
1775
  description: "Neural networks, search algorithms, knowledge representation, reinforcement learning, and LLMs.",
709
1776
  coreTopics: ["Supervised Learning", "Deep Learning & Transformers", "Search & Optimization", "AI Ethics & Alignment"]
710
1777
  },
1778
+ DM: {
1779
+ code: "DM",
1780
+ name: "Data Management",
1781
+ description: "Relational databases, NoSQL, query optimization, data modeling, and transactions.",
1782
+ coreTopics: ["Relational & SQL Modeling", "Indexing & Query Optimization", "Transactions & ACID", "Distributed Data Stores"]
1783
+ },
1784
+ GIT: {
1785
+ code: "GIT",
1786
+ name: "Graphics and Interactive Techniques",
1787
+ description: "Game engines, 2D/3D rendering, physics simulation, collision detection, and interactive media.",
1788
+ coreTopics: ["Game Loop & Frame Timing", "2D/3D Coordinate Systems & Physics", "Collision Detection & State Machines", "Shader & Rendering Pipelines"]
1789
+ },
1790
+ HCI: {
1791
+ code: "HCI",
1792
+ name: "Human-Computer Interaction",
1793
+ description: "User interface design, usability, accessibility, state-driven UI frameworks, and interaction loops.",
1794
+ coreTopics: ["Declarative & State-Driven UI", "Usability & Cognitive Ergonomics", "Accessibility Standards", "Interactive Prototyping"]
1795
+ },
711
1796
  OS: {
712
1797
  code: "OS",
713
1798
  name: "Operating Systems",
@@ -720,6 +1805,12 @@ var CS2023_KNOWLEDGE_AREAS = {
720
1805
  description: "Multi-threading, actor model, async programming, distributed consensus, and cloud scaling.",
721
1806
  coreTopics: ["Async/Await & Futures", "Actor Model & Message Passing", "Shared Memory vs Message Passing", "Distributed Systems & Consistency"]
722
1807
  },
1808
+ SDF: {
1809
+ code: "SDF",
1810
+ name: "Software Development Fundamentals",
1811
+ description: "Core programming constructs, variables, control flow, data structures, debugging, and modularization.",
1812
+ coreTopics: ["Variables, Loops & Control Flow", "Data Structures (Arrays, Lists, Maps)", "Systematic Debugging & Testing", "Modular & Procedural Design"]
1813
+ },
723
1814
  SE: {
724
1815
  code: "SE",
725
1816
  name: "Software Engineering",
@@ -731,38 +1822,47 @@ var CS2023_KNOWLEDGE_AREAS = {
731
1822
  name: "Security & Cryptography",
732
1823
  description: "Authentication, cryptography, secure coding, threat modeling, and network defense.",
733
1824
  coreTopics: ["Symmetric & Asymmetric Encryption", "OAuth & JWT Auth", "Secure Coding & Vulnerabilities", "Identity & Access Management"]
734
- },
735
- SDF: {
736
- code: "SDF",
737
- name: "Software Development Fundamentals",
738
- description: "Core programming constructs, data structures, debugging, and modularization.",
739
- coreTopics: ["Variables, Loops & Control Flow", "Data Structures (Arrays, Lists, Trees)", "Debugging & Version Control", "OOP & Modular Design"]
1825
+ }};
1826
+ function mapTopicToCS2023(topic, context) {
1827
+ const combined = [
1828
+ topic,
1829
+ context?.courseTitle,
1830
+ context?.positioning,
1831
+ context?.hardwarePlatform,
1832
+ context?.softwareEnv,
1833
+ ...context?.inScopeTopics || []
1834
+ ].filter(Boolean).join(" ").toLowerCase();
1835
+ if (/\b(concurren|async|actor|thread|distributed|parallel|đồng thời|bất đồng bộ|đa luồng)\b/.test(combined)) {
1836
+ return { area: CS2023_KNOWLEDGE_AREAS.PDC, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.95 };
740
1837
  }
741
- };
742
- function mapTopicToCS2023(topic) {
743
- const t = topic.toLowerCase();
744
- if (t.includes("concurren") || t.includes("async") || t.includes("actor") || t.includes("thread") || t.includes("distributed")) {
745
- return { area: CS2023_KNOWLEDGE_AREAS.PDC, matchConfidence: 0.95 };
1838
+ if (/\b(ai|machine learning|deep learning|neural|llm|computer vision|trí tuệ nhân tạo|học máy|mạng nơ-ron)\b/.test(combined)) {
1839
+ return { area: CS2023_KNOWLEDGE_AREAS.AI, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.95 };
746
1840
  }
747
- if (t.includes("ai") || t.includes("machine learning") || t.includes("deep learning") || t.includes("robot") || t.includes("neural")) {
748
- return { area: CS2023_KNOWLEDGE_AREAS.AI, matchConfidence: 0.95 };
1841
+ if (/\b(arduino|esp32|raspberry|micro:?bit|mblock|makecode|lego|sensor|circuit|gpio|iot|hardware|robotics|cảm biến|vi điều khiển|mạch điện|điện tử)\b/.test(combined)) {
1842
+ return { area: CS2023_KNOWLEDGE_AREAS.AR, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.92 };
749
1843
  }
750
- if (t.includes("os") || t.includes("operating system") || t.includes("kernel") || t.includes("memory") || t.includes("linux")) {
751
- return { area: CS2023_KNOWLEDGE_AREAS.OS, matchConfidence: 0.9 };
1844
+ if (/\b(roblox|unity|godot|pygame|game|2d|3d|sprite|collision|physics|shader|trò chơi|làm game)\b/.test(combined)) {
1845
+ return { area: CS2023_KNOWLEDGE_AREAS.GIT, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.92 };
752
1846
  }
753
- if (t.includes("security") || t.includes("auth") || t.includes("crypto") || t.includes("cyber")) {
754
- return { area: CS2023_KNOWLEDGE_AREAS.SEC, matchConfidence: 0.9 };
1847
+ if (/\b(swiftui|ui|ux|interface|frontend|accessibility|figma|giao diện|tương tác người-máy)\b/.test(combined)) {
1848
+ return { area: CS2023_KNOWLEDGE_AREAS.HCI, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
755
1849
  }
756
- if (t.includes("architecture") || t.includes("pattern") || t.includes("software engineer") || t.includes("clean code")) {
757
- return { area: CS2023_KNOWLEDGE_AREAS.SE, matchConfidence: 0.85 };
1850
+ if (/\b(os|operating system|kernel|virtual memory|linux|hệ điều hành)\b/.test(combined)) {
1851
+ return { area: CS2023_KNOWLEDGE_AREAS.OS, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.9 };
758
1852
  }
759
- if (t.includes("algorithm") || t.includes("data structure") || t.includes("tree") || t.includes("graph")) {
760
- return { area: CS2023_KNOWLEDGE_AREAS.AL, matchConfidence: 0.9 };
1853
+ if (/\b(security|auth|crypto|cyber|jwt|oauth|bảo mật|an toàn thông tin|mã hóa)\b/.test(combined)) {
1854
+ return { area: CS2023_KNOWLEDGE_AREAS.SEC, k12Strand: CSTA_K12_STRANDS.NI, matchConfidence: 0.9 };
761
1855
  }
762
- if (t.includes("arduino") || t.includes("esp32") || t.includes("raspberry") || t.includes("hardware") || t.includes("circuit")) {
763
- return { area: CS2023_KNOWLEDGE_AREAS.AR, matchConfidence: 0.9 };
1856
+ if (/\b(sql|database|postgres|supabase|mongodb|data model|cơ sở dữ liệu|truy vấn)\b/.test(combined)) {
1857
+ return { area: CS2023_KNOWLEDGE_AREAS.DM, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.9 };
764
1858
  }
765
- return { area: CS2023_KNOWLEDGE_AREAS.SDF, matchConfidence: 0.7 };
1859
+ if (/\b(architecture|design pattern|software engineer|clean code|testing|tdd|ci\/cd|kiến trúc phần mềm|kiểm thử)\b/.test(combined)) {
1860
+ return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88 };
1861
+ }
1862
+ if (/\b(algorithm|data structure|tree|graph|dynamic programming|sorting|thuật toán|cấu trúc dữ liệu|đồ thị|quy hoạch động)\b/.test(combined)) {
1863
+ return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
1864
+ }
1865
+ return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.8 };
766
1866
  }
767
1867
 
768
1868
  // src/academic/citationManager.ts
@@ -814,65 +1914,144 @@ var CitationManager = class {
814
1914
 
815
1915
  // src/academic/literatureReview.ts
816
1916
  var LiteratureReviewSynthesizer = class {
817
- synthesize(topic, papers, cs2023Area) {
1917
+ synthesize(topic, papers, cs2023Area, k12Strand) {
818
1918
  const citationManager = new CitationManager();
819
1919
  const sotaMatrix = [];
1920
+ const foundations = [];
820
1921
  for (const paper of papers) {
821
1922
  citationManager.addPaper(paper);
1923
+ const summaryText = paper.tldr?.trim() || (paper.abstract ? paper.abstract.trim().split(/(?<=[.!?])\s+/)[0] || paper.abstract.slice(0, 160) : "");
822
1924
  sotaMatrix.push({
823
- approach: paper.title.split(":")[0] || paper.title.slice(0, 30),
824
- sourcePaper: `${paper.authors[0] || "Unknown"} (${paper.year || "n.d."})`,
825
- keyStrengths: paper.tldr || (paper.abstract ? paper.abstract.slice(0, 120) + "..." : "Foundational theoretical concept"),
826
- limitations: "Context-specific or hardware-constrained",
827
- relevanceToCourse: `Informs pedagogical sequencing and theoretical grounding for ${topic}.`
1925
+ approach: paper.title.split(":")[0]?.trim() || paper.title.slice(0, 45),
1926
+ sourcePaper: `${paper.authors[0] || "Unknown"} (${paper.year || "n.d."})${paper.venue ? ` \u2014 ${paper.venue}` : ""}`,
1927
+ keyStrengths: summaryText ? summaryText.slice(0, 180) : `Peer-reviewed study (${paper.citationCount} citations)`,
1928
+ limitations: paper.year && paper.year < 2020 ? `Published in ${paper.year}; verify against modern tooling` : "Requires pedagogical scaffolding for target learners",
1929
+ relevanceToCourse: `Grounds "${topic}" in peer-reviewed evidence (${paper.fieldsOfStudy.slice(0, 2).join(", ") || "CS"}).`
828
1930
  });
1931
+ if (summaryText && foundations.length < 4) {
1932
+ foundations.push(`[${paper.authors[0] || "Ref"} ${paper.year || ""}] ${summaryText}`);
1933
+ }
1934
+ }
1935
+ if (cs2023Area) {
1936
+ foundations.push(
1937
+ `IEEE/ACM CS2023 (${cs2023Area.code} - ${cs2023Area.name}): ${cs2023Area.coreTopics.join(", ")}`
1938
+ );
1939
+ }
1940
+ if (k12Strand) {
1941
+ foundations.push(
1942
+ `CSTA K-12 (${k12Strand.code} - ${k12Strand.name} / ${k12Strand.nameVi}): ${k12Strand.subconcepts.join(", ")}`
1943
+ );
829
1944
  }
1945
+ const openQuestions = [
1946
+ `How to scaffold core concepts of "${topic}" (${cs2023Area?.code || "CS"}) so learners master mental models before syntax memorization?`,
1947
+ `What observable debugging and verification tasks best diagnose learner misconceptions in "${topic}"?`
1948
+ ];
830
1949
  const mdLines = [
831
1950
  `# \u{1F393} Academic Literature Review & Theoretical Ground Truth`,
832
1951
  `> **Topic:** ${topic}`,
833
- cs2023Area ? `> **IEEE/ACM CS2023 Standard Area:** \`${cs2023Area.code}\` \u2014 *${cs2023Area.name}*
1952
+ cs2023Area ? `> **IEEE/ACM CS2023 Standard Area:** \`${cs2023Area.code}\` \u2014 *${cs2023Area.name}*` : "",
1953
+ k12Strand ? `> **CSTA K-12 Strand:** \`${k12Strand.code}\` \u2014 *${k12Strand.name} (${k12Strand.nameVi})*
834
1954
  ` : "",
835
1955
  `## 1. Executive Summary`,
836
- `Synthesized from **${papers.length} peer-reviewed works** and international curricular standards directly relevant to \`${topic}\`.`,
1956
+ papers.length > 0 ? `Synthesized from **${papers.length} peer-reviewed works** and curricular standards directly relevant to \`${topic}\`.` : `No peer-reviewed papers matched directly; aligned to curricular taxonomy standards for \`${topic}\`.`,
837
1957
  `
838
1958
  ## 2. State-of-the-Art (SOTA) Comparison Matrix`,
839
- `| Paradigm / Approach | Source Reference | Core Strengths | Constraints & Limitations | Pedagogical Application |`,
840
- `|---|---|---|---|---|`,
841
- ...sotaMatrix.map((r) => `| **${r.approach}** | ${r.sourcePaper} | ${r.keyStrengths} | ${r.limitations} | ${r.relevanceToCourse} |`),
1959
+ ...sotaMatrix.length > 0 ? [
1960
+ `| Paradigm / Approach | Source Reference | Core Findings / Abstract | Pedagogical Note | Course Application |`,
1961
+ `|---|---|---|---|---|`,
1962
+ ...sotaMatrix.map(
1963
+ (r) => `| **${r.approach.replace(/\|/g, "/")}** | ${r.sourcePaper.replace(/\|/g, "/")} | ${r.keyStrengths.replace(/\|/g, "/")} | ${r.limitations.replace(/\|/g, "/")} | ${r.relevanceToCourse.replace(/\|/g, "/")} |`
1964
+ )
1965
+ ] : ["*No peer-reviewed papers returned from academic engines for this query.*"],
842
1966
  `
843
- ## 3. Core Theoretical Foundations`,
844
- `- **Academic Rigor:** Curriculum models strictly adhere to peer-reviewed principles.`,
845
- `- **CS2023 Competency Alignment:** Maps directly to IEEE/ACM recommended graduate competencies.`,
846
- `- **Theory-to-Practice Bridge:** Direct translational pipeline from formal papers to executable lab operations.`,
1967
+ ## 3. Core Theoretical & Curricular Foundations`,
1968
+ ...foundations.map((f) => `- ${f}`),
847
1969
  `
848
- ## 4. Open Pedagogical & Research Questions`,
849
- `- How to distill complex formal mechanisms for high-school / undergraduate learners without diluting accuracy?`,
850
- `- What automated verification setups validate theoretical guarantees in real-world student code?`,
1970
+ ## 4. Open Pedagogical & Curriculum Design Questions`,
1971
+ ...openQuestions.map((q) => `- ${q}`),
851
1972
  `
852
1973
  ${citationManager.getMarkdownFootnotes()}`
853
- ];
1974
+ ].filter((line) => line !== "");
854
1975
  return {
855
1976
  topic,
856
1977
  cs2023Area,
1978
+ k12Strand,
857
1979
  totalPapersFound: papers.length,
858
1980
  papers,
859
1981
  sotaMatrix,
860
- keyTheoreticalFoundations: [
861
- "Immutable state guarantees and memory safety semantics",
862
- "Dataflow and message-passing distributed concurrency models",
863
- "Empirical latency and throughput benchmark validation"
864
- ],
865
- openResearchQuestions: [
866
- "How to distill complex formal mechanisms for high-school / undergraduate learners without diluting accuracy?",
867
- "What automated verification setups validate theoretical guarantees in real-world student code?"
868
- ],
1982
+ keyTheoreticalFoundations: foundations,
1983
+ openResearchQuestions: openQuestions,
869
1984
  markdownReport: mdLines.join("\n"),
870
1985
  bibtex: citationManager.getBibTeX()
871
1986
  };
872
1987
  }
1988
+ /**
1989
+ * Optional LLM semantic synthesis of dynamic open research questions and pedagogical takeaways
1990
+ * based on the retrieved academic literature.
1991
+ */
1992
+ async synthesizeWithLlm(topic, papers, cs2023Area, k12Strand, options = {}) {
1993
+ const base = this.synthesize(topic, papers, cs2023Area, k12Strand);
1994
+ if (!options.llm || papers.length === 0) {
1995
+ return base;
1996
+ }
1997
+ try {
1998
+ const abstractsSummary = papers.map((p, i) => `[Paper ${i + 1}] ${p.title} (${p.authors[0]} ${p.year}): ${p.tldr || p.abstract?.slice(0, 300) || "No abstract"}`).join("\n\n");
1999
+ const prompt = [
2000
+ `You are a Professor of Computing Education. Review these academic research papers retrieved for the course topic "${topic}".`,
2001
+ cs2023Area ? `IEEE/ACM CS2023 Area: ${cs2023Area.code} - ${cs2023Area.name}` : "",
2002
+ k12Strand ? `CSTA K-12 Strand: ${k12Strand.code} - ${k12Strand.name}` : "",
2003
+ options.context?.ageBand ? `Target Age: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
2004
+ `
2005
+ Retrieved Academic Papers:
2006
+ ${abstractsSummary}`,
2007
+ `
2008
+ Generate 2-3 deep, actionable pedagogical research & curriculum design questions directly inspired by these papers.`,
2009
+ `Return ONLY a JSON array of strings: ["Question 1", "Question 2"]`
2010
+ ].filter(Boolean).join("\n");
2011
+ const raw = await options.llm(prompt, {
2012
+ system: "You are an educational research synthesis specialist. Output strictly a JSON array of strings.",
2013
+ temperature: 0.2,
2014
+ jsonMode: true
2015
+ });
2016
+ const match = raw.match(/\[[\s\S]*\]/);
2017
+ if (match) {
2018
+ const parsed = JSON.parse(match[0]);
2019
+ if (Array.isArray(parsed) && parsed.length > 0) {
2020
+ const dynamicQuestions = parsed.map(String).filter((q) => q.length > 10);
2021
+ if (dynamicQuestions.length > 0) {
2022
+ base.openResearchQuestions = dynamicQuestions;
2023
+ base.markdownReport = base.markdownReport.replace(
2024
+ /## 4\. Open Pedagogical & Curriculum Design Questions[\s\S]*?(?=\n\n\[\^|\n\n$|$)/,
2025
+ `## 4. Open Pedagogical & Curriculum Design Questions
2026
+ ${dynamicQuestions.map((q) => `- ${q}`).join("\n")}`
2027
+ );
2028
+ }
2029
+ }
2030
+ }
2031
+ } catch (e) {
2032
+ console.warn("[LiteratureReviewSynthesizer] LLM question synthesis failed, using baseline:", e?.message || e);
2033
+ }
2034
+ return base;
2035
+ }
873
2036
  };
874
2037
 
875
2038
  // src/pipeline/harvestPipeline.ts
2039
+ function normalizeUrlForDedup(rawUrl) {
2040
+ try {
2041
+ const u = new URL(rawUrl.trim());
2042
+ u.hash = "";
2043
+ for (const key of [...u.searchParams.keys()]) {
2044
+ if (/^(utm_|fbclid$|gclid$|ref$|source$)/i.test(key)) {
2045
+ u.searchParams.delete(key);
2046
+ }
2047
+ }
2048
+ const path = u.pathname.replace(/\/+$/, "") || "/";
2049
+ const query = u.searchParams.toString();
2050
+ return `${u.protocol}//${u.hostname.toLowerCase()}${path}${query ? `?${query}` : ""}`;
2051
+ } catch {
2052
+ return rawUrl.trim().toLowerCase().replace(/\/+$/, "");
2053
+ }
2054
+ }
876
2055
  var HarvestPipeline = class {
877
2056
  searchEngine;
878
2057
  scraper;
@@ -884,14 +2063,27 @@ var HarvestPipeline = class {
884
2063
  litReviewSynthesizer;
885
2064
  planner;
886
2065
  refPackBuilder;
2066
+ termExtractor;
2067
+ customSynthesizer;
2068
+ defaultLlm;
887
2069
  constructor(options = {}) {
2070
+ const allowEnv = options.allowEnvFallback ?? true;
2071
+ const keys = {
2072
+ tavily: options.apiKeys?.tavily || options.tavilyKey,
2073
+ exa: options.apiKeys?.exa || options.exaKey,
2074
+ jina: options.apiKeys?.jina || options.jinaKey,
2075
+ serper: options.apiKeys?.serper || options.serperKey,
2076
+ semanticScholar: options.apiKeys?.semanticScholar || options.semanticScholarKey
2077
+ };
888
2078
  this.searchEngine = new MultiSourceSearchEngine({
889
- tavilyKey: options.tavilyKey,
890
- exaKey: options.exaKey,
2079
+ apiKeys: keys,
2080
+ allowEnvFallback: allowEnv,
891
2081
  customAdapter: options.customSearchAdapter
892
2082
  });
893
2083
  this.scraper = new WebScraper({
894
2084
  crawl4aiHost: options.crawl4aiHost,
2085
+ jinaKey: keys.jina,
2086
+ allowEnvFallback: allowEnv,
895
2087
  customAdapter: options.customCrawlerAdapter
896
2088
  });
897
2089
  this.auditor = new HealthAuditor();
@@ -899,15 +2091,29 @@ var HarvestPipeline = class {
899
2091
  this.catalogBuilder = new CatalogBuilder();
900
2092
  this.provenanceTracker = new ProvenanceTracker();
901
2093
  this.academicSearch = new AcademicSearchEngine({
2094
+ apiKey: keys.semanticScholar,
2095
+ allowEnvFallback: allowEnv,
902
2096
  customAdapter: options.customAcademicSearchAdapter
903
2097
  });
904
2098
  this.litReviewSynthesizer = new LiteratureReviewSynthesizer();
905
2099
  this.planner = new CampaignPlanner();
906
2100
  this.refPackBuilder = new ReferencePackBuilder();
2101
+ this.termExtractor = new PedagogicalTermExtractor();
2102
+ this.customSynthesizer = options.customSynthesizerAdapter;
2103
+ this.defaultLlm = options.llm;
907
2104
  }
908
2105
  async run(topic, options = {}) {
909
2106
  const notify = options.onProgress || (() => {
910
2107
  });
2108
+ const llm = options.llm || this.defaultLlm;
2109
+ const synthesizer = options.customSynthesizerAdapter || this.customSynthesizer;
2110
+ const mergedKeys = {
2111
+ tavily: options.apiKeys?.tavily || options.tavilyKey,
2112
+ exa: options.apiKeys?.exa || options.exaKey,
2113
+ jina: options.apiKeys?.jina || options.jinaKey,
2114
+ serper: options.apiKeys?.serper || options.serperKey,
2115
+ semanticScholar: options.apiKeys?.semanticScholar || options.semanticScholarKey
2116
+ };
911
2117
  if (options.customSearchAdapter) {
912
2118
  this.searchEngine.setCustomAdapter(options.customSearchAdapter);
913
2119
  }
@@ -918,32 +2124,59 @@ var HarvestPipeline = class {
918
2124
  this.academicSearch.setCustomAdapter(options.customAcademicSearchAdapter);
919
2125
  }
920
2126
  notify("PLANNING", `Classifying domain and creating Master Research Plan for "${topic}"...`);
921
- const domainResult = classifyTechnicalDomain(topic, options.domain);
2127
+ const domainResult = classifyTechnicalDomain(topic, options.domain, options.context);
922
2128
  const domain = domainResult.domain;
923
- const plan = this.planner.createPlan(topic, domain, { maxRounds: options.maxRounds, isDeep: options.isDeep });
924
- notify("ACADEMIC", `Querying academic literature & mapping to IEEE/ACM CS2023 standards...`);
925
- const cs2023Match = mapTopicToCS2023(topic);
2129
+ const plan = llm ? await this.planner.createPlanWithLlm(topic, domain, {
2130
+ maxRounds: options.maxRounds,
2131
+ isDeep: options.isDeep,
2132
+ context: options.context,
2133
+ llm
2134
+ }) : this.planner.createPlan(topic, domain, {
2135
+ maxRounds: options.maxRounds,
2136
+ isDeep: options.isDeep,
2137
+ context: options.context
2138
+ });
2139
+ notify("ACADEMIC", `Querying academic literature & mapping to IEEE/ACM CS2023 and CSTA K-12 standards...`);
2140
+ const taxonomyMatch = mapTopicToCS2023(topic, options.context);
926
2141
  const papers = await this.academicSearch.searchPapers(topic, {
927
2142
  limit: 4,
928
- apiKey: options.semanticScholarKey,
2143
+ apiKey: mergedKeys.semanticScholar,
929
2144
  customAdapter: options.customAcademicSearchAdapter
930
2145
  });
931
- const litReview = this.litReviewSynthesizer.synthesize(topic, papers, cs2023Match.area);
2146
+ const litReview = llm ? await this.litReviewSynthesizer.synthesizeWithLlm(
2147
+ topic,
2148
+ papers,
2149
+ taxonomyMatch.area,
2150
+ taxonomyMatch.k12Strand,
2151
+ { llm, context: options.context }
2152
+ ) : this.litReviewSynthesizer.synthesize(
2153
+ topic,
2154
+ papers,
2155
+ taxonomyMatch.area,
2156
+ taxonomyMatch.k12Strand
2157
+ );
932
2158
  notify("DISCOVERY", `Executing multi-source search queries across configured adapters...`);
933
2159
  const scrapedDocs = [];
934
2160
  const provenanceRecords = [];
2161
+ const visitedUrls = /* @__PURE__ */ new Set();
935
2162
  for (const query of plan.plannedQueries) {
936
2163
  notify("SEARCHING", `Searching: ${query}`);
937
2164
  const hits = await this.searchEngine.search(query, {
938
2165
  limit: 2,
939
- tavilyKey: options.tavilyKey,
940
- exaKey: options.exaKey,
2166
+ apiKeys: mergedKeys,
941
2167
  customAdapter: options.customSearchAdapter
942
2168
  });
943
2169
  for (const hit of hits) {
2170
+ if (!hit?.url) continue;
2171
+ const normUrl = normalizeUrlForDedup(hit.url);
2172
+ if (visitedUrls.has(normUrl)) continue;
2173
+ visitedUrls.add(normUrl);
944
2174
  try {
945
2175
  notify("SCRAPING", `Extracting clean markdown from: ${hit.url}`);
946
- const doc = await this.scraper.scrapeUrl(hit.url, { customAdapter: options.customCrawlerAdapter });
2176
+ const doc = await this.scraper.scrapeUrl(hit.url, {
2177
+ customAdapter: options.customCrawlerAdapter,
2178
+ jinaKey: mergedKeys.jina
2179
+ });
947
2180
  const audit = this.auditor.auditDocument(doc);
948
2181
  if (audit.passed) {
949
2182
  const scored = this.scorer.scoreDocument(doc, domain, topic);
@@ -951,7 +2184,8 @@ var HarvestPipeline = class {
951
2184
  scrapedDocs.push({
952
2185
  doc,
953
2186
  score: scored.score,
954
- summary: scored.summary
2187
+ summary: scored.summary,
2188
+ keyTakeaways: scored.keyTakeaways
955
2189
  });
956
2190
  provenanceRecords.push({
957
2191
  resourceId: `RES-${String(scrapedDocs.length).padStart(2, "0")}`,
@@ -968,13 +2202,36 @@ var HarvestPipeline = class {
968
2202
  }
969
2203
  }
970
2204
  }
2205
+ notify("TERMS", `Extracting bilingual technical concepts and prerequisite DAG...`);
2206
+ const docsWithIds = scrapedDocs.map((item, idx) => ({
2207
+ resourceId: `RES-${String(idx + 1).padStart(2, "0")}`,
2208
+ doc: item.doc
2209
+ }));
2210
+ const termBundle = await this.termExtractor.extractWithOptionalLlm(topic, domain, docsWithIds, {
2211
+ context: options.context,
2212
+ llm
2213
+ });
2214
+ notify("SYNTHESIS", `Synthesizing pedagogical takeaways, misconceptions, and reference pack...`);
2215
+ const synthesizedInsights = await this.synthesizeInsights(topic, domain, scrapedDocs, {
2216
+ context: options.context,
2217
+ llm,
2218
+ synthesizer,
2219
+ concepts: termBundle.concepts
2220
+ });
971
2221
  notify("CATALOGING", `Compiling Resource Index, Catalog, and Distilled Knowledge Graph...`);
972
- const catalog = this.catalogBuilder.buildBundle(topic, domain, scrapedDocs);
973
- notify("SYNTHESIS", `Building canonical REFERENCE_PACK.md & REFERENCE_REGISTRY.json...`);
2222
+ const catalog = this.catalogBuilder.buildBundle(topic, domain, scrapedDocs, {
2223
+ context: options.context,
2224
+ concepts: termBundle.concepts,
2225
+ conceptDagTsv: termBundle.conceptDagTsv
2226
+ });
974
2227
  const referencePack = this.refPackBuilder.buildReferencePack(topic, domain, {
975
- cs2023Area: cs2023Match.area,
2228
+ cs2023Area: taxonomyMatch.area,
2229
+ k12Strand: taxonomyMatch.k12Strand,
2230
+ context: options.context,
976
2231
  scrapedDocs,
977
- academicPapers: papers
2232
+ academicPapers: papers,
2233
+ concepts: termBundle.concepts,
2234
+ synthesizedInsights
978
2235
  });
979
2236
  notify("PROVENANCE", `Generating audit trail and provenance sidecar...`);
980
2237
  const provenance = this.provenanceTracker.generateSidecar(topic, provenanceRecords);
@@ -987,13 +2244,85 @@ var HarvestPipeline = class {
987
2244
  literatureReview: litReview,
988
2245
  referencePack,
989
2246
  provenance,
2247
+ concepts: termBundle.concepts,
2248
+ conceptDagTsv: termBundle.conceptDagTsv,
2249
+ synthesizedInsights,
990
2250
  totalDocumentsScraped: scrapedDocs.length
991
2251
  };
992
2252
  }
2253
+ async synthesizeInsights(topic, domain, scrapedDocs, options) {
2254
+ const excerpts = scrapedDocs.map((d, idx) => ({
2255
+ resourceId: `RES-${String(idx + 1).padStart(2, "0")}`,
2256
+ title: d.doc.title,
2257
+ url: d.doc.url,
2258
+ excerpt: d.keyTakeaways.join(" ") || d.summary,
2259
+ codeSnippets: d.doc.extractedCodeSnippets.slice(0, 2)
2260
+ }));
2261
+ if (options.synthesizer) {
2262
+ try {
2263
+ return await options.synthesizer.synthesize({
2264
+ topic,
2265
+ domain,
2266
+ context: options.context,
2267
+ excerpts
2268
+ });
2269
+ } catch (e) {
2270
+ console.warn("[HarvestPipeline] CustomSynthesizerAdapter failed, falling back:", e?.message || e);
2271
+ }
2272
+ }
2273
+ if (options.llm && excerpts.length > 0) {
2274
+ try {
2275
+ const prompt = [
2276
+ `Synthesize pedagogical and technical ground truth for the course topic "${topic}" (Domain: ${domain}).`,
2277
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
2278
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : "",
2279
+ options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}` : "",
2280
+ `
2281
+ Verified Harvested Sources:
2282
+ ` + excerpts.map((e) => `[${e.resourceId}] ${e.title} (${e.url}): ${e.excerpt}`).join("\n\n"),
2283
+ `
2284
+ Return ONLY a valid JSON object with this structure:`,
2285
+ `{"executiveSummary": "2-3 sentence synthesis anchored to sources", "keyTakeaways": ["[RES-01] ...", "[RES-02] ..."], "commonMisconceptionsAndPitfalls": ["Specific beginner bug or misconception 1", "Misconception 2"], "recommendedExemplars": ["Concrete project/activity exemplar 1", "Exemplar 2"]}`
2286
+ ].filter(Boolean).join("\n");
2287
+ const raw = await options.llm(prompt, {
2288
+ system: "You are a Principal Curriculum Researcher. Output strictly valid JSON grounded in the provided sources.",
2289
+ temperature: 0.2,
2290
+ jsonMode: true
2291
+ });
2292
+ const match = raw.match(/\{[\s\S]*\}/);
2293
+ if (match) {
2294
+ const parsed = JSON.parse(match[0]);
2295
+ return {
2296
+ executiveSummary: String(parsed.executiveSummary || ""),
2297
+ keyTakeaways: Array.isArray(parsed.keyTakeaways) ? parsed.keyTakeaways.map(String) : [],
2298
+ commonMisconceptionsAndPitfalls: Array.isArray(parsed.commonMisconceptionsAndPitfalls) ? parsed.commonMisconceptionsAndPitfalls.map(String) : [],
2299
+ recommendedExemplars: Array.isArray(parsed.recommendedExemplars) ? parsed.recommendedExemplars.map(String) : [],
2300
+ extractedConcepts: options.concepts
2301
+ };
2302
+ }
2303
+ } catch (e) {
2304
+ console.warn("[HarvestPipeline] Host LLM synthesis failed, using deterministic synthesis:", e?.message || e);
2305
+ }
2306
+ }
2307
+ const keyTakeaways = excerpts.filter((e) => e.excerpt).slice(0, 6).map((e) => `[${e.resourceId}] ${e.title}: ${e.excerpt}`);
2308
+ const pitfalls = scrapedDocs.flatMap(
2309
+ (d) => d.doc.markdownContent.split(/(?<=[.!?。\n])\s+/).map((s) => s.trim()).filter(
2310
+ (s) => s.length >= 35 && s.length <= 240 && /\b(pitfall|mistake|error|bug|warning|caution|avoid|race condition|deadlock|lỗi|nhầm lẫn|lưu ý|cảnh báo|tránh)\b/i.test(s)
2311
+ )
2312
+ ).slice(0, 4);
2313
+ return {
2314
+ executiveSummary: scrapedDocs.length > 0 ? `Synthesized from ${scrapedDocs.length} verified primary sources for "${topic}" (${domain.toUpperCase()}).` : `No external web documents passed quality threshold for "${topic}".`,
2315
+ keyTakeaways,
2316
+ commonMisconceptionsAndPitfalls: pitfalls,
2317
+ recommendedExemplars: scrapedDocs.slice(0, 3).map((d) => `${d.doc.title} (${d.doc.url})`),
2318
+ extractedConcepts: options.concepts
2319
+ };
2320
+ }
993
2321
  };
994
2322
 
995
2323
  exports.CampaignPlanner = CampaignPlanner;
996
2324
  exports.HarvestPipeline = HarvestPipeline;
997
2325
  exports.ReferencePackBuilder = ReferencePackBuilder;
2326
+ exports.normalizeUrlForDedup = normalizeUrlForDedup;
998
2327
  //# sourceMappingURL=index.cjs.map
999
2328
  //# sourceMappingURL=index.cjs.map