@thanh01.pmt/research-kit 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +213 -0
  3. package/dist/academic/index.cjs +332 -127
  4. package/dist/academic/index.cjs.map +1 -1
  5. package/dist/academic/index.d.cts +3 -3
  6. package/dist/academic/index.d.ts +3 -3
  7. package/dist/academic/index.js +2 -2
  8. package/dist/{catalogBuilder-8A-Dg_A-.d.ts → catalogBuilder-CmmQzUnf.d.ts} +11 -3
  9. package/dist/{catalogBuilder-feU79wQQ.d.cts → catalogBuilder-DgXYxL2F.d.cts} +11 -3
  10. package/dist/chunk-2JEVBCIF.js +3 -0
  11. package/dist/{chunk-KGCDYFFD.js.map → chunk-2JEVBCIF.js.map} +1 -1
  12. package/dist/chunk-2XIEWCUE.js +244 -0
  13. package/dist/chunk-2XIEWCUE.js.map +1 -0
  14. package/dist/chunk-AUGVVZTL.js +514 -0
  15. package/dist/chunk-AUGVVZTL.js.map +1 -0
  16. package/dist/chunk-DOXSPRSP.js +559 -0
  17. package/dist/chunk-DOXSPRSP.js.map +1 -0
  18. package/dist/chunk-GKF3NDEX.js +162 -0
  19. package/dist/chunk-GKF3NDEX.js.map +1 -0
  20. package/dist/chunk-J3VCGH7Z.js +3 -0
  21. package/dist/chunk-J3VCGH7Z.js.map +1 -0
  22. package/dist/{chunk-67V2GJGK.js → chunk-J7XKDMGL.js} +7 -5
  23. package/dist/chunk-J7XKDMGL.js.map +1 -0
  24. package/dist/chunk-KAFQVI2R.js +333 -0
  25. package/dist/chunk-KAFQVI2R.js.map +1 -0
  26. package/dist/chunk-XIF5YNI7.js +799 -0
  27. package/dist/chunk-XIF5YNI7.js.map +1 -0
  28. package/dist/config/index.cjs +5 -3
  29. package/dist/config/index.cjs.map +1 -1
  30. package/dist/config/index.d.cts +1 -1
  31. package/dist/config/index.d.ts +1 -1
  32. package/dist/config/index.js +1 -1
  33. package/dist/evidence/index.cjs +337 -21
  34. package/dist/evidence/index.cjs.map +1 -1
  35. package/dist/evidence/index.d.cts +44 -7
  36. package/dist/evidence/index.d.ts +44 -7
  37. package/dist/evidence/index.js +3 -2
  38. package/dist/harvester/index.cjs +805 -64
  39. package/dist/harvester/index.cjs.map +1 -1
  40. package/dist/harvester/index.d.cts +79 -8
  41. package/dist/harvester/index.d.ts +79 -8
  42. package/dist/harvester/index.js +3 -2
  43. package/dist/{index-BU5n1ip_.d.ts → index-t8pahlJ4.d.cts} +104 -15
  44. package/dist/{index-BU5n1ip_.d.cts → index-t8pahlJ4.d.ts} +104 -15
  45. package/dist/index.cjs +2001 -252
  46. package/dist/index.cjs.map +1 -1
  47. package/dist/index.d.cts +156 -8
  48. package/dist/index.d.ts +156 -8
  49. package/dist/index.js +237 -8
  50. package/dist/index.js.map +1 -1
  51. package/dist/literatureReview-BTPFe7eb.d.cts +64 -0
  52. package/dist/literatureReview-hGbfrBmw.d.ts +64 -0
  53. package/dist/pipeline/index.cjs +1570 -241
  54. package/dist/pipeline/index.cjs.map +1 -1
  55. package/dist/pipeline/index.d.cts +49 -9
  56. package/dist/pipeline/index.d.ts +49 -9
  57. package/dist/pipeline/index.js +5 -4
  58. package/dist/webScraper-BhiQ8s9x.d.ts +68 -0
  59. package/dist/webScraper-D9Ldus9p.d.cts +68 -0
  60. package/package.json +8 -9
  61. package/dist/chunk-47L7ZHMY.js +0 -316
  62. package/dist/chunk-47L7ZHMY.js.map +0 -1
  63. package/dist/chunk-63X2HPCI.js +0 -47
  64. package/dist/chunk-63X2HPCI.js.map +0 -1
  65. package/dist/chunk-67V2GJGK.js.map +0 -1
  66. package/dist/chunk-6Q2XCUG7.js +0 -40
  67. package/dist/chunk-6Q2XCUG7.js.map +0 -1
  68. package/dist/chunk-KGCDYFFD.js +0 -3
  69. package/dist/chunk-MHAYD555.js +0 -239
  70. package/dist/chunk-MHAYD555.js.map +0 -1
  71. package/dist/chunk-MPLDSHFM.js +0 -396
  72. package/dist/chunk-MPLDSHFM.js.map +0 -1
  73. package/dist/chunk-QZNLB5X4.js +0 -65
  74. package/dist/chunk-QZNLB5X4.js.map +0 -1
  75. package/dist/literatureReview-C4AgVljZ.d.ts +0 -43
  76. package/dist/literatureReview-Dlods4Aw.d.cts +0 -43
  77. package/dist/webScraper-pKzdXhfI.d.ts +0 -46
  78. package/dist/webScraper-yBcYOHoL.d.cts +0 -46
@@ -1,46 +1,95 @@
1
1
  'use strict';
2
2
 
3
3
  // src/harvester/domainClassifier.ts
4
- function classifyTechnicalDomain(topic, explicitDomain) {
4
+ var HARDWARE_PATTERNS = [
5
+ /\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
6
+ /\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
7
+ /(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
8
+ ];
9
+ var PLATFORM_PATTERNS = [
10
+ /\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
11
+ /(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
12
+ ];
13
+ var ACADEMIC_PATTERNS = [
14
+ /\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
15
+ /(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
16
+ ];
17
+ function inferPrimaryCodeLanguage(topic, context, domain) {
18
+ const combined = [
19
+ topic,
20
+ context?.courseTitle,
21
+ context?.hardwarePlatform,
22
+ context?.softwareEnv,
23
+ ...context?.inScopeTopics || []
24
+ ].filter(Boolean).join(" ").toLowerCase();
25
+ if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
26
+ if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
27
+ if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
28
+ if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
29
+ if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
30
+ if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
31
+ if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
32
+ if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
33
+ if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
34
+ if (/\b(rust|cargo)\b/.test(combined)) return "rust";
35
+ if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
36
+ if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
37
+ if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
38
+ if (domain === "hardware") return "cpp";
39
+ return "text";
40
+ }
41
+ function classifyTechnicalDomain(topic, explicitDomain, context) {
5
42
  if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
6
43
  const domain = explicitDomain.toLowerCase();
7
44
  return {
8
45
  domain,
9
46
  confidence: 1,
10
47
  rationale: "Explicitly configured by user flag.",
11
- recommendedExtractors: getExtractorsForDomain(domain)
48
+ recommendedExtractors: getExtractorsForDomain(domain),
49
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
12
50
  };
13
51
  }
14
- const t = topic.toLowerCase();
15
- if (t.includes("esp32") || t.includes("arduino") || t.includes("raspberry") || t.includes("stm32") || t.includes("sensor") || t.includes("circuit") || t.includes("gpio") || t.includes("iot hardware")) {
52
+ const combined = [
53
+ topic,
54
+ context?.courseTitle,
55
+ context?.positioning,
56
+ context?.hardwarePlatform,
57
+ context?.softwareEnv,
58
+ ...context?.inScopeTopics || []
59
+ ].filter(Boolean).join(" ");
60
+ if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
16
61
  return {
17
62
  domain: "hardware",
18
63
  confidence: 0.95,
19
- rationale: "Topic contains microcontrollers, sensors, or physical electronics keywords.",
20
- recommendedExtractors: ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"]
64
+ rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
65
+ recommendedExtractors: getExtractorsForDomain("hardware"),
66
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
21
67
  };
22
68
  }
23
- if (t.includes("supabase") || t.includes("firebase") || t.includes("aws") || t.includes("cloud") || t.includes("vercel") || t.includes("backend-as-a-service") || t.includes("docker") || t.includes("k8s")) {
69
+ if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
24
70
  return {
25
71
  domain: "platform",
26
72
  confidence: 0.9,
27
- rationale: "Topic represents a cloud infrastructure, hosting, or BaaS platform.",
28
- recommendedExtractors: ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"]
73
+ rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
74
+ recommendedExtractors: getExtractorsForDomain("platform"),
75
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
29
76
  };
30
77
  }
31
- if (t.includes("theorem") || t.includes("complexity") || t.includes("pedagogy") || t.includes("bloom") || t.includes("constructivism") || t.includes("formal verification") || t.includes("computational theory")) {
78
+ if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
32
79
  return {
33
80
  domain: "academic",
34
81
  confidence: 0.9,
35
82
  rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
36
- recommendedExtractors: ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"]
83
+ recommendedExtractors: getExtractorsForDomain("academic"),
84
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
37
85
  };
38
86
  }
39
87
  return {
40
88
  domain: "software",
41
89
  confidence: 0.85,
42
- rationale: "Topic is a programming language, client/server library, or software framework.",
43
- recommendedExtractors: ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"]
90
+ rationale: "Topic is a programming language, game engine, client/server library, or software application.",
91
+ recommendedExtractors: getExtractorsForDomain("software"),
92
+ primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
44
93
  };
45
94
  }
46
95
  function getExtractorsForDomain(domain) {
@@ -61,15 +110,24 @@ function getExtractorsForDomain(domain) {
61
110
  var MultiSourceSearchEngine = class {
62
111
  tavilyKey;
63
112
  exaKey;
113
+ serperKey;
114
+ allowEnvFallback;
64
115
  customAdapter;
65
116
  constructor(options = {}) {
66
- this.tavilyKey = options.tavilyKey || process.env.TAVILY_API_KEY;
67
- this.exaKey = options.exaKey || process.env.EXA_API_KEY;
117
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
118
+ this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
119
+ this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
120
+ this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
68
121
  this.customAdapter = options.customAdapter;
69
122
  }
70
123
  setCustomAdapter(adapter) {
71
124
  this.customAdapter = adapter;
72
125
  }
126
+ setApiKeys(keys) {
127
+ if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
128
+ if (keys.exa !== void 0) this.exaKey = keys.exa;
129
+ if (keys.serper !== void 0) this.serperKey = keys.serper;
130
+ }
73
131
  async search(query, options = {}) {
74
132
  const limit = options.limit ?? 5;
75
133
  const adapter = options.customAdapter || this.customAdapter;
@@ -90,28 +148,37 @@ var MultiSourceSearchEngine = class {
90
148
  console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
91
149
  }
92
150
  }
93
- const tavilyKey = options.tavilyKey || this.tavilyKey;
94
- const exaKey = options.exaKey || this.exaKey;
151
+ const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
152
+ const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
153
+ const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
95
154
  if (preferred === "exa" && exaKey) {
96
- const exaHits = await this.queryExa(query, exaKey, limit);
155
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
97
156
  if (exaHits.length > 0) return exaHits;
98
157
  }
158
+ if (preferred === "serper" && serperKey) {
159
+ const serperHits = await this.querySerper(query, serperKey, limit);
160
+ if (serperHits.length > 0) return serperHits;
161
+ }
99
162
  if (preferred === "duckduckgo") {
100
163
  const ddgHits2 = await this.queryDuckDuckGo(query, limit);
101
164
  if (ddgHits2.length > 0) return ddgHits2;
102
165
  }
103
166
  if (tavilyKey && (!preferred || preferred === "tavily")) {
104
167
  try {
168
+ const body = {
169
+ api_key: tavilyKey,
170
+ query,
171
+ search_depth: "advanced",
172
+ include_answer: true,
173
+ max_results: limit
174
+ };
175
+ if (options.domainFilter && options.domainFilter.length > 0) {
176
+ body.include_domains = options.domainFilter;
177
+ }
105
178
  const res = await fetch("https://api.tavily.com/search", {
106
179
  method: "POST",
107
180
  headers: { "Content-Type": "application/json" },
108
- body: JSON.stringify({
109
- api_key: tavilyKey,
110
- query,
111
- search_depth: "advanced",
112
- include_answer: true,
113
- max_results: limit
114
- }),
181
+ body: JSON.stringify(body),
115
182
  signal: AbortSignal.timeout(12e3)
116
183
  });
117
184
  if (res.ok) {
@@ -131,26 +198,34 @@ var MultiSourceSearchEngine = class {
131
198
  }
132
199
  }
133
200
  if (exaKey) {
134
- const exaHits = await this.queryExa(query, exaKey, limit);
201
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
135
202
  if (exaHits.length > 0) return exaHits;
136
203
  }
204
+ if (serperKey) {
205
+ const serperHits = await this.querySerper(query, serperKey, limit);
206
+ if (serperHits.length > 0) return serperHits;
207
+ }
137
208
  const ddgHits = await this.queryDuckDuckGo(query, limit);
138
209
  if (ddgHits.length > 0) return ddgHits;
139
210
  return [];
140
211
  }
141
- async queryExa(query, exaKey, limit) {
212
+ async queryExa(query, exaKey, limit, domainFilter) {
142
213
  try {
214
+ const body = {
215
+ query,
216
+ numResults: limit,
217
+ useAutoprompt: true
218
+ };
219
+ if (domainFilter && domainFilter.length > 0) {
220
+ body.includeDomains = domainFilter;
221
+ }
143
222
  const res = await fetch("https://api.exa.ai/search", {
144
223
  method: "POST",
145
224
  headers: {
146
225
  "x-api-key": exaKey,
147
226
  "Content-Type": "application/json"
148
227
  },
149
- body: JSON.stringify({
150
- query,
151
- numResults: limit,
152
- useAutoprompt: true
153
- }),
228
+ body: JSON.stringify(body),
154
229
  signal: AbortSignal.timeout(1e4)
155
230
  });
156
231
  if (res.ok) {
@@ -170,6 +245,33 @@ var MultiSourceSearchEngine = class {
170
245
  }
171
246
  return [];
172
247
  }
248
+ async querySerper(query, serperKey, limit) {
249
+ try {
250
+ const res = await fetch("https://google.serper.dev/search", {
251
+ method: "POST",
252
+ headers: {
253
+ "X-API-KEY": serperKey,
254
+ "Content-Type": "application/json"
255
+ },
256
+ body: JSON.stringify({ q: query, num: limit }),
257
+ signal: AbortSignal.timeout(1e4)
258
+ });
259
+ if (res.ok) {
260
+ const data = await res.json();
261
+ if (Array.isArray(data?.organic) && data.organic.length > 0) {
262
+ return data.organic.slice(0, limit).map((r) => ({
263
+ title: r.title || "Untitled",
264
+ url: r.link,
265
+ content: r.snippet || "",
266
+ engine: "serper"
267
+ }));
268
+ }
269
+ }
270
+ } catch (e) {
271
+ console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
272
+ }
273
+ return [];
274
+ }
173
275
  async queryDuckDuckGo(query, limit) {
174
276
  try {
175
277
  const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
@@ -180,15 +282,16 @@ var MultiSourceSearchEngine = class {
180
282
  if (res.ok) {
181
283
  const html = await res.text();
182
284
  const hits = [];
183
- const resultRegex = /<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
285
+ const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
184
286
  let match;
185
287
  while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
186
- const rawUrl = match[1]?.trim();
288
+ const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
289
+ const rawUrl = match[2]?.trim();
187
290
  const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
188
- const snippet = match[2]?.replace(/<[^>]+>/g, "").trim() || "";
291
+ const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
189
292
  if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
190
293
  hits.push({
191
- title: query,
294
+ title: rawTitle,
192
295
  url: cleanUrl,
193
296
  content: snippet,
194
297
  engine: "duckduckgo"
@@ -204,15 +307,210 @@ var MultiSourceSearchEngine = class {
204
307
  }
205
308
  };
206
309
 
310
+ // src/harvester/youtubeTranscript.ts
311
+ var SPONSOR_NOISE_PATTERNS = [
312
+ /\b(sponsored by|this video is brought to you by|use (?:promo )?code\b|don't forget to like and subscribe|hit the bell icon|link in the description)\b/i,
313
+ /(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\s+này)?\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i
314
+ ];
315
+ function extractYouTubeVideoId(urlOrId) {
316
+ const trimmed = urlOrId.trim();
317
+ if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;
318
+ try {
319
+ const u = new URL(trimmed);
320
+ const host = u.hostname.replace(/^www\./, "").toLowerCase();
321
+ if (host === "youtu.be") {
322
+ const id = u.pathname.split("/").filter(Boolean)[0];
323
+ return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;
324
+ }
325
+ if (host === "youtube.com" || host === "m.youtube.com" || host === "youtube-nocookie.com") {
326
+ const v = u.searchParams.get("v");
327
+ if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;
328
+ const parts = u.pathname.split("/").filter(Boolean);
329
+ if ((parts[0] === "embed" || parts[0] === "shorts" || parts[0] === "v") && parts[1]) {
330
+ return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;
331
+ }
332
+ }
333
+ } catch {
334
+ }
335
+ return null;
336
+ }
337
+ function isYouTubeUrl(url) {
338
+ return extractYouTubeVideoId(url) !== null;
339
+ }
340
+ function formatTimestamp(seconds) {
341
+ const total = Math.max(0, Math.floor(seconds));
342
+ const hrs = Math.floor(total / 3600);
343
+ const mins = Math.floor(total % 3600 / 60);
344
+ const secs = total % 60;
345
+ if (hrs > 0) {
346
+ return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
347
+ }
348
+ return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
349
+ }
350
+ function decodeHtmlEntities(str) {
351
+ return str.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&#x27;/g, "'").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
352
+ }
353
+ function stitchCaptionSegments(segments) {
354
+ const cleaned = segments.map((s) => ({
355
+ startSec: s.startSec,
356
+ durationSec: s.durationSec,
357
+ text: decodeHtmlEntities(s.text)
358
+ })).filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));
359
+ if (cleaned.length === 0) return [];
360
+ const stitched = [];
361
+ let bucketStart = cleaned[0].startSec;
362
+ let bucketEnd = cleaned[0].startSec + cleaned[0].durationSec;
363
+ let bucketTexts = [];
364
+ for (const seg of cleaned) {
365
+ bucketTexts.push(seg.text);
366
+ bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);
367
+ const joined = bucketTexts.join(" ");
368
+ const elapsed = seg.startSec - bucketStart;
369
+ const endsSentence = /[.!?…]$/.test(seg.text);
370
+ if (elapsed >= 30 && endsSentence || elapsed >= 50 || joined.length >= 360) {
371
+ stitched.push({
372
+ startSec: bucketStart,
373
+ durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
374
+ timestamp: formatTimestamp(bucketStart),
375
+ text: joined
376
+ });
377
+ bucketTexts = [];
378
+ bucketStart = seg.startSec + seg.durationSec;
379
+ }
380
+ }
381
+ if (bucketTexts.length > 0) {
382
+ stitched.push({
383
+ startSec: bucketStart,
384
+ durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
385
+ timestamp: formatTimestamp(bucketStart),
386
+ text: bucketTexts.join(" ")
387
+ });
388
+ }
389
+ return stitched;
390
+ }
391
+ var YouTubeIngestionService = class {
392
+ async fetchTranscript(urlOrId, preferredLanguages = ["vi", "en"]) {
393
+ const videoId = extractYouTubeVideoId(urlOrId);
394
+ if (!videoId) {
395
+ throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);
396
+ }
397
+ const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;
398
+ const res = await fetch(canonicalUrl, {
399
+ headers: {
400
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
401
+ "Accept-Language": "vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7"
402
+ },
403
+ signal: AbortSignal.timeout(12e3)
404
+ });
405
+ if (!res.ok) {
406
+ throw new Error(`Failed to fetch YouTube watch page (${res.status})`);
407
+ }
408
+ const html = await res.text();
409
+ const playerMatch = html.match(/ytInitialPlayerResponse\s*=\s*(\{[\s\S]*?\});\s*(?:var\s|window\.|<\/script>)/);
410
+ let title = `YouTube Video (${videoId})`;
411
+ let author;
412
+ let shortDescription = "";
413
+ let rawSegments = [];
414
+ let selectedLang = "en";
415
+ let isAutoGenerated = false;
416
+ if (playerMatch?.[1]) {
417
+ try {
418
+ const playerData = JSON.parse(playerMatch[1]);
419
+ title = playerData?.videoDetails?.title || title;
420
+ author = playerData?.videoDetails?.author;
421
+ shortDescription = playerData?.videoDetails?.shortDescription || "";
422
+ const tracks = playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];
423
+ if (tracks.length > 0) {
424
+ let chosenTrack = tracks.find(
425
+ (t) => preferredLanguages.includes(t.languageCode) && t.kind !== "asr"
426
+ ) || tracks.find((t) => preferredLanguages.includes(t.languageCode)) || tracks[0];
427
+ if (chosenTrack?.baseUrl) {
428
+ selectedLang = chosenTrack.languageCode || "en";
429
+ isAutoGenerated = chosenTrack.kind === "asr";
430
+ const trackRes = await fetch(chosenTrack.baseUrl, {
431
+ signal: AbortSignal.timeout(1e4)
432
+ });
433
+ if (trackRes.ok) {
434
+ const xml = await trackRes.text();
435
+ const textMatches = [
436
+ ...xml.matchAll(/<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g)
437
+ ];
438
+ for (const m of textMatches) {
439
+ rawSegments.push({
440
+ startSec: parseFloat(m[1] || "0"),
441
+ durationSec: parseFloat(m[2] || "2"),
442
+ text: (m[3] || "").replace(/<[^>]+>/g, " ")
443
+ });
444
+ }
445
+ }
446
+ }
447
+ }
448
+ } catch {
449
+ }
450
+ }
451
+ if (title === `YouTube Video (${videoId})`) {
452
+ const titleMeta = html.match(/<title>([\s\S]*?)<\/title>/i);
453
+ if (titleMeta?.[1]) {
454
+ title = decodeHtmlEntities(titleMeta[1].replace(/\s*-\s*YouTube$/i, ""));
455
+ }
456
+ }
457
+ const stitched = stitchCaptionSegments(rawSegments);
458
+ const mdLines = [
459
+ `# ${title}`,
460
+ `> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ""} | **Language:** \`${selectedLang}\`${isAutoGenerated ? " (Auto-caption)" : ""}`
461
+ ];
462
+ if (shortDescription.trim()) {
463
+ mdLines.push(`
464
+ ## Video Overview & Description
465
+ ${shortDescription.trim().slice(0, 1500)}`);
466
+ }
467
+ if (stitched.length > 0) {
468
+ mdLines.push(`
469
+ ## Timestamped Transcript`);
470
+ for (const seg of stitched) {
471
+ mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);
472
+ }
473
+ }
474
+ return {
475
+ videoId,
476
+ url: canonicalUrl,
477
+ title,
478
+ author,
479
+ language: selectedLang,
480
+ isAutoGenerated,
481
+ segments: stitched,
482
+ stitchedMarkdown: mdLines.join("\n")
483
+ };
484
+ }
485
+ };
486
+
207
487
  // src/harvester/webScraper.ts
488
+ function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
489
+ const withoutCode = markdown.replace(/```[\s\S]*?```/g, "\n");
490
+ const blocks = withoutCode.split(/\n{2,}/).map((b) => b.replace(/\s+/g, " ").trim()).filter((b) => {
491
+ if (b.length < 60 || b.length > 900) return false;
492
+ if (b.startsWith("#") || b.startsWith("|")) return false;
493
+ const linkRatio = (b.match(/\[[^\]]+\]\([^)]+\)/g) || []).join("").length / b.length;
494
+ if (linkRatio > 0.45) return false;
495
+ if (/\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\b/i.test(b)) {
496
+ return false;
497
+ }
498
+ return true;
499
+ });
500
+ return blocks.slice(0, maxExcerpts);
501
+ }
208
502
  var WebScraper = class {
209
503
  crawl4aiHost;
210
504
  jinaKey;
505
+ allowEnvFallback;
211
506
  customAdapter;
507
+ youtubeService;
212
508
  constructor(options = {}) {
213
- this.crawl4aiHost = options.crawl4aiHost || process.env.CRAWL4AI_HOST || "http://localhost:11235";
214
- this.jinaKey = options.jinaKey || process.env.JINA_API_KEY;
509
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
510
+ this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
511
+ this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
215
512
  this.customAdapter = options.customAdapter;
513
+ this.youtubeService = new YouTubeIngestionService();
216
514
  }
217
515
  setCustomAdapter(adapter) {
218
516
  this.customAdapter = adapter;
@@ -226,12 +524,22 @@ var WebScraper = class {
226
524
  try {
227
525
  const result = await adapter.crawl(targetUrl);
228
526
  if (result && result.markdownContent) {
229
- return this.packageDocument(targetUrl, result.markdownContent, "custom");
527
+ return this.packageDocument(targetUrl, result.markdownContent, "custom", result.title);
230
528
  }
231
529
  } catch (e) {
232
530
  console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);
233
531
  }
234
532
  }
533
+ if (isYouTubeUrl(targetUrl)) {
534
+ try {
535
+ const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);
536
+ if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {
537
+ return this.packageDocument(yt.url, yt.stitchedMarkdown, "youtube", yt.title);
538
+ }
539
+ } catch (e) {
540
+ console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
541
+ }
542
+ }
235
543
  try {
236
544
  const jinaUrl = `https://r.jina.ai/${targetUrl}`;
237
545
  const jinaKey = options.jinaKey || this.jinaKey;
@@ -282,35 +590,435 @@ var WebScraper = class {
282
590
  signal: AbortSignal.timeout(1e4)
283
591
  });
284
592
  if (res.ok) {
285
- const rawText = await res.text();
286
- const clean = rawText.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
287
- return this.packageDocument(targetUrl, clean, "direct");
593
+ const rawHtml = await res.text();
594
+ const titleTag = rawHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
595
+ const clean = rawHtml.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "").replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, "").replace(/<h([1-3])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, lvl, txt) => `
596
+ ${"#".repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, "").trim()}
597
+ `).replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => `
598
+ \`\`\`
599
+ ${code.replace(/<[^>]+>/g, "").trim()}
600
+ \`\`\`
601
+ `).replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, p) => `
602
+ ${p.replace(/<[^>]+>/g, " ").trim()}
603
+ `).replace(/<[^>]+>/g, " ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
604
+ return this.packageDocument(targetUrl, clean, "direct", titleTag);
288
605
  }
289
606
  } catch (e) {
290
607
  console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);
291
608
  }
292
609
  throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
293
610
  }
294
- packageDocument(url, markdown, provider) {
295
- const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/Title:\s*(.+)/i);
296
- const title = titleMatch ? titleMatch[1].trim() : url.split("/").pop() || "Document";
297
- const codeMatches = [...markdown.matchAll(/```(?:[a-z0-9_-]+)?\n([\s\S]*?)```/gi)].map((m) => m[1]?.trim()).filter(Boolean);
298
- const tableMatches = markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
611
+ packageDocument(url, markdown, provider, explicitTitle) {
612
+ const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
613
+ const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
614
+ const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```/g)];
615
+ const extractedCodeBlocks = codeBlockMatches.map((m) => ({
616
+ language: (m[1] || "").trim().toLowerCase(),
617
+ code: (m[2] || "").trim()
618
+ })).filter((b) => b.code.length > 0);
619
+ const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);
620
+ const tableBlocks = markdown.match(/(?:^\|.+\|\s*\n)+^\|[-:| ]+\|\s*\n(?:^\|.+\|\s*\n?)+/gm) || markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
621
+ const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);
622
+ const extractedHeadings = [...markdown.matchAll(/^#{1,3}\s+(.+)$/gm)].map((m) => m[1].trim()).filter((h) => h.length > 2 && h.length < 120);
623
+ const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);
299
624
  return {
300
625
  url,
301
626
  title,
302
627
  markdownContent: markdown,
303
628
  byteSize: Buffer.byteLength(markdown, "utf-8"),
304
- extractedCodeSnippets: codeMatches,
305
- tablesCount: tableMatches.length,
629
+ extractedCodeSnippets,
630
+ extractedCodeBlocks,
631
+ extractedTables,
632
+ extractedHeadings,
633
+ keyExcerpts,
634
+ tablesCount: extractedTables.length,
306
635
  fetchProvider: provider
307
636
  };
308
637
  }
309
638
  };
310
639
 
640
+ // src/harvester/termExtractor.ts
641
+ var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
642
+ "l\xE0",
643
+ "c\u1EE7a",
644
+ "trong",
645
+ "\u0111\u01B0\u1EE3c",
646
+ "nh\u1EEFng",
647
+ "c\xE1c",
648
+ "cho",
649
+ "v\u1EDBi",
650
+ "khi",
651
+ "n\u1EBFu",
652
+ "th\xEC",
653
+ "v\xE0",
654
+ "ho\u1EB7c",
655
+ "\u0111\u1EC3",
656
+ "t\u1EEB",
657
+ "\u0111\u1EBFn",
658
+ "v\u1EC1",
659
+ "n\xE0y",
660
+ "\u0111\xF3",
661
+ "m\u1ED9t",
662
+ "nh\u01B0",
663
+ "s\u1EBD",
664
+ "\u0111ang",
665
+ "\u0111\xE3",
666
+ "kh\xF4ng",
667
+ "c\xF3",
668
+ "ra",
669
+ "v\xE0o",
670
+ "b\u1EB1ng",
671
+ "theo",
672
+ "tr\xEAn",
673
+ "d\u01B0\u1EDBi",
674
+ "qua",
675
+ "nh\u1EDD",
676
+ "v\xEC",
677
+ "do",
678
+ "t\u1EA1i",
679
+ "b\u1EDFi",
680
+ "nh\u01B0ng",
681
+ "m\xE0",
682
+ "c\u0169ng",
683
+ "v\u1EABn",
684
+ "r\u1EA5t",
685
+ "qu\xE1",
686
+ "h\u01A1n",
687
+ "nh\u1EA5t",
688
+ "c\u1EA7n",
689
+ "ph\u1EA3i",
690
+ "n\xEAn",
691
+ "th\u1EC3",
692
+ "vi\u1EC7c",
693
+ "c\xE1ch",
694
+ "ph\u1EA7n",
695
+ "b\xE0i",
696
+ "bu\u1ED5i",
697
+ "h\u1ECDc",
698
+ "sinh",
699
+ "gi\xE1o",
700
+ "vi\xEAn",
701
+ "ng\u01B0\u1EDDi",
702
+ "d\xF9ng",
703
+ "ch\xFAng",
704
+ "ta",
705
+ "b\u1EA1n",
706
+ "em",
707
+ "gi\xFAp",
708
+ "l\xE0m",
709
+ "t\u1EA1o",
710
+ "s\u1EED",
711
+ "d\u1EE5ng",
712
+ "th\u1EF1c",
713
+ "hi\u1EC7n",
714
+ "hi\u1EC3u",
715
+ "bi\u1EBFt",
716
+ "xem",
717
+ "th\xEAm",
718
+ "sau",
719
+ "tr\u01B0\u1EDBc"
720
+ ]);
721
+ var ENGLISH_STOPWORDS = /* @__PURE__ */ new Set([
722
+ "the",
723
+ "and",
724
+ "for",
725
+ "with",
726
+ "that",
727
+ "this",
728
+ "from",
729
+ "into",
730
+ "your",
731
+ "you",
732
+ "are",
733
+ "was",
734
+ "were",
735
+ "have",
736
+ "has",
737
+ "had",
738
+ "will",
739
+ "would",
740
+ "can",
741
+ "could",
742
+ "should",
743
+ "about",
744
+ "more",
745
+ "other",
746
+ "such",
747
+ "than",
748
+ "then",
749
+ "them",
750
+ "these",
751
+ "those",
752
+ "their",
753
+ "what",
754
+ "when",
755
+ "where",
756
+ "which",
757
+ "while",
758
+ "who",
759
+ "how",
760
+ "why",
761
+ "each",
762
+ "some",
763
+ "many",
764
+ "most",
765
+ "also",
766
+ "only",
767
+ "just",
768
+ "very",
769
+ "using",
770
+ "used",
771
+ "use",
772
+ "overview",
773
+ "introduction",
774
+ "getting",
775
+ "started",
776
+ "guide",
777
+ "tutorial",
778
+ "example",
779
+ "examples",
780
+ "documentation"
781
+ ]);
782
+ var VIETNAMESE_TECHNICAL_HEADS = /^(cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|chuỗi|đối tượng|lớp|phương thức|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|mạng|giao thức|khái niệm|mô hình|kiến trúc|kiểm thử|gỡ lỗi)\b/i;
783
+ var SYNTAX_OR_API_PATTERN = /(?:`[^`]+`|\b[a-z]+[A-Z][a-zA-Z0-9]*\b|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b|@[A-Za-z]+|\b[a-z_]+\(\)|\b(?:pinMode|digitalWrite|digitalRead|analogRead|analogWrite|async|await|actor|struct|class|func|def|let|const|var)\b)/;
784
+ var HARDWARE_COMPONENT_PATTERN = /\b(esp32|arduino|micro:?bit|raspberry|sensor|led|servo|ultrasonic|ldr|dht11|dht22|breadboard|gpio|pwm|i2c|spi|uart|cảm biến|động cơ|mạch|đèn led|vi điều khiển)\b/i;
785
+ function normalizeTermKey(raw) {
786
+ return raw.normalize("NFC").toLowerCase().replace(/[`*_~#"']/g, "").replace(/\s+/g, " ").trim();
787
+ }
788
+ function cleanDisplayTerm(raw) {
789
+ return raw.normalize("NFC").replace(/^[-*•#\d.)\s]+/, "").replace(/[`*_~]+/g, "").replace(/\s+/g, " ").trim();
790
+ }
791
+ function isNoiseTerm(term) {
792
+ const clean = normalizeTermKey(term);
793
+ if (clean.length < 2 || clean.length > 65) return true;
794
+ if (/^\d+$/.test(clean)) return true;
795
+ if (/^https?:\/\//i.test(clean)) return true;
796
+ if (ENGLISH_STOPWORDS.has(clean) || VIETNAMESE_STOPWORDS.has(clean)) return true;
797
+ const words = clean.split(" ");
798
+ if (words.length > 6) return true;
799
+ if (words.length === 1 && clean.length < 3 && !/^(ai|ui|ux|os|io|db|id|ip)$/i.test(clean)) return true;
800
+ return false;
801
+ }
802
+ function classifyMarrAndCategory(term, domain) {
803
+ if (HARDWARE_COMPONENT_PATTERN.test(term)) {
804
+ return { category: "hardware_component", marrLayer: "CIO", bloomLevel: "apply" };
805
+ }
806
+ if (SYNTAX_OR_API_PATTERN.test(term)) {
807
+ return { category: "api_syntax", marrLayer: "SIO", bloomLevel: "apply" };
808
+ }
809
+ if (/\b(architecture|pattern|model|design|debugging|verification|mô hình|kiến trúc|tư duy|quy trình)\b/i.test(term)) {
810
+ return { category: "pedagogical_pattern", marrLayer: "ULO", bloomLevel: "analyze" };
811
+ }
812
+ return {
813
+ category: "core_concept",
814
+ marrLayer: domain === "academic" ? "ULO" : "CIO",
815
+ bloomLevel: "understand"
816
+ };
817
+ }
818
+ function findSentenceDefinition(term, markdown) {
819
+ const clean = cleanDisplayTerm(term);
820
+ const escaped = clean.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
821
+ const sentences = markdown.replace(/```[\s\S]*?```/g, " ").split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").trim()).filter((s) => s.length >= 25 && s.length <= 280);
822
+ const re = new RegExp(escaped, "i");
823
+ const match = sentences.find((s) => re.test(s));
824
+ return match || `Core concept in ${clean} extracted from primary technical reference.`;
825
+ }
826
+ var PedagogicalTermExtractor = class {
827
+ extractDeterministic(topic, domain, docs, maxTerms = 15) {
828
+ const candidates = /* @__PURE__ */ new Map();
829
+ let orderCounter = 0;
830
+ const upsertCandidate = (rawTerm, weight, resourceId, markdown, pair) => {
831
+ const display = cleanDisplayTerm(rawTerm);
832
+ if (isNoiseTerm(display)) return;
833
+ const key = normalizeTermKey(display);
834
+ const existing = candidates.get(key);
835
+ if (existing) {
836
+ existing.score += weight;
837
+ existing.sourceIds.add(resourceId);
838
+ if (pair?.termVi && !existing.termVi) existing.termVi = pair.termVi;
839
+ if (pair?.termEn && !existing.termEn) existing.termEn = pair.termEn;
840
+ } else {
841
+ candidates.set(key, {
842
+ display,
843
+ termVi: pair?.termVi,
844
+ termEn: pair?.termEn,
845
+ score: weight,
846
+ firstSeenOrder: orderCounter++,
847
+ sourceIds: /* @__PURE__ */ new Set([resourceId]),
848
+ definition: findSentenceDefinition(display, markdown)
849
+ });
850
+ }
851
+ };
852
+ for (const { resourceId, doc } of docs) {
853
+ const md = doc.markdownContent || "";
854
+ const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
855
+ const bilingualMatches = mdWithoutCode.matchAll(
856
+ /([A-ZÀÁẠẢÃÂẦẤẬẨẪĂẰẮẶẲẴÈÉẸẺẼÊỀẾỆỂỄÌÍỊỈĨÒÓỌỎÕÔỒỐỘỔỖƠỜỚỢỞỠÙÚỤỦŨƯỪỨỰỬỮỲÝỴỶỸĐ][a-zA-Z0-9\u00C0-\u1EF9\s/-]{2,40}?)\s*\(([A-Za-z0-9\u00C0-\u1EF9\s/_:-]{2,40})\)/g
857
+ );
858
+ for (const m of bilingualMatches) {
859
+ const left = cleanDisplayTerm(m[1] || "");
860
+ const right = cleanDisplayTerm(m[2] || "");
861
+ if (!isNoiseTerm(left) && !isNoiseTerm(right)) {
862
+ const leftHasVi = /[\u00C0-\u1EF9]/.test(left);
863
+ const rightHasVi = /[\u00C0-\u1EF9]/.test(right);
864
+ if (leftHasVi && !rightHasVi) {
865
+ upsertCandidate(`${left} (${right})`, 6, resourceId, mdWithoutCode, { termVi: left, termEn: right });
866
+ } else if (!leftHasVi && rightHasVi) {
867
+ upsertCandidate(`${right} (${left})`, 6, resourceId, mdWithoutCode, { termVi: right, termEn: left });
868
+ } else {
869
+ upsertCandidate(left, 4, resourceId, mdWithoutCode);
870
+ }
871
+ }
872
+ }
873
+ for (const heading of doc.extractedHeadings || []) {
874
+ const cleaned = cleanDisplayTerm(heading.replace(/^\d+(\.\d+)*\s*/, ""));
875
+ if (!isNoiseTerm(cleaned)) {
876
+ upsertCandidate(cleaned, 4, resourceId, mdWithoutCode);
877
+ }
878
+ }
879
+ for (const m of mdWithoutCode.matchAll(/\*\*([^*\n]{3,45})\*\*/g)) {
880
+ upsertCandidate(m[1] || "", 3.5, resourceId, mdWithoutCode);
881
+ }
882
+ for (const m of mdWithoutCode.matchAll(/`([a-zA-Z_@][a-zA-Z0-9_.:()-]{2,35})`/g)) {
883
+ upsertCandidate(m[1] || "", 3, resourceId, mdWithoutCode);
884
+ }
885
+ for (const line of mdWithoutCode.split("\n")) {
886
+ const clauses = line.split(/[.,;:!?()[\]{}"“”‘’/\\|]+/);
887
+ for (const clause of clauses) {
888
+ const trimmed = clause.trim();
889
+ const viMatch = trimmed.match(
890
+ /\b((?:cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|đối tượng|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|giao thức|mô hình|kiến trúc|kiểm thử|gỡ lỗi)(?:\s+[a-zA-Z0-9\u00C0-\u1EF9_-]+){1,3})\b/i
891
+ );
892
+ if (viMatch?.[1]) {
893
+ const tokens = viMatch[1].trim().split(/\s+/);
894
+ while (tokens.length > 1 && VIETNAMESE_STOPWORDS.has(tokens[tokens.length - 1].toLowerCase())) {
895
+ tokens.pop();
896
+ }
897
+ const phrase = tokens.join(" ");
898
+ if (tokens.length >= 2 && VIETNAMESE_TECHNICAL_HEADS.test(phrase)) {
899
+ upsertCandidate(phrase, 3.5, resourceId, mdWithoutCode, { termVi: phrase });
900
+ }
901
+ }
902
+ }
903
+ }
904
+ }
905
+ const sorted = [...candidates.values()].sort((a, b) => b.score - a.score || a.firstSeenOrder - b.firstSeenOrder).slice(0, maxTerms);
906
+ const orderedByAppearance = [...sorted].sort((a, b) => a.firstSeenOrder - b.firstSeenOrder);
907
+ const maxScore = Math.max(1, sorted[0]?.score || 1);
908
+ const concepts = orderedByAppearance.map((item, idx) => {
909
+ const id = `C_${String(idx + 1).padStart(2, "0")}`;
910
+ const { category, marrLayer, bloomLevel } = classifyMarrAndCategory(item.display, domain);
911
+ const prerequisites = [];
912
+ return {
913
+ id,
914
+ term: item.display,
915
+ termVi: item.termVi,
916
+ termEn: item.termEn,
917
+ category,
918
+ marrLayer,
919
+ bloomLevel,
920
+ salienceScore: Number(Math.min(10, Math.max(1, item.score / maxScore * 10)).toFixed(1)),
921
+ definition: item.definition,
922
+ prerequisites,
923
+ sourceIds: [...item.sourceIds]
924
+ };
925
+ });
926
+ return {
927
+ concepts,
928
+ conceptDagTsv: this.formatConceptDagTsv(concepts),
929
+ isLlmRefined: false,
930
+ unrefinedCandidatesCount: candidates.size
931
+ };
932
+ }
933
+ async extractWithOptionalLlm(topic, domain, docs, options = {}) {
934
+ const detBundle = this.extractDeterministic(topic, domain, docs, options.maxTerms ?? 15);
935
+ if (!options.llm || docs.length === 0) {
936
+ return detBundle;
937
+ }
938
+ try {
939
+ const corpusPreview = docs.slice(0, 6).map(
940
+ (d) => `[${d.resourceId}] ${d.doc.title} (${d.doc.url}):
941
+ ${(d.doc.keyExcerpts || []).slice(0, 3).join("\n")}`
942
+ ).join("\n\n");
943
+ const candidateSeed = detBundle.concepts.slice(0, 12).map((c) => `${c.term} (${c.sourceIds.join(",")})`).join("; ");
944
+ const prompt = [
945
+ `Analyze the harvested technical & pedagogical corpus for topic "${topic}" (Domain: ${domain}).`,
946
+ options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]} years old.` : "",
947
+ options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}.` : "",
948
+ options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}.` : "",
949
+ `Deterministic Candidate Terms: ${candidateSeed}`,
950
+ `
951
+ Harvested Source Excerpts:
952
+ ${corpusPreview}`,
953
+ `
954
+ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
955
+ `IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
956
+ `Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
957
+ ].filter(Boolean).join("\n");
958
+ const rawResponse = await options.llm(prompt, {
959
+ system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
960
+ temperature: 0.2,
961
+ jsonMode: true
962
+ });
963
+ const jsonMatch = rawResponse.match(/\[[\s\S]*\]/);
964
+ if (jsonMatch) {
965
+ const parsed = JSON.parse(jsonMatch[0]);
966
+ if (Array.isArray(parsed) && parsed.length > 0) {
967
+ const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
968
+ const concepts = parsed.map((item, idx) => {
969
+ const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
970
+ const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
971
+ return {
972
+ id,
973
+ term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
974
+ termVi: item.termVi ? String(item.termVi) : void 0,
975
+ termEn: item.termEn ? String(item.termEn) : void 0,
976
+ category: ["core_concept", "api_syntax", "hardware_component", "pedagogical_pattern"].includes(item.category) ? item.category : "core_concept",
977
+ marrLayer: ["ULO", "CIO", "SIO"].includes(item.marrLayer) ? item.marrLayer : "CIO",
978
+ bloomLevel: ["remember", "understand", "apply", "analyze", "evaluate", "create"].includes(item.bloomLevel) ? item.bloomLevel : "understand",
979
+ salienceScore: Number(item.salienceScore) || 8,
980
+ definition: String(item.definition || ""),
981
+ prerequisites: prereqs,
982
+ sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
983
+ };
984
+ });
985
+ return {
986
+ concepts,
987
+ conceptDagTsv: this.formatConceptDagTsv(concepts),
988
+ isLlmRefined: true,
989
+ unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
990
+ };
991
+ }
992
+ }
993
+ } catch (e) {
994
+ console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
995
+ }
996
+ return detBundle;
997
+ }
998
+ formatConceptDagTsv(concepts) {
999
+ const header = "id term term_vi term_en marr_layer bloom_level salience prerequisites sources definition";
1000
+ const rows = concepts.map(
1001
+ (c) => [
1002
+ c.id,
1003
+ c.term.replace(/\t|\n/g, " "),
1004
+ (c.termVi || "").replace(/\t|\n/g, " "),
1005
+ (c.termEn || "").replace(/\t|\n/g, " "),
1006
+ c.marrLayer,
1007
+ c.bloomLevel,
1008
+ c.salienceScore,
1009
+ c.prerequisites.join(","),
1010
+ c.sourceIds.join(","),
1011
+ c.definition.replace(/\t|\n/g, " ")
1012
+ ].join(" ")
1013
+ );
1014
+ return [header, ...rows].join("\n");
1015
+ }
1016
+ };
1017
+
311
1018
  // src/harvester/catalogBuilder.ts
312
1019
  var CatalogBuilder = class {
313
- buildBundle(topic, domain, items) {
1020
+ buildBundle(topic, domain, items, options = {}) {
1021
+ const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
314
1022
  const indexList = items.map((item, idx) => ({
315
1023
  id: `RES-${String(idx + 1).padStart(2, "0")}`,
316
1024
  title: item.doc.title,
@@ -320,7 +1028,8 @@ var CatalogBuilder = class {
320
1028
  byteSize: item.doc.byteSize,
321
1029
  tablesCount: item.doc.tablesCount,
322
1030
  codeSnippetsCount: item.doc.extractedCodeSnippets.length,
323
- summary: item.summary
1031
+ summary: item.summary,
1032
+ keyTakeaways: item.keyTakeaways || item.doc.keyExcerpts?.slice(0, 3) || []
324
1033
  }));
325
1034
  const resourceIndexJson = JSON.stringify(
326
1035
  {
@@ -339,36 +1048,60 @@ var CatalogBuilder = class {
339
1048
  `
340
1049
  | ID | Resource Title | Relevance (1-10) | Code Snippets | Tables | Source Link |`,
341
1050
  `|---|---|---|---|---|---|`,
342
- ...indexList.map((r) => `| **${r.id}** | ${r.title} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`),
1051
+ ...indexList.map(
1052
+ (r) => `| **${r.id}** | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`
1053
+ ),
343
1054
  `
344
1055
  ## Resource Summaries & Technical Ground Truth
345
1056
  `,
346
- ...indexList.map((r) => `### ${r.id}: ${r.title}
1057
+ ...indexList.map((r) => {
1058
+ const takeaways = r.keyTakeaways && r.keyTakeaways.length > 0 ? r.keyTakeaways.map((t) => ` - ${t}`).join("\n") : ` - ${r.summary}`;
1059
+ return `### ${r.id}: ${r.title}
347
1060
  - **URL:** ${r.url}
348
- - **Summary:** ${r.summary}
349
1061
  - **Relevance:** ${r.relevanceScore}/10
350
- `)
1062
+ - **Key Excerpts:**
1063
+ ${takeaways}
1064
+ `;
1065
+ })
351
1066
  ];
1067
+ const codeSections = items.flatMap((item) => {
1068
+ if (item.doc.extractedCodeBlocks && item.doc.extractedCodeBlocks.length > 0) {
1069
+ return item.doc.extractedCodeBlocks.slice(0, 2).map((b) => {
1070
+ const lang = b.language || defaultLang;
1071
+ return `\`\`\`${lang}
1072
+ ${b.code}
1073
+ \`\`\``;
1074
+ });
1075
+ }
1076
+ return item.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
1077
+ ${c}
1078
+ \`\`\``);
1079
+ });
1080
+ const conceptLines = options.concepts && options.concepts.length > 0 ? [
1081
+ `| ID | Concept / Term | Vietnamese | English | Marr | Bloom | Prereqs | Sources |`,
1082
+ `|---|---|---|---|---|---|---|---|`,
1083
+ ...options.concepts.map(
1084
+ (c) => `| \`${c.id}\` | **${c.term.replace(/\|/g, "/")}** | ${c.termVi || "\u2014"} | ${c.termEn || "\u2014"} | \`${c.marrLayer}\` | \`${c.bloomLevel}\` | ${c.prerequisites.join(", ") || "\u2014"} | ${c.sourceIds.join(", ")} |`
1085
+ )
1086
+ ] : [`- Canonical technical breakdown for **${topic}** derived from ${items.length} verified primary sources.`];
352
1087
  const kgLines = [
353
1088
  `# \u{1F9EC} Distilled Knowledge Graph: ${topic}`,
354
1089
  `> **Domain:** \`${domain.toUpperCase()}\` | **Ground Truth Verification:** PASSED`,
355
1090
  `
356
- ## 1. Architectural Blueprint & Component Decomposition`,
357
- `- Canonical technical breakdown for **${topic}** derived from ${items.length} peer-reviewed and verified primary sources.`,
1091
+ ## 1. Architectural Blueprint & Concept Hierarchy (Marr & Bloom)`,
1092
+ ...conceptLines,
358
1093
  `
359
1094
  ## 2. Core APIs & Code Primitives`,
360
- ...items.flatMap((i) => i.doc.extractedCodeSnippets.slice(0, 2)).map((c) => `\`\`\`${domain === "hardware" ? "cpp" : "swift"}
361
- ${c}
362
- \`\`\``),
1095
+ ...codeSections.length > 0 ? codeSections : ["*No explicit code snippets captured in harvested sources.*"],
363
1096
  `
364
- ## 3. Technical Constraints & Common Pitfalls`,
365
- `- Strict compliance with ${domain.toUpperCase()} safety and concurrency contracts.`,
366
- `- Proactively prevent documented anti-patterns and performance regressions.`
1097
+ ## 3. Extracted Reference Tables`,
1098
+ ...items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)).length > 0 ? items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)) : ["*No markdown reference tables extracted.*"]
367
1099
  ];
368
1100
  return {
369
1101
  resourceIndexJson,
370
1102
  resourceCatalogMd: catalogLines.join("\n"),
371
- knowledgeGraphMd: kgLines.join("\n")
1103
+ knowledgeGraphMd: kgLines.join("\n"),
1104
+ conceptDagTsv: options.conceptDagTsv
372
1105
  };
373
1106
  }
374
1107
  };
@@ -396,7 +1129,15 @@ var HealthAuditor = class {
396
1129
  exports.CatalogBuilder = CatalogBuilder;
397
1130
  exports.HealthAuditor = HealthAuditor;
398
1131
  exports.MultiSourceSearchEngine = MultiSourceSearchEngine;
1132
+ exports.PedagogicalTermExtractor = PedagogicalTermExtractor;
399
1133
  exports.WebScraper = WebScraper;
1134
+ exports.YouTubeIngestionService = YouTubeIngestionService;
400
1135
  exports.classifyTechnicalDomain = classifyTechnicalDomain;
1136
+ exports.extractSubstantiveExcerpts = extractSubstantiveExcerpts;
1137
+ exports.extractYouTubeVideoId = extractYouTubeVideoId;
1138
+ exports.formatTimestamp = formatTimestamp;
1139
+ exports.inferPrimaryCodeLanguage = inferPrimaryCodeLanguage;
1140
+ exports.isYouTubeUrl = isYouTubeUrl;
1141
+ exports.stitchCaptionSegments = stitchCaptionSegments;
401
1142
  //# sourceMappingURL=index.cjs.map
402
1143
  //# sourceMappingURL=index.cjs.map