@thanh01.pmt/research-kit 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +213 -0
- package/dist/academic/index.cjs +332 -127
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +2 -2
- package/dist/{catalogBuilder-8A-Dg_A-.d.ts → catalogBuilder-CmmQzUnf.d.ts} +11 -3
- package/dist/{catalogBuilder-feU79wQQ.d.cts → catalogBuilder-DgXYxL2F.d.cts} +11 -3
- package/dist/chunk-2JEVBCIF.js +3 -0
- package/dist/{chunk-KGCDYFFD.js.map → chunk-2JEVBCIF.js.map} +1 -1
- package/dist/chunk-2XIEWCUE.js +244 -0
- package/dist/chunk-2XIEWCUE.js.map +1 -0
- package/dist/chunk-AUGVVZTL.js +514 -0
- package/dist/chunk-AUGVVZTL.js.map +1 -0
- package/dist/chunk-DOXSPRSP.js +559 -0
- package/dist/chunk-DOXSPRSP.js.map +1 -0
- package/dist/chunk-GKF3NDEX.js +162 -0
- package/dist/chunk-GKF3NDEX.js.map +1 -0
- package/dist/chunk-J3VCGH7Z.js +3 -0
- package/dist/chunk-J3VCGH7Z.js.map +1 -0
- package/dist/{chunk-67V2GJGK.js → chunk-J7XKDMGL.js} +7 -5
- package/dist/chunk-J7XKDMGL.js.map +1 -0
- package/dist/chunk-KAFQVI2R.js +333 -0
- package/dist/chunk-KAFQVI2R.js.map +1 -0
- package/dist/chunk-XIF5YNI7.js +799 -0
- package/dist/chunk-XIF5YNI7.js.map +1 -0
- package/dist/config/index.cjs +5 -3
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs +337 -21
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +44 -7
- package/dist/evidence/index.d.ts +44 -7
- package/dist/evidence/index.js +3 -2
- package/dist/harvester/index.cjs +805 -64
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +79 -8
- package/dist/harvester/index.d.ts +79 -8
- package/dist/harvester/index.js +3 -2
- package/dist/{index-BU5n1ip_.d.ts → index-t8pahlJ4.d.cts} +104 -15
- package/dist/{index-BU5n1ip_.d.cts → index-t8pahlJ4.d.ts} +104 -15
- package/dist/index.cjs +2001 -252
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +156 -8
- package/dist/index.d.ts +156 -8
- package/dist/index.js +237 -8
- package/dist/index.js.map +1 -1
- package/dist/literatureReview-BTPFe7eb.d.cts +64 -0
- package/dist/literatureReview-hGbfrBmw.d.ts +64 -0
- package/dist/pipeline/index.cjs +1570 -241
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -9
- package/dist/pipeline/index.d.ts +49 -9
- package/dist/pipeline/index.js +5 -4
- package/dist/webScraper-BhiQ8s9x.d.ts +68 -0
- package/dist/webScraper-D9Ldus9p.d.cts +68 -0
- package/package.json +8 -9
- package/dist/chunk-47L7ZHMY.js +0 -316
- package/dist/chunk-47L7ZHMY.js.map +0 -1
- package/dist/chunk-63X2HPCI.js +0 -47
- package/dist/chunk-63X2HPCI.js.map +0 -1
- package/dist/chunk-67V2GJGK.js.map +0 -1
- package/dist/chunk-6Q2XCUG7.js +0 -40
- package/dist/chunk-6Q2XCUG7.js.map +0 -1
- package/dist/chunk-KGCDYFFD.js +0 -3
- package/dist/chunk-MHAYD555.js +0 -239
- package/dist/chunk-MHAYD555.js.map +0 -1
- package/dist/chunk-MPLDSHFM.js +0 -396
- package/dist/chunk-MPLDSHFM.js.map +0 -1
- package/dist/chunk-QZNLB5X4.js +0 -65
- package/dist/chunk-QZNLB5X4.js.map +0 -1
- package/dist/literatureReview-C4AgVljZ.d.ts +0 -43
- package/dist/literatureReview-Dlods4Aw.d.cts +0 -43
- package/dist/webScraper-pKzdXhfI.d.ts +0 -46
- package/dist/webScraper-yBcYOHoL.d.cts +0 -46
package/dist/harvester/index.cjs
CHANGED
|
@@ -1,46 +1,95 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
// src/harvester/domainClassifier.ts
|
|
4
|
-
|
|
4
|
+
var HARDWARE_PATTERNS = [
|
|
5
|
+
/\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
|
|
6
|
+
/\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
|
|
7
|
+
/(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
|
|
8
|
+
];
|
|
9
|
+
var PLATFORM_PATTERNS = [
|
|
10
|
+
/\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
|
|
11
|
+
/(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
|
|
12
|
+
];
|
|
13
|
+
var ACADEMIC_PATTERNS = [
|
|
14
|
+
/\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
|
|
15
|
+
/(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
|
|
16
|
+
];
|
|
17
|
+
function inferPrimaryCodeLanguage(topic, context, domain) {
|
|
18
|
+
const combined = [
|
|
19
|
+
topic,
|
|
20
|
+
context?.courseTitle,
|
|
21
|
+
context?.hardwarePlatform,
|
|
22
|
+
context?.softwareEnv,
|
|
23
|
+
...context?.inScopeTopics || []
|
|
24
|
+
].filter(Boolean).join(" ").toLowerCase();
|
|
25
|
+
if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
|
|
26
|
+
if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
|
|
27
|
+
if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
|
|
28
|
+
if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
|
|
29
|
+
if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
|
|
30
|
+
if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
|
|
31
|
+
if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
|
|
32
|
+
if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
|
|
33
|
+
if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
|
|
34
|
+
if (/\b(rust|cargo)\b/.test(combined)) return "rust";
|
|
35
|
+
if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
|
|
36
|
+
if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
|
|
37
|
+
if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
|
|
38
|
+
if (domain === "hardware") return "cpp";
|
|
39
|
+
return "text";
|
|
40
|
+
}
|
|
41
|
+
function classifyTechnicalDomain(topic, explicitDomain, context) {
|
|
5
42
|
if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
|
|
6
43
|
const domain = explicitDomain.toLowerCase();
|
|
7
44
|
return {
|
|
8
45
|
domain,
|
|
9
46
|
confidence: 1,
|
|
10
47
|
rationale: "Explicitly configured by user flag.",
|
|
11
|
-
recommendedExtractors: getExtractorsForDomain(domain)
|
|
48
|
+
recommendedExtractors: getExtractorsForDomain(domain),
|
|
49
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
|
|
12
50
|
};
|
|
13
51
|
}
|
|
14
|
-
const
|
|
15
|
-
|
|
52
|
+
const combined = [
|
|
53
|
+
topic,
|
|
54
|
+
context?.courseTitle,
|
|
55
|
+
context?.positioning,
|
|
56
|
+
context?.hardwarePlatform,
|
|
57
|
+
context?.softwareEnv,
|
|
58
|
+
...context?.inScopeTopics || []
|
|
59
|
+
].filter(Boolean).join(" ");
|
|
60
|
+
if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
|
|
16
61
|
return {
|
|
17
62
|
domain: "hardware",
|
|
18
63
|
confidence: 0.95,
|
|
19
|
-
rationale: "Topic contains microcontrollers, sensors, or physical electronics keywords.",
|
|
20
|
-
recommendedExtractors:
|
|
64
|
+
rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
|
|
65
|
+
recommendedExtractors: getExtractorsForDomain("hardware"),
|
|
66
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
|
|
21
67
|
};
|
|
22
68
|
}
|
|
23
|
-
if (
|
|
69
|
+
if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
|
|
24
70
|
return {
|
|
25
71
|
domain: "platform",
|
|
26
72
|
confidence: 0.9,
|
|
27
|
-
rationale: "Topic represents a cloud infrastructure, hosting, or BaaS platform.",
|
|
28
|
-
recommendedExtractors:
|
|
73
|
+
rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
|
|
74
|
+
recommendedExtractors: getExtractorsForDomain("platform"),
|
|
75
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
|
|
29
76
|
};
|
|
30
77
|
}
|
|
31
|
-
if (
|
|
78
|
+
if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
|
|
32
79
|
return {
|
|
33
80
|
domain: "academic",
|
|
34
81
|
confidence: 0.9,
|
|
35
82
|
rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
|
|
36
|
-
recommendedExtractors:
|
|
83
|
+
recommendedExtractors: getExtractorsForDomain("academic"),
|
|
84
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
|
|
37
85
|
};
|
|
38
86
|
}
|
|
39
87
|
return {
|
|
40
88
|
domain: "software",
|
|
41
89
|
confidence: 0.85,
|
|
42
|
-
rationale: "Topic is a programming language, client/server library, or software
|
|
43
|
-
recommendedExtractors:
|
|
90
|
+
rationale: "Topic is a programming language, game engine, client/server library, or software application.",
|
|
91
|
+
recommendedExtractors: getExtractorsForDomain("software"),
|
|
92
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
|
|
44
93
|
};
|
|
45
94
|
}
|
|
46
95
|
function getExtractorsForDomain(domain) {
|
|
@@ -61,15 +110,24 @@ function getExtractorsForDomain(domain) {
|
|
|
61
110
|
var MultiSourceSearchEngine = class {
|
|
62
111
|
tavilyKey;
|
|
63
112
|
exaKey;
|
|
113
|
+
serperKey;
|
|
114
|
+
allowEnvFallback;
|
|
64
115
|
customAdapter;
|
|
65
116
|
constructor(options = {}) {
|
|
66
|
-
this.
|
|
67
|
-
this.
|
|
117
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
118
|
+
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
119
|
+
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
120
|
+
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
68
121
|
this.customAdapter = options.customAdapter;
|
|
69
122
|
}
|
|
70
123
|
setCustomAdapter(adapter) {
|
|
71
124
|
this.customAdapter = adapter;
|
|
72
125
|
}
|
|
126
|
+
setApiKeys(keys) {
|
|
127
|
+
if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
|
|
128
|
+
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
129
|
+
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
130
|
+
}
|
|
73
131
|
async search(query, options = {}) {
|
|
74
132
|
const limit = options.limit ?? 5;
|
|
75
133
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -90,28 +148,37 @@ var MultiSourceSearchEngine = class {
|
|
|
90
148
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
91
149
|
}
|
|
92
150
|
}
|
|
93
|
-
const tavilyKey = options.tavilyKey || this.tavilyKey;
|
|
94
|
-
const exaKey = options.exaKey || this.exaKey;
|
|
151
|
+
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
152
|
+
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
153
|
+
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
95
154
|
if (preferred === "exa" && exaKey) {
|
|
96
|
-
const exaHits = await this.queryExa(query, exaKey, limit);
|
|
155
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
97
156
|
if (exaHits.length > 0) return exaHits;
|
|
98
157
|
}
|
|
158
|
+
if (preferred === "serper" && serperKey) {
|
|
159
|
+
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
160
|
+
if (serperHits.length > 0) return serperHits;
|
|
161
|
+
}
|
|
99
162
|
if (preferred === "duckduckgo") {
|
|
100
163
|
const ddgHits2 = await this.queryDuckDuckGo(query, limit);
|
|
101
164
|
if (ddgHits2.length > 0) return ddgHits2;
|
|
102
165
|
}
|
|
103
166
|
if (tavilyKey && (!preferred || preferred === "tavily")) {
|
|
104
167
|
try {
|
|
168
|
+
const body = {
|
|
169
|
+
api_key: tavilyKey,
|
|
170
|
+
query,
|
|
171
|
+
search_depth: "advanced",
|
|
172
|
+
include_answer: true,
|
|
173
|
+
max_results: limit
|
|
174
|
+
};
|
|
175
|
+
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
176
|
+
body.include_domains = options.domainFilter;
|
|
177
|
+
}
|
|
105
178
|
const res = await fetch("https://api.tavily.com/search", {
|
|
106
179
|
method: "POST",
|
|
107
180
|
headers: { "Content-Type": "application/json" },
|
|
108
|
-
body: JSON.stringify(
|
|
109
|
-
api_key: tavilyKey,
|
|
110
|
-
query,
|
|
111
|
-
search_depth: "advanced",
|
|
112
|
-
include_answer: true,
|
|
113
|
-
max_results: limit
|
|
114
|
-
}),
|
|
181
|
+
body: JSON.stringify(body),
|
|
115
182
|
signal: AbortSignal.timeout(12e3)
|
|
116
183
|
});
|
|
117
184
|
if (res.ok) {
|
|
@@ -131,26 +198,34 @@ var MultiSourceSearchEngine = class {
|
|
|
131
198
|
}
|
|
132
199
|
}
|
|
133
200
|
if (exaKey) {
|
|
134
|
-
const exaHits = await this.queryExa(query, exaKey, limit);
|
|
201
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
135
202
|
if (exaHits.length > 0) return exaHits;
|
|
136
203
|
}
|
|
204
|
+
if (serperKey) {
|
|
205
|
+
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
206
|
+
if (serperHits.length > 0) return serperHits;
|
|
207
|
+
}
|
|
137
208
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
138
209
|
if (ddgHits.length > 0) return ddgHits;
|
|
139
210
|
return [];
|
|
140
211
|
}
|
|
141
|
-
async queryExa(query, exaKey, limit) {
|
|
212
|
+
async queryExa(query, exaKey, limit, domainFilter) {
|
|
142
213
|
try {
|
|
214
|
+
const body = {
|
|
215
|
+
query,
|
|
216
|
+
numResults: limit,
|
|
217
|
+
useAutoprompt: true
|
|
218
|
+
};
|
|
219
|
+
if (domainFilter && domainFilter.length > 0) {
|
|
220
|
+
body.includeDomains = domainFilter;
|
|
221
|
+
}
|
|
143
222
|
const res = await fetch("https://api.exa.ai/search", {
|
|
144
223
|
method: "POST",
|
|
145
224
|
headers: {
|
|
146
225
|
"x-api-key": exaKey,
|
|
147
226
|
"Content-Type": "application/json"
|
|
148
227
|
},
|
|
149
|
-
body: JSON.stringify(
|
|
150
|
-
query,
|
|
151
|
-
numResults: limit,
|
|
152
|
-
useAutoprompt: true
|
|
153
|
-
}),
|
|
228
|
+
body: JSON.stringify(body),
|
|
154
229
|
signal: AbortSignal.timeout(1e4)
|
|
155
230
|
});
|
|
156
231
|
if (res.ok) {
|
|
@@ -170,6 +245,33 @@ var MultiSourceSearchEngine = class {
|
|
|
170
245
|
}
|
|
171
246
|
return [];
|
|
172
247
|
}
|
|
248
|
+
async querySerper(query, serperKey, limit) {
|
|
249
|
+
try {
|
|
250
|
+
const res = await fetch("https://google.serper.dev/search", {
|
|
251
|
+
method: "POST",
|
|
252
|
+
headers: {
|
|
253
|
+
"X-API-KEY": serperKey,
|
|
254
|
+
"Content-Type": "application/json"
|
|
255
|
+
},
|
|
256
|
+
body: JSON.stringify({ q: query, num: limit }),
|
|
257
|
+
signal: AbortSignal.timeout(1e4)
|
|
258
|
+
});
|
|
259
|
+
if (res.ok) {
|
|
260
|
+
const data = await res.json();
|
|
261
|
+
if (Array.isArray(data?.organic) && data.organic.length > 0) {
|
|
262
|
+
return data.organic.slice(0, limit).map((r) => ({
|
|
263
|
+
title: r.title || "Untitled",
|
|
264
|
+
url: r.link,
|
|
265
|
+
content: r.snippet || "",
|
|
266
|
+
engine: "serper"
|
|
267
|
+
}));
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
} catch (e) {
|
|
271
|
+
console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
|
|
272
|
+
}
|
|
273
|
+
return [];
|
|
274
|
+
}
|
|
173
275
|
async queryDuckDuckGo(query, limit) {
|
|
174
276
|
try {
|
|
175
277
|
const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
@@ -180,15 +282,16 @@ var MultiSourceSearchEngine = class {
|
|
|
180
282
|
if (res.ok) {
|
|
181
283
|
const html = await res.text();
|
|
182
284
|
const hits = [];
|
|
183
|
-
const resultRegex = /<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
285
|
+
const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
184
286
|
let match;
|
|
185
287
|
while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
|
|
186
|
-
const
|
|
288
|
+
const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
289
|
+
const rawUrl = match[2]?.trim();
|
|
187
290
|
const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
|
|
188
|
-
const snippet = match[
|
|
291
|
+
const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
|
|
189
292
|
if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
|
|
190
293
|
hits.push({
|
|
191
|
-
title:
|
|
294
|
+
title: rawTitle,
|
|
192
295
|
url: cleanUrl,
|
|
193
296
|
content: snippet,
|
|
194
297
|
engine: "duckduckgo"
|
|
@@ -204,15 +307,210 @@ var MultiSourceSearchEngine = class {
|
|
|
204
307
|
}
|
|
205
308
|
};
|
|
206
309
|
|
|
310
|
+
// src/harvester/youtubeTranscript.ts
|
|
311
|
+
var SPONSOR_NOISE_PATTERNS = [
|
|
312
|
+
/\b(sponsored by|this video is brought to you by|use (?:promo )?code\b|don't forget to like and subscribe|hit the bell icon|link in the description)\b/i,
|
|
313
|
+
/(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\s+này)?\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i
|
|
314
|
+
];
|
|
315
|
+
function extractYouTubeVideoId(urlOrId) {
|
|
316
|
+
const trimmed = urlOrId.trim();
|
|
317
|
+
if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;
|
|
318
|
+
try {
|
|
319
|
+
const u = new URL(trimmed);
|
|
320
|
+
const host = u.hostname.replace(/^www\./, "").toLowerCase();
|
|
321
|
+
if (host === "youtu.be") {
|
|
322
|
+
const id = u.pathname.split("/").filter(Boolean)[0];
|
|
323
|
+
return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;
|
|
324
|
+
}
|
|
325
|
+
if (host === "youtube.com" || host === "m.youtube.com" || host === "youtube-nocookie.com") {
|
|
326
|
+
const v = u.searchParams.get("v");
|
|
327
|
+
if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;
|
|
328
|
+
const parts = u.pathname.split("/").filter(Boolean);
|
|
329
|
+
if ((parts[0] === "embed" || parts[0] === "shorts" || parts[0] === "v") && parts[1]) {
|
|
330
|
+
return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
} catch {
|
|
334
|
+
}
|
|
335
|
+
return null;
|
|
336
|
+
}
|
|
337
|
+
function isYouTubeUrl(url) {
|
|
338
|
+
return extractYouTubeVideoId(url) !== null;
|
|
339
|
+
}
|
|
340
|
+
function formatTimestamp(seconds) {
|
|
341
|
+
const total = Math.max(0, Math.floor(seconds));
|
|
342
|
+
const hrs = Math.floor(total / 3600);
|
|
343
|
+
const mins = Math.floor(total % 3600 / 60);
|
|
344
|
+
const secs = total % 60;
|
|
345
|
+
if (hrs > 0) {
|
|
346
|
+
return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
|
|
347
|
+
}
|
|
348
|
+
return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
|
|
349
|
+
}
|
|
350
|
+
function decodeHtmlEntities(str) {
|
|
351
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/'/g, "'").replace(/ /g, " ").replace(/\s+/g, " ").trim();
|
|
352
|
+
}
|
|
353
|
+
function stitchCaptionSegments(segments) {
|
|
354
|
+
const cleaned = segments.map((s) => ({
|
|
355
|
+
startSec: s.startSec,
|
|
356
|
+
durationSec: s.durationSec,
|
|
357
|
+
text: decodeHtmlEntities(s.text)
|
|
358
|
+
})).filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));
|
|
359
|
+
if (cleaned.length === 0) return [];
|
|
360
|
+
const stitched = [];
|
|
361
|
+
let bucketStart = cleaned[0].startSec;
|
|
362
|
+
let bucketEnd = cleaned[0].startSec + cleaned[0].durationSec;
|
|
363
|
+
let bucketTexts = [];
|
|
364
|
+
for (const seg of cleaned) {
|
|
365
|
+
bucketTexts.push(seg.text);
|
|
366
|
+
bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);
|
|
367
|
+
const joined = bucketTexts.join(" ");
|
|
368
|
+
const elapsed = seg.startSec - bucketStart;
|
|
369
|
+
const endsSentence = /[.!?…]$/.test(seg.text);
|
|
370
|
+
if (elapsed >= 30 && endsSentence || elapsed >= 50 || joined.length >= 360) {
|
|
371
|
+
stitched.push({
|
|
372
|
+
startSec: bucketStart,
|
|
373
|
+
durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
|
|
374
|
+
timestamp: formatTimestamp(bucketStart),
|
|
375
|
+
text: joined
|
|
376
|
+
});
|
|
377
|
+
bucketTexts = [];
|
|
378
|
+
bucketStart = seg.startSec + seg.durationSec;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
if (bucketTexts.length > 0) {
|
|
382
|
+
stitched.push({
|
|
383
|
+
startSec: bucketStart,
|
|
384
|
+
durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
|
|
385
|
+
timestamp: formatTimestamp(bucketStart),
|
|
386
|
+
text: bucketTexts.join(" ")
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
return stitched;
|
|
390
|
+
}
|
|
391
|
+
var YouTubeIngestionService = class {
|
|
392
|
+
async fetchTranscript(urlOrId, preferredLanguages = ["vi", "en"]) {
|
|
393
|
+
const videoId = extractYouTubeVideoId(urlOrId);
|
|
394
|
+
if (!videoId) {
|
|
395
|
+
throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);
|
|
396
|
+
}
|
|
397
|
+
const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
|
398
|
+
const res = await fetch(canonicalUrl, {
|
|
399
|
+
headers: {
|
|
400
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
401
|
+
"Accept-Language": "vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7"
|
|
402
|
+
},
|
|
403
|
+
signal: AbortSignal.timeout(12e3)
|
|
404
|
+
});
|
|
405
|
+
if (!res.ok) {
|
|
406
|
+
throw new Error(`Failed to fetch YouTube watch page (${res.status})`);
|
|
407
|
+
}
|
|
408
|
+
const html = await res.text();
|
|
409
|
+
const playerMatch = html.match(/ytInitialPlayerResponse\s*=\s*(\{[\s\S]*?\});\s*(?:var\s|window\.|<\/script>)/);
|
|
410
|
+
let title = `YouTube Video (${videoId})`;
|
|
411
|
+
let author;
|
|
412
|
+
let shortDescription = "";
|
|
413
|
+
let rawSegments = [];
|
|
414
|
+
let selectedLang = "en";
|
|
415
|
+
let isAutoGenerated = false;
|
|
416
|
+
if (playerMatch?.[1]) {
|
|
417
|
+
try {
|
|
418
|
+
const playerData = JSON.parse(playerMatch[1]);
|
|
419
|
+
title = playerData?.videoDetails?.title || title;
|
|
420
|
+
author = playerData?.videoDetails?.author;
|
|
421
|
+
shortDescription = playerData?.videoDetails?.shortDescription || "";
|
|
422
|
+
const tracks = playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];
|
|
423
|
+
if (tracks.length > 0) {
|
|
424
|
+
let chosenTrack = tracks.find(
|
|
425
|
+
(t) => preferredLanguages.includes(t.languageCode) && t.kind !== "asr"
|
|
426
|
+
) || tracks.find((t) => preferredLanguages.includes(t.languageCode)) || tracks[0];
|
|
427
|
+
if (chosenTrack?.baseUrl) {
|
|
428
|
+
selectedLang = chosenTrack.languageCode || "en";
|
|
429
|
+
isAutoGenerated = chosenTrack.kind === "asr";
|
|
430
|
+
const trackRes = await fetch(chosenTrack.baseUrl, {
|
|
431
|
+
signal: AbortSignal.timeout(1e4)
|
|
432
|
+
});
|
|
433
|
+
if (trackRes.ok) {
|
|
434
|
+
const xml = await trackRes.text();
|
|
435
|
+
const textMatches = [
|
|
436
|
+
...xml.matchAll(/<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g)
|
|
437
|
+
];
|
|
438
|
+
for (const m of textMatches) {
|
|
439
|
+
rawSegments.push({
|
|
440
|
+
startSec: parseFloat(m[1] || "0"),
|
|
441
|
+
durationSec: parseFloat(m[2] || "2"),
|
|
442
|
+
text: (m[3] || "").replace(/<[^>]+>/g, " ")
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
} catch {
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
if (title === `YouTube Video (${videoId})`) {
|
|
452
|
+
const titleMeta = html.match(/<title>([\s\S]*?)<\/title>/i);
|
|
453
|
+
if (titleMeta?.[1]) {
|
|
454
|
+
title = decodeHtmlEntities(titleMeta[1].replace(/\s*-\s*YouTube$/i, ""));
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
const stitched = stitchCaptionSegments(rawSegments);
|
|
458
|
+
const mdLines = [
|
|
459
|
+
`# ${title}`,
|
|
460
|
+
`> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ""} | **Language:** \`${selectedLang}\`${isAutoGenerated ? " (Auto-caption)" : ""}`
|
|
461
|
+
];
|
|
462
|
+
if (shortDescription.trim()) {
|
|
463
|
+
mdLines.push(`
|
|
464
|
+
## Video Overview & Description
|
|
465
|
+
${shortDescription.trim().slice(0, 1500)}`);
|
|
466
|
+
}
|
|
467
|
+
if (stitched.length > 0) {
|
|
468
|
+
mdLines.push(`
|
|
469
|
+
## Timestamped Transcript`);
|
|
470
|
+
for (const seg of stitched) {
|
|
471
|
+
mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
return {
|
|
475
|
+
videoId,
|
|
476
|
+
url: canonicalUrl,
|
|
477
|
+
title,
|
|
478
|
+
author,
|
|
479
|
+
language: selectedLang,
|
|
480
|
+
isAutoGenerated,
|
|
481
|
+
segments: stitched,
|
|
482
|
+
stitchedMarkdown: mdLines.join("\n")
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
};
|
|
486
|
+
|
|
207
487
|
// src/harvester/webScraper.ts
|
|
488
|
+
function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
|
|
489
|
+
const withoutCode = markdown.replace(/```[\s\S]*?```/g, "\n");
|
|
490
|
+
const blocks = withoutCode.split(/\n{2,}/).map((b) => b.replace(/\s+/g, " ").trim()).filter((b) => {
|
|
491
|
+
if (b.length < 60 || b.length > 900) return false;
|
|
492
|
+
if (b.startsWith("#") || b.startsWith("|")) return false;
|
|
493
|
+
const linkRatio = (b.match(/\[[^\]]+\]\([^)]+\)/g) || []).join("").length / b.length;
|
|
494
|
+
if (linkRatio > 0.45) return false;
|
|
495
|
+
if (/\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\b/i.test(b)) {
|
|
496
|
+
return false;
|
|
497
|
+
}
|
|
498
|
+
return true;
|
|
499
|
+
});
|
|
500
|
+
return blocks.slice(0, maxExcerpts);
|
|
501
|
+
}
|
|
208
502
|
var WebScraper = class {
|
|
209
503
|
crawl4aiHost;
|
|
210
504
|
jinaKey;
|
|
505
|
+
allowEnvFallback;
|
|
211
506
|
customAdapter;
|
|
507
|
+
youtubeService;
|
|
212
508
|
constructor(options = {}) {
|
|
213
|
-
this.
|
|
214
|
-
this.
|
|
509
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
510
|
+
this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
|
|
511
|
+
this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
|
|
215
512
|
this.customAdapter = options.customAdapter;
|
|
513
|
+
this.youtubeService = new YouTubeIngestionService();
|
|
216
514
|
}
|
|
217
515
|
setCustomAdapter(adapter) {
|
|
218
516
|
this.customAdapter = adapter;
|
|
@@ -226,12 +524,22 @@ var WebScraper = class {
|
|
|
226
524
|
try {
|
|
227
525
|
const result = await adapter.crawl(targetUrl);
|
|
228
526
|
if (result && result.markdownContent) {
|
|
229
|
-
return this.packageDocument(targetUrl, result.markdownContent, "custom");
|
|
527
|
+
return this.packageDocument(targetUrl, result.markdownContent, "custom", result.title);
|
|
230
528
|
}
|
|
231
529
|
} catch (e) {
|
|
232
530
|
console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);
|
|
233
531
|
}
|
|
234
532
|
}
|
|
533
|
+
if (isYouTubeUrl(targetUrl)) {
|
|
534
|
+
try {
|
|
535
|
+
const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);
|
|
536
|
+
if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {
|
|
537
|
+
return this.packageDocument(yt.url, yt.stitchedMarkdown, "youtube", yt.title);
|
|
538
|
+
}
|
|
539
|
+
} catch (e) {
|
|
540
|
+
console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
235
543
|
try {
|
|
236
544
|
const jinaUrl = `https://r.jina.ai/${targetUrl}`;
|
|
237
545
|
const jinaKey = options.jinaKey || this.jinaKey;
|
|
@@ -282,35 +590,435 @@ var WebScraper = class {
|
|
|
282
590
|
signal: AbortSignal.timeout(1e4)
|
|
283
591
|
});
|
|
284
592
|
if (res.ok) {
|
|
285
|
-
const
|
|
286
|
-
const
|
|
287
|
-
|
|
593
|
+
const rawHtml = await res.text();
|
|
594
|
+
const titleTag = rawHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
|
|
595
|
+
const clean = rawHtml.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "").replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, "").replace(/<h([1-3])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, lvl, txt) => `
|
|
596
|
+
${"#".repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, "").trim()}
|
|
597
|
+
`).replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => `
|
|
598
|
+
\`\`\`
|
|
599
|
+
${code.replace(/<[^>]+>/g, "").trim()}
|
|
600
|
+
\`\`\`
|
|
601
|
+
`).replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, p) => `
|
|
602
|
+
${p.replace(/<[^>]+>/g, " ").trim()}
|
|
603
|
+
`).replace(/<[^>]+>/g, " ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
604
|
+
return this.packageDocument(targetUrl, clean, "direct", titleTag);
|
|
288
605
|
}
|
|
289
606
|
} catch (e) {
|
|
290
607
|
console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);
|
|
291
608
|
}
|
|
292
609
|
throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
|
|
293
610
|
}
|
|
294
|
-
packageDocument(url, markdown, provider) {
|
|
295
|
-
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(
|
|
296
|
-
const title = titleMatch ? titleMatch[1].trim() : url.split("/").pop() || "Document";
|
|
297
|
-
const
|
|
298
|
-
const
|
|
611
|
+
packageDocument(url, markdown, provider, explicitTitle) {
|
|
612
|
+
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
|
|
613
|
+
const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
|
|
614
|
+
const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```/g)];
|
|
615
|
+
const extractedCodeBlocks = codeBlockMatches.map((m) => ({
|
|
616
|
+
language: (m[1] || "").trim().toLowerCase(),
|
|
617
|
+
code: (m[2] || "").trim()
|
|
618
|
+
})).filter((b) => b.code.length > 0);
|
|
619
|
+
const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);
|
|
620
|
+
const tableBlocks = markdown.match(/(?:^\|.+\|\s*\n)+^\|[-:| ]+\|\s*\n(?:^\|.+\|\s*\n?)+/gm) || markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
|
|
621
|
+
const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);
|
|
622
|
+
const extractedHeadings = [...markdown.matchAll(/^#{1,3}\s+(.+)$/gm)].map((m) => m[1].trim()).filter((h) => h.length > 2 && h.length < 120);
|
|
623
|
+
const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);
|
|
299
624
|
return {
|
|
300
625
|
url,
|
|
301
626
|
title,
|
|
302
627
|
markdownContent: markdown,
|
|
303
628
|
byteSize: Buffer.byteLength(markdown, "utf-8"),
|
|
304
|
-
extractedCodeSnippets
|
|
305
|
-
|
|
629
|
+
extractedCodeSnippets,
|
|
630
|
+
extractedCodeBlocks,
|
|
631
|
+
extractedTables,
|
|
632
|
+
extractedHeadings,
|
|
633
|
+
keyExcerpts,
|
|
634
|
+
tablesCount: extractedTables.length,
|
|
306
635
|
fetchProvider: provider
|
|
307
636
|
};
|
|
308
637
|
}
|
|
309
638
|
};
|
|
310
639
|
|
|
640
|
+
// src/harvester/termExtractor.ts
|
|
641
|
+
var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
642
|
+
"l\xE0",
|
|
643
|
+
"c\u1EE7a",
|
|
644
|
+
"trong",
|
|
645
|
+
"\u0111\u01B0\u1EE3c",
|
|
646
|
+
"nh\u1EEFng",
|
|
647
|
+
"c\xE1c",
|
|
648
|
+
"cho",
|
|
649
|
+
"v\u1EDBi",
|
|
650
|
+
"khi",
|
|
651
|
+
"n\u1EBFu",
|
|
652
|
+
"th\xEC",
|
|
653
|
+
"v\xE0",
|
|
654
|
+
"ho\u1EB7c",
|
|
655
|
+
"\u0111\u1EC3",
|
|
656
|
+
"t\u1EEB",
|
|
657
|
+
"\u0111\u1EBFn",
|
|
658
|
+
"v\u1EC1",
|
|
659
|
+
"n\xE0y",
|
|
660
|
+
"\u0111\xF3",
|
|
661
|
+
"m\u1ED9t",
|
|
662
|
+
"nh\u01B0",
|
|
663
|
+
"s\u1EBD",
|
|
664
|
+
"\u0111ang",
|
|
665
|
+
"\u0111\xE3",
|
|
666
|
+
"kh\xF4ng",
|
|
667
|
+
"c\xF3",
|
|
668
|
+
"ra",
|
|
669
|
+
"v\xE0o",
|
|
670
|
+
"b\u1EB1ng",
|
|
671
|
+
"theo",
|
|
672
|
+
"tr\xEAn",
|
|
673
|
+
"d\u01B0\u1EDBi",
|
|
674
|
+
"qua",
|
|
675
|
+
"nh\u1EDD",
|
|
676
|
+
"v\xEC",
|
|
677
|
+
"do",
|
|
678
|
+
"t\u1EA1i",
|
|
679
|
+
"b\u1EDFi",
|
|
680
|
+
"nh\u01B0ng",
|
|
681
|
+
"m\xE0",
|
|
682
|
+
"c\u0169ng",
|
|
683
|
+
"v\u1EABn",
|
|
684
|
+
"r\u1EA5t",
|
|
685
|
+
"qu\xE1",
|
|
686
|
+
"h\u01A1n",
|
|
687
|
+
"nh\u1EA5t",
|
|
688
|
+
"c\u1EA7n",
|
|
689
|
+
"ph\u1EA3i",
|
|
690
|
+
"n\xEAn",
|
|
691
|
+
"th\u1EC3",
|
|
692
|
+
"vi\u1EC7c",
|
|
693
|
+
"c\xE1ch",
|
|
694
|
+
"ph\u1EA7n",
|
|
695
|
+
"b\xE0i",
|
|
696
|
+
"bu\u1ED5i",
|
|
697
|
+
"h\u1ECDc",
|
|
698
|
+
"sinh",
|
|
699
|
+
"gi\xE1o",
|
|
700
|
+
"vi\xEAn",
|
|
701
|
+
"ng\u01B0\u1EDDi",
|
|
702
|
+
"d\xF9ng",
|
|
703
|
+
"ch\xFAng",
|
|
704
|
+
"ta",
|
|
705
|
+
"b\u1EA1n",
|
|
706
|
+
"em",
|
|
707
|
+
"gi\xFAp",
|
|
708
|
+
"l\xE0m",
|
|
709
|
+
"t\u1EA1o",
|
|
710
|
+
"s\u1EED",
|
|
711
|
+
"d\u1EE5ng",
|
|
712
|
+
"th\u1EF1c",
|
|
713
|
+
"hi\u1EC7n",
|
|
714
|
+
"hi\u1EC3u",
|
|
715
|
+
"bi\u1EBFt",
|
|
716
|
+
"xem",
|
|
717
|
+
"th\xEAm",
|
|
718
|
+
"sau",
|
|
719
|
+
"tr\u01B0\u1EDBc"
|
|
720
|
+
]);
|
|
721
|
+
var ENGLISH_STOPWORDS = /* @__PURE__ */ new Set([
|
|
722
|
+
"the",
|
|
723
|
+
"and",
|
|
724
|
+
"for",
|
|
725
|
+
"with",
|
|
726
|
+
"that",
|
|
727
|
+
"this",
|
|
728
|
+
"from",
|
|
729
|
+
"into",
|
|
730
|
+
"your",
|
|
731
|
+
"you",
|
|
732
|
+
"are",
|
|
733
|
+
"was",
|
|
734
|
+
"were",
|
|
735
|
+
"have",
|
|
736
|
+
"has",
|
|
737
|
+
"had",
|
|
738
|
+
"will",
|
|
739
|
+
"would",
|
|
740
|
+
"can",
|
|
741
|
+
"could",
|
|
742
|
+
"should",
|
|
743
|
+
"about",
|
|
744
|
+
"more",
|
|
745
|
+
"other",
|
|
746
|
+
"such",
|
|
747
|
+
"than",
|
|
748
|
+
"then",
|
|
749
|
+
"them",
|
|
750
|
+
"these",
|
|
751
|
+
"those",
|
|
752
|
+
"their",
|
|
753
|
+
"what",
|
|
754
|
+
"when",
|
|
755
|
+
"where",
|
|
756
|
+
"which",
|
|
757
|
+
"while",
|
|
758
|
+
"who",
|
|
759
|
+
"how",
|
|
760
|
+
"why",
|
|
761
|
+
"each",
|
|
762
|
+
"some",
|
|
763
|
+
"many",
|
|
764
|
+
"most",
|
|
765
|
+
"also",
|
|
766
|
+
"only",
|
|
767
|
+
"just",
|
|
768
|
+
"very",
|
|
769
|
+
"using",
|
|
770
|
+
"used",
|
|
771
|
+
"use",
|
|
772
|
+
"overview",
|
|
773
|
+
"introduction",
|
|
774
|
+
"getting",
|
|
775
|
+
"started",
|
|
776
|
+
"guide",
|
|
777
|
+
"tutorial",
|
|
778
|
+
"example",
|
|
779
|
+
"examples",
|
|
780
|
+
"documentation"
|
|
781
|
+
]);
|
|
782
|
+
var VIETNAMESE_TECHNICAL_HEADS = /^(cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|chuỗi|đối tượng|lớp|phương thức|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|mạng|giao thức|khái niệm|mô hình|kiến trúc|kiểm thử|gỡ lỗi)\b/i;
|
|
783
|
+
var SYNTAX_OR_API_PATTERN = /(?:`[^`]+`|\b[a-z]+[A-Z][a-zA-Z0-9]*\b|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b|@[A-Za-z]+|\b[a-z_]+\(\)|\b(?:pinMode|digitalWrite|digitalRead|analogRead|analogWrite|async|await|actor|struct|class|func|def|let|const|var)\b)/;
|
|
784
|
+
var HARDWARE_COMPONENT_PATTERN = /\b(esp32|arduino|micro:?bit|raspberry|sensor|led|servo|ultrasonic|ldr|dht11|dht22|breadboard|gpio|pwm|i2c|spi|uart|cảm biến|động cơ|mạch|đèn led|vi điều khiển)\b/i;
|
|
785
|
+
function normalizeTermKey(raw) {
|
|
786
|
+
return raw.normalize("NFC").toLowerCase().replace(/[`*_~#"']/g, "").replace(/\s+/g, " ").trim();
|
|
787
|
+
}
|
|
788
|
+
function cleanDisplayTerm(raw) {
|
|
789
|
+
return raw.normalize("NFC").replace(/^[-*•#\d.)\s]+/, "").replace(/[`*_~]+/g, "").replace(/\s+/g, " ").trim();
|
|
790
|
+
}
|
|
791
|
+
function isNoiseTerm(term) {
|
|
792
|
+
const clean = normalizeTermKey(term);
|
|
793
|
+
if (clean.length < 2 || clean.length > 65) return true;
|
|
794
|
+
if (/^\d+$/.test(clean)) return true;
|
|
795
|
+
if (/^https?:\/\//i.test(clean)) return true;
|
|
796
|
+
if (ENGLISH_STOPWORDS.has(clean) || VIETNAMESE_STOPWORDS.has(clean)) return true;
|
|
797
|
+
const words = clean.split(" ");
|
|
798
|
+
if (words.length > 6) return true;
|
|
799
|
+
if (words.length === 1 && clean.length < 3 && !/^(ai|ui|ux|os|io|db|id|ip)$/i.test(clean)) return true;
|
|
800
|
+
return false;
|
|
801
|
+
}
|
|
802
|
+
function classifyMarrAndCategory(term, domain) {
|
|
803
|
+
if (HARDWARE_COMPONENT_PATTERN.test(term)) {
|
|
804
|
+
return { category: "hardware_component", marrLayer: "CIO", bloomLevel: "apply" };
|
|
805
|
+
}
|
|
806
|
+
if (SYNTAX_OR_API_PATTERN.test(term)) {
|
|
807
|
+
return { category: "api_syntax", marrLayer: "SIO", bloomLevel: "apply" };
|
|
808
|
+
}
|
|
809
|
+
if (/\b(architecture|pattern|model|design|debugging|verification|mô hình|kiến trúc|tư duy|quy trình)\b/i.test(term)) {
|
|
810
|
+
return { category: "pedagogical_pattern", marrLayer: "ULO", bloomLevel: "analyze" };
|
|
811
|
+
}
|
|
812
|
+
return {
|
|
813
|
+
category: "core_concept",
|
|
814
|
+
marrLayer: domain === "academic" ? "ULO" : "CIO",
|
|
815
|
+
bloomLevel: "understand"
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
function findSentenceDefinition(term, markdown) {
|
|
819
|
+
const clean = cleanDisplayTerm(term);
|
|
820
|
+
const escaped = clean.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
821
|
+
const sentences = markdown.replace(/```[\s\S]*?```/g, " ").split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").trim()).filter((s) => s.length >= 25 && s.length <= 280);
|
|
822
|
+
const re = new RegExp(escaped, "i");
|
|
823
|
+
const match = sentences.find((s) => re.test(s));
|
|
824
|
+
return match || `Core concept in ${clean} extracted from primary technical reference.`;
|
|
825
|
+
}
|
|
826
|
+
var PedagogicalTermExtractor = class {
|
|
827
|
+
extractDeterministic(topic, domain, docs, maxTerms = 15) {
|
|
828
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
829
|
+
let orderCounter = 0;
|
|
830
|
+
const upsertCandidate = (rawTerm, weight, resourceId, markdown, pair) => {
|
|
831
|
+
const display = cleanDisplayTerm(rawTerm);
|
|
832
|
+
if (isNoiseTerm(display)) return;
|
|
833
|
+
const key = normalizeTermKey(display);
|
|
834
|
+
const existing = candidates.get(key);
|
|
835
|
+
if (existing) {
|
|
836
|
+
existing.score += weight;
|
|
837
|
+
existing.sourceIds.add(resourceId);
|
|
838
|
+
if (pair?.termVi && !existing.termVi) existing.termVi = pair.termVi;
|
|
839
|
+
if (pair?.termEn && !existing.termEn) existing.termEn = pair.termEn;
|
|
840
|
+
} else {
|
|
841
|
+
candidates.set(key, {
|
|
842
|
+
display,
|
|
843
|
+
termVi: pair?.termVi,
|
|
844
|
+
termEn: pair?.termEn,
|
|
845
|
+
score: weight,
|
|
846
|
+
firstSeenOrder: orderCounter++,
|
|
847
|
+
sourceIds: /* @__PURE__ */ new Set([resourceId]),
|
|
848
|
+
definition: findSentenceDefinition(display, markdown)
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
};
|
|
852
|
+
for (const { resourceId, doc } of docs) {
|
|
853
|
+
const md = doc.markdownContent || "";
|
|
854
|
+
const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
|
|
855
|
+
const bilingualMatches = mdWithoutCode.matchAll(
|
|
856
|
+
/([A-ZÀÁẠẢÃÂẦẤẬẨẪĂẰẮẶẲẴÈÉẸẺẼÊỀẾỆỂỄÌÍỊỈĨÒÓỌỎÕÔỒỐỘỔỖƠỜỚỢỞỠÙÚỤỦŨƯỪỨỰỬỮỲÝỴỶỸĐ][a-zA-Z0-9\u00C0-\u1EF9\s/-]{2,40}?)\s*\(([A-Za-z0-9\u00C0-\u1EF9\s/_:-]{2,40})\)/g
|
|
857
|
+
);
|
|
858
|
+
for (const m of bilingualMatches) {
|
|
859
|
+
const left = cleanDisplayTerm(m[1] || "");
|
|
860
|
+
const right = cleanDisplayTerm(m[2] || "");
|
|
861
|
+
if (!isNoiseTerm(left) && !isNoiseTerm(right)) {
|
|
862
|
+
const leftHasVi = /[\u00C0-\u1EF9]/.test(left);
|
|
863
|
+
const rightHasVi = /[\u00C0-\u1EF9]/.test(right);
|
|
864
|
+
if (leftHasVi && !rightHasVi) {
|
|
865
|
+
upsertCandidate(`${left} (${right})`, 6, resourceId, mdWithoutCode, { termVi: left, termEn: right });
|
|
866
|
+
} else if (!leftHasVi && rightHasVi) {
|
|
867
|
+
upsertCandidate(`${right} (${left})`, 6, resourceId, mdWithoutCode, { termVi: right, termEn: left });
|
|
868
|
+
} else {
|
|
869
|
+
upsertCandidate(left, 4, resourceId, mdWithoutCode);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
for (const heading of doc.extractedHeadings || []) {
|
|
874
|
+
const cleaned = cleanDisplayTerm(heading.replace(/^\d+(\.\d+)*\s*/, ""));
|
|
875
|
+
if (!isNoiseTerm(cleaned)) {
|
|
876
|
+
upsertCandidate(cleaned, 4, resourceId, mdWithoutCode);
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
for (const m of mdWithoutCode.matchAll(/\*\*([^*\n]{3,45})\*\*/g)) {
|
|
880
|
+
upsertCandidate(m[1] || "", 3.5, resourceId, mdWithoutCode);
|
|
881
|
+
}
|
|
882
|
+
for (const m of mdWithoutCode.matchAll(/`([a-zA-Z_@][a-zA-Z0-9_.:()-]{2,35})`/g)) {
|
|
883
|
+
upsertCandidate(m[1] || "", 3, resourceId, mdWithoutCode);
|
|
884
|
+
}
|
|
885
|
+
for (const line of mdWithoutCode.split("\n")) {
|
|
886
|
+
const clauses = line.split(/[.,;:!?()[\]{}"“”‘’/\\|]+/);
|
|
887
|
+
for (const clause of clauses) {
|
|
888
|
+
const trimmed = clause.trim();
|
|
889
|
+
const viMatch = trimmed.match(
|
|
890
|
+
/\b((?:cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|đối tượng|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|giao thức|mô hình|kiến trúc|kiểm thử|gỡ lỗi)(?:\s+[a-zA-Z0-9\u00C0-\u1EF9_-]+){1,3})\b/i
|
|
891
|
+
);
|
|
892
|
+
if (viMatch?.[1]) {
|
|
893
|
+
const tokens = viMatch[1].trim().split(/\s+/);
|
|
894
|
+
while (tokens.length > 1 && VIETNAMESE_STOPWORDS.has(tokens[tokens.length - 1].toLowerCase())) {
|
|
895
|
+
tokens.pop();
|
|
896
|
+
}
|
|
897
|
+
const phrase = tokens.join(" ");
|
|
898
|
+
if (tokens.length >= 2 && VIETNAMESE_TECHNICAL_HEADS.test(phrase)) {
|
|
899
|
+
upsertCandidate(phrase, 3.5, resourceId, mdWithoutCode, { termVi: phrase });
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
const sorted = [...candidates.values()].sort((a, b) => b.score - a.score || a.firstSeenOrder - b.firstSeenOrder).slice(0, maxTerms);
|
|
906
|
+
const orderedByAppearance = [...sorted].sort((a, b) => a.firstSeenOrder - b.firstSeenOrder);
|
|
907
|
+
const maxScore = Math.max(1, sorted[0]?.score || 1);
|
|
908
|
+
const concepts = orderedByAppearance.map((item, idx) => {
|
|
909
|
+
const id = `C_${String(idx + 1).padStart(2, "0")}`;
|
|
910
|
+
const { category, marrLayer, bloomLevel } = classifyMarrAndCategory(item.display, domain);
|
|
911
|
+
const prerequisites = [];
|
|
912
|
+
return {
|
|
913
|
+
id,
|
|
914
|
+
term: item.display,
|
|
915
|
+
termVi: item.termVi,
|
|
916
|
+
termEn: item.termEn,
|
|
917
|
+
category,
|
|
918
|
+
marrLayer,
|
|
919
|
+
bloomLevel,
|
|
920
|
+
salienceScore: Number(Math.min(10, Math.max(1, item.score / maxScore * 10)).toFixed(1)),
|
|
921
|
+
definition: item.definition,
|
|
922
|
+
prerequisites,
|
|
923
|
+
sourceIds: [...item.sourceIds]
|
|
924
|
+
};
|
|
925
|
+
});
|
|
926
|
+
return {
|
|
927
|
+
concepts,
|
|
928
|
+
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
929
|
+
isLlmRefined: false,
|
|
930
|
+
unrefinedCandidatesCount: candidates.size
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
async extractWithOptionalLlm(topic, domain, docs, options = {}) {
|
|
934
|
+
const detBundle = this.extractDeterministic(topic, domain, docs, options.maxTerms ?? 15);
|
|
935
|
+
if (!options.llm || docs.length === 0) {
|
|
936
|
+
return detBundle;
|
|
937
|
+
}
|
|
938
|
+
try {
|
|
939
|
+
const corpusPreview = docs.slice(0, 6).map(
|
|
940
|
+
(d) => `[${d.resourceId}] ${d.doc.title} (${d.doc.url}):
|
|
941
|
+
${(d.doc.keyExcerpts || []).slice(0, 3).join("\n")}`
|
|
942
|
+
).join("\n\n");
|
|
943
|
+
const candidateSeed = detBundle.concepts.slice(0, 12).map((c) => `${c.term} (${c.sourceIds.join(",")})`).join("; ");
|
|
944
|
+
const prompt = [
|
|
945
|
+
`Analyze the harvested technical & pedagogical corpus for topic "${topic}" (Domain: ${domain}).`,
|
|
946
|
+
options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]} years old.` : "",
|
|
947
|
+
options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}.` : "",
|
|
948
|
+
options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}.` : "",
|
|
949
|
+
`Deterministic Candidate Terms: ${candidateSeed}`,
|
|
950
|
+
`
|
|
951
|
+
Harvested Source Excerpts:
|
|
952
|
+
${corpusPreview}`,
|
|
953
|
+
`
|
|
954
|
+
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
955
|
+
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
956
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
|
|
957
|
+
].filter(Boolean).join("\n");
|
|
958
|
+
const rawResponse = await options.llm(prompt, {
|
|
959
|
+
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
960
|
+
temperature: 0.2,
|
|
961
|
+
jsonMode: true
|
|
962
|
+
});
|
|
963
|
+
const jsonMatch = rawResponse.match(/\[[\s\S]*\]/);
|
|
964
|
+
if (jsonMatch) {
|
|
965
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
966
|
+
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
967
|
+
const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
|
|
968
|
+
const concepts = parsed.map((item, idx) => {
|
|
969
|
+
const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
|
|
970
|
+
const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
|
|
971
|
+
return {
|
|
972
|
+
id,
|
|
973
|
+
term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
|
|
974
|
+
termVi: item.termVi ? String(item.termVi) : void 0,
|
|
975
|
+
termEn: item.termEn ? String(item.termEn) : void 0,
|
|
976
|
+
category: ["core_concept", "api_syntax", "hardware_component", "pedagogical_pattern"].includes(item.category) ? item.category : "core_concept",
|
|
977
|
+
marrLayer: ["ULO", "CIO", "SIO"].includes(item.marrLayer) ? item.marrLayer : "CIO",
|
|
978
|
+
bloomLevel: ["remember", "understand", "apply", "analyze", "evaluate", "create"].includes(item.bloomLevel) ? item.bloomLevel : "understand",
|
|
979
|
+
salienceScore: Number(item.salienceScore) || 8,
|
|
980
|
+
definition: String(item.definition || ""),
|
|
981
|
+
prerequisites: prereqs,
|
|
982
|
+
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
|
|
983
|
+
};
|
|
984
|
+
});
|
|
985
|
+
return {
|
|
986
|
+
concepts,
|
|
987
|
+
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
988
|
+
isLlmRefined: true,
|
|
989
|
+
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
} catch (e) {
|
|
994
|
+
console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
|
|
995
|
+
}
|
|
996
|
+
return detBundle;
|
|
997
|
+
}
|
|
998
|
+
formatConceptDagTsv(concepts) {
|
|
999
|
+
const header = "id term term_vi term_en marr_layer bloom_level salience prerequisites sources definition";
|
|
1000
|
+
const rows = concepts.map(
|
|
1001
|
+
(c) => [
|
|
1002
|
+
c.id,
|
|
1003
|
+
c.term.replace(/\t|\n/g, " "),
|
|
1004
|
+
(c.termVi || "").replace(/\t|\n/g, " "),
|
|
1005
|
+
(c.termEn || "").replace(/\t|\n/g, " "),
|
|
1006
|
+
c.marrLayer,
|
|
1007
|
+
c.bloomLevel,
|
|
1008
|
+
c.salienceScore,
|
|
1009
|
+
c.prerequisites.join(","),
|
|
1010
|
+
c.sourceIds.join(","),
|
|
1011
|
+
c.definition.replace(/\t|\n/g, " ")
|
|
1012
|
+
].join(" ")
|
|
1013
|
+
);
|
|
1014
|
+
return [header, ...rows].join("\n");
|
|
1015
|
+
}
|
|
1016
|
+
};
|
|
1017
|
+
|
|
311
1018
|
// src/harvester/catalogBuilder.ts
|
|
312
1019
|
var CatalogBuilder = class {
|
|
313
|
-
buildBundle(topic, domain, items) {
|
|
1020
|
+
buildBundle(topic, domain, items, options = {}) {
|
|
1021
|
+
const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
|
|
314
1022
|
const indexList = items.map((item, idx) => ({
|
|
315
1023
|
id: `RES-${String(idx + 1).padStart(2, "0")}`,
|
|
316
1024
|
title: item.doc.title,
|
|
@@ -320,7 +1028,8 @@ var CatalogBuilder = class {
|
|
|
320
1028
|
byteSize: item.doc.byteSize,
|
|
321
1029
|
tablesCount: item.doc.tablesCount,
|
|
322
1030
|
codeSnippetsCount: item.doc.extractedCodeSnippets.length,
|
|
323
|
-
summary: item.summary
|
|
1031
|
+
summary: item.summary,
|
|
1032
|
+
keyTakeaways: item.keyTakeaways || item.doc.keyExcerpts?.slice(0, 3) || []
|
|
324
1033
|
}));
|
|
325
1034
|
const resourceIndexJson = JSON.stringify(
|
|
326
1035
|
{
|
|
@@ -339,36 +1048,60 @@ var CatalogBuilder = class {
|
|
|
339
1048
|
`
|
|
340
1049
|
| ID | Resource Title | Relevance (1-10) | Code Snippets | Tables | Source Link |`,
|
|
341
1050
|
`|---|---|---|---|---|---|`,
|
|
342
|
-
...indexList.map(
|
|
1051
|
+
...indexList.map(
|
|
1052
|
+
(r) => `| **${r.id}** | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`
|
|
1053
|
+
),
|
|
343
1054
|
`
|
|
344
1055
|
## Resource Summaries & Technical Ground Truth
|
|
345
1056
|
`,
|
|
346
|
-
...indexList.map((r) =>
|
|
1057
|
+
...indexList.map((r) => {
|
|
1058
|
+
const takeaways = r.keyTakeaways && r.keyTakeaways.length > 0 ? r.keyTakeaways.map((t) => ` - ${t}`).join("\n") : ` - ${r.summary}`;
|
|
1059
|
+
return `### ${r.id}: ${r.title}
|
|
347
1060
|
- **URL:** ${r.url}
|
|
348
|
-
- **Summary:** ${r.summary}
|
|
349
1061
|
- **Relevance:** ${r.relevanceScore}/10
|
|
350
|
-
|
|
1062
|
+
- **Key Excerpts:**
|
|
1063
|
+
${takeaways}
|
|
1064
|
+
`;
|
|
1065
|
+
})
|
|
351
1066
|
];
|
|
1067
|
+
const codeSections = items.flatMap((item) => {
|
|
1068
|
+
if (item.doc.extractedCodeBlocks && item.doc.extractedCodeBlocks.length > 0) {
|
|
1069
|
+
return item.doc.extractedCodeBlocks.slice(0, 2).map((b) => {
|
|
1070
|
+
const lang = b.language || defaultLang;
|
|
1071
|
+
return `\`\`\`${lang}
|
|
1072
|
+
${b.code}
|
|
1073
|
+
\`\`\``;
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
return item.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
|
|
1077
|
+
${c}
|
|
1078
|
+
\`\`\``);
|
|
1079
|
+
});
|
|
1080
|
+
const conceptLines = options.concepts && options.concepts.length > 0 ? [
|
|
1081
|
+
`| ID | Concept / Term | Vietnamese | English | Marr | Bloom | Prereqs | Sources |`,
|
|
1082
|
+
`|---|---|---|---|---|---|---|---|`,
|
|
1083
|
+
...options.concepts.map(
|
|
1084
|
+
(c) => `| \`${c.id}\` | **${c.term.replace(/\|/g, "/")}** | ${c.termVi || "\u2014"} | ${c.termEn || "\u2014"} | \`${c.marrLayer}\` | \`${c.bloomLevel}\` | ${c.prerequisites.join(", ") || "\u2014"} | ${c.sourceIds.join(", ")} |`
|
|
1085
|
+
)
|
|
1086
|
+
] : [`- Canonical technical breakdown for **${topic}** derived from ${items.length} verified primary sources.`];
|
|
352
1087
|
const kgLines = [
|
|
353
1088
|
`# \u{1F9EC} Distilled Knowledge Graph: ${topic}`,
|
|
354
1089
|
`> **Domain:** \`${domain.toUpperCase()}\` | **Ground Truth Verification:** PASSED`,
|
|
355
1090
|
`
|
|
356
|
-
## 1. Architectural Blueprint &
|
|
357
|
-
|
|
1091
|
+
## 1. Architectural Blueprint & Concept Hierarchy (Marr & Bloom)`,
|
|
1092
|
+
...conceptLines,
|
|
358
1093
|
`
|
|
359
1094
|
## 2. Core APIs & Code Primitives`,
|
|
360
|
-
...
|
|
361
|
-
${c}
|
|
362
|
-
\`\`\``),
|
|
1095
|
+
...codeSections.length > 0 ? codeSections : ["*No explicit code snippets captured in harvested sources.*"],
|
|
363
1096
|
`
|
|
364
|
-
## 3.
|
|
365
|
-
|
|
366
|
-
`- Proactively prevent documented anti-patterns and performance regressions.`
|
|
1097
|
+
## 3. Extracted Reference Tables`,
|
|
1098
|
+
...items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)).length > 0 ? items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)) : ["*No markdown reference tables extracted.*"]
|
|
367
1099
|
];
|
|
368
1100
|
return {
|
|
369
1101
|
resourceIndexJson,
|
|
370
1102
|
resourceCatalogMd: catalogLines.join("\n"),
|
|
371
|
-
knowledgeGraphMd: kgLines.join("\n")
|
|
1103
|
+
knowledgeGraphMd: kgLines.join("\n"),
|
|
1104
|
+
conceptDagTsv: options.conceptDagTsv
|
|
372
1105
|
};
|
|
373
1106
|
}
|
|
374
1107
|
};
|
|
@@ -396,7 +1129,15 @@ var HealthAuditor = class {
|
|
|
396
1129
|
exports.CatalogBuilder = CatalogBuilder;
|
|
397
1130
|
exports.HealthAuditor = HealthAuditor;
|
|
398
1131
|
exports.MultiSourceSearchEngine = MultiSourceSearchEngine;
|
|
1132
|
+
exports.PedagogicalTermExtractor = PedagogicalTermExtractor;
|
|
399
1133
|
exports.WebScraper = WebScraper;
|
|
1134
|
+
exports.YouTubeIngestionService = YouTubeIngestionService;
|
|
400
1135
|
exports.classifyTechnicalDomain = classifyTechnicalDomain;
|
|
1136
|
+
exports.extractSubstantiveExcerpts = extractSubstantiveExcerpts;
|
|
1137
|
+
exports.extractYouTubeVideoId = extractYouTubeVideoId;
|
|
1138
|
+
exports.formatTimestamp = formatTimestamp;
|
|
1139
|
+
exports.inferPrimaryCodeLanguage = inferPrimaryCodeLanguage;
|
|
1140
|
+
exports.isYouTubeUrl = isYouTubeUrl;
|
|
1141
|
+
exports.stitchCaptionSegments = stitchCaptionSegments;
|
|
401
1142
|
//# sourceMappingURL=index.cjs.map
|
|
402
1143
|
//# sourceMappingURL=index.cjs.map
|