@thanh01.pmt/research-kit 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +213 -0
- package/dist/academic/index.cjs +332 -127
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +2 -2
- package/dist/{catalogBuilder-8A-Dg_A-.d.ts → catalogBuilder-CmmQzUnf.d.ts} +11 -3
- package/dist/{catalogBuilder-feU79wQQ.d.cts → catalogBuilder-DgXYxL2F.d.cts} +11 -3
- package/dist/chunk-2JEVBCIF.js +3 -0
- package/dist/{chunk-KGCDYFFD.js.map → chunk-2JEVBCIF.js.map} +1 -1
- package/dist/chunk-2XIEWCUE.js +244 -0
- package/dist/chunk-2XIEWCUE.js.map +1 -0
- package/dist/chunk-AUGVVZTL.js +514 -0
- package/dist/chunk-AUGVVZTL.js.map +1 -0
- package/dist/chunk-DOXSPRSP.js +559 -0
- package/dist/chunk-DOXSPRSP.js.map +1 -0
- package/dist/chunk-GKF3NDEX.js +162 -0
- package/dist/chunk-GKF3NDEX.js.map +1 -0
- package/dist/chunk-J3VCGH7Z.js +3 -0
- package/dist/chunk-J3VCGH7Z.js.map +1 -0
- package/dist/{chunk-67V2GJGK.js → chunk-J7XKDMGL.js} +7 -5
- package/dist/chunk-J7XKDMGL.js.map +1 -0
- package/dist/chunk-KAFQVI2R.js +333 -0
- package/dist/chunk-KAFQVI2R.js.map +1 -0
- package/dist/chunk-XIF5YNI7.js +799 -0
- package/dist/chunk-XIF5YNI7.js.map +1 -0
- package/dist/config/index.cjs +5 -3
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs +337 -21
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +44 -7
- package/dist/evidence/index.d.ts +44 -7
- package/dist/evidence/index.js +3 -2
- package/dist/harvester/index.cjs +805 -64
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +79 -8
- package/dist/harvester/index.d.ts +79 -8
- package/dist/harvester/index.js +3 -2
- package/dist/{index-BU5n1ip_.d.ts → index-t8pahlJ4.d.cts} +104 -15
- package/dist/{index-BU5n1ip_.d.cts → index-t8pahlJ4.d.ts} +104 -15
- package/dist/index.cjs +2001 -252
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +156 -8
- package/dist/index.d.ts +156 -8
- package/dist/index.js +237 -8
- package/dist/index.js.map +1 -1
- package/dist/literatureReview-BTPFe7eb.d.cts +64 -0
- package/dist/literatureReview-hGbfrBmw.d.ts +64 -0
- package/dist/pipeline/index.cjs +1570 -241
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -9
- package/dist/pipeline/index.d.ts +49 -9
- package/dist/pipeline/index.js +5 -4
- package/dist/webScraper-BhiQ8s9x.d.ts +68 -0
- package/dist/webScraper-D9Ldus9p.d.cts +68 -0
- package/package.json +8 -9
- package/dist/chunk-47L7ZHMY.js +0 -316
- package/dist/chunk-47L7ZHMY.js.map +0 -1
- package/dist/chunk-63X2HPCI.js +0 -47
- package/dist/chunk-63X2HPCI.js.map +0 -1
- package/dist/chunk-67V2GJGK.js.map +0 -1
- package/dist/chunk-6Q2XCUG7.js +0 -40
- package/dist/chunk-6Q2XCUG7.js.map +0 -1
- package/dist/chunk-KGCDYFFD.js +0 -3
- package/dist/chunk-MHAYD555.js +0 -239
- package/dist/chunk-MHAYD555.js.map +0 -1
- package/dist/chunk-MPLDSHFM.js +0 -396
- package/dist/chunk-MPLDSHFM.js.map +0 -1
- package/dist/chunk-QZNLB5X4.js +0 -65
- package/dist/chunk-QZNLB5X4.js.map +0 -1
- package/dist/literatureReview-C4AgVljZ.d.ts +0 -43
- package/dist/literatureReview-Dlods4Aw.d.cts +0 -43
- package/dist/webScraper-pKzdXhfI.d.ts +0 -46
- package/dist/webScraper-yBcYOHoL.d.cts +0 -46
package/dist/index.cjs
CHANGED
|
@@ -5,11 +5,13 @@ var zod = require('zod');
|
|
|
5
5
|
// src/config/researchConfig.ts
|
|
6
6
|
var ResearchConfigSchema = zod.z.object({
|
|
7
7
|
search: zod.z.object({
|
|
8
|
-
preferredProvider: zod.z.enum(["tavily", "exa", "searxng", "duckduckgo", "semantic_scholar", "openalex", "arxiv", "custom"]).default("tavily"),
|
|
9
|
-
fallbackChain: zod.z.array(zod.z.string()).default(["tavily", "exa", "duckduckgo"]),
|
|
8
|
+
preferredProvider: zod.z.enum(["tavily", "exa", "serper", "searxng", "duckduckgo", "semantic_scholar", "openalex", "arxiv", "custom"]).default("tavily"),
|
|
9
|
+
fallbackChain: zod.z.array(zod.z.string()).default(["tavily", "exa", "serper", "duckduckgo"]),
|
|
10
10
|
apiKeys: zod.z.object({
|
|
11
11
|
tavily: zod.z.string().optional(),
|
|
12
12
|
exa: zod.z.string().optional(),
|
|
13
|
+
jina: zod.z.string().optional(),
|
|
14
|
+
serper: zod.z.string().optional(),
|
|
13
15
|
semanticScholar: zod.z.string().optional()
|
|
14
16
|
}).default({}),
|
|
15
17
|
searxngHost: zod.z.string().optional(),
|
|
@@ -17,7 +19,7 @@ var ResearchConfigSchema = zod.z.object({
|
|
|
17
19
|
maxResultsPerQuery: zod.z.number().default(5)
|
|
18
20
|
}).default({}),
|
|
19
21
|
crawler: zod.z.object({
|
|
20
|
-
preferredProvider: zod.z.enum(["jina", "crawl4ai", "defuddle", "direct", "custom"]).default("jina"),
|
|
22
|
+
preferredProvider: zod.z.enum(["jina", "crawl4ai", "youtube", "defuddle", "direct", "custom"]).default("jina"),
|
|
21
23
|
fallbackChain: zod.z.array(zod.z.string()).default(["jina", "crawl4ai", "direct"]),
|
|
22
24
|
crawl4aiHost: zod.z.string().default("http://localhost:11235"),
|
|
23
25
|
timeoutMs: zod.z.number().default(15e3),
|
|
@@ -30,6 +32,52 @@ var ResearchConfigSchema = zod.z.object({
|
|
|
30
32
|
}).default({})
|
|
31
33
|
});
|
|
32
34
|
var DEFAULT_RESEARCH_CONFIG = ResearchConfigSchema.parse({});
|
|
35
|
+
|
|
36
|
+
// src/academic/arxivClient.ts
|
|
37
|
+
var ArXivSearchEngine = class {
|
|
38
|
+
/**
|
|
39
|
+
* Search ArXiv repository using the public atom XML API
|
|
40
|
+
*/
|
|
41
|
+
async search(query, maxResults = 5) {
|
|
42
|
+
const papers = [];
|
|
43
|
+
try {
|
|
44
|
+
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
45
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
46
|
+
if (res.ok) {
|
|
47
|
+
const xmlText = await res.text();
|
|
48
|
+
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
49
|
+
for (const entry of entryMatches) {
|
|
50
|
+
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
51
|
+
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
52
|
+
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
53
|
+
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
54
|
+
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
55
|
+
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
56
|
+
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
57
|
+
if (titleMatch) {
|
|
58
|
+
papers.push({
|
|
59
|
+
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
60
|
+
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
61
|
+
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
62
|
+
authors: authorMatches,
|
|
63
|
+
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
64
|
+
citationCount: 0,
|
|
65
|
+
venue: "arXiv preprint",
|
|
66
|
+
url: arxivUrl,
|
|
67
|
+
pdfUrl,
|
|
68
|
+
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
} catch (e) {
|
|
74
|
+
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
75
|
+
}
|
|
76
|
+
return papers;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
// src/academic/semanticScholar.ts
|
|
33
81
|
var AcademicPaperSchema = zod.z.object({
|
|
34
82
|
id: zod.z.string(),
|
|
35
83
|
title: zod.z.string(),
|
|
@@ -44,22 +92,56 @@ var AcademicPaperSchema = zod.z.object({
|
|
|
44
92
|
tldr: zod.z.string().optional(),
|
|
45
93
|
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
46
94
|
});
|
|
95
|
+
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
96
|
+
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
97
|
+
const entries = Object.entries(invertedIndex);
|
|
98
|
+
if (entries.length === 0) return void 0;
|
|
99
|
+
let maxIndex = 0;
|
|
100
|
+
for (const [, positions] of entries) {
|
|
101
|
+
if (Array.isArray(positions)) {
|
|
102
|
+
for (const p of positions) {
|
|
103
|
+
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
108
|
+
const words = new Array(maxIndex + 1);
|
|
109
|
+
for (const [word, positions] of entries) {
|
|
110
|
+
if (Array.isArray(positions)) {
|
|
111
|
+
for (const p of positions) {
|
|
112
|
+
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
113
|
+
words[p] = word;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
119
|
+
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
120
|
+
}
|
|
47
121
|
var AcademicSearchEngine = class {
|
|
48
122
|
apiKey;
|
|
123
|
+
allowEnvFallback;
|
|
49
124
|
customAdapter;
|
|
125
|
+
arxivEngine;
|
|
50
126
|
constructor(options = {}) {
|
|
127
|
+
this.arxivEngine = new ArXivSearchEngine();
|
|
51
128
|
if (typeof options === "string") {
|
|
52
129
|
this.apiKey = options;
|
|
130
|
+
this.allowEnvFallback = true;
|
|
53
131
|
} else {
|
|
54
|
-
this.
|
|
132
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
133
|
+
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
55
134
|
this.customAdapter = options.customAdapter;
|
|
56
135
|
}
|
|
57
136
|
}
|
|
58
137
|
setCustomAdapter(adapter) {
|
|
59
138
|
this.customAdapter = adapter;
|
|
60
139
|
}
|
|
140
|
+
setApiKey(apiKey) {
|
|
141
|
+
this.apiKey = apiKey;
|
|
142
|
+
}
|
|
61
143
|
/**
|
|
62
|
-
* Search academic papers across Semantic Scholar with OpenAlex fallback or Custom Adapter
|
|
144
|
+
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
63
145
|
*/
|
|
64
146
|
async searchPapers(query, options = {}) {
|
|
65
147
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -75,6 +157,13 @@ var AcademicSearchEngine = class {
|
|
|
75
157
|
}
|
|
76
158
|
const limit = options.limit ?? 5;
|
|
77
159
|
const papers = [];
|
|
160
|
+
const seenTitles = /* @__PURE__ */ new Set();
|
|
161
|
+
const pushUnique = (p) => {
|
|
162
|
+
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
163
|
+
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
164
|
+
seenTitles.add(normTitle);
|
|
165
|
+
papers.push(p);
|
|
166
|
+
};
|
|
78
167
|
try {
|
|
79
168
|
const apiKey = options.apiKey || this.apiKey;
|
|
80
169
|
const headers = { "Content-Type": "application/json" };
|
|
@@ -97,104 +186,111 @@ var AcademicSearchEngine = class {
|
|
|
97
186
|
const data = await res.json();
|
|
98
187
|
if (Array.isArray(data?.data)) {
|
|
99
188
|
for (const item of data.data) {
|
|
100
|
-
|
|
189
|
+
pushUnique({
|
|
101
190
|
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
102
191
|
title: item.title || "Untitled",
|
|
103
|
-
abstract: item.abstract,
|
|
104
|
-
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name) : [],
|
|
192
|
+
abstract: item.abstract || void 0,
|
|
193
|
+
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
105
194
|
year: item.year,
|
|
106
195
|
citationCount: item.citationCount || 0,
|
|
107
|
-
venue: item.venue,
|
|
196
|
+
venue: item.venue || void 0,
|
|
108
197
|
doi: item.externalIds?.DOI,
|
|
109
|
-
url: item.url,
|
|
198
|
+
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
110
199
|
pdfUrl: item.openAccessPdf?.url,
|
|
111
200
|
tldr: item.tldr?.text,
|
|
112
|
-
fieldsOfStudy: item.fieldsOfStudy
|
|
201
|
+
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
113
202
|
});
|
|
114
203
|
}
|
|
115
|
-
if (papers.length
|
|
204
|
+
if (papers.length >= limit) return papers.slice(0, limit);
|
|
116
205
|
}
|
|
117
206
|
}
|
|
118
207
|
} catch (e) {
|
|
119
208
|
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
120
209
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
210
|
+
if (papers.length < limit) {
|
|
211
|
+
try {
|
|
212
|
+
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
213
|
+
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
214
|
+
signal: AbortSignal.timeout(6e3)
|
|
215
|
+
});
|
|
216
|
+
if (res.ok) {
|
|
217
|
+
const data = await res.json();
|
|
218
|
+
if (Array.isArray(data?.results)) {
|
|
219
|
+
for (const item of data.results) {
|
|
220
|
+
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
221
|
+
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
222
|
+
pushUnique({
|
|
223
|
+
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
224
|
+
title: item.title || "Untitled",
|
|
225
|
+
abstract: abstractText,
|
|
226
|
+
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
227
|
+
year: item.publication_year,
|
|
228
|
+
citationCount: item.cited_by_count || 0,
|
|
229
|
+
venue: item.primary_location?.source?.display_name,
|
|
230
|
+
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
231
|
+
url: item.doi || item.id,
|
|
232
|
+
pdfUrl: item.open_access?.oa_url,
|
|
233
|
+
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
234
|
+
});
|
|
235
|
+
}
|
|
143
236
|
}
|
|
144
237
|
}
|
|
238
|
+
} catch (e) {
|
|
239
|
+
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
145
240
|
}
|
|
146
|
-
} catch (e) {
|
|
147
|
-
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
148
241
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
var ArXivSearchEngine = class {
|
|
155
|
-
/**
|
|
156
|
-
* Search ArXiv repository using the public atom XML API
|
|
157
|
-
*/
|
|
158
|
-
async search(query, maxResults = 5) {
|
|
159
|
-
const papers = [];
|
|
160
|
-
try {
|
|
161
|
-
const url = `http://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
162
|
-
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
163
|
-
if (res.ok) {
|
|
164
|
-
const xmlText = await res.text();
|
|
165
|
-
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
166
|
-
for (const entry of entryMatches) {
|
|
167
|
-
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
168
|
-
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
169
|
-
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
170
|
-
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
171
|
-
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
172
|
-
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
173
|
-
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
174
|
-
if (titleMatch) {
|
|
175
|
-
papers.push({
|
|
176
|
-
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
177
|
-
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
178
|
-
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
179
|
-
authors: authorMatches,
|
|
180
|
-
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
181
|
-
citationCount: 0,
|
|
182
|
-
venue: "arXiv preprint",
|
|
183
|
-
url: arxivUrl,
|
|
184
|
-
pdfUrl,
|
|
185
|
-
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
186
|
-
});
|
|
187
|
-
}
|
|
242
|
+
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
243
|
+
try {
|
|
244
|
+
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
245
|
+
for (const p of arxivPapers) {
|
|
246
|
+
pushUnique(p);
|
|
188
247
|
}
|
|
248
|
+
} catch (e) {
|
|
249
|
+
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
189
250
|
}
|
|
190
|
-
} catch (e) {
|
|
191
|
-
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
192
251
|
}
|
|
193
|
-
return papers;
|
|
252
|
+
return papers.slice(0, limit);
|
|
194
253
|
}
|
|
195
254
|
};
|
|
196
255
|
|
|
197
256
|
// src/academic/cs2023Taxonomy.ts
|
|
257
|
+
var CSTA_K12_STRANDS = {
|
|
258
|
+
CS: {
|
|
259
|
+
code: "CS",
|
|
260
|
+
name: "Computing Systems",
|
|
261
|
+
nameVi: "H\u1EC7 th\u1ED1ng M\xE1y t\xEDnh & Ph\u1EA7n c\u1EE9ng",
|
|
262
|
+
description: "Devices, hardware and software integration, sensors, actuators, and troubleshooting.",
|
|
263
|
+
subconcepts: ["Devices", "Hardware & Software", "Troubleshooting"]
|
|
264
|
+
},
|
|
265
|
+
NI: {
|
|
266
|
+
code: "NI",
|
|
267
|
+
name: "Networks and the Internet",
|
|
268
|
+
nameVi: "M\u1EA1ng m\xE1y t\xEDnh & An to\xE0n th\xF4ng tin",
|
|
269
|
+
description: "Network communication, protocols, IoT connectivity, and cybersecurity.",
|
|
270
|
+
subconcepts: ["Network Communication & Organization", "Cybersecurity"]
|
|
271
|
+
},
|
|
272
|
+
DA: {
|
|
273
|
+
code: "DA",
|
|
274
|
+
name: "Data and Analysis",
|
|
275
|
+
nameVi: "D\u1EEF li\u1EC7u & Ph\xE2n t\xEDch",
|
|
276
|
+
description: "Storage, collection, visualization, and transformation of sensor or application data.",
|
|
277
|
+
subconcepts: ["Storage", "Collection, Visualization & Transformation", "Inference & Models"]
|
|
278
|
+
},
|
|
279
|
+
AP: {
|
|
280
|
+
code: "AP",
|
|
281
|
+
name: "Algorithms and Programming",
|
|
282
|
+
nameVi: "Thu\u1EADt to\xE1n & L\u1EADp tr\xECnh",
|
|
283
|
+
description: "Algorithms, variables, control structures, modularity, and iterative program development.",
|
|
284
|
+
subconcepts: ["Algorithms", "Variables", "Control", "Modularity", "Program Development"]
|
|
285
|
+
},
|
|
286
|
+
IC: {
|
|
287
|
+
code: "IC",
|
|
288
|
+
name: "Impacts of Computing",
|
|
289
|
+
nameVi: "T\xE1c \u0111\u1ED9ng c\u1EE7a Tin h\u1ECDc & AI",
|
|
290
|
+
description: "Culture, social interactions, safety, law, and ethics of computing and AI systems.",
|
|
291
|
+
subconcepts: ["Culture", "Social Interactions", "Safety, Law, & Ethics"]
|
|
292
|
+
}
|
|
293
|
+
};
|
|
198
294
|
var CS2023_KNOWLEDGE_AREAS = {
|
|
199
295
|
AL: {
|
|
200
296
|
code: "AL",
|
|
@@ -205,8 +301,8 @@ var CS2023_KNOWLEDGE_AREAS = {
|
|
|
205
301
|
AR: {
|
|
206
302
|
code: "AR",
|
|
207
303
|
name: "Architecture and Organization",
|
|
208
|
-
description: "Hardware architecture,
|
|
209
|
-
coreTopics: ["Digital Logic", "Memory Hierarchy & Cache", "
|
|
304
|
+
description: "Hardware architecture, microcontrollers, sensors, memory hierarchy, and interfacing.",
|
|
305
|
+
coreTopics: ["Digital Logic", "Memory Hierarchy & Cache", "Microcontrollers & Embedded I/O", "Sensor & Actuator Interfacing"]
|
|
210
306
|
},
|
|
211
307
|
AI: {
|
|
212
308
|
code: "AI",
|
|
@@ -214,6 +310,30 @@ var CS2023_KNOWLEDGE_AREAS = {
|
|
|
214
310
|
description: "Neural networks, search algorithms, knowledge representation, reinforcement learning, and LLMs.",
|
|
215
311
|
coreTopics: ["Supervised Learning", "Deep Learning & Transformers", "Search & Optimization", "AI Ethics & Alignment"]
|
|
216
312
|
},
|
|
313
|
+
DM: {
|
|
314
|
+
code: "DM",
|
|
315
|
+
name: "Data Management",
|
|
316
|
+
description: "Relational databases, NoSQL, query optimization, data modeling, and transactions.",
|
|
317
|
+
coreTopics: ["Relational & SQL Modeling", "Indexing & Query Optimization", "Transactions & ACID", "Distributed Data Stores"]
|
|
318
|
+
},
|
|
319
|
+
FPL: {
|
|
320
|
+
code: "FPL",
|
|
321
|
+
name: "Foundations of Programming Languages",
|
|
322
|
+
description: "Language paradigms, type systems, event-driven programming, memory safety, and compilers.",
|
|
323
|
+
coreTopics: ["Event-Driven & Reactive Programming", "Functional & OOP Paradigms", "Static & Dynamic Typing", "Memory Safety & Ownership"]
|
|
324
|
+
},
|
|
325
|
+
GIT: {
|
|
326
|
+
code: "GIT",
|
|
327
|
+
name: "Graphics and Interactive Techniques",
|
|
328
|
+
description: "Game engines, 2D/3D rendering, physics simulation, collision detection, and interactive media.",
|
|
329
|
+
coreTopics: ["Game Loop & Frame Timing", "2D/3D Coordinate Systems & Physics", "Collision Detection & State Machines", "Shader & Rendering Pipelines"]
|
|
330
|
+
},
|
|
331
|
+
HCI: {
|
|
332
|
+
code: "HCI",
|
|
333
|
+
name: "Human-Computer Interaction",
|
|
334
|
+
description: "User interface design, usability, accessibility, state-driven UI frameworks, and interaction loops.",
|
|
335
|
+
coreTopics: ["Declarative & State-Driven UI", "Usability & Cognitive Ergonomics", "Accessibility Standards", "Interactive Prototyping"]
|
|
336
|
+
},
|
|
217
337
|
OS: {
|
|
218
338
|
code: "OS",
|
|
219
339
|
name: "Operating Systems",
|
|
@@ -232,6 +352,12 @@ var CS2023_KNOWLEDGE_AREAS = {
|
|
|
232
352
|
description: "Language paradigms, type systems, memory safety, compilers, and runtime environments.",
|
|
233
353
|
coreTopics: ["Functional & OOP Paradigms", "Static & Dynamic Typing", "Memory Safety & Ownership", "Language Runtimes"]
|
|
234
354
|
},
|
|
355
|
+
SDF: {
|
|
356
|
+
code: "SDF",
|
|
357
|
+
name: "Software Development Fundamentals",
|
|
358
|
+
description: "Core programming constructs, variables, control flow, data structures, debugging, and modularization.",
|
|
359
|
+
coreTopics: ["Variables, Loops & Control Flow", "Data Structures (Arrays, Lists, Maps)", "Systematic Debugging & Testing", "Modular & Procedural Design"]
|
|
360
|
+
},
|
|
235
361
|
SE: {
|
|
236
362
|
code: "SE",
|
|
237
363
|
name: "Software Engineering",
|
|
@@ -244,37 +370,53 @@ var CS2023_KNOWLEDGE_AREAS = {
|
|
|
244
370
|
description: "Authentication, cryptography, secure coding, threat modeling, and network defense.",
|
|
245
371
|
coreTopics: ["Symmetric & Asymmetric Encryption", "OAuth & JWT Auth", "Secure Coding & Vulnerabilities", "Identity & Access Management"]
|
|
246
372
|
},
|
|
247
|
-
|
|
248
|
-
code: "
|
|
249
|
-
name: "
|
|
250
|
-
description: "
|
|
251
|
-
coreTopics: ["
|
|
373
|
+
SPD: {
|
|
374
|
+
code: "SPD",
|
|
375
|
+
name: "Specialized Platform Development",
|
|
376
|
+
description: "Web, mobile, cloud, IoT, and robotics platform engineering.",
|
|
377
|
+
coreTopics: ["Mobile & Desktop App Lifecycle", "Web Full-Stack Architecture", "IoT & Edge Device Connectivity", "Platform SDK Integration"]
|
|
252
378
|
}
|
|
253
379
|
};
|
|
254
|
-
function mapTopicToCS2023(topic) {
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
380
|
+
function mapTopicToCS2023(topic, context) {
|
|
381
|
+
const combined = [
|
|
382
|
+
topic,
|
|
383
|
+
context?.courseTitle,
|
|
384
|
+
context?.positioning,
|
|
385
|
+
context?.hardwarePlatform,
|
|
386
|
+
context?.softwareEnv,
|
|
387
|
+
...context?.inScopeTopics || []
|
|
388
|
+
].filter(Boolean).join(" ").toLowerCase();
|
|
389
|
+
if (/\b(concurren|async|actor|thread|distributed|parallel|đồng thời|bất đồng bộ|đa luồng)\b/.test(combined)) {
|
|
390
|
+
return { area: CS2023_KNOWLEDGE_AREAS.PDC, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.95 };
|
|
391
|
+
}
|
|
392
|
+
if (/\b(ai|machine learning|deep learning|neural|llm|computer vision|trí tuệ nhân tạo|học máy|mạng nơ-ron)\b/.test(combined)) {
|
|
393
|
+
return { area: CS2023_KNOWLEDGE_AREAS.AI, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.95 };
|
|
258
394
|
}
|
|
259
|
-
if (
|
|
260
|
-
return { area: CS2023_KNOWLEDGE_AREAS.
|
|
395
|
+
if (/\b(arduino|esp32|raspberry|micro:?bit|mblock|makecode|lego|sensor|circuit|gpio|iot|hardware|robotics|cảm biến|vi điều khiển|mạch điện|điện tử)\b/.test(combined)) {
|
|
396
|
+
return { area: CS2023_KNOWLEDGE_AREAS.AR, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.92 };
|
|
261
397
|
}
|
|
262
|
-
if (
|
|
263
|
-
return { area: CS2023_KNOWLEDGE_AREAS.
|
|
398
|
+
if (/\b(roblox|unity|godot|pygame|game|2d|3d|sprite|collision|physics|shader|trò chơi|làm game)\b/.test(combined)) {
|
|
399
|
+
return { area: CS2023_KNOWLEDGE_AREAS.GIT, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.92 };
|
|
264
400
|
}
|
|
265
|
-
if (
|
|
266
|
-
return { area: CS2023_KNOWLEDGE_AREAS.
|
|
401
|
+
if (/\b(swiftui|ui|ux|interface|frontend|accessibility|figma|giao diện|tương tác người-máy)\b/.test(combined)) {
|
|
402
|
+
return { area: CS2023_KNOWLEDGE_AREAS.HCI, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
|
|
267
403
|
}
|
|
268
|
-
if (
|
|
269
|
-
return { area: CS2023_KNOWLEDGE_AREAS.
|
|
404
|
+
if (/\b(os|operating system|kernel|virtual memory|linux|hệ điều hành)\b/.test(combined)) {
|
|
405
|
+
return { area: CS2023_KNOWLEDGE_AREAS.OS, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.9 };
|
|
270
406
|
}
|
|
271
|
-
if (
|
|
272
|
-
return { area: CS2023_KNOWLEDGE_AREAS.
|
|
407
|
+
if (/\b(security|auth|crypto|cyber|jwt|oauth|bảo mật|an toàn thông tin|mã hóa)\b/.test(combined)) {
|
|
408
|
+
return { area: CS2023_KNOWLEDGE_AREAS.SEC, k12Strand: CSTA_K12_STRANDS.NI, matchConfidence: 0.9 };
|
|
273
409
|
}
|
|
274
|
-
if (
|
|
275
|
-
return { area: CS2023_KNOWLEDGE_AREAS.
|
|
410
|
+
if (/\b(sql|database|postgres|supabase|mongodb|data model|cơ sở dữ liệu|truy vấn)\b/.test(combined)) {
|
|
411
|
+
return { area: CS2023_KNOWLEDGE_AREAS.DM, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.9 };
|
|
276
412
|
}
|
|
277
|
-
|
|
413
|
+
if (/\b(architecture|design pattern|software engineer|clean code|testing|tdd|ci\/cd|kiến trúc phần mềm|kiểm thử)\b/.test(combined)) {
|
|
414
|
+
return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88 };
|
|
415
|
+
}
|
|
416
|
+
if (/\b(algorithm|data structure|tree|graph|dynamic programming|sorting|thuật toán|cấu trúc dữ liệu|đồ thị|quy hoạch động)\b/.test(combined)) {
|
|
417
|
+
return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
|
|
418
|
+
}
|
|
419
|
+
return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.8 };
|
|
278
420
|
}
|
|
279
421
|
|
|
280
422
|
// src/academic/citationManager.ts
|
|
@@ -326,105 +468,217 @@ var CitationManager = class {
|
|
|
326
468
|
|
|
327
469
|
// src/academic/literatureReview.ts
|
|
328
470
|
var LiteratureReviewSynthesizer = class {
|
|
329
|
-
synthesize(topic, papers, cs2023Area) {
|
|
471
|
+
synthesize(topic, papers, cs2023Area, k12Strand) {
|
|
330
472
|
const citationManager = new CitationManager();
|
|
331
473
|
const sotaMatrix = [];
|
|
474
|
+
const foundations = [];
|
|
332
475
|
for (const paper of papers) {
|
|
333
476
|
citationManager.addPaper(paper);
|
|
477
|
+
const summaryText = paper.tldr?.trim() || (paper.abstract ? paper.abstract.trim().split(/(?<=[.!?])\s+/)[0] || paper.abstract.slice(0, 160) : "");
|
|
334
478
|
sotaMatrix.push({
|
|
335
|
-
approach: paper.title.split(":")[0] || paper.title.slice(0,
|
|
336
|
-
sourcePaper: `${paper.authors[0] || "Unknown"} (${paper.year || "n.d."})`,
|
|
337
|
-
keyStrengths:
|
|
338
|
-
limitations: "
|
|
339
|
-
relevanceToCourse: `
|
|
479
|
+
approach: paper.title.split(":")[0]?.trim() || paper.title.slice(0, 45),
|
|
480
|
+
sourcePaper: `${paper.authors[0] || "Unknown"} (${paper.year || "n.d."})${paper.venue ? ` \u2014 ${paper.venue}` : ""}`,
|
|
481
|
+
keyStrengths: summaryText ? summaryText.slice(0, 180) : `Peer-reviewed study (${paper.citationCount} citations)`,
|
|
482
|
+
limitations: paper.year && paper.year < 2020 ? `Published in ${paper.year}; verify against modern tooling` : "Requires pedagogical scaffolding for target learners",
|
|
483
|
+
relevanceToCourse: `Grounds "${topic}" in peer-reviewed evidence (${paper.fieldsOfStudy.slice(0, 2).join(", ") || "CS"}).`
|
|
340
484
|
});
|
|
485
|
+
if (summaryText && foundations.length < 4) {
|
|
486
|
+
foundations.push(`[${paper.authors[0] || "Ref"} ${paper.year || ""}] ${summaryText}`);
|
|
487
|
+
}
|
|
341
488
|
}
|
|
489
|
+
if (cs2023Area) {
|
|
490
|
+
foundations.push(
|
|
491
|
+
`IEEE/ACM CS2023 (${cs2023Area.code} - ${cs2023Area.name}): ${cs2023Area.coreTopics.join(", ")}`
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
if (k12Strand) {
|
|
495
|
+
foundations.push(
|
|
496
|
+
`CSTA K-12 (${k12Strand.code} - ${k12Strand.name} / ${k12Strand.nameVi}): ${k12Strand.subconcepts.join(", ")}`
|
|
497
|
+
);
|
|
498
|
+
}
|
|
499
|
+
const openQuestions = [
|
|
500
|
+
`How to scaffold core concepts of "${topic}" (${cs2023Area?.code || "CS"}) so learners master mental models before syntax memorization?`,
|
|
501
|
+
`What observable debugging and verification tasks best diagnose learner misconceptions in "${topic}"?`
|
|
502
|
+
];
|
|
342
503
|
const mdLines = [
|
|
343
504
|
`# \u{1F393} Academic Literature Review & Theoretical Ground Truth`,
|
|
344
505
|
`> **Topic:** ${topic}`,
|
|
345
|
-
cs2023Area ? `> **IEEE/ACM CS2023 Standard Area:** \`${cs2023Area.code}\` \u2014 *${cs2023Area.name}
|
|
506
|
+
cs2023Area ? `> **IEEE/ACM CS2023 Standard Area:** \`${cs2023Area.code}\` \u2014 *${cs2023Area.name}*` : "",
|
|
507
|
+
k12Strand ? `> **CSTA K-12 Strand:** \`${k12Strand.code}\` \u2014 *${k12Strand.name} (${k12Strand.nameVi})*
|
|
346
508
|
` : "",
|
|
347
509
|
`## 1. Executive Summary`,
|
|
348
|
-
`Synthesized from **${papers.length} peer-reviewed works** and
|
|
510
|
+
papers.length > 0 ? `Synthesized from **${papers.length} peer-reviewed works** and curricular standards directly relevant to \`${topic}\`.` : `No peer-reviewed papers matched directly; aligned to curricular taxonomy standards for \`${topic}\`.`,
|
|
349
511
|
`
|
|
350
512
|
## 2. State-of-the-Art (SOTA) Comparison Matrix`,
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
513
|
+
...sotaMatrix.length > 0 ? [
|
|
514
|
+
`| Paradigm / Approach | Source Reference | Core Findings / Abstract | Pedagogical Note | Course Application |`,
|
|
515
|
+
`|---|---|---|---|---|`,
|
|
516
|
+
...sotaMatrix.map(
|
|
517
|
+
(r) => `| **${r.approach.replace(/\|/g, "/")}** | ${r.sourcePaper.replace(/\|/g, "/")} | ${r.keyStrengths.replace(/\|/g, "/")} | ${r.limitations.replace(/\|/g, "/")} | ${r.relevanceToCourse.replace(/\|/g, "/")} |`
|
|
518
|
+
)
|
|
519
|
+
] : ["*No peer-reviewed papers returned from academic engines for this query.*"],
|
|
354
520
|
`
|
|
355
|
-
## 3. Core Theoretical Foundations`,
|
|
356
|
-
|
|
357
|
-
`- **CS2023 Competency Alignment:** Maps directly to IEEE/ACM recommended graduate competencies.`,
|
|
358
|
-
`- **Theory-to-Practice Bridge:** Direct translational pipeline from formal papers to executable lab operations.`,
|
|
521
|
+
## 3. Core Theoretical & Curricular Foundations`,
|
|
522
|
+
...foundations.map((f) => `- ${f}`),
|
|
359
523
|
`
|
|
360
|
-
## 4. Open Pedagogical &
|
|
361
|
-
|
|
362
|
-
`- What automated verification setups validate theoretical guarantees in real-world student code?`,
|
|
524
|
+
## 4. Open Pedagogical & Curriculum Design Questions`,
|
|
525
|
+
...openQuestions.map((q) => `- ${q}`),
|
|
363
526
|
`
|
|
364
527
|
${citationManager.getMarkdownFootnotes()}`
|
|
365
|
-
];
|
|
528
|
+
].filter((line) => line !== "");
|
|
366
529
|
return {
|
|
367
530
|
topic,
|
|
368
531
|
cs2023Area,
|
|
532
|
+
k12Strand,
|
|
369
533
|
totalPapersFound: papers.length,
|
|
370
534
|
papers,
|
|
371
535
|
sotaMatrix,
|
|
372
|
-
keyTheoreticalFoundations:
|
|
373
|
-
|
|
374
|
-
"Dataflow and message-passing distributed concurrency models",
|
|
375
|
-
"Empirical latency and throughput benchmark validation"
|
|
376
|
-
],
|
|
377
|
-
openResearchQuestions: [
|
|
378
|
-
"How to distill complex formal mechanisms for high-school / undergraduate learners without diluting accuracy?",
|
|
379
|
-
"What automated verification setups validate theoretical guarantees in real-world student code?"
|
|
380
|
-
],
|
|
536
|
+
keyTheoreticalFoundations: foundations,
|
|
537
|
+
openResearchQuestions: openQuestions,
|
|
381
538
|
markdownReport: mdLines.join("\n"),
|
|
382
539
|
bibtex: citationManager.getBibTeX()
|
|
383
540
|
};
|
|
384
541
|
}
|
|
542
|
+
/**
|
|
543
|
+
* Optional LLM semantic synthesis of dynamic open research questions and pedagogical takeaways
|
|
544
|
+
* based on the retrieved academic literature.
|
|
545
|
+
*/
|
|
546
|
+
async synthesizeWithLlm(topic, papers, cs2023Area, k12Strand, options = {}) {
|
|
547
|
+
const base = this.synthesize(topic, papers, cs2023Area, k12Strand);
|
|
548
|
+
if (!options.llm || papers.length === 0) {
|
|
549
|
+
return base;
|
|
550
|
+
}
|
|
551
|
+
try {
|
|
552
|
+
const abstractsSummary = papers.map((p, i) => `[Paper ${i + 1}] ${p.title} (${p.authors[0]} ${p.year}): ${p.tldr || p.abstract?.slice(0, 300) || "No abstract"}`).join("\n\n");
|
|
553
|
+
const prompt = [
|
|
554
|
+
`You are a Professor of Computing Education. Review these academic research papers retrieved for the course topic "${topic}".`,
|
|
555
|
+
cs2023Area ? `IEEE/ACM CS2023 Area: ${cs2023Area.code} - ${cs2023Area.name}` : "",
|
|
556
|
+
k12Strand ? `CSTA K-12 Strand: ${k12Strand.code} - ${k12Strand.name}` : "",
|
|
557
|
+
options.context?.ageBand ? `Target Age: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
|
|
558
|
+
`
|
|
559
|
+
Retrieved Academic Papers:
|
|
560
|
+
${abstractsSummary}`,
|
|
561
|
+
`
|
|
562
|
+
Generate 2-3 deep, actionable pedagogical research & curriculum design questions directly inspired by these papers.`,
|
|
563
|
+
`Return ONLY a JSON array of strings: ["Question 1", "Question 2"]`
|
|
564
|
+
].filter(Boolean).join("\n");
|
|
565
|
+
const raw = await options.llm(prompt, {
|
|
566
|
+
system: "You are an educational research synthesis specialist. Output strictly a JSON array of strings.",
|
|
567
|
+
temperature: 0.2,
|
|
568
|
+
jsonMode: true
|
|
569
|
+
});
|
|
570
|
+
const match = raw.match(/\[[\s\S]*\]/);
|
|
571
|
+
if (match) {
|
|
572
|
+
const parsed = JSON.parse(match[0]);
|
|
573
|
+
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
574
|
+
const dynamicQuestions = parsed.map(String).filter((q) => q.length > 10);
|
|
575
|
+
if (dynamicQuestions.length > 0) {
|
|
576
|
+
base.openResearchQuestions = dynamicQuestions;
|
|
577
|
+
base.markdownReport = base.markdownReport.replace(
|
|
578
|
+
/## 4\. Open Pedagogical & Curriculum Design Questions[\s\S]*?(?=\n\n\[\^|\n\n$|$)/,
|
|
579
|
+
`## 4. Open Pedagogical & Curriculum Design Questions
|
|
580
|
+
${dynamicQuestions.map((q) => `- ${q}`).join("\n")}`
|
|
581
|
+
);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
} catch (e) {
|
|
586
|
+
console.warn("[LiteratureReviewSynthesizer] LLM question synthesis failed, using baseline:", e?.message || e);
|
|
587
|
+
}
|
|
588
|
+
return base;
|
|
589
|
+
}
|
|
385
590
|
};
|
|
386
591
|
|
|
387
592
|
// src/harvester/domainClassifier.ts
|
|
388
|
-
|
|
593
|
+
var HARDWARE_PATTERNS = [
|
|
594
|
+
/\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
|
|
595
|
+
/\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
|
|
596
|
+
/(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
|
|
597
|
+
];
|
|
598
|
+
var PLATFORM_PATTERNS = [
|
|
599
|
+
/\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
|
|
600
|
+
/(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
|
|
601
|
+
];
|
|
602
|
+
var ACADEMIC_PATTERNS = [
|
|
603
|
+
/\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
|
|
604
|
+
/(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
|
|
605
|
+
];
|
|
606
|
+
function inferPrimaryCodeLanguage(topic, context, domain) {
|
|
607
|
+
const combined = [
|
|
608
|
+
topic,
|
|
609
|
+
context?.courseTitle,
|
|
610
|
+
context?.hardwarePlatform,
|
|
611
|
+
context?.softwareEnv,
|
|
612
|
+
...context?.inScopeTopics || []
|
|
613
|
+
].filter(Boolean).join(" ").toLowerCase();
|
|
614
|
+
if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
|
|
615
|
+
if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
|
|
616
|
+
if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
|
|
617
|
+
if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
|
|
618
|
+
if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
|
|
619
|
+
if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
|
|
620
|
+
if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
|
|
621
|
+
if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
|
|
622
|
+
if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
|
|
623
|
+
if (/\b(rust|cargo)\b/.test(combined)) return "rust";
|
|
624
|
+
if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
|
|
625
|
+
if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
|
|
626
|
+
if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
|
|
627
|
+
if (domain === "hardware") return "cpp";
|
|
628
|
+
return "text";
|
|
629
|
+
}
|
|
630
|
+
function classifyTechnicalDomain(topic, explicitDomain, context) {
|
|
389
631
|
if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
|
|
390
632
|
const domain = explicitDomain.toLowerCase();
|
|
391
633
|
return {
|
|
392
634
|
domain,
|
|
393
635
|
confidence: 1,
|
|
394
636
|
rationale: "Explicitly configured by user flag.",
|
|
395
|
-
recommendedExtractors: getExtractorsForDomain(domain)
|
|
637
|
+
recommendedExtractors: getExtractorsForDomain(domain),
|
|
638
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
|
|
396
639
|
};
|
|
397
640
|
}
|
|
398
|
-
const
|
|
399
|
-
|
|
641
|
+
const combined = [
|
|
642
|
+
topic,
|
|
643
|
+
context?.courseTitle,
|
|
644
|
+
context?.positioning,
|
|
645
|
+
context?.hardwarePlatform,
|
|
646
|
+
context?.softwareEnv,
|
|
647
|
+
...context?.inScopeTopics || []
|
|
648
|
+
].filter(Boolean).join(" ");
|
|
649
|
+
if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
|
|
400
650
|
return {
|
|
401
651
|
domain: "hardware",
|
|
402
652
|
confidence: 0.95,
|
|
403
|
-
rationale: "Topic contains microcontrollers, sensors, or physical electronics keywords.",
|
|
404
|
-
recommendedExtractors:
|
|
653
|
+
rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
|
|
654
|
+
recommendedExtractors: getExtractorsForDomain("hardware"),
|
|
655
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
|
|
405
656
|
};
|
|
406
657
|
}
|
|
407
|
-
if (
|
|
658
|
+
if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
|
|
408
659
|
return {
|
|
409
660
|
domain: "platform",
|
|
410
661
|
confidence: 0.9,
|
|
411
|
-
rationale: "Topic represents a cloud infrastructure, hosting, or BaaS platform.",
|
|
412
|
-
recommendedExtractors:
|
|
662
|
+
rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
|
|
663
|
+
recommendedExtractors: getExtractorsForDomain("platform"),
|
|
664
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
|
|
413
665
|
};
|
|
414
666
|
}
|
|
415
|
-
if (
|
|
667
|
+
if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
|
|
416
668
|
return {
|
|
417
669
|
domain: "academic",
|
|
418
670
|
confidence: 0.9,
|
|
419
671
|
rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
|
|
420
|
-
recommendedExtractors:
|
|
672
|
+
recommendedExtractors: getExtractorsForDomain("academic"),
|
|
673
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
|
|
421
674
|
};
|
|
422
675
|
}
|
|
423
676
|
return {
|
|
424
677
|
domain: "software",
|
|
425
678
|
confidence: 0.85,
|
|
426
|
-
rationale: "Topic is a programming language, client/server library, or software
|
|
427
|
-
recommendedExtractors:
|
|
679
|
+
rationale: "Topic is a programming language, game engine, client/server library, or software application.",
|
|
680
|
+
recommendedExtractors: getExtractorsForDomain("software"),
|
|
681
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
|
|
428
682
|
};
|
|
429
683
|
}
|
|
430
684
|
function getExtractorsForDomain(domain) {
|
|
@@ -445,15 +699,24 @@ function getExtractorsForDomain(domain) {
|
|
|
445
699
|
var MultiSourceSearchEngine = class {
|
|
446
700
|
tavilyKey;
|
|
447
701
|
exaKey;
|
|
702
|
+
serperKey;
|
|
703
|
+
allowEnvFallback;
|
|
448
704
|
customAdapter;
|
|
449
705
|
constructor(options = {}) {
|
|
450
|
-
this.
|
|
451
|
-
this.
|
|
706
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
707
|
+
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
708
|
+
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
709
|
+
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
452
710
|
this.customAdapter = options.customAdapter;
|
|
453
711
|
}
|
|
454
712
|
setCustomAdapter(adapter) {
|
|
455
713
|
this.customAdapter = adapter;
|
|
456
714
|
}
|
|
715
|
+
setApiKeys(keys) {
|
|
716
|
+
if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
|
|
717
|
+
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
718
|
+
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
719
|
+
}
|
|
457
720
|
async search(query, options = {}) {
|
|
458
721
|
const limit = options.limit ?? 5;
|
|
459
722
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -474,28 +737,37 @@ var MultiSourceSearchEngine = class {
|
|
|
474
737
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
475
738
|
}
|
|
476
739
|
}
|
|
477
|
-
const tavilyKey = options.tavilyKey || this.tavilyKey;
|
|
478
|
-
const exaKey = options.exaKey || this.exaKey;
|
|
740
|
+
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
741
|
+
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
742
|
+
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
479
743
|
if (preferred === "exa" && exaKey) {
|
|
480
|
-
const exaHits = await this.queryExa(query, exaKey, limit);
|
|
744
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
481
745
|
if (exaHits.length > 0) return exaHits;
|
|
482
746
|
}
|
|
747
|
+
if (preferred === "serper" && serperKey) {
|
|
748
|
+
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
749
|
+
if (serperHits.length > 0) return serperHits;
|
|
750
|
+
}
|
|
483
751
|
if (preferred === "duckduckgo") {
|
|
484
752
|
const ddgHits2 = await this.queryDuckDuckGo(query, limit);
|
|
485
753
|
if (ddgHits2.length > 0) return ddgHits2;
|
|
486
754
|
}
|
|
487
755
|
if (tavilyKey && (!preferred || preferred === "tavily")) {
|
|
488
756
|
try {
|
|
757
|
+
const body = {
|
|
758
|
+
api_key: tavilyKey,
|
|
759
|
+
query,
|
|
760
|
+
search_depth: "advanced",
|
|
761
|
+
include_answer: true,
|
|
762
|
+
max_results: limit
|
|
763
|
+
};
|
|
764
|
+
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
765
|
+
body.include_domains = options.domainFilter;
|
|
766
|
+
}
|
|
489
767
|
const res = await fetch("https://api.tavily.com/search", {
|
|
490
768
|
method: "POST",
|
|
491
769
|
headers: { "Content-Type": "application/json" },
|
|
492
|
-
body: JSON.stringify(
|
|
493
|
-
api_key: tavilyKey,
|
|
494
|
-
query,
|
|
495
|
-
search_depth: "advanced",
|
|
496
|
-
include_answer: true,
|
|
497
|
-
max_results: limit
|
|
498
|
-
}),
|
|
770
|
+
body: JSON.stringify(body),
|
|
499
771
|
signal: AbortSignal.timeout(12e3)
|
|
500
772
|
});
|
|
501
773
|
if (res.ok) {
|
|
@@ -515,26 +787,34 @@ var MultiSourceSearchEngine = class {
|
|
|
515
787
|
}
|
|
516
788
|
}
|
|
517
789
|
if (exaKey) {
|
|
518
|
-
const exaHits = await this.queryExa(query, exaKey, limit);
|
|
790
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
519
791
|
if (exaHits.length > 0) return exaHits;
|
|
520
792
|
}
|
|
793
|
+
if (serperKey) {
|
|
794
|
+
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
795
|
+
if (serperHits.length > 0) return serperHits;
|
|
796
|
+
}
|
|
521
797
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
522
798
|
if (ddgHits.length > 0) return ddgHits;
|
|
523
799
|
return [];
|
|
524
800
|
}
|
|
525
|
-
async queryExa(query, exaKey, limit) {
|
|
801
|
+
async queryExa(query, exaKey, limit, domainFilter) {
|
|
526
802
|
try {
|
|
803
|
+
const body = {
|
|
804
|
+
query,
|
|
805
|
+
numResults: limit,
|
|
806
|
+
useAutoprompt: true
|
|
807
|
+
};
|
|
808
|
+
if (domainFilter && domainFilter.length > 0) {
|
|
809
|
+
body.includeDomains = domainFilter;
|
|
810
|
+
}
|
|
527
811
|
const res = await fetch("https://api.exa.ai/search", {
|
|
528
812
|
method: "POST",
|
|
529
813
|
headers: {
|
|
530
814
|
"x-api-key": exaKey,
|
|
531
815
|
"Content-Type": "application/json"
|
|
532
816
|
},
|
|
533
|
-
body: JSON.stringify(
|
|
534
|
-
query,
|
|
535
|
-
numResults: limit,
|
|
536
|
-
useAutoprompt: true
|
|
537
|
-
}),
|
|
817
|
+
body: JSON.stringify(body),
|
|
538
818
|
signal: AbortSignal.timeout(1e4)
|
|
539
819
|
});
|
|
540
820
|
if (res.ok) {
|
|
@@ -554,6 +834,33 @@ var MultiSourceSearchEngine = class {
|
|
|
554
834
|
}
|
|
555
835
|
return [];
|
|
556
836
|
}
|
|
837
|
+
async querySerper(query, serperKey, limit) {
|
|
838
|
+
try {
|
|
839
|
+
const res = await fetch("https://google.serper.dev/search", {
|
|
840
|
+
method: "POST",
|
|
841
|
+
headers: {
|
|
842
|
+
"X-API-KEY": serperKey,
|
|
843
|
+
"Content-Type": "application/json"
|
|
844
|
+
},
|
|
845
|
+
body: JSON.stringify({ q: query, num: limit }),
|
|
846
|
+
signal: AbortSignal.timeout(1e4)
|
|
847
|
+
});
|
|
848
|
+
if (res.ok) {
|
|
849
|
+
const data = await res.json();
|
|
850
|
+
if (Array.isArray(data?.organic) && data.organic.length > 0) {
|
|
851
|
+
return data.organic.slice(0, limit).map((r) => ({
|
|
852
|
+
title: r.title || "Untitled",
|
|
853
|
+
url: r.link,
|
|
854
|
+
content: r.snippet || "",
|
|
855
|
+
engine: "serper"
|
|
856
|
+
}));
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
} catch (e) {
|
|
860
|
+
console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
|
|
861
|
+
}
|
|
862
|
+
return [];
|
|
863
|
+
}
|
|
557
864
|
async queryDuckDuckGo(query, limit) {
|
|
558
865
|
try {
|
|
559
866
|
const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
@@ -564,15 +871,16 @@ var MultiSourceSearchEngine = class {
|
|
|
564
871
|
if (res.ok) {
|
|
565
872
|
const html = await res.text();
|
|
566
873
|
const hits = [];
|
|
567
|
-
const resultRegex = /<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
874
|
+
const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
568
875
|
let match;
|
|
569
876
|
while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
|
|
570
|
-
const
|
|
877
|
+
const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
878
|
+
const rawUrl = match[2]?.trim();
|
|
571
879
|
const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
|
|
572
|
-
const snippet = match[
|
|
880
|
+
const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
|
|
573
881
|
if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
|
|
574
882
|
hits.push({
|
|
575
|
-
title:
|
|
883
|
+
title: rawTitle,
|
|
576
884
|
url: cleanUrl,
|
|
577
885
|
content: snippet,
|
|
578
886
|
engine: "duckduckgo"
|
|
@@ -588,15 +896,210 @@ var MultiSourceSearchEngine = class {
|
|
|
588
896
|
}
|
|
589
897
|
};
|
|
590
898
|
|
|
899
|
+
// src/harvester/youtubeTranscript.ts
|
|
900
|
+
var SPONSOR_NOISE_PATTERNS = [
|
|
901
|
+
/\b(sponsored by|this video is brought to you by|use (?:promo )?code\b|don't forget to like and subscribe|hit the bell icon|link in the description)\b/i,
|
|
902
|
+
/(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\s+này)?\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i
|
|
903
|
+
];
|
|
904
|
+
function extractYouTubeVideoId(urlOrId) {
|
|
905
|
+
const trimmed = urlOrId.trim();
|
|
906
|
+
if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;
|
|
907
|
+
try {
|
|
908
|
+
const u = new URL(trimmed);
|
|
909
|
+
const host = u.hostname.replace(/^www\./, "").toLowerCase();
|
|
910
|
+
if (host === "youtu.be") {
|
|
911
|
+
const id = u.pathname.split("/").filter(Boolean)[0];
|
|
912
|
+
return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;
|
|
913
|
+
}
|
|
914
|
+
if (host === "youtube.com" || host === "m.youtube.com" || host === "youtube-nocookie.com") {
|
|
915
|
+
const v = u.searchParams.get("v");
|
|
916
|
+
if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;
|
|
917
|
+
const parts = u.pathname.split("/").filter(Boolean);
|
|
918
|
+
if ((parts[0] === "embed" || parts[0] === "shorts" || parts[0] === "v") && parts[1]) {
|
|
919
|
+
return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
} catch {
|
|
923
|
+
}
|
|
924
|
+
return null;
|
|
925
|
+
}
|
|
926
|
+
function isYouTubeUrl(url) {
|
|
927
|
+
return extractYouTubeVideoId(url) !== null;
|
|
928
|
+
}
|
|
929
|
+
function formatTimestamp(seconds) {
|
|
930
|
+
const total = Math.max(0, Math.floor(seconds));
|
|
931
|
+
const hrs = Math.floor(total / 3600);
|
|
932
|
+
const mins = Math.floor(total % 3600 / 60);
|
|
933
|
+
const secs = total % 60;
|
|
934
|
+
if (hrs > 0) {
|
|
935
|
+
return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
|
|
936
|
+
}
|
|
937
|
+
return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
|
|
938
|
+
}
|
|
939
|
+
function decodeHtmlEntities(str) {
|
|
940
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/'/g, "'").replace(/ /g, " ").replace(/\s+/g, " ").trim();
|
|
941
|
+
}
|
|
942
|
+
function stitchCaptionSegments(segments) {
|
|
943
|
+
const cleaned = segments.map((s) => ({
|
|
944
|
+
startSec: s.startSec,
|
|
945
|
+
durationSec: s.durationSec,
|
|
946
|
+
text: decodeHtmlEntities(s.text)
|
|
947
|
+
})).filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));
|
|
948
|
+
if (cleaned.length === 0) return [];
|
|
949
|
+
const stitched = [];
|
|
950
|
+
let bucketStart = cleaned[0].startSec;
|
|
951
|
+
let bucketEnd = cleaned[0].startSec + cleaned[0].durationSec;
|
|
952
|
+
let bucketTexts = [];
|
|
953
|
+
for (const seg of cleaned) {
|
|
954
|
+
bucketTexts.push(seg.text);
|
|
955
|
+
bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);
|
|
956
|
+
const joined = bucketTexts.join(" ");
|
|
957
|
+
const elapsed = seg.startSec - bucketStart;
|
|
958
|
+
const endsSentence = /[.!?…]$/.test(seg.text);
|
|
959
|
+
if (elapsed >= 30 && endsSentence || elapsed >= 50 || joined.length >= 360) {
|
|
960
|
+
stitched.push({
|
|
961
|
+
startSec: bucketStart,
|
|
962
|
+
durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
|
|
963
|
+
timestamp: formatTimestamp(bucketStart),
|
|
964
|
+
text: joined
|
|
965
|
+
});
|
|
966
|
+
bucketTexts = [];
|
|
967
|
+
bucketStart = seg.startSec + seg.durationSec;
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
if (bucketTexts.length > 0) {
|
|
971
|
+
stitched.push({
|
|
972
|
+
startSec: bucketStart,
|
|
973
|
+
durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),
|
|
974
|
+
timestamp: formatTimestamp(bucketStart),
|
|
975
|
+
text: bucketTexts.join(" ")
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
return stitched;
|
|
979
|
+
}
|
|
980
|
+
var YouTubeIngestionService = class {
|
|
981
|
+
async fetchTranscript(urlOrId, preferredLanguages = ["vi", "en"]) {
|
|
982
|
+
const videoId = extractYouTubeVideoId(urlOrId);
|
|
983
|
+
if (!videoId) {
|
|
984
|
+
throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);
|
|
985
|
+
}
|
|
986
|
+
const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
|
987
|
+
const res = await fetch(canonicalUrl, {
|
|
988
|
+
headers: {
|
|
989
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
990
|
+
"Accept-Language": "vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7"
|
|
991
|
+
},
|
|
992
|
+
signal: AbortSignal.timeout(12e3)
|
|
993
|
+
});
|
|
994
|
+
if (!res.ok) {
|
|
995
|
+
throw new Error(`Failed to fetch YouTube watch page (${res.status})`);
|
|
996
|
+
}
|
|
997
|
+
const html = await res.text();
|
|
998
|
+
const playerMatch = html.match(/ytInitialPlayerResponse\s*=\s*(\{[\s\S]*?\});\s*(?:var\s|window\.|<\/script>)/);
|
|
999
|
+
let title = `YouTube Video (${videoId})`;
|
|
1000
|
+
let author;
|
|
1001
|
+
let shortDescription = "";
|
|
1002
|
+
let rawSegments = [];
|
|
1003
|
+
let selectedLang = "en";
|
|
1004
|
+
let isAutoGenerated = false;
|
|
1005
|
+
if (playerMatch?.[1]) {
|
|
1006
|
+
try {
|
|
1007
|
+
const playerData = JSON.parse(playerMatch[1]);
|
|
1008
|
+
title = playerData?.videoDetails?.title || title;
|
|
1009
|
+
author = playerData?.videoDetails?.author;
|
|
1010
|
+
shortDescription = playerData?.videoDetails?.shortDescription || "";
|
|
1011
|
+
const tracks = playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];
|
|
1012
|
+
if (tracks.length > 0) {
|
|
1013
|
+
let chosenTrack = tracks.find(
|
|
1014
|
+
(t) => preferredLanguages.includes(t.languageCode) && t.kind !== "asr"
|
|
1015
|
+
) || tracks.find((t) => preferredLanguages.includes(t.languageCode)) || tracks[0];
|
|
1016
|
+
if (chosenTrack?.baseUrl) {
|
|
1017
|
+
selectedLang = chosenTrack.languageCode || "en";
|
|
1018
|
+
isAutoGenerated = chosenTrack.kind === "asr";
|
|
1019
|
+
const trackRes = await fetch(chosenTrack.baseUrl, {
|
|
1020
|
+
signal: AbortSignal.timeout(1e4)
|
|
1021
|
+
});
|
|
1022
|
+
if (trackRes.ok) {
|
|
1023
|
+
const xml = await trackRes.text();
|
|
1024
|
+
const textMatches = [
|
|
1025
|
+
...xml.matchAll(/<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g)
|
|
1026
|
+
];
|
|
1027
|
+
for (const m of textMatches) {
|
|
1028
|
+
rawSegments.push({
|
|
1029
|
+
startSec: parseFloat(m[1] || "0"),
|
|
1030
|
+
durationSec: parseFloat(m[2] || "2"),
|
|
1031
|
+
text: (m[3] || "").replace(/<[^>]+>/g, " ")
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
} catch {
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
if (title === `YouTube Video (${videoId})`) {
|
|
1041
|
+
const titleMeta = html.match(/<title>([\s\S]*?)<\/title>/i);
|
|
1042
|
+
if (titleMeta?.[1]) {
|
|
1043
|
+
title = decodeHtmlEntities(titleMeta[1].replace(/\s*-\s*YouTube$/i, ""));
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
const stitched = stitchCaptionSegments(rawSegments);
|
|
1047
|
+
const mdLines = [
|
|
1048
|
+
`# ${title}`,
|
|
1049
|
+
`> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ""} | **Language:** \`${selectedLang}\`${isAutoGenerated ? " (Auto-caption)" : ""}`
|
|
1050
|
+
];
|
|
1051
|
+
if (shortDescription.trim()) {
|
|
1052
|
+
mdLines.push(`
|
|
1053
|
+
## Video Overview & Description
|
|
1054
|
+
${shortDescription.trim().slice(0, 1500)}`);
|
|
1055
|
+
}
|
|
1056
|
+
if (stitched.length > 0) {
|
|
1057
|
+
mdLines.push(`
|
|
1058
|
+
## Timestamped Transcript`);
|
|
1059
|
+
for (const seg of stitched) {
|
|
1060
|
+
mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
return {
|
|
1064
|
+
videoId,
|
|
1065
|
+
url: canonicalUrl,
|
|
1066
|
+
title,
|
|
1067
|
+
author,
|
|
1068
|
+
language: selectedLang,
|
|
1069
|
+
isAutoGenerated,
|
|
1070
|
+
segments: stitched,
|
|
1071
|
+
stitchedMarkdown: mdLines.join("\n")
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
};
|
|
1075
|
+
|
|
591
1076
|
// src/harvester/webScraper.ts
|
|
1077
|
+
function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
|
|
1078
|
+
const withoutCode = markdown.replace(/```[\s\S]*?```/g, "\n");
|
|
1079
|
+
const blocks = withoutCode.split(/\n{2,}/).map((b) => b.replace(/\s+/g, " ").trim()).filter((b) => {
|
|
1080
|
+
if (b.length < 60 || b.length > 900) return false;
|
|
1081
|
+
if (b.startsWith("#") || b.startsWith("|")) return false;
|
|
1082
|
+
const linkRatio = (b.match(/\[[^\]]+\]\([^)]+\)/g) || []).join("").length / b.length;
|
|
1083
|
+
if (linkRatio > 0.45) return false;
|
|
1084
|
+
if (/\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\b/i.test(b)) {
|
|
1085
|
+
return false;
|
|
1086
|
+
}
|
|
1087
|
+
return true;
|
|
1088
|
+
});
|
|
1089
|
+
return blocks.slice(0, maxExcerpts);
|
|
1090
|
+
}
|
|
592
1091
|
var WebScraper = class {
|
|
593
1092
|
crawl4aiHost;
|
|
594
1093
|
jinaKey;
|
|
1094
|
+
allowEnvFallback;
|
|
595
1095
|
customAdapter;
|
|
1096
|
+
youtubeService;
|
|
596
1097
|
constructor(options = {}) {
|
|
597
|
-
this.
|
|
598
|
-
this.
|
|
1098
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
1099
|
+
this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
|
|
1100
|
+
this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
|
|
599
1101
|
this.customAdapter = options.customAdapter;
|
|
1102
|
+
this.youtubeService = new YouTubeIngestionService();
|
|
600
1103
|
}
|
|
601
1104
|
setCustomAdapter(adapter) {
|
|
602
1105
|
this.customAdapter = adapter;
|
|
@@ -610,12 +1113,22 @@ var WebScraper = class {
|
|
|
610
1113
|
try {
|
|
611
1114
|
const result = await adapter.crawl(targetUrl);
|
|
612
1115
|
if (result && result.markdownContent) {
|
|
613
|
-
return this.packageDocument(targetUrl, result.markdownContent, "custom");
|
|
1116
|
+
return this.packageDocument(targetUrl, result.markdownContent, "custom", result.title);
|
|
614
1117
|
}
|
|
615
1118
|
} catch (e) {
|
|
616
1119
|
console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);
|
|
617
1120
|
}
|
|
618
1121
|
}
|
|
1122
|
+
if (isYouTubeUrl(targetUrl)) {
|
|
1123
|
+
try {
|
|
1124
|
+
const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);
|
|
1125
|
+
if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {
|
|
1126
|
+
return this.packageDocument(yt.url, yt.stitchedMarkdown, "youtube", yt.title);
|
|
1127
|
+
}
|
|
1128
|
+
} catch (e) {
|
|
1129
|
+
console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
619
1132
|
try {
|
|
620
1133
|
const jinaUrl = `https://r.jina.ai/${targetUrl}`;
|
|
621
1134
|
const jinaKey = options.jinaKey || this.jinaKey;
|
|
@@ -666,35 +1179,435 @@ var WebScraper = class {
|
|
|
666
1179
|
signal: AbortSignal.timeout(1e4)
|
|
667
1180
|
});
|
|
668
1181
|
if (res.ok) {
|
|
669
|
-
const
|
|
670
|
-
const
|
|
671
|
-
|
|
1182
|
+
const rawHtml = await res.text();
|
|
1183
|
+
const titleTag = rawHtml.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
|
|
1184
|
+
const clean = rawHtml.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "").replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, "").replace(/<h([1-3])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, lvl, txt) => `
|
|
1185
|
+
${"#".repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, "").trim()}
|
|
1186
|
+
`).replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => `
|
|
1187
|
+
\`\`\`
|
|
1188
|
+
${code.replace(/<[^>]+>/g, "").trim()}
|
|
1189
|
+
\`\`\`
|
|
1190
|
+
`).replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, p) => `
|
|
1191
|
+
${p.replace(/<[^>]+>/g, " ").trim()}
|
|
1192
|
+
`).replace(/<[^>]+>/g, " ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
1193
|
+
return this.packageDocument(targetUrl, clean, "direct", titleTag);
|
|
672
1194
|
}
|
|
673
1195
|
} catch (e) {
|
|
674
1196
|
console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);
|
|
675
1197
|
}
|
|
676
1198
|
throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
|
|
677
1199
|
}
|
|
678
|
-
packageDocument(url, markdown, provider) {
|
|
679
|
-
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(
|
|
680
|
-
const title = titleMatch ? titleMatch[1].trim() : url.split("/").pop() || "Document";
|
|
681
|
-
const
|
|
682
|
-
const
|
|
1200
|
+
packageDocument(url, markdown, provider, explicitTitle) {
|
|
1201
|
+
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
|
|
1202
|
+
const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
|
|
1203
|
+
const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```/g)];
|
|
1204
|
+
const extractedCodeBlocks = codeBlockMatches.map((m) => ({
|
|
1205
|
+
language: (m[1] || "").trim().toLowerCase(),
|
|
1206
|
+
code: (m[2] || "").trim()
|
|
1207
|
+
})).filter((b) => b.code.length > 0);
|
|
1208
|
+
const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);
|
|
1209
|
+
const tableBlocks = markdown.match(/(?:^\|.+\|\s*\n)+^\|[-:| ]+\|\s*\n(?:^\|.+\|\s*\n?)+/gm) || markdown.match(/\|[\s\S]*?\|[\s\S]*?\|/g) || [];
|
|
1210
|
+
const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);
|
|
1211
|
+
const extractedHeadings = [...markdown.matchAll(/^#{1,3}\s+(.+)$/gm)].map((m) => m[1].trim()).filter((h) => h.length > 2 && h.length < 120);
|
|
1212
|
+
const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);
|
|
683
1213
|
return {
|
|
684
1214
|
url,
|
|
685
1215
|
title,
|
|
686
1216
|
markdownContent: markdown,
|
|
687
1217
|
byteSize: Buffer.byteLength(markdown, "utf-8"),
|
|
688
|
-
extractedCodeSnippets
|
|
689
|
-
|
|
1218
|
+
extractedCodeSnippets,
|
|
1219
|
+
extractedCodeBlocks,
|
|
1220
|
+
extractedTables,
|
|
1221
|
+
extractedHeadings,
|
|
1222
|
+
keyExcerpts,
|
|
1223
|
+
tablesCount: extractedTables.length,
|
|
690
1224
|
fetchProvider: provider
|
|
691
1225
|
};
|
|
692
1226
|
}
|
|
693
1227
|
};
|
|
694
1228
|
|
|
1229
|
+
// src/harvester/termExtractor.ts
|
|
1230
|
+
var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
1231
|
+
"l\xE0",
|
|
1232
|
+
"c\u1EE7a",
|
|
1233
|
+
"trong",
|
|
1234
|
+
"\u0111\u01B0\u1EE3c",
|
|
1235
|
+
"nh\u1EEFng",
|
|
1236
|
+
"c\xE1c",
|
|
1237
|
+
"cho",
|
|
1238
|
+
"v\u1EDBi",
|
|
1239
|
+
"khi",
|
|
1240
|
+
"n\u1EBFu",
|
|
1241
|
+
"th\xEC",
|
|
1242
|
+
"v\xE0",
|
|
1243
|
+
"ho\u1EB7c",
|
|
1244
|
+
"\u0111\u1EC3",
|
|
1245
|
+
"t\u1EEB",
|
|
1246
|
+
"\u0111\u1EBFn",
|
|
1247
|
+
"v\u1EC1",
|
|
1248
|
+
"n\xE0y",
|
|
1249
|
+
"\u0111\xF3",
|
|
1250
|
+
"m\u1ED9t",
|
|
1251
|
+
"nh\u01B0",
|
|
1252
|
+
"s\u1EBD",
|
|
1253
|
+
"\u0111ang",
|
|
1254
|
+
"\u0111\xE3",
|
|
1255
|
+
"kh\xF4ng",
|
|
1256
|
+
"c\xF3",
|
|
1257
|
+
"ra",
|
|
1258
|
+
"v\xE0o",
|
|
1259
|
+
"b\u1EB1ng",
|
|
1260
|
+
"theo",
|
|
1261
|
+
"tr\xEAn",
|
|
1262
|
+
"d\u01B0\u1EDBi",
|
|
1263
|
+
"qua",
|
|
1264
|
+
"nh\u1EDD",
|
|
1265
|
+
"v\xEC",
|
|
1266
|
+
"do",
|
|
1267
|
+
"t\u1EA1i",
|
|
1268
|
+
"b\u1EDFi",
|
|
1269
|
+
"nh\u01B0ng",
|
|
1270
|
+
"m\xE0",
|
|
1271
|
+
"c\u0169ng",
|
|
1272
|
+
"v\u1EABn",
|
|
1273
|
+
"r\u1EA5t",
|
|
1274
|
+
"qu\xE1",
|
|
1275
|
+
"h\u01A1n",
|
|
1276
|
+
"nh\u1EA5t",
|
|
1277
|
+
"c\u1EA7n",
|
|
1278
|
+
"ph\u1EA3i",
|
|
1279
|
+
"n\xEAn",
|
|
1280
|
+
"th\u1EC3",
|
|
1281
|
+
"vi\u1EC7c",
|
|
1282
|
+
"c\xE1ch",
|
|
1283
|
+
"ph\u1EA7n",
|
|
1284
|
+
"b\xE0i",
|
|
1285
|
+
"bu\u1ED5i",
|
|
1286
|
+
"h\u1ECDc",
|
|
1287
|
+
"sinh",
|
|
1288
|
+
"gi\xE1o",
|
|
1289
|
+
"vi\xEAn",
|
|
1290
|
+
"ng\u01B0\u1EDDi",
|
|
1291
|
+
"d\xF9ng",
|
|
1292
|
+
"ch\xFAng",
|
|
1293
|
+
"ta",
|
|
1294
|
+
"b\u1EA1n",
|
|
1295
|
+
"em",
|
|
1296
|
+
"gi\xFAp",
|
|
1297
|
+
"l\xE0m",
|
|
1298
|
+
"t\u1EA1o",
|
|
1299
|
+
"s\u1EED",
|
|
1300
|
+
"d\u1EE5ng",
|
|
1301
|
+
"th\u1EF1c",
|
|
1302
|
+
"hi\u1EC7n",
|
|
1303
|
+
"hi\u1EC3u",
|
|
1304
|
+
"bi\u1EBFt",
|
|
1305
|
+
"xem",
|
|
1306
|
+
"th\xEAm",
|
|
1307
|
+
"sau",
|
|
1308
|
+
"tr\u01B0\u1EDBc"
|
|
1309
|
+
]);
|
|
1310
|
+
var ENGLISH_STOPWORDS = /* @__PURE__ */ new Set([
|
|
1311
|
+
"the",
|
|
1312
|
+
"and",
|
|
1313
|
+
"for",
|
|
1314
|
+
"with",
|
|
1315
|
+
"that",
|
|
1316
|
+
"this",
|
|
1317
|
+
"from",
|
|
1318
|
+
"into",
|
|
1319
|
+
"your",
|
|
1320
|
+
"you",
|
|
1321
|
+
"are",
|
|
1322
|
+
"was",
|
|
1323
|
+
"were",
|
|
1324
|
+
"have",
|
|
1325
|
+
"has",
|
|
1326
|
+
"had",
|
|
1327
|
+
"will",
|
|
1328
|
+
"would",
|
|
1329
|
+
"can",
|
|
1330
|
+
"could",
|
|
1331
|
+
"should",
|
|
1332
|
+
"about",
|
|
1333
|
+
"more",
|
|
1334
|
+
"other",
|
|
1335
|
+
"such",
|
|
1336
|
+
"than",
|
|
1337
|
+
"then",
|
|
1338
|
+
"them",
|
|
1339
|
+
"these",
|
|
1340
|
+
"those",
|
|
1341
|
+
"their",
|
|
1342
|
+
"what",
|
|
1343
|
+
"when",
|
|
1344
|
+
"where",
|
|
1345
|
+
"which",
|
|
1346
|
+
"while",
|
|
1347
|
+
"who",
|
|
1348
|
+
"how",
|
|
1349
|
+
"why",
|
|
1350
|
+
"each",
|
|
1351
|
+
"some",
|
|
1352
|
+
"many",
|
|
1353
|
+
"most",
|
|
1354
|
+
"also",
|
|
1355
|
+
"only",
|
|
1356
|
+
"just",
|
|
1357
|
+
"very",
|
|
1358
|
+
"using",
|
|
1359
|
+
"used",
|
|
1360
|
+
"use",
|
|
1361
|
+
"overview",
|
|
1362
|
+
"introduction",
|
|
1363
|
+
"getting",
|
|
1364
|
+
"started",
|
|
1365
|
+
"guide",
|
|
1366
|
+
"tutorial",
|
|
1367
|
+
"example",
|
|
1368
|
+
"examples",
|
|
1369
|
+
"documentation"
|
|
1370
|
+
]);
|
|
1371
|
+
var VIETNAMESE_TECHNICAL_HEADS = /^(cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|chuỗi|đối tượng|lớp|phương thức|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|mạng|giao thức|khái niệm|mô hình|kiến trúc|kiểm thử|gỡ lỗi)\b/i;
|
|
1372
|
+
var SYNTAX_OR_API_PATTERN = /(?:`[^`]+`|\b[a-z]+[A-Z][a-zA-Z0-9]*\b|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b|@[A-Za-z]+|\b[a-z_]+\(\)|\b(?:pinMode|digitalWrite|digitalRead|analogRead|analogWrite|async|await|actor|struct|class|func|def|let|const|var)\b)/;
|
|
1373
|
+
var HARDWARE_COMPONENT_PATTERN = /\b(esp32|arduino|micro:?bit|raspberry|sensor|led|servo|ultrasonic|ldr|dht11|dht22|breadboard|gpio|pwm|i2c|spi|uart|cảm biến|động cơ|mạch|đèn led|vi điều khiển)\b/i;
|
|
1374
|
+
function normalizeTermKey(raw) {
|
|
1375
|
+
return raw.normalize("NFC").toLowerCase().replace(/[`*_~#"']/g, "").replace(/\s+/g, " ").trim();
|
|
1376
|
+
}
|
|
1377
|
+
function cleanDisplayTerm(raw) {
|
|
1378
|
+
return raw.normalize("NFC").replace(/^[-*•#\d.)\s]+/, "").replace(/[`*_~]+/g, "").replace(/\s+/g, " ").trim();
|
|
1379
|
+
}
|
|
1380
|
+
function isNoiseTerm(term) {
|
|
1381
|
+
const clean = normalizeTermKey(term);
|
|
1382
|
+
if (clean.length < 2 || clean.length > 65) return true;
|
|
1383
|
+
if (/^\d+$/.test(clean)) return true;
|
|
1384
|
+
if (/^https?:\/\//i.test(clean)) return true;
|
|
1385
|
+
if (ENGLISH_STOPWORDS.has(clean) || VIETNAMESE_STOPWORDS.has(clean)) return true;
|
|
1386
|
+
const words = clean.split(" ");
|
|
1387
|
+
if (words.length > 6) return true;
|
|
1388
|
+
if (words.length === 1 && clean.length < 3 && !/^(ai|ui|ux|os|io|db|id|ip)$/i.test(clean)) return true;
|
|
1389
|
+
return false;
|
|
1390
|
+
}
|
|
1391
|
+
function classifyMarrAndCategory(term, domain) {
|
|
1392
|
+
if (HARDWARE_COMPONENT_PATTERN.test(term)) {
|
|
1393
|
+
return { category: "hardware_component", marrLayer: "CIO", bloomLevel: "apply" };
|
|
1394
|
+
}
|
|
1395
|
+
if (SYNTAX_OR_API_PATTERN.test(term)) {
|
|
1396
|
+
return { category: "api_syntax", marrLayer: "SIO", bloomLevel: "apply" };
|
|
1397
|
+
}
|
|
1398
|
+
if (/\b(architecture|pattern|model|design|debugging|verification|mô hình|kiến trúc|tư duy|quy trình)\b/i.test(term)) {
|
|
1399
|
+
return { category: "pedagogical_pattern", marrLayer: "ULO", bloomLevel: "analyze" };
|
|
1400
|
+
}
|
|
1401
|
+
return {
|
|
1402
|
+
category: "core_concept",
|
|
1403
|
+
marrLayer: domain === "academic" ? "ULO" : "CIO",
|
|
1404
|
+
bloomLevel: "understand"
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
function findSentenceDefinition(term, markdown) {
|
|
1408
|
+
const clean = cleanDisplayTerm(term);
|
|
1409
|
+
const escaped = clean.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1410
|
+
const sentences = markdown.replace(/```[\s\S]*?```/g, " ").split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").trim()).filter((s) => s.length >= 25 && s.length <= 280);
|
|
1411
|
+
const re = new RegExp(escaped, "i");
|
|
1412
|
+
const match = sentences.find((s) => re.test(s));
|
|
1413
|
+
return match || `Core concept in ${clean} extracted from primary technical reference.`;
|
|
1414
|
+
}
|
|
1415
|
+
var PedagogicalTermExtractor = class {
|
|
1416
|
+
extractDeterministic(topic, domain, docs, maxTerms = 15) {
|
|
1417
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
1418
|
+
let orderCounter = 0;
|
|
1419
|
+
const upsertCandidate = (rawTerm, weight, resourceId, markdown, pair) => {
|
|
1420
|
+
const display = cleanDisplayTerm(rawTerm);
|
|
1421
|
+
if (isNoiseTerm(display)) return;
|
|
1422
|
+
const key = normalizeTermKey(display);
|
|
1423
|
+
const existing = candidates.get(key);
|
|
1424
|
+
if (existing) {
|
|
1425
|
+
existing.score += weight;
|
|
1426
|
+
existing.sourceIds.add(resourceId);
|
|
1427
|
+
if (pair?.termVi && !existing.termVi) existing.termVi = pair.termVi;
|
|
1428
|
+
if (pair?.termEn && !existing.termEn) existing.termEn = pair.termEn;
|
|
1429
|
+
} else {
|
|
1430
|
+
candidates.set(key, {
|
|
1431
|
+
display,
|
|
1432
|
+
termVi: pair?.termVi,
|
|
1433
|
+
termEn: pair?.termEn,
|
|
1434
|
+
score: weight,
|
|
1435
|
+
firstSeenOrder: orderCounter++,
|
|
1436
|
+
sourceIds: /* @__PURE__ */ new Set([resourceId]),
|
|
1437
|
+
definition: findSentenceDefinition(display, markdown)
|
|
1438
|
+
});
|
|
1439
|
+
}
|
|
1440
|
+
};
|
|
1441
|
+
for (const { resourceId, doc } of docs) {
|
|
1442
|
+
const md = doc.markdownContent || "";
|
|
1443
|
+
const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
|
|
1444
|
+
const bilingualMatches = mdWithoutCode.matchAll(
|
|
1445
|
+
/([A-ZÀÁẠẢÃÂẦẤẬẨẪĂẰẮẶẲẴÈÉẸẺẼÊỀẾỆỂỄÌÍỊỈĨÒÓỌỎÕÔỒỐỘỔỖƠỜỚỢỞỠÙÚỤỦŨƯỪỨỰỬỮỲÝỴỶỸĐ][a-zA-Z0-9\u00C0-\u1EF9\s/-]{2,40}?)\s*\(([A-Za-z0-9\u00C0-\u1EF9\s/_:-]{2,40})\)/g
|
|
1446
|
+
);
|
|
1447
|
+
for (const m of bilingualMatches) {
|
|
1448
|
+
const left = cleanDisplayTerm(m[1] || "");
|
|
1449
|
+
const right = cleanDisplayTerm(m[2] || "");
|
|
1450
|
+
if (!isNoiseTerm(left) && !isNoiseTerm(right)) {
|
|
1451
|
+
const leftHasVi = /[\u00C0-\u1EF9]/.test(left);
|
|
1452
|
+
const rightHasVi = /[\u00C0-\u1EF9]/.test(right);
|
|
1453
|
+
if (leftHasVi && !rightHasVi) {
|
|
1454
|
+
upsertCandidate(`${left} (${right})`, 6, resourceId, mdWithoutCode, { termVi: left, termEn: right });
|
|
1455
|
+
} else if (!leftHasVi && rightHasVi) {
|
|
1456
|
+
upsertCandidate(`${right} (${left})`, 6, resourceId, mdWithoutCode, { termVi: right, termEn: left });
|
|
1457
|
+
} else {
|
|
1458
|
+
upsertCandidate(left, 4, resourceId, mdWithoutCode);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
for (const heading of doc.extractedHeadings || []) {
|
|
1463
|
+
const cleaned = cleanDisplayTerm(heading.replace(/^\d+(\.\d+)*\s*/, ""));
|
|
1464
|
+
if (!isNoiseTerm(cleaned)) {
|
|
1465
|
+
upsertCandidate(cleaned, 4, resourceId, mdWithoutCode);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
for (const m of mdWithoutCode.matchAll(/\*\*([^*\n]{3,45})\*\*/g)) {
|
|
1469
|
+
upsertCandidate(m[1] || "", 3.5, resourceId, mdWithoutCode);
|
|
1470
|
+
}
|
|
1471
|
+
for (const m of mdWithoutCode.matchAll(/`([a-zA-Z_@][a-zA-Z0-9_.:()-]{2,35})`/g)) {
|
|
1472
|
+
upsertCandidate(m[1] || "", 3, resourceId, mdWithoutCode);
|
|
1473
|
+
}
|
|
1474
|
+
for (const line of mdWithoutCode.split("\n")) {
|
|
1475
|
+
const clauses = line.split(/[.,;:!?()[\]{}"“”‘’/\\|]+/);
|
|
1476
|
+
for (const clause of clauses) {
|
|
1477
|
+
const trimmed = clause.trim();
|
|
1478
|
+
const viMatch = trimmed.match(
|
|
1479
|
+
/\b((?:cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|đối tượng|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|giao thức|mô hình|kiến trúc|kiểm thử|gỡ lỗi)(?:\s+[a-zA-Z0-9\u00C0-\u1EF9_-]+){1,3})\b/i
|
|
1480
|
+
);
|
|
1481
|
+
if (viMatch?.[1]) {
|
|
1482
|
+
const tokens = viMatch[1].trim().split(/\s+/);
|
|
1483
|
+
while (tokens.length > 1 && VIETNAMESE_STOPWORDS.has(tokens[tokens.length - 1].toLowerCase())) {
|
|
1484
|
+
tokens.pop();
|
|
1485
|
+
}
|
|
1486
|
+
const phrase = tokens.join(" ");
|
|
1487
|
+
if (tokens.length >= 2 && VIETNAMESE_TECHNICAL_HEADS.test(phrase)) {
|
|
1488
|
+
upsertCandidate(phrase, 3.5, resourceId, mdWithoutCode, { termVi: phrase });
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
const sorted = [...candidates.values()].sort((a, b) => b.score - a.score || a.firstSeenOrder - b.firstSeenOrder).slice(0, maxTerms);
|
|
1495
|
+
const orderedByAppearance = [...sorted].sort((a, b) => a.firstSeenOrder - b.firstSeenOrder);
|
|
1496
|
+
const maxScore = Math.max(1, sorted[0]?.score || 1);
|
|
1497
|
+
const concepts = orderedByAppearance.map((item, idx) => {
|
|
1498
|
+
const id = `C_${String(idx + 1).padStart(2, "0")}`;
|
|
1499
|
+
const { category, marrLayer, bloomLevel } = classifyMarrAndCategory(item.display, domain);
|
|
1500
|
+
const prerequisites = [];
|
|
1501
|
+
return {
|
|
1502
|
+
id,
|
|
1503
|
+
term: item.display,
|
|
1504
|
+
termVi: item.termVi,
|
|
1505
|
+
termEn: item.termEn,
|
|
1506
|
+
category,
|
|
1507
|
+
marrLayer,
|
|
1508
|
+
bloomLevel,
|
|
1509
|
+
salienceScore: Number(Math.min(10, Math.max(1, item.score / maxScore * 10)).toFixed(1)),
|
|
1510
|
+
definition: item.definition,
|
|
1511
|
+
prerequisites,
|
|
1512
|
+
sourceIds: [...item.sourceIds]
|
|
1513
|
+
};
|
|
1514
|
+
});
|
|
1515
|
+
return {
|
|
1516
|
+
concepts,
|
|
1517
|
+
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
1518
|
+
isLlmRefined: false,
|
|
1519
|
+
unrefinedCandidatesCount: candidates.size
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
async extractWithOptionalLlm(topic, domain, docs, options = {}) {
|
|
1523
|
+
const detBundle = this.extractDeterministic(topic, domain, docs, options.maxTerms ?? 15);
|
|
1524
|
+
if (!options.llm || docs.length === 0) {
|
|
1525
|
+
return detBundle;
|
|
1526
|
+
}
|
|
1527
|
+
try {
|
|
1528
|
+
const corpusPreview = docs.slice(0, 6).map(
|
|
1529
|
+
(d) => `[${d.resourceId}] ${d.doc.title} (${d.doc.url}):
|
|
1530
|
+
${(d.doc.keyExcerpts || []).slice(0, 3).join("\n")}`
|
|
1531
|
+
).join("\n\n");
|
|
1532
|
+
const candidateSeed = detBundle.concepts.slice(0, 12).map((c) => `${c.term} (${c.sourceIds.join(",")})`).join("; ");
|
|
1533
|
+
const prompt = [
|
|
1534
|
+
`Analyze the harvested technical & pedagogical corpus for topic "${topic}" (Domain: ${domain}).`,
|
|
1535
|
+
options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]} years old.` : "",
|
|
1536
|
+
options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}.` : "",
|
|
1537
|
+
options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}.` : "",
|
|
1538
|
+
`Deterministic Candidate Terms: ${candidateSeed}`,
|
|
1539
|
+
`
|
|
1540
|
+
Harvested Source Excerpts:
|
|
1541
|
+
${corpusPreview}`,
|
|
1542
|
+
`
|
|
1543
|
+
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
1544
|
+
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
1545
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
|
|
1546
|
+
].filter(Boolean).join("\n");
|
|
1547
|
+
const rawResponse = await options.llm(prompt, {
|
|
1548
|
+
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
1549
|
+
temperature: 0.2,
|
|
1550
|
+
jsonMode: true
|
|
1551
|
+
});
|
|
1552
|
+
const jsonMatch = rawResponse.match(/\[[\s\S]*\]/);
|
|
1553
|
+
if (jsonMatch) {
|
|
1554
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
1555
|
+
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
1556
|
+
const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
|
|
1557
|
+
const concepts = parsed.map((item, idx) => {
|
|
1558
|
+
const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
|
|
1559
|
+
const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
|
|
1560
|
+
return {
|
|
1561
|
+
id,
|
|
1562
|
+
term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
|
|
1563
|
+
termVi: item.termVi ? String(item.termVi) : void 0,
|
|
1564
|
+
termEn: item.termEn ? String(item.termEn) : void 0,
|
|
1565
|
+
category: ["core_concept", "api_syntax", "hardware_component", "pedagogical_pattern"].includes(item.category) ? item.category : "core_concept",
|
|
1566
|
+
marrLayer: ["ULO", "CIO", "SIO"].includes(item.marrLayer) ? item.marrLayer : "CIO",
|
|
1567
|
+
bloomLevel: ["remember", "understand", "apply", "analyze", "evaluate", "create"].includes(item.bloomLevel) ? item.bloomLevel : "understand",
|
|
1568
|
+
salienceScore: Number(item.salienceScore) || 8,
|
|
1569
|
+
definition: String(item.definition || ""),
|
|
1570
|
+
prerequisites: prereqs,
|
|
1571
|
+
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
|
|
1572
|
+
};
|
|
1573
|
+
});
|
|
1574
|
+
return {
|
|
1575
|
+
concepts,
|
|
1576
|
+
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
1577
|
+
isLlmRefined: true,
|
|
1578
|
+
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
1579
|
+
};
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
} catch (e) {
|
|
1583
|
+
console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
|
|
1584
|
+
}
|
|
1585
|
+
return detBundle;
|
|
1586
|
+
}
|
|
1587
|
+
formatConceptDagTsv(concepts) {
|
|
1588
|
+
const header = "id term term_vi term_en marr_layer bloom_level salience prerequisites sources definition";
|
|
1589
|
+
const rows = concepts.map(
|
|
1590
|
+
(c) => [
|
|
1591
|
+
c.id,
|
|
1592
|
+
c.term.replace(/\t|\n/g, " "),
|
|
1593
|
+
(c.termVi || "").replace(/\t|\n/g, " "),
|
|
1594
|
+
(c.termEn || "").replace(/\t|\n/g, " "),
|
|
1595
|
+
c.marrLayer,
|
|
1596
|
+
c.bloomLevel,
|
|
1597
|
+
c.salienceScore,
|
|
1598
|
+
c.prerequisites.join(","),
|
|
1599
|
+
c.sourceIds.join(","),
|
|
1600
|
+
c.definition.replace(/\t|\n/g, " ")
|
|
1601
|
+
].join(" ")
|
|
1602
|
+
);
|
|
1603
|
+
return [header, ...rows].join("\n");
|
|
1604
|
+
}
|
|
1605
|
+
};
|
|
1606
|
+
|
|
695
1607
|
// src/harvester/catalogBuilder.ts
|
|
696
1608
|
var CatalogBuilder = class {
|
|
697
|
-
buildBundle(topic, domain, items) {
|
|
1609
|
+
buildBundle(topic, domain, items, options = {}) {
|
|
1610
|
+
const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
|
|
698
1611
|
const indexList = items.map((item, idx) => ({
|
|
699
1612
|
id: `RES-${String(idx + 1).padStart(2, "0")}`,
|
|
700
1613
|
title: item.doc.title,
|
|
@@ -704,7 +1617,8 @@ var CatalogBuilder = class {
|
|
|
704
1617
|
byteSize: item.doc.byteSize,
|
|
705
1618
|
tablesCount: item.doc.tablesCount,
|
|
706
1619
|
codeSnippetsCount: item.doc.extractedCodeSnippets.length,
|
|
707
|
-
summary: item.summary
|
|
1620
|
+
summary: item.summary,
|
|
1621
|
+
keyTakeaways: item.keyTakeaways || item.doc.keyExcerpts?.slice(0, 3) || []
|
|
708
1622
|
}));
|
|
709
1623
|
const resourceIndexJson = JSON.stringify(
|
|
710
1624
|
{
|
|
@@ -723,36 +1637,60 @@ var CatalogBuilder = class {
|
|
|
723
1637
|
`
|
|
724
1638
|
| ID | Resource Title | Relevance (1-10) | Code Snippets | Tables | Source Link |`,
|
|
725
1639
|
`|---|---|---|---|---|---|`,
|
|
726
|
-
...indexList.map(
|
|
1640
|
+
...indexList.map(
|
|
1641
|
+
(r) => `| **${r.id}** | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`
|
|
1642
|
+
),
|
|
727
1643
|
`
|
|
728
1644
|
## Resource Summaries & Technical Ground Truth
|
|
729
1645
|
`,
|
|
730
|
-
...indexList.map((r) =>
|
|
1646
|
+
...indexList.map((r) => {
|
|
1647
|
+
const takeaways = r.keyTakeaways && r.keyTakeaways.length > 0 ? r.keyTakeaways.map((t) => ` - ${t}`).join("\n") : ` - ${r.summary}`;
|
|
1648
|
+
return `### ${r.id}: ${r.title}
|
|
731
1649
|
- **URL:** ${r.url}
|
|
732
|
-
- **Summary:** ${r.summary}
|
|
733
1650
|
- **Relevance:** ${r.relevanceScore}/10
|
|
734
|
-
|
|
1651
|
+
- **Key Excerpts:**
|
|
1652
|
+
${takeaways}
|
|
1653
|
+
`;
|
|
1654
|
+
})
|
|
735
1655
|
];
|
|
1656
|
+
const codeSections = items.flatMap((item) => {
|
|
1657
|
+
if (item.doc.extractedCodeBlocks && item.doc.extractedCodeBlocks.length > 0) {
|
|
1658
|
+
return item.doc.extractedCodeBlocks.slice(0, 2).map((b) => {
|
|
1659
|
+
const lang = b.language || defaultLang;
|
|
1660
|
+
return `\`\`\`${lang}
|
|
1661
|
+
${b.code}
|
|
1662
|
+
\`\`\``;
|
|
1663
|
+
});
|
|
1664
|
+
}
|
|
1665
|
+
return item.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
|
|
1666
|
+
${c}
|
|
1667
|
+
\`\`\``);
|
|
1668
|
+
});
|
|
1669
|
+
const conceptLines = options.concepts && options.concepts.length > 0 ? [
|
|
1670
|
+
`| ID | Concept / Term | Vietnamese | English | Marr | Bloom | Prereqs | Sources |`,
|
|
1671
|
+
`|---|---|---|---|---|---|---|---|`,
|
|
1672
|
+
...options.concepts.map(
|
|
1673
|
+
(c) => `| \`${c.id}\` | **${c.term.replace(/\|/g, "/")}** | ${c.termVi || "\u2014"} | ${c.termEn || "\u2014"} | \`${c.marrLayer}\` | \`${c.bloomLevel}\` | ${c.prerequisites.join(", ") || "\u2014"} | ${c.sourceIds.join(", ")} |`
|
|
1674
|
+
)
|
|
1675
|
+
] : [`- Canonical technical breakdown for **${topic}** derived from ${items.length} verified primary sources.`];
|
|
736
1676
|
const kgLines = [
|
|
737
1677
|
`# \u{1F9EC} Distilled Knowledge Graph: ${topic}`,
|
|
738
1678
|
`> **Domain:** \`${domain.toUpperCase()}\` | **Ground Truth Verification:** PASSED`,
|
|
739
1679
|
`
|
|
740
|
-
## 1. Architectural Blueprint &
|
|
741
|
-
|
|
1680
|
+
## 1. Architectural Blueprint & Concept Hierarchy (Marr & Bloom)`,
|
|
1681
|
+
...conceptLines,
|
|
742
1682
|
`
|
|
743
1683
|
## 2. Core APIs & Code Primitives`,
|
|
744
|
-
...
|
|
745
|
-
${c}
|
|
746
|
-
\`\`\``),
|
|
1684
|
+
...codeSections.length > 0 ? codeSections : ["*No explicit code snippets captured in harvested sources.*"],
|
|
747
1685
|
`
|
|
748
|
-
## 3.
|
|
749
|
-
|
|
750
|
-
`- Proactively prevent documented anti-patterns and performance regressions.`
|
|
1686
|
+
## 3. Extracted Reference Tables`,
|
|
1687
|
+
...items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)).length > 0 ? items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)) : ["*No markdown reference tables extracted.*"]
|
|
751
1688
|
];
|
|
752
1689
|
return {
|
|
753
1690
|
resourceIndexJson,
|
|
754
1691
|
resourceCatalogMd: catalogLines.join("\n"),
|
|
755
|
-
knowledgeGraphMd: kgLines.join("\n")
|
|
1692
|
+
knowledgeGraphMd: kgLines.join("\n"),
|
|
1693
|
+
conceptDagTsv: options.conceptDagTsv
|
|
756
1694
|
};
|
|
757
1695
|
}
|
|
758
1696
|
};
|
|
@@ -778,73 +1716,372 @@ var HealthAuditor = class {
|
|
|
778
1716
|
};
|
|
779
1717
|
|
|
780
1718
|
// src/evidence/evidenceScorer.ts
|
|
1719
|
+
function tokenizeBilingualKeywords(text) {
|
|
1720
|
+
return text.normalize("NFC").toLowerCase().replace(/[.,;:!?()[\]{}"“”‘’/\\|`*_~#-]+/g, " ").split(/\s+/).filter((w) => w.length >= 2 && !/^(and|the|for|with|from|into|của|trong|được|những|các|cho|với|khi|và|hoặc|để|từ|đến|về)$/.test(w));
|
|
1721
|
+
}
|
|
1722
|
+
function extractTopSentences(markdown, targetTopic, maxSentences = 3) {
|
|
1723
|
+
const keywords = tokenizeBilingualKeywords(targetTopic);
|
|
1724
|
+
const withoutCode = markdown.replace(/```[\s\S]*?```/g, " ");
|
|
1725
|
+
const rawSentences = withoutCode.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => {
|
|
1726
|
+
if (s.length < 45 || s.length > 320) return false;
|
|
1727
|
+
if (s.startsWith("|") || s.startsWith("http")) return false;
|
|
1728
|
+
if (/\b(cookie|privacy policy|sign in|subscribe|all rights reserved)\b/i.test(s)) return false;
|
|
1729
|
+
return true;
|
|
1730
|
+
});
|
|
1731
|
+
if (rawSentences.length === 0) return [];
|
|
1732
|
+
const scored = rawSentences.map((sentence, idx) => {
|
|
1733
|
+
const lower = sentence.normalize("NFC").toLowerCase();
|
|
1734
|
+
const matchedCount = keywords.filter((k) => lower.includes(k)).length;
|
|
1735
|
+
const positionBonus = idx < 8 ? 1.5 : idx < 20 ? 0.5 : 0;
|
|
1736
|
+
const technicalBonus = /(`[^`]+`|\b(?:api|function|class|actor|async|sensor|pin|loop|state|biến|hàm|cảm biến|vòng lặp|mạch)\b)/i.test(
|
|
1737
|
+
sentence
|
|
1738
|
+
) ? 1.5 : 0;
|
|
1739
|
+
return {
|
|
1740
|
+
sentence,
|
|
1741
|
+
idx,
|
|
1742
|
+
score: matchedCount * 2 + positionBonus + technicalBonus
|
|
1743
|
+
};
|
|
1744
|
+
});
|
|
1745
|
+
return scored.sort((a, b) => b.score - a.score || a.idx - b.idx).slice(0, maxSentences).sort((a, b) => a.idx - b.idx).map((s) => s.sentence);
|
|
1746
|
+
}
|
|
781
1747
|
var EvidenceScorer = class {
|
|
782
1748
|
scoreDocument(doc, domain, targetTopic) {
|
|
783
1749
|
let depthScore = 5;
|
|
784
|
-
|
|
1750
|
+
const hasCode = doc.extractedCodeSnippets.length > 0;
|
|
1751
|
+
let codeScore = hasCode ? Math.min(10, 7 + Math.min(3, doc.extractedCodeSnippets.length)) : 5;
|
|
785
1752
|
let noiseScore = 2;
|
|
786
|
-
const lowerContent = doc.markdownContent.toLowerCase();
|
|
787
|
-
const targetKeywords = targetTopic
|
|
1753
|
+
const lowerContent = doc.markdownContent.normalize("NFC").toLowerCase();
|
|
1754
|
+
const targetKeywords = tokenizeBilingualKeywords(targetTopic);
|
|
788
1755
|
const keywordMatches = targetKeywords.filter((k) => lowerContent.includes(k)).length;
|
|
789
|
-
|
|
1756
|
+
const matchRatio = targetKeywords.length > 0 ? keywordMatches / targetKeywords.length : 1;
|
|
1757
|
+
if (matchRatio >= 0.75) depthScore += 3;
|
|
1758
|
+
else if (matchRatio >= 0.4) depthScore += 2;
|
|
1759
|
+
else if (keywordMatches >= 1) depthScore += 1;
|
|
790
1760
|
if (doc.tablesCount > 0) depthScore += 1;
|
|
791
|
-
if (doc.byteSize >
|
|
1761
|
+
if (doc.byteSize > 1500) depthScore += 1;
|
|
1762
|
+
if ((doc.extractedHeadings?.length || 0) >= 2) depthScore += 1;
|
|
1763
|
+
depthScore = Math.min(10, depthScore);
|
|
1764
|
+
if (!hasCode && doc.byteSize > 1200 && matchRatio >= 0.5) {
|
|
1765
|
+
codeScore = 6;
|
|
1766
|
+
}
|
|
792
1767
|
if (lowerContent.includes("subscribe") && lowerContent.includes("advertisement")) {
|
|
793
1768
|
noiseScore += 3;
|
|
794
1769
|
}
|
|
795
|
-
const rawTotal = depthScore * 0.
|
|
1770
|
+
const rawTotal = depthScore * 0.55 + codeScore * 0.35 - noiseScore * 0.1 + 0.5;
|
|
796
1771
|
const score = Math.max(1, Math.min(10, Math.round(rawTotal)));
|
|
797
1772
|
const isApproved = score >= 6;
|
|
798
|
-
const
|
|
1773
|
+
const topSentences = extractTopSentences(doc.markdownContent, targetTopic, 3);
|
|
1774
|
+
const excerpts = doc.keyExcerpts?.length ? doc.keyExcerpts : extractSubstantiveExcerpts(doc.markdownContent, 3);
|
|
1775
|
+
const keyTakeaways = topSentences.length > 0 ? topSentences : excerpts.slice(0, 2);
|
|
1776
|
+
const summary = keyTakeaways.length > 0 ? `${keyTakeaways[0]} [Evidence ${score}/10 \xB7 ${doc.extractedCodeSnippets.length} code blocks \xB7 ${doc.tablesCount} tables]` : `Evidence scored ${score}/10 (${isApproved ? "APPROVED" : "REJECTED"}) for ${domain} (${doc.byteSize} bytes).`;
|
|
799
1777
|
return {
|
|
800
1778
|
doc,
|
|
801
1779
|
score,
|
|
802
1780
|
isApproved,
|
|
1781
|
+
isLlmEvaluated: false,
|
|
803
1782
|
breakdown: {
|
|
804
1783
|
technicalDepth: depthScore,
|
|
1784
|
+
structuralRichness: depthScore,
|
|
805
1785
|
codeSnippetQuality: codeScore,
|
|
806
1786
|
noiseFactor: noiseScore
|
|
807
1787
|
},
|
|
808
|
-
summary
|
|
1788
|
+
summary,
|
|
1789
|
+
keyTakeaways
|
|
809
1790
|
};
|
|
810
1791
|
}
|
|
1792
|
+
/**
|
|
1793
|
+
* Evaluates evidence with host LLM for genuine pedagogical relevance, conceptual depth,
|
|
1794
|
+
* and accuracy for the target learner age band.
|
|
1795
|
+
*/
|
|
1796
|
+
async scoreDocumentWithLlm(doc, domain, targetTopic, options = {}) {
|
|
1797
|
+
const baseline = this.scoreDocument(doc, domain, targetTopic);
|
|
1798
|
+
if (!options.llm) {
|
|
1799
|
+
return baseline;
|
|
1800
|
+
}
|
|
1801
|
+
try {
|
|
1802
|
+
const prompt = [
|
|
1803
|
+
`You are a Senior Curriculum Researcher. Evaluate this harvested web document for the course topic "${targetTopic}" (Domain: ${domain}).`,
|
|
1804
|
+
options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
|
|
1805
|
+
options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : "",
|
|
1806
|
+
options.context?.softwareEnv ? `Software / Tools: ${options.context.softwareEnv}` : "",
|
|
1807
|
+
`
|
|
1808
|
+
Document Title: ${doc.title}`,
|
|
1809
|
+
`Document URL: ${doc.url}`,
|
|
1810
|
+
`Sample Excerpts:
|
|
1811
|
+
${(doc.keyExcerpts || []).slice(0, 3).join("\n")}`,
|
|
1812
|
+
`Code Blocks Found: ${doc.extractedCodeSnippets.length}`,
|
|
1813
|
+
`
|
|
1814
|
+
Return ONLY a JSON object:`,
|
|
1815
|
+
`{"score": 8, "isApproved": true, "pedagogicalRelevance": 9, "summary": "1-sentence evaluative summary", "keyTakeaways": ["Takeaway 1 grounded in text", "Takeaway 2"]}`
|
|
1816
|
+
].filter(Boolean).join("\n");
|
|
1817
|
+
const raw = await options.llm(prompt, {
|
|
1818
|
+
system: "You are an educational curriculum auditor. Output strictly valid JSON.",
|
|
1819
|
+
temperature: 0.2,
|
|
1820
|
+
jsonMode: true
|
|
1821
|
+
});
|
|
1822
|
+
const match = raw.match(/\{[\s\S]*\}/);
|
|
1823
|
+
if (match) {
|
|
1824
|
+
const parsed = JSON.parse(match[0]);
|
|
1825
|
+
const score = Math.max(1, Math.min(10, Number(parsed.score) || baseline.score));
|
|
1826
|
+
return {
|
|
1827
|
+
doc,
|
|
1828
|
+
score,
|
|
1829
|
+
isApproved: score >= 6,
|
|
1830
|
+
isLlmEvaluated: true,
|
|
1831
|
+
breakdown: {
|
|
1832
|
+
...baseline.breakdown,
|
|
1833
|
+
pedagogicalRelevance: Number(parsed.pedagogicalRelevance) || score
|
|
1834
|
+
},
|
|
1835
|
+
summary: String(parsed.summary || baseline.summary),
|
|
1836
|
+
keyTakeaways: Array.isArray(parsed.keyTakeaways) && parsed.keyTakeaways.length > 0 ? parsed.keyTakeaways.map(String) : baseline.keyTakeaways
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
} catch (e) {
|
|
1840
|
+
console.warn("[EvidenceScorer] LLM evidence evaluation failed, using structural baseline:", e?.message || e);
|
|
1841
|
+
}
|
|
1842
|
+
return baseline;
|
|
1843
|
+
}
|
|
811
1844
|
};
|
|
812
1845
|
|
|
813
1846
|
// src/evidence/claimVerifier.ts
|
|
1847
|
+
var CLAIM_STOPWORDS = /* @__PURE__ */ new Set([
|
|
1848
|
+
"the",
|
|
1849
|
+
"and",
|
|
1850
|
+
"for",
|
|
1851
|
+
"with",
|
|
1852
|
+
"that",
|
|
1853
|
+
"this",
|
|
1854
|
+
"from",
|
|
1855
|
+
"into",
|
|
1856
|
+
"are",
|
|
1857
|
+
"was",
|
|
1858
|
+
"were",
|
|
1859
|
+
"have",
|
|
1860
|
+
"has",
|
|
1861
|
+
"will",
|
|
1862
|
+
"would",
|
|
1863
|
+
"through",
|
|
1864
|
+
"their",
|
|
1865
|
+
"these",
|
|
1866
|
+
"those",
|
|
1867
|
+
"which",
|
|
1868
|
+
"while",
|
|
1869
|
+
"c\u1EE7a",
|
|
1870
|
+
"trong",
|
|
1871
|
+
"\u0111\u01B0\u1EE3c",
|
|
1872
|
+
"nh\u1EEFng",
|
|
1873
|
+
"c\xE1c",
|
|
1874
|
+
"cho",
|
|
1875
|
+
"v\u1EDBi",
|
|
1876
|
+
"v\xE0",
|
|
1877
|
+
"ho\u1EB7c",
|
|
1878
|
+
"\u0111\u1EC3",
|
|
1879
|
+
"t\u1EEB",
|
|
1880
|
+
"\u0111\u1EBFn",
|
|
1881
|
+
"v\u1EC1",
|
|
1882
|
+
"n\xE0y",
|
|
1883
|
+
"\u0111\xF3",
|
|
1884
|
+
"m\u1ED9t",
|
|
1885
|
+
"nh\u01B0",
|
|
1886
|
+
"s\u1EBD",
|
|
1887
|
+
"\u0111ang",
|
|
1888
|
+
"\u0111\xE3",
|
|
1889
|
+
"ra",
|
|
1890
|
+
"v\xE0o",
|
|
1891
|
+
"l\xE0"
|
|
1892
|
+
]);
|
|
1893
|
+
function extractSignificantTokens(text) {
|
|
1894
|
+
return text.normalize("NFC").toLowerCase().replace(/[.,;:!?()[\]{}"“”‘’/\\|`*_~#-]+/g, " ").split(/\s+/).filter((w) => w.length >= 3 && !CLAIM_STOPWORDS.has(w));
|
|
1895
|
+
}
|
|
1896
|
+
function tokenMatchesCorpus(token, lowerCorpus) {
|
|
1897
|
+
if (lowerCorpus.includes(token)) return true;
|
|
1898
|
+
if (token.length >= 6) {
|
|
1899
|
+
const stem = token.replace(/(?:ing|tion|ions|ment|ments|ed|es|s)$/, "");
|
|
1900
|
+
if (stem.length >= 4 && lowerCorpus.includes(stem)) return true;
|
|
1901
|
+
}
|
|
1902
|
+
return false;
|
|
1903
|
+
}
|
|
814
1904
|
var ClaimVerifier = class {
|
|
815
|
-
|
|
1905
|
+
/**
|
|
1906
|
+
* Deterministic lexical match baseline.
|
|
1907
|
+
* Explicitly marks verdict as 'lexical_match_only' to prevent false sense of semantic verification.
|
|
1908
|
+
*/
|
|
1909
|
+
verifyClaimsDeterministic(claims, sourceCorpus) {
|
|
816
1910
|
const verifiedList = [];
|
|
817
|
-
const
|
|
1911
|
+
const normCorpus = sourceCorpus.normalize("NFC");
|
|
1912
|
+
const lowerCorpus = normCorpus.toLowerCase();
|
|
1913
|
+
const corpusSentences = normCorpus.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => {
|
|
1914
|
+
if (s.length < 25) return false;
|
|
1915
|
+
if (/^(title:|url source:|final url:|http|!\[)/i.test(s)) return false;
|
|
1916
|
+
return true;
|
|
1917
|
+
});
|
|
818
1918
|
for (const claim of claims) {
|
|
819
|
-
const
|
|
820
|
-
const matchCount =
|
|
821
|
-
const matchRatio =
|
|
822
|
-
|
|
1919
|
+
const claimTokens = extractSignificantTokens(claim);
|
|
1920
|
+
const matchCount = claimTokens.filter((t) => tokenMatchesCorpus(t, lowerCorpus)).length;
|
|
1921
|
+
const matchRatio = claimTokens.length > 0 ? matchCount / claimTokens.length : 0;
|
|
1922
|
+
let bestQuote;
|
|
1923
|
+
let bestSentenceRatio = 0;
|
|
1924
|
+
for (const sent of corpusSentences) {
|
|
1925
|
+
const lowerSent = sent.toLowerCase();
|
|
1926
|
+
const sentMatches = claimTokens.filter((t) => tokenMatchesCorpus(t, lowerSent)).length;
|
|
1927
|
+
const sentRatio = claimTokens.length > 0 ? sentMatches / claimTokens.length : 0;
|
|
1928
|
+
if (sentRatio > bestSentenceRatio) {
|
|
1929
|
+
bestSentenceRatio = sentRatio;
|
|
1930
|
+
bestQuote = sent.slice(0, 240);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
const effectiveRatio = Math.max(matchRatio, bestSentenceRatio);
|
|
1934
|
+
const hasLexicalEvidence = effectiveRatio >= 0.55;
|
|
823
1935
|
verifiedList.push({
|
|
824
1936
|
claimText: claim,
|
|
825
|
-
|
|
826
|
-
|
|
1937
|
+
verdict: "lexical_match_only",
|
|
1938
|
+
isSupported: hasLexicalEvidence,
|
|
1939
|
+
isLlmVerified: false,
|
|
1940
|
+
confidenceScore: Math.round(effectiveRatio * 100),
|
|
1941
|
+
extractedQuote: hasLexicalEvidence ? bestQuote : void 0,
|
|
1942
|
+
explanation: hasLexicalEvidence ? `Lexical token match found (${Math.round(effectiveRatio * 100)}% token overlap). Semantic entailment requires LLM verification.` : `Insufficient lexical overlap in harvest corpus (${Math.round(effectiveRatio * 100)}%).`
|
|
827
1943
|
});
|
|
828
1944
|
}
|
|
829
1945
|
const supportedCount = verifiedList.filter((c) => c.isSupported).length;
|
|
830
1946
|
const supportPct = claims.length > 0 ? supportedCount / claims.length * 100 : 100;
|
|
831
1947
|
const hallucinationRisk = supportPct >= 80 ? "low" : supportPct >= 50 ? "medium" : "high";
|
|
832
1948
|
const mdLines = [
|
|
833
|
-
`# \u{1F50D} Claim-Support Sweep & Verification Report`,
|
|
834
|
-
`> **
|
|
1949
|
+
`# \u{1F50D} Claim-Support Sweep & Verification Report (Lexical Scan Mode)`,
|
|
1950
|
+
`> **Notice:** *Scored via deterministic lexical token matching. True semantic entailment requires an LLM call.*`,
|
|
1951
|
+
`> **Lexical Coverage:** \`${supportPct.toFixed(1)}%\` (${supportedCount}/${claims.length} claims with token matches) | **Hallucination Risk (Lexical):** \`${hallucinationRisk.toUpperCase()}\``,
|
|
835
1952
|
`
|
|
836
|
-
| Pedagogical / Technical Claim | Evidence Status |
|
|
837
|
-
|
|
838
|
-
...verifiedList.map(
|
|
1953
|
+
| Pedagogical / Technical Claim | Evidence Status | Token Match | Supporting Quote | Semantic Status |`,
|
|
1954
|
+
`|---|---|---|---|---|`,
|
|
1955
|
+
...verifiedList.map(
|
|
1956
|
+
(c) => `| ${c.claimText.replace(/\|/g, "/")} | ${c.isSupported ? "\u{1F7E1} **LEXICAL MATCH**" : "\u26A0\uFE0F *UNVERIFIED / GAP*"} | \`${c.confidenceScore}%\` | ${(c.extractedQuote || "\u2014").replace(/\|/g, "/")} | *Awaiting LLM Entailment* |`
|
|
1957
|
+
)
|
|
839
1958
|
];
|
|
840
1959
|
return {
|
|
841
1960
|
totalClaimsChecked: claims.length,
|
|
842
1961
|
supportedClaimsCount: supportedCount,
|
|
1962
|
+
refutedClaimsCount: 0,
|
|
1963
|
+
unverifiedClaimsCount: claims.length - supportedCount,
|
|
843
1964
|
hallucinationRisk,
|
|
1965
|
+
isLlmVerified: false,
|
|
844
1966
|
claims: verifiedList,
|
|
845
1967
|
reportMd: mdLines.join("\n")
|
|
846
1968
|
};
|
|
847
1969
|
}
|
|
1970
|
+
/**
|
|
1971
|
+
* Primary verification interface.
|
|
1972
|
+
* If an LLM callback is provided, performs genuine Semantic Entailment (NLI).
|
|
1973
|
+
* Otherwise falls back to deterministic lexical retrieval with honest labeling.
|
|
1974
|
+
*/
|
|
1975
|
+
async verifyClaims(claims, sourceCorpus, options = {}) {
|
|
1976
|
+
const detReport = this.verifyClaimsDeterministic(claims, sourceCorpus);
|
|
1977
|
+
if (!options.llm || claims.length === 0) {
|
|
1978
|
+
return detReport;
|
|
1979
|
+
}
|
|
1980
|
+
try {
|
|
1981
|
+
const normCorpus = sourceCorpus.normalize("NFC");
|
|
1982
|
+
const cleanSubstantiveText = normCorpus.replace(/```[\s\S]*?```/g, "\n").split("\n").filter((l) => !/^(title:|url source:|final url:|http|!\[|#)/i.test(l.trim())).join("\n").trim();
|
|
1983
|
+
const corpusSentences = cleanSubstantiveText.split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").replace(/\s+/g, " ").trim()).filter((s) => s.length >= 25 && !/^(cookie|privacy|sign in|all rights reserved)/i.test(s));
|
|
1984
|
+
const fallbackExcerpts = corpusSentences.slice(0, 8).join("\n");
|
|
1985
|
+
const claimContexts = claims.map((claim, idx) => {
|
|
1986
|
+
const tokens = extractSignificantTokens(claim);
|
|
1987
|
+
const topSentences = corpusSentences.map((sent) => ({
|
|
1988
|
+
sent,
|
|
1989
|
+
overlap: tokens.filter((t) => tokenMatchesCorpus(t, sent.toLowerCase())).length
|
|
1990
|
+
})).filter((item) => item.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, 5).map((item) => item.sent);
|
|
1991
|
+
const candidates = topSentences.length > 0 ? `${topSentences.join("\n")}
|
|
1992
|
+
|
|
1993
|
+
[General Reference Excerpts]:
|
|
1994
|
+
${fallbackExcerpts}` : fallbackExcerpts || "No substantive text found.";
|
|
1995
|
+
return {
|
|
1996
|
+
idx,
|
|
1997
|
+
claim,
|
|
1998
|
+
candidatePassages: candidates
|
|
1999
|
+
};
|
|
2000
|
+
});
|
|
2001
|
+
const prompt = [
|
|
2002
|
+
`You are a Rigorous Scientific & Pedagogical Fact-Checking System (Natural Language Inference).`,
|
|
2003
|
+
`Evaluate whether each technical/pedagogical claim is strictly SUPPORTED, directly REFUTED, or has NOT_ENOUGH_EVIDENCE in the harvested technical reference text.`,
|
|
2004
|
+
`Pay careful attention to NEGATION words ("kh\xF4ng", "never", "tr\xE1nh", "do not"), causal relationships, hardware pin constraints, and edge cases. Do not assume truth without textual evidence.`,
|
|
2005
|
+
`
|
|
2006
|
+
Harvested Reference Excerpts for Claims:
|
|
2007
|
+
`,
|
|
2008
|
+
claimContexts.map((c) => `[Claim ${c.idx + 1}]: "${c.claim}"
|
|
2009
|
+
[Passage Candidates]: ${c.candidatePassages}`).join("\n\n"),
|
|
2010
|
+
`
|
|
2011
|
+
Return ONLY a valid JSON array of objects with the exact structure:`,
|
|
2012
|
+
`[{"claimIndex": 1, "verdict": "supported"|"refuted"|"not_enough_evidence", "confidenceScore": 95, "extractedQuote": "exact quote from passage", "explanation": "1-sentence semantic reason"}]`
|
|
2013
|
+
].join("\n");
|
|
2014
|
+
const raw = await options.llm(prompt, {
|
|
2015
|
+
system: "You are an exacting academic fact-checker. Output strictly valid JSON array without commentary.",
|
|
2016
|
+
temperature: 0.1,
|
|
2017
|
+
jsonMode: true
|
|
2018
|
+
});
|
|
2019
|
+
let parsed = [];
|
|
2020
|
+
const arrayMatch = raw.match(/\[[\s\S]*\]/);
|
|
2021
|
+
if (arrayMatch) {
|
|
2022
|
+
try {
|
|
2023
|
+
const res = JSON.parse(arrayMatch[0]);
|
|
2024
|
+
if (Array.isArray(res)) parsed = res;
|
|
2025
|
+
} catch {
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
if (parsed.length === 0) {
|
|
2029
|
+
const objMatch = raw.match(/\{[\s\S]*\}/);
|
|
2030
|
+
if (objMatch) {
|
|
2031
|
+
try {
|
|
2032
|
+
const obj = JSON.parse(objMatch[0]);
|
|
2033
|
+
const candidateList = obj.claims || obj.results || obj.verifications || Object.values(obj).find(Array.isArray);
|
|
2034
|
+
if (Array.isArray(candidateList)) parsed = candidateList;
|
|
2035
|
+
} catch {
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
if (parsed.length > 0) {
|
|
2040
|
+
const verifiedList = claims.map((claim, idx) => {
|
|
2041
|
+
const item = parsed.find((p) => p.claimIndex === idx + 1) || parsed[idx];
|
|
2042
|
+
const verdict = item && ["supported", "refuted", "not_enough_evidence"].includes(item.verdict) ? item.verdict : detReport.claims[idx]?.isSupported ? "supported" : "not_enough_evidence";
|
|
2043
|
+
const isSupported = verdict === "supported";
|
|
2044
|
+
return {
|
|
2045
|
+
claimText: claim,
|
|
2046
|
+
verdict,
|
|
2047
|
+
isSupported,
|
|
2048
|
+
isLlmVerified: true,
|
|
2049
|
+
confidenceScore: Number(item?.confidenceScore) || (isSupported ? 90 : 30),
|
|
2050
|
+
extractedQuote: item?.extractedQuote || detReport.claims[idx]?.extractedQuote,
|
|
2051
|
+
explanation: item?.explanation || (isSupported ? "Semantically entailed by reference corpus." : "Not verified.")
|
|
2052
|
+
};
|
|
2053
|
+
});
|
|
2054
|
+
const supportedCount = verifiedList.filter((c) => c.verdict === "supported").length;
|
|
2055
|
+
const refutedCount = verifiedList.filter((c) => c.verdict === "refuted").length;
|
|
2056
|
+
const unverifiedCount = claims.length - supportedCount;
|
|
2057
|
+
const supportPct = claims.length > 0 ? supportedCount / claims.length * 100 : 100;
|
|
2058
|
+
const hallucinationRisk = refutedCount > 0 ? "high" : supportPct >= 75 ? "low" : "medium";
|
|
2059
|
+
const mdLines = [
|
|
2060
|
+
`# \u{1F50D} Claim-Support Sweep & Semantic Verification Report (LLM NLI Evaluated)`,
|
|
2061
|
+
`> **Semantic Verification Rate:** \`${supportPct.toFixed(1)}%\` (${supportedCount}/${claims.length} claims supported) | **Refuted / Contradictions:** \`${refutedCount}\` | **Hallucination Risk:** \`${hallucinationRisk.toUpperCase()}\``,
|
|
2062
|
+
`
|
|
2063
|
+
| Pedagogical / Technical Claim | Semantic Verdict | Confidence | Supporting Quote / Evidence | Explanation |`,
|
|
2064
|
+
`|---|---|---|---|---|`,
|
|
2065
|
+
...verifiedList.map(
|
|
2066
|
+
(c) => `| ${c.claimText.replace(/\|/g, "/")} | ${c.verdict === "supported" ? "\u2705 **SUPPORTED**" : c.verdict === "refuted" ? "\u274C **REFUTED**" : "\u26A0\uFE0F *NOT ENOUGH EVIDENCE*"} | \`${c.confidenceScore}%\` | ${(c.extractedQuote || "\u2014").replace(/\|/g, "/")} | ${(c.explanation || "").replace(/\|/g, "/")} |`
|
|
2067
|
+
)
|
|
2068
|
+
];
|
|
2069
|
+
return {
|
|
2070
|
+
totalClaimsChecked: claims.length,
|
|
2071
|
+
supportedClaimsCount: supportedCount,
|
|
2072
|
+
refutedClaimsCount: refutedCount,
|
|
2073
|
+
unverifiedClaimsCount: unverifiedCount,
|
|
2074
|
+
hallucinationRisk,
|
|
2075
|
+
isLlmVerified: true,
|
|
2076
|
+
claims: verifiedList,
|
|
2077
|
+
reportMd: mdLines.join("\n")
|
|
2078
|
+
};
|
|
2079
|
+
}
|
|
2080
|
+
} catch (e) {
|
|
2081
|
+
console.warn("[ClaimVerifier] LLM semantic verification failed, falling back to deterministic lexical report:", e?.message || e);
|
|
2082
|
+
}
|
|
2083
|
+
return detReport;
|
|
2084
|
+
}
|
|
848
2085
|
};
|
|
849
2086
|
|
|
850
2087
|
// src/evidence/provenanceTracker.ts
|
|
@@ -879,16 +2116,33 @@ var ProvenanceTracker = class {
|
|
|
879
2116
|
// src/pipeline/campaignPlanner.ts
|
|
880
2117
|
var CampaignPlanner = class {
|
|
881
2118
|
createPlan(topic, domain, options = {}) {
|
|
882
|
-
const slug = topic.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40);
|
|
2119
|
+
const slug = topic.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40) || "research-topic";
|
|
883
2120
|
const scaleMode = options.isDeep ? "deep" : domain === "academic" ? "academic_survey" : "direct";
|
|
884
2121
|
const maxRounds = options.maxRounds ?? (scaleMode === "deep" ? 3 : 1);
|
|
885
|
-
const
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
2122
|
+
const ctx = options.context;
|
|
2123
|
+
const toolAnchor = [ctx?.hardwarePlatform, ctx?.softwareEnv].filter(Boolean).join(" ").trim();
|
|
2124
|
+
const coreSubject = toolAnchor ? `${toolAnchor} ${topic}` : topic;
|
|
2125
|
+
const plannedQueries = [];
|
|
2126
|
+
if (ctx && (ctx.hardwarePlatform || ctx.softwareEnv || ctx.inScopeTopics && ctx.inScopeTopics.length > 0)) {
|
|
2127
|
+
plannedQueries.push(
|
|
2128
|
+
`${toolAnchor || topic} official documentation getting started guide ${ctx.inScopeTopics?.slice(0, 2).join(" ") || ""}`.trim()
|
|
2129
|
+
);
|
|
2130
|
+
const ageHint = ctx.ageBand ? `ages ${ctx.ageBand[0]}-${ctx.ageBand[1]}` : "beginner project";
|
|
2131
|
+
plannedQueries.push(
|
|
2132
|
+
`${toolAnchor || topic} hands-on project tutorial curriculum ${ageHint}`.trim()
|
|
2133
|
+
);
|
|
2134
|
+
plannedQueries.push(
|
|
2135
|
+
`${toolAnchor || topic} common mistakes pitfalls debugging troubleshooting`.trim()
|
|
2136
|
+
);
|
|
2137
|
+
} else {
|
|
2138
|
+
plannedQueries.push(
|
|
2139
|
+
`${coreSubject} official documentation getting started`,
|
|
2140
|
+
`${coreSubject} architecture design best practices`,
|
|
2141
|
+
`${coreSubject} practical code examples and pitfalls`
|
|
2142
|
+
);
|
|
2143
|
+
}
|
|
890
2144
|
if (domain === "hardware") {
|
|
891
|
-
plannedQueries.push(`${topic} pinout schematic
|
|
2145
|
+
plannedQueries.push(`${ctx?.hardwarePlatform || topic} pinout schematic sensor wiring specifications`);
|
|
892
2146
|
} else if (domain === "academic") {
|
|
893
2147
|
plannedQueries.push(`${topic} state of the art benchmarks survey paper`);
|
|
894
2148
|
}
|
|
@@ -897,14 +2151,14 @@ var CampaignPlanner = class {
|
|
|
897
2151
|
`> **Slug:** \`${slug}\` | **Domain:** \`${domain.toUpperCase()}\` | **Scale Mode:** \`${scaleMode.toUpperCase()}\` | **Max Rounds:** \`${maxRounds}\``,
|
|
898
2152
|
`
|
|
899
2153
|
## 1. Campaign Objective`,
|
|
900
|
-
`- Exhaustively harvest and verify Ground Truth for **${topic}** to anchor all downstream SOT artifacts in Curriculum OS.`,
|
|
2154
|
+
`- Exhaustively harvest and verify Technical & Pedagogical Ground Truth for **${topic}** to anchor all downstream SOT artifacts in Curriculum OS.`,
|
|
901
2155
|
`
|
|
902
2156
|
## 2. Planned Multi-Source Queries`,
|
|
903
2157
|
...plannedQueries.map((q, i) => `${i + 1}. \`${q}\``),
|
|
904
2158
|
`
|
|
905
2159
|
## 3. Acceptance Criteria & Quality Gates`,
|
|
906
2160
|
`- Minimum 3 primary technical resources with Relevancy Score >= 6/10.`,
|
|
907
|
-
`- Extract
|
|
2161
|
+
`- Extract verified code blocks, reference tables, and bilingual domain concepts.`,
|
|
908
2162
|
`- Produce the complete 3-layer catalog: \`RESOURCE_INDEX.json\`, \`RESOURCE_CATALOG.md\`, and \`KNOWLEDGE_GRAPH.md\`.`
|
|
909
2163
|
];
|
|
910
2164
|
return {
|
|
@@ -917,33 +2171,75 @@ var CampaignPlanner = class {
|
|
|
917
2171
|
planMd: mdLines.join("\n")
|
|
918
2172
|
};
|
|
919
2173
|
}
|
|
2174
|
+
/**
|
|
2175
|
+
* Optional LLM-augmented query planner when the host app provides an `llm` callback.
|
|
2176
|
+
*/
|
|
2177
|
+
async createPlanWithLlm(topic, domain, options = {}) {
|
|
2178
|
+
const basePlan = this.createPlan(topic, domain, options);
|
|
2179
|
+
if (!options.llm) return basePlan;
|
|
2180
|
+
try {
|
|
2181
|
+
const ctx = options.context;
|
|
2182
|
+
const prompt = [
|
|
2183
|
+
`Generate 4 high-precision search engine queries (in English or bilingual as best suited for technical documentation and curriculum discovery) for researching this course:`,
|
|
2184
|
+
`- Topic / Title: ${topic}`,
|
|
2185
|
+
ctx?.positioning ? `- Positioning: ${ctx.positioning}` : "",
|
|
2186
|
+
ctx?.hardwarePlatform ? `- Hardware Platform: ${ctx.hardwarePlatform}` : "",
|
|
2187
|
+
ctx?.softwareEnv ? `- Software / Tools: ${ctx.softwareEnv}` : "",
|
|
2188
|
+
ctx?.ageBand ? `- Learner Age Band: ${ctx.ageBand[0]}-${ctx.ageBand[1]}` : "",
|
|
2189
|
+
ctx?.inScopeTopics?.length ? `- In-Scope Topics: ${ctx.inScopeTopics.join(", ")}` : "",
|
|
2190
|
+
`The 4 queries must cover: (1) Official technical/API/hardware documentation, (2) Real-world project/course exemplars for this age group, (3) Common beginner misconceptions & debugging pitfalls, (4) Best pedagogical scaffolding practices.`,
|
|
2191
|
+
`Return ONLY a JSON array of 4 query strings: ["query 1", "query 2", "query 3", "query 4"]`
|
|
2192
|
+
].filter(Boolean).join("\n");
|
|
2193
|
+
const raw = await options.llm(prompt, {
|
|
2194
|
+
system: "You are a technical research librarian. Return only a valid JSON array of 4 search query strings.",
|
|
2195
|
+
temperature: 0.2,
|
|
2196
|
+
jsonMode: true
|
|
2197
|
+
});
|
|
2198
|
+
const match = raw.match(/\[[\s\S]*\]/);
|
|
2199
|
+
if (match) {
|
|
2200
|
+
const parsed = JSON.parse(match[0]);
|
|
2201
|
+
if (Array.isArray(parsed) && parsed.length >= 2) {
|
|
2202
|
+
const queries = parsed.map((q) => String(q).trim()).filter((q) => q.length > 4).slice(0, 5);
|
|
2203
|
+
if (queries.length >= 2) {
|
|
2204
|
+
basePlan.plannedQueries = queries;
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
2208
|
+
} catch (e) {
|
|
2209
|
+
console.warn("[CampaignPlanner] LLM query expansion failed, using deterministic queries:", e?.message || e);
|
|
2210
|
+
}
|
|
2211
|
+
return basePlan;
|
|
2212
|
+
}
|
|
920
2213
|
};
|
|
921
2214
|
|
|
922
2215
|
// src/pipeline/referencePackBuilder.ts
|
|
923
2216
|
var ReferencePackBuilder = class {
|
|
924
2217
|
buildReferencePack(topic, domain, options = {}) {
|
|
925
2218
|
const registry = [];
|
|
2219
|
+
const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
|
|
926
2220
|
if (options.academicPapers) {
|
|
927
2221
|
options.academicPapers.forEach((p, idx) => {
|
|
2222
|
+
const takeaway = p.tldr?.trim() || (p.abstract ? p.abstract.slice(0, 220).trim() : `Peer-reviewed publication (${p.year || "n.d."}, ${p.citationCount} citations).`);
|
|
928
2223
|
registry.push({
|
|
929
2224
|
id: `REF-ACAD-${String(idx + 1).padStart(2, "0")}`,
|
|
930
2225
|
title: p.title,
|
|
931
2226
|
sourceType: "academic_paper",
|
|
932
|
-
url: p.url || `https://doi.org/${p.doi
|
|
2227
|
+
url: p.url || (p.doi ? `https://doi.org/${p.doi}` : ""),
|
|
933
2228
|
relevanceScore: 9,
|
|
934
|
-
extractedKeyTakeaways: [
|
|
2229
|
+
extractedKeyTakeaways: [takeaway]
|
|
935
2230
|
});
|
|
936
2231
|
});
|
|
937
2232
|
}
|
|
938
2233
|
if (options.scrapedDocs) {
|
|
939
2234
|
options.scrapedDocs.forEach((d, idx) => {
|
|
2235
|
+
const takeaways = d.keyTakeaways && d.keyTakeaways.length > 0 ? d.keyTakeaways : d.doc.keyExcerpts && d.doc.keyExcerpts.length > 0 ? d.doc.keyExcerpts.slice(0, 3) : [d.summary];
|
|
940
2236
|
registry.push({
|
|
941
2237
|
id: `REF-DOC-${String(idx + 1).padStart(2, "0")}`,
|
|
942
2238
|
title: d.doc.title,
|
|
943
|
-
sourceType: "official_docs",
|
|
2239
|
+
sourceType: d.doc.fetchProvider === "youtube" ? "video_transcript" : "official_docs",
|
|
944
2240
|
url: d.doc.url,
|
|
945
2241
|
relevanceScore: d.score,
|
|
946
|
-
extractedKeyTakeaways:
|
|
2242
|
+
extractedKeyTakeaways: takeaways
|
|
947
2243
|
});
|
|
948
2244
|
});
|
|
949
2245
|
}
|
|
@@ -953,33 +2249,83 @@ var ReferencePackBuilder = class {
|
|
|
953
2249
|
domain,
|
|
954
2250
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
955
2251
|
totalReferences: registry.length,
|
|
956
|
-
cs2023StandardArea: options.cs2023Area?.code || "
|
|
2252
|
+
cs2023StandardArea: options.cs2023Area?.code || "SDF",
|
|
2253
|
+
cstaK12Strand: options.k12Strand?.code || "AP",
|
|
2254
|
+
synthesizedInsights: options.synthesizedInsights,
|
|
2255
|
+
concepts: options.concepts || [],
|
|
957
2256
|
registry
|
|
958
2257
|
},
|
|
959
2258
|
null,
|
|
960
2259
|
2
|
|
961
2260
|
);
|
|
2261
|
+
const codeBlocks = options.scrapedDocs?.flatMap((d) => {
|
|
2262
|
+
if (d.doc.extractedCodeBlocks && d.doc.extractedCodeBlocks.length > 0) {
|
|
2263
|
+
return d.doc.extractedCodeBlocks.slice(0, 2).map((b) => `\`\`\`${b.language || defaultLang}
|
|
2264
|
+
${b.code}
|
|
2265
|
+
\`\`\``);
|
|
2266
|
+
}
|
|
2267
|
+
return d.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
|
|
2268
|
+
${c}
|
|
2269
|
+
\`\`\``);
|
|
2270
|
+
}) || [];
|
|
2271
|
+
const insightsSection = [];
|
|
2272
|
+
if (options.synthesizedInsights) {
|
|
2273
|
+
if (options.synthesizedInsights.executiveSummary) {
|
|
2274
|
+
insightsSection.push(`
|
|
2275
|
+
### Synthesized Executive Summary
|
|
2276
|
+
${options.synthesizedInsights.executiveSummary}`);
|
|
2277
|
+
}
|
|
2278
|
+
if (options.synthesizedInsights.keyTakeaways?.length > 0) {
|
|
2279
|
+
insightsSection.push(
|
|
2280
|
+
`
|
|
2281
|
+
### Key Technical & Pedagogical Takeaways
|
|
2282
|
+
` + options.synthesizedInsights.keyTakeaways.map((t) => `- ${t}`).join("\n")
|
|
2283
|
+
);
|
|
2284
|
+
}
|
|
2285
|
+
if (options.synthesizedInsights.commonMisconceptionsAndPitfalls?.length > 0) {
|
|
2286
|
+
insightsSection.push(
|
|
2287
|
+
`
|
|
2288
|
+
### Documented Learner Misconceptions & Debugging Pitfalls
|
|
2289
|
+
` + options.synthesizedInsights.commonMisconceptionsAndPitfalls.map((m) => `- ${m}`).join("\n")
|
|
2290
|
+
);
|
|
2291
|
+
}
|
|
2292
|
+
if (options.synthesizedInsights.recommendedExemplars?.length > 0) {
|
|
2293
|
+
insightsSection.push(
|
|
2294
|
+
`
|
|
2295
|
+
### Reference Project & Curriculum Exemplars
|
|
2296
|
+
` + options.synthesizedInsights.recommendedExemplars.map((e) => `- ${e}`).join("\n")
|
|
2297
|
+
);
|
|
2298
|
+
}
|
|
2299
|
+
} else if (registry.length > 0) {
|
|
2300
|
+
const topTakeaways = registry.flatMap((r) => r.extractedKeyTakeaways.slice(0, 2).map((t) => `- **[${r.id}]** ${t}`));
|
|
2301
|
+
if (topTakeaways.length > 0) {
|
|
2302
|
+
insightsSection.push(`
|
|
2303
|
+
### Extracted Source Takeaways
|
|
2304
|
+
` + topTakeaways.join("\n"));
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
962
2307
|
const mdLines = [
|
|
963
2308
|
`# \u{1F4DA} REFERENCE_PACK.md: ${topic}`,
|
|
964
|
-
`> **Domain:** \`${domain.toUpperCase()}\` | **CS2023 Area:** \`${options.cs2023Area?.code || "SE"}\` \u2014 *${options.cs2023Area?.name || "Software Engineering"}
|
|
2309
|
+
`> **Domain:** \`${domain.toUpperCase()}\` | **CS2023 Area:** \`${options.cs2023Area?.code || "SE"}\` \u2014 *${options.cs2023Area?.name || "Software Engineering"}*${options.k12Strand ? ` | **CSTA K-12:** \`${options.k12Strand.code}\` (${options.k12Strand.name})` : ""}`,
|
|
965
2310
|
`> **Curriculum OS Status:** \`<!-- [SME_APPROVED] -->\``,
|
|
966
2311
|
`
|
|
967
2312
|
## 1. Executive Summary & Ground Truth Mandate`,
|
|
968
2313
|
`This reference pack serves as the immutable ground truth for all downstream lesson plans, code labs, and rubrics for **${topic}**.`,
|
|
2314
|
+
...insightsSection,
|
|
969
2315
|
`
|
|
970
2316
|
## 2. Technical Ground Truth Registry`,
|
|
971
2317
|
`| Reference ID | Type | Title | Relevance | Source Link |`,
|
|
972
2318
|
`|---|---|---|---|---|`,
|
|
973
|
-
...registry.map(
|
|
2319
|
+
...registry.map(
|
|
2320
|
+
(r) => `| **${r.id}** | \`${r.sourceType}\` | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | [Access](${r.url}) |`
|
|
2321
|
+
),
|
|
974
2322
|
`
|
|
975
2323
|
## 3. Extracted Code Signatures & Primitives`,
|
|
976
|
-
...
|
|
977
|
-
${c}
|
|
978
|
-
\`\`\``) || ["*No explicit code signatures captured.*"],
|
|
2324
|
+
...codeBlocks.length > 0 ? codeBlocks : ["*No explicit code signatures captured.*"],
|
|
979
2325
|
`
|
|
980
2326
|
## 4. Academic Alignment & Standard Citations`,
|
|
981
|
-
`- Aligns directly with IEEE/ACM Computer Science Curricula standards.`,
|
|
982
|
-
`- All code examples
|
|
2327
|
+
`- Aligns directly with IEEE/ACM Computer Science Curricula (${options.cs2023Area?.code || "SDF"}) and CSTA K-12 (${options.k12Strand?.code || "AP"}) standards.`,
|
|
2328
|
+
`- All claims and code examples are anchored to physical primary sources listed in the registry above.`
|
|
983
2329
|
];
|
|
984
2330
|
return {
|
|
985
2331
|
referenceRegistryJson,
|
|
@@ -989,6 +2335,22 @@ ${c}
|
|
|
989
2335
|
};
|
|
990
2336
|
|
|
991
2337
|
// src/pipeline/harvestPipeline.ts
|
|
2338
|
+
function normalizeUrlForDedup(rawUrl) {
|
|
2339
|
+
try {
|
|
2340
|
+
const u = new URL(rawUrl.trim());
|
|
2341
|
+
u.hash = "";
|
|
2342
|
+
for (const key of [...u.searchParams.keys()]) {
|
|
2343
|
+
if (/^(utm_|fbclid$|gclid$|ref$|source$)/i.test(key)) {
|
|
2344
|
+
u.searchParams.delete(key);
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
const path = u.pathname.replace(/\/+$/, "") || "/";
|
|
2348
|
+
const query = u.searchParams.toString();
|
|
2349
|
+
return `${u.protocol}//${u.hostname.toLowerCase()}${path}${query ? `?${query}` : ""}`;
|
|
2350
|
+
} catch {
|
|
2351
|
+
return rawUrl.trim().toLowerCase().replace(/\/+$/, "");
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
992
2354
|
var HarvestPipeline = class {
|
|
993
2355
|
searchEngine;
|
|
994
2356
|
scraper;
|
|
@@ -1000,14 +2362,27 @@ var HarvestPipeline = class {
|
|
|
1000
2362
|
litReviewSynthesizer;
|
|
1001
2363
|
planner;
|
|
1002
2364
|
refPackBuilder;
|
|
2365
|
+
termExtractor;
|
|
2366
|
+
customSynthesizer;
|
|
2367
|
+
defaultLlm;
|
|
1003
2368
|
constructor(options = {}) {
|
|
2369
|
+
const allowEnv = options.allowEnvFallback ?? true;
|
|
2370
|
+
const keys = {
|
|
2371
|
+
tavily: options.apiKeys?.tavily || options.tavilyKey,
|
|
2372
|
+
exa: options.apiKeys?.exa || options.exaKey,
|
|
2373
|
+
jina: options.apiKeys?.jina || options.jinaKey,
|
|
2374
|
+
serper: options.apiKeys?.serper || options.serperKey,
|
|
2375
|
+
semanticScholar: options.apiKeys?.semanticScholar || options.semanticScholarKey
|
|
2376
|
+
};
|
|
1004
2377
|
this.searchEngine = new MultiSourceSearchEngine({
|
|
1005
|
-
|
|
1006
|
-
|
|
2378
|
+
apiKeys: keys,
|
|
2379
|
+
allowEnvFallback: allowEnv,
|
|
1007
2380
|
customAdapter: options.customSearchAdapter
|
|
1008
2381
|
});
|
|
1009
2382
|
this.scraper = new WebScraper({
|
|
1010
2383
|
crawl4aiHost: options.crawl4aiHost,
|
|
2384
|
+
jinaKey: keys.jina,
|
|
2385
|
+
allowEnvFallback: allowEnv,
|
|
1011
2386
|
customAdapter: options.customCrawlerAdapter
|
|
1012
2387
|
});
|
|
1013
2388
|
this.auditor = new HealthAuditor();
|
|
@@ -1015,15 +2390,29 @@ var HarvestPipeline = class {
|
|
|
1015
2390
|
this.catalogBuilder = new CatalogBuilder();
|
|
1016
2391
|
this.provenanceTracker = new ProvenanceTracker();
|
|
1017
2392
|
this.academicSearch = new AcademicSearchEngine({
|
|
2393
|
+
apiKey: keys.semanticScholar,
|
|
2394
|
+
allowEnvFallback: allowEnv,
|
|
1018
2395
|
customAdapter: options.customAcademicSearchAdapter
|
|
1019
2396
|
});
|
|
1020
2397
|
this.litReviewSynthesizer = new LiteratureReviewSynthesizer();
|
|
1021
2398
|
this.planner = new CampaignPlanner();
|
|
1022
2399
|
this.refPackBuilder = new ReferencePackBuilder();
|
|
2400
|
+
this.termExtractor = new PedagogicalTermExtractor();
|
|
2401
|
+
this.customSynthesizer = options.customSynthesizerAdapter;
|
|
2402
|
+
this.defaultLlm = options.llm;
|
|
1023
2403
|
}
|
|
1024
2404
|
async run(topic, options = {}) {
|
|
1025
2405
|
const notify = options.onProgress || (() => {
|
|
1026
2406
|
});
|
|
2407
|
+
const llm = options.llm || this.defaultLlm;
|
|
2408
|
+
const synthesizer = options.customSynthesizerAdapter || this.customSynthesizer;
|
|
2409
|
+
const mergedKeys = {
|
|
2410
|
+
tavily: options.apiKeys?.tavily || options.tavilyKey,
|
|
2411
|
+
exa: options.apiKeys?.exa || options.exaKey,
|
|
2412
|
+
jina: options.apiKeys?.jina || options.jinaKey,
|
|
2413
|
+
serper: options.apiKeys?.serper || options.serperKey,
|
|
2414
|
+
semanticScholar: options.apiKeys?.semanticScholar || options.semanticScholarKey
|
|
2415
|
+
};
|
|
1027
2416
|
if (options.customSearchAdapter) {
|
|
1028
2417
|
this.searchEngine.setCustomAdapter(options.customSearchAdapter);
|
|
1029
2418
|
}
|
|
@@ -1034,32 +2423,59 @@ var HarvestPipeline = class {
|
|
|
1034
2423
|
this.academicSearch.setCustomAdapter(options.customAcademicSearchAdapter);
|
|
1035
2424
|
}
|
|
1036
2425
|
notify("PLANNING", `Classifying domain and creating Master Research Plan for "${topic}"...`);
|
|
1037
|
-
const domainResult = classifyTechnicalDomain(topic, options.domain);
|
|
2426
|
+
const domainResult = classifyTechnicalDomain(topic, options.domain, options.context);
|
|
1038
2427
|
const domain = domainResult.domain;
|
|
1039
|
-
const plan = this.planner.
|
|
1040
|
-
|
|
1041
|
-
|
|
2428
|
+
const plan = llm ? await this.planner.createPlanWithLlm(topic, domain, {
|
|
2429
|
+
maxRounds: options.maxRounds,
|
|
2430
|
+
isDeep: options.isDeep,
|
|
2431
|
+
context: options.context,
|
|
2432
|
+
llm
|
|
2433
|
+
}) : this.planner.createPlan(topic, domain, {
|
|
2434
|
+
maxRounds: options.maxRounds,
|
|
2435
|
+
isDeep: options.isDeep,
|
|
2436
|
+
context: options.context
|
|
2437
|
+
});
|
|
2438
|
+
notify("ACADEMIC", `Querying academic literature & mapping to IEEE/ACM CS2023 and CSTA K-12 standards...`);
|
|
2439
|
+
const taxonomyMatch = mapTopicToCS2023(topic, options.context);
|
|
1042
2440
|
const papers = await this.academicSearch.searchPapers(topic, {
|
|
1043
2441
|
limit: 4,
|
|
1044
|
-
apiKey:
|
|
2442
|
+
apiKey: mergedKeys.semanticScholar,
|
|
1045
2443
|
customAdapter: options.customAcademicSearchAdapter
|
|
1046
2444
|
});
|
|
1047
|
-
const litReview = this.litReviewSynthesizer.
|
|
2445
|
+
const litReview = llm ? await this.litReviewSynthesizer.synthesizeWithLlm(
|
|
2446
|
+
topic,
|
|
2447
|
+
papers,
|
|
2448
|
+
taxonomyMatch.area,
|
|
2449
|
+
taxonomyMatch.k12Strand,
|
|
2450
|
+
{ llm, context: options.context }
|
|
2451
|
+
) : this.litReviewSynthesizer.synthesize(
|
|
2452
|
+
topic,
|
|
2453
|
+
papers,
|
|
2454
|
+
taxonomyMatch.area,
|
|
2455
|
+
taxonomyMatch.k12Strand
|
|
2456
|
+
);
|
|
1048
2457
|
notify("DISCOVERY", `Executing multi-source search queries across configured adapters...`);
|
|
1049
2458
|
const scrapedDocs = [];
|
|
1050
2459
|
const provenanceRecords = [];
|
|
2460
|
+
const visitedUrls = /* @__PURE__ */ new Set();
|
|
1051
2461
|
for (const query of plan.plannedQueries) {
|
|
1052
2462
|
notify("SEARCHING", `Searching: ${query}`);
|
|
1053
2463
|
const hits = await this.searchEngine.search(query, {
|
|
1054
2464
|
limit: 2,
|
|
1055
|
-
|
|
1056
|
-
exaKey: options.exaKey,
|
|
2465
|
+
apiKeys: mergedKeys,
|
|
1057
2466
|
customAdapter: options.customSearchAdapter
|
|
1058
2467
|
});
|
|
1059
2468
|
for (const hit of hits) {
|
|
2469
|
+
if (!hit?.url) continue;
|
|
2470
|
+
const normUrl = normalizeUrlForDedup(hit.url);
|
|
2471
|
+
if (visitedUrls.has(normUrl)) continue;
|
|
2472
|
+
visitedUrls.add(normUrl);
|
|
1060
2473
|
try {
|
|
1061
2474
|
notify("SCRAPING", `Extracting clean markdown from: ${hit.url}`);
|
|
1062
|
-
const doc = await this.scraper.scrapeUrl(hit.url, {
|
|
2475
|
+
const doc = await this.scraper.scrapeUrl(hit.url, {
|
|
2476
|
+
customAdapter: options.customCrawlerAdapter,
|
|
2477
|
+
jinaKey: mergedKeys.jina
|
|
2478
|
+
});
|
|
1063
2479
|
const audit = this.auditor.auditDocument(doc);
|
|
1064
2480
|
if (audit.passed) {
|
|
1065
2481
|
const scored = this.scorer.scoreDocument(doc, domain, topic);
|
|
@@ -1067,7 +2483,8 @@ var HarvestPipeline = class {
|
|
|
1067
2483
|
scrapedDocs.push({
|
|
1068
2484
|
doc,
|
|
1069
2485
|
score: scored.score,
|
|
1070
|
-
summary: scored.summary
|
|
2486
|
+
summary: scored.summary,
|
|
2487
|
+
keyTakeaways: scored.keyTakeaways
|
|
1071
2488
|
});
|
|
1072
2489
|
provenanceRecords.push({
|
|
1073
2490
|
resourceId: `RES-${String(scrapedDocs.length).padStart(2, "0")}`,
|
|
@@ -1084,13 +2501,36 @@ var HarvestPipeline = class {
|
|
|
1084
2501
|
}
|
|
1085
2502
|
}
|
|
1086
2503
|
}
|
|
2504
|
+
notify("TERMS", `Extracting bilingual technical concepts and prerequisite DAG...`);
|
|
2505
|
+
const docsWithIds = scrapedDocs.map((item, idx) => ({
|
|
2506
|
+
resourceId: `RES-${String(idx + 1).padStart(2, "0")}`,
|
|
2507
|
+
doc: item.doc
|
|
2508
|
+
}));
|
|
2509
|
+
const termBundle = await this.termExtractor.extractWithOptionalLlm(topic, domain, docsWithIds, {
|
|
2510
|
+
context: options.context,
|
|
2511
|
+
llm
|
|
2512
|
+
});
|
|
2513
|
+
notify("SYNTHESIS", `Synthesizing pedagogical takeaways, misconceptions, and reference pack...`);
|
|
2514
|
+
const synthesizedInsights = await this.synthesizeInsights(topic, domain, scrapedDocs, {
|
|
2515
|
+
context: options.context,
|
|
2516
|
+
llm,
|
|
2517
|
+
synthesizer,
|
|
2518
|
+
concepts: termBundle.concepts
|
|
2519
|
+
});
|
|
1087
2520
|
notify("CATALOGING", `Compiling Resource Index, Catalog, and Distilled Knowledge Graph...`);
|
|
1088
|
-
const catalog = this.catalogBuilder.buildBundle(topic, domain, scrapedDocs
|
|
1089
|
-
|
|
2521
|
+
const catalog = this.catalogBuilder.buildBundle(topic, domain, scrapedDocs, {
|
|
2522
|
+
context: options.context,
|
|
2523
|
+
concepts: termBundle.concepts,
|
|
2524
|
+
conceptDagTsv: termBundle.conceptDagTsv
|
|
2525
|
+
});
|
|
1090
2526
|
const referencePack = this.refPackBuilder.buildReferencePack(topic, domain, {
|
|
1091
|
-
cs2023Area:
|
|
2527
|
+
cs2023Area: taxonomyMatch.area,
|
|
2528
|
+
k12Strand: taxonomyMatch.k12Strand,
|
|
2529
|
+
context: options.context,
|
|
1092
2530
|
scrapedDocs,
|
|
1093
|
-
academicPapers: papers
|
|
2531
|
+
academicPapers: papers,
|
|
2532
|
+
concepts: termBundle.concepts,
|
|
2533
|
+
synthesizedInsights
|
|
1094
2534
|
});
|
|
1095
2535
|
notify("PROVENANCE", `Generating audit trail and provenance sidecar...`);
|
|
1096
2536
|
const provenance = this.provenanceTracker.generateSidecar(topic, provenanceRecords);
|
|
@@ -1103,30 +2543,339 @@ var HarvestPipeline = class {
|
|
|
1103
2543
|
literatureReview: litReview,
|
|
1104
2544
|
referencePack,
|
|
1105
2545
|
provenance,
|
|
2546
|
+
concepts: termBundle.concepts,
|
|
2547
|
+
conceptDagTsv: termBundle.conceptDagTsv,
|
|
2548
|
+
synthesizedInsights,
|
|
1106
2549
|
totalDocumentsScraped: scrapedDocs.length
|
|
1107
2550
|
};
|
|
1108
2551
|
}
|
|
2552
|
+
async synthesizeInsights(topic, domain, scrapedDocs, options) {
|
|
2553
|
+
const excerpts = scrapedDocs.map((d, idx) => ({
|
|
2554
|
+
resourceId: `RES-${String(idx + 1).padStart(2, "0")}`,
|
|
2555
|
+
title: d.doc.title,
|
|
2556
|
+
url: d.doc.url,
|
|
2557
|
+
excerpt: d.keyTakeaways.join(" ") || d.summary,
|
|
2558
|
+
codeSnippets: d.doc.extractedCodeSnippets.slice(0, 2)
|
|
2559
|
+
}));
|
|
2560
|
+
if (options.synthesizer) {
|
|
2561
|
+
try {
|
|
2562
|
+
return await options.synthesizer.synthesize({
|
|
2563
|
+
topic,
|
|
2564
|
+
domain,
|
|
2565
|
+
context: options.context,
|
|
2566
|
+
excerpts
|
|
2567
|
+
});
|
|
2568
|
+
} catch (e) {
|
|
2569
|
+
console.warn("[HarvestPipeline] CustomSynthesizerAdapter failed, falling back:", e?.message || e);
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
if (options.llm && excerpts.length > 0) {
|
|
2573
|
+
try {
|
|
2574
|
+
const prompt = [
|
|
2575
|
+
`Synthesize pedagogical and technical ground truth for the course topic "${topic}" (Domain: ${domain}).`,
|
|
2576
|
+
options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : "",
|
|
2577
|
+
options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : "",
|
|
2578
|
+
options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}` : "",
|
|
2579
|
+
`
|
|
2580
|
+
Verified Harvested Sources:
|
|
2581
|
+
` + excerpts.map((e) => `[${e.resourceId}] ${e.title} (${e.url}): ${e.excerpt}`).join("\n\n"),
|
|
2582
|
+
`
|
|
2583
|
+
Return ONLY a valid JSON object with this structure:`,
|
|
2584
|
+
`{"executiveSummary": "2-3 sentence synthesis anchored to sources", "keyTakeaways": ["[RES-01] ...", "[RES-02] ..."], "commonMisconceptionsAndPitfalls": ["Specific beginner bug or misconception 1", "Misconception 2"], "recommendedExemplars": ["Concrete project/activity exemplar 1", "Exemplar 2"]}`
|
|
2585
|
+
].filter(Boolean).join("\n");
|
|
2586
|
+
const raw = await options.llm(prompt, {
|
|
2587
|
+
system: "You are a Principal Curriculum Researcher. Output strictly valid JSON grounded in the provided sources.",
|
|
2588
|
+
temperature: 0.2,
|
|
2589
|
+
jsonMode: true
|
|
2590
|
+
});
|
|
2591
|
+
const match = raw.match(/\{[\s\S]*\}/);
|
|
2592
|
+
if (match) {
|
|
2593
|
+
const parsed = JSON.parse(match[0]);
|
|
2594
|
+
return {
|
|
2595
|
+
executiveSummary: String(parsed.executiveSummary || ""),
|
|
2596
|
+
keyTakeaways: Array.isArray(parsed.keyTakeaways) ? parsed.keyTakeaways.map(String) : [],
|
|
2597
|
+
commonMisconceptionsAndPitfalls: Array.isArray(parsed.commonMisconceptionsAndPitfalls) ? parsed.commonMisconceptionsAndPitfalls.map(String) : [],
|
|
2598
|
+
recommendedExemplars: Array.isArray(parsed.recommendedExemplars) ? parsed.recommendedExemplars.map(String) : [],
|
|
2599
|
+
extractedConcepts: options.concepts
|
|
2600
|
+
};
|
|
2601
|
+
}
|
|
2602
|
+
} catch (e) {
|
|
2603
|
+
console.warn("[HarvestPipeline] Host LLM synthesis failed, using deterministic synthesis:", e?.message || e);
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
const keyTakeaways = excerpts.filter((e) => e.excerpt).slice(0, 6).map((e) => `[${e.resourceId}] ${e.title}: ${e.excerpt}`);
|
|
2607
|
+
const pitfalls = scrapedDocs.flatMap(
|
|
2608
|
+
(d) => d.doc.markdownContent.split(/(?<=[.!?。\n])\s+/).map((s) => s.trim()).filter(
|
|
2609
|
+
(s) => s.length >= 35 && s.length <= 240 && /\b(pitfall|mistake|error|bug|warning|caution|avoid|race condition|deadlock|lỗi|nhầm lẫn|lưu ý|cảnh báo|tránh)\b/i.test(s)
|
|
2610
|
+
)
|
|
2611
|
+
).slice(0, 4);
|
|
2612
|
+
return {
|
|
2613
|
+
executiveSummary: scrapedDocs.length > 0 ? `Synthesized from ${scrapedDocs.length} verified primary sources for "${topic}" (${domain.toUpperCase()}).` : `No external web documents passed quality threshold for "${topic}".`,
|
|
2614
|
+
keyTakeaways,
|
|
2615
|
+
commonMisconceptionsAndPitfalls: pitfalls,
|
|
2616
|
+
recommendedExemplars: scrapedDocs.slice(0, 3).map((d) => `${d.doc.title} (${d.doc.url})`),
|
|
2617
|
+
extractedConcepts: options.concepts
|
|
2618
|
+
};
|
|
2619
|
+
}
|
|
2620
|
+
};
|
|
2621
|
+
|
|
2622
|
+
// src/services/researchKitClient.ts
|
|
2623
|
+
var KnowledgeDiscoveryService = class {
|
|
2624
|
+
constructor(config) {
|
|
2625
|
+
this.config = config;
|
|
2626
|
+
this.searchEngine = new MultiSourceSearchEngine({
|
|
2627
|
+
apiKeys: config.apiKeys,
|
|
2628
|
+
allowEnvFallback: config.allowEnvFallback,
|
|
2629
|
+
customAdapter: config.customSearchAdapter
|
|
2630
|
+
});
|
|
2631
|
+
this.webScraper = new WebScraper({
|
|
2632
|
+
crawl4aiHost: config.crawl4aiHost,
|
|
2633
|
+
jinaKey: config.apiKeys?.jina,
|
|
2634
|
+
allowEnvFallback: config.allowEnvFallback,
|
|
2635
|
+
customAdapter: config.customCrawlerAdapter
|
|
2636
|
+
});
|
|
2637
|
+
this.youtubeService = new YouTubeIngestionService();
|
|
2638
|
+
}
|
|
2639
|
+
searchEngine;
|
|
2640
|
+
webScraper;
|
|
2641
|
+
youtubeService;
|
|
2642
|
+
async search(query, limit = 5) {
|
|
2643
|
+
return this.searchEngine.search(query, {
|
|
2644
|
+
limit,
|
|
2645
|
+
apiKeys: this.config.apiKeys,
|
|
2646
|
+
customAdapter: this.config.customSearchAdapter
|
|
2647
|
+
});
|
|
2648
|
+
}
|
|
2649
|
+
async scrapeUrl(url) {
|
|
2650
|
+
return this.webScraper.scrapeUrl(url, {
|
|
2651
|
+
customAdapter: this.config.customCrawlerAdapter,
|
|
2652
|
+
jinaKey: this.config.apiKeys?.jina
|
|
2653
|
+
});
|
|
2654
|
+
}
|
|
2655
|
+
async ingestYouTube(urlOrVideoId, preferredLanguages = ["vi", "en"]) {
|
|
2656
|
+
return this.youtubeService.fetchTranscript(urlOrVideoId, preferredLanguages);
|
|
2657
|
+
}
|
|
2658
|
+
};
|
|
2659
|
+
var TerminologyGraphService = class {
|
|
2660
|
+
constructor(config) {
|
|
2661
|
+
this.config = config;
|
|
2662
|
+
}
|
|
2663
|
+
extractor = new PedagogicalTermExtractor();
|
|
2664
|
+
extractConceptsDeterministic(topic, docs, domain) {
|
|
2665
|
+
const resolvedDomain = domain ?? classifyTechnicalDomain(topic, void 0, this.config.defaultCourseContext).domain;
|
|
2666
|
+
const normalizedDocs = typeof docs === "string" ? [
|
|
2667
|
+
{
|
|
2668
|
+
resourceId: "RES-01",
|
|
2669
|
+
doc: {
|
|
2670
|
+
url: "inline://corpus",
|
|
2671
|
+
title: topic,
|
|
2672
|
+
markdownContent: docs,
|
|
2673
|
+
byteSize: Buffer.byteLength(docs, "utf-8"),
|
|
2674
|
+
extractedCodeSnippets: [],
|
|
2675
|
+
tablesCount: 0,
|
|
2676
|
+
fetchProvider: "direct"
|
|
2677
|
+
}
|
|
2678
|
+
}
|
|
2679
|
+
] : docs;
|
|
2680
|
+
return this.extractor.extractDeterministic(topic, resolvedDomain, normalizedDocs);
|
|
2681
|
+
}
|
|
2682
|
+
async extractConcepts(topic, docs, options) {
|
|
2683
|
+
const resolvedDomain = options?.domain ?? classifyTechnicalDomain(topic, void 0, options?.context ?? this.config.defaultCourseContext).domain;
|
|
2684
|
+
const normalizedDocs = typeof docs === "string" ? [
|
|
2685
|
+
{
|
|
2686
|
+
resourceId: "RES-01",
|
|
2687
|
+
doc: {
|
|
2688
|
+
url: "inline://corpus",
|
|
2689
|
+
title: topic,
|
|
2690
|
+
markdownContent: docs,
|
|
2691
|
+
byteSize: Buffer.byteLength(docs, "utf-8"),
|
|
2692
|
+
extractedCodeSnippets: [],
|
|
2693
|
+
keyExcerpts: [docs.slice(0, 600)],
|
|
2694
|
+
tablesCount: 0,
|
|
2695
|
+
fetchProvider: "direct"
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
] : docs;
|
|
2699
|
+
return this.extractor.extractWithOptionalLlm(topic, resolvedDomain, normalizedDocs, {
|
|
2700
|
+
context: options?.context ?? this.config.defaultCourseContext,
|
|
2701
|
+
llm: options?.llm ?? this.config.llm,
|
|
2702
|
+
maxTerms: options?.maxTerms
|
|
2703
|
+
});
|
|
2704
|
+
}
|
|
2705
|
+
formatConceptDagTsv(concepts) {
|
|
2706
|
+
return this.extractor.formatConceptDagTsv(concepts);
|
|
2707
|
+
}
|
|
2708
|
+
};
|
|
2709
|
+
var AcademicStandardsService = class {
|
|
2710
|
+
constructor(config) {
|
|
2711
|
+
this.config = config;
|
|
2712
|
+
this.academicSearch = new AcademicSearchEngine({
|
|
2713
|
+
apiKey: config.apiKeys?.semanticScholar,
|
|
2714
|
+
allowEnvFallback: config.allowEnvFallback,
|
|
2715
|
+
customAdapter: config.customAcademicSearchAdapter
|
|
2716
|
+
});
|
|
2717
|
+
}
|
|
2718
|
+
academicSearch;
|
|
2719
|
+
litSynthesizer = new LiteratureReviewSynthesizer();
|
|
2720
|
+
alignStandards(topic, context) {
|
|
2721
|
+
return mapTopicToCS2023(topic, context ?? this.config.defaultCourseContext);
|
|
2722
|
+
}
|
|
2723
|
+
async searchPapers(query, limit = 5) {
|
|
2724
|
+
return this.academicSearch.searchPapers(query, {
|
|
2725
|
+
limit,
|
|
2726
|
+
apiKey: this.config.apiKeys?.semanticScholar,
|
|
2727
|
+
customAdapter: this.config.customAcademicSearchAdapter
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
synthesizeLiteratureReview(topic, papers, context) {
|
|
2731
|
+
const alignment = this.alignStandards(topic, context);
|
|
2732
|
+
return this.litSynthesizer.synthesize(topic, papers, alignment.area, alignment.k12Strand);
|
|
2733
|
+
}
|
|
2734
|
+
async synthesizeLiteratureReviewWithLlm(topic, papers, context, options) {
|
|
2735
|
+
const alignment = this.alignStandards(topic, context);
|
|
2736
|
+
return this.litSynthesizer.synthesizeWithLlm(
|
|
2737
|
+
topic,
|
|
2738
|
+
papers,
|
|
2739
|
+
alignment.area,
|
|
2740
|
+
alignment.k12Strand,
|
|
2741
|
+
{
|
|
2742
|
+
llm: options?.llm ?? this.config.llm,
|
|
2743
|
+
context: context ?? this.config.defaultCourseContext
|
|
2744
|
+
}
|
|
2745
|
+
);
|
|
2746
|
+
}
|
|
2747
|
+
};
|
|
2748
|
+
var EvidenceVerificationService = class {
|
|
2749
|
+
constructor(config = {}) {
|
|
2750
|
+
this.config = config;
|
|
2751
|
+
}
|
|
2752
|
+
healthAuditor = new HealthAuditor();
|
|
2753
|
+
evidenceScorer = new EvidenceScorer();
|
|
2754
|
+
claimVerifier = new ClaimVerifier();
|
|
2755
|
+
provenanceTracker = new ProvenanceTracker();
|
|
2756
|
+
auditDocument(doc) {
|
|
2757
|
+
return this.healthAuditor.auditDocument(doc);
|
|
2758
|
+
}
|
|
2759
|
+
scoreDocument(doc, domain, topic) {
|
|
2760
|
+
return this.evidenceScorer.scoreDocument(doc, domain, topic);
|
|
2761
|
+
}
|
|
2762
|
+
async scoreDocumentWithLlm(doc, domain, topic, options) {
|
|
2763
|
+
return this.evidenceScorer.scoreDocumentWithLlm(doc, domain, topic, {
|
|
2764
|
+
llm: options?.llm ?? this.config.llm,
|
|
2765
|
+
context: options?.context ?? this.config.defaultCourseContext
|
|
2766
|
+
});
|
|
2767
|
+
}
|
|
2768
|
+
verifyClaimsDeterministic(claimsToVerify, sourceCorpusText) {
|
|
2769
|
+
return this.claimVerifier.verifyClaimsDeterministic(claimsToVerify, sourceCorpusText);
|
|
2770
|
+
}
|
|
2771
|
+
async verifyClaims(claimsToVerify, sourceCorpusText, options) {
|
|
2772
|
+
return this.claimVerifier.verifyClaims(claimsToVerify, sourceCorpusText, {
|
|
2773
|
+
llm: options?.llm ?? this.config.llm
|
|
2774
|
+
});
|
|
2775
|
+
}
|
|
2776
|
+
buildProvenance(topic, records) {
|
|
2777
|
+
return this.provenanceTracker.generateSidecar(topic, records);
|
|
2778
|
+
}
|
|
2779
|
+
};
|
|
2780
|
+
var PedagogicalSynthesisService = class {
|
|
2781
|
+
constructor(config) {
|
|
2782
|
+
this.config = config;
|
|
2783
|
+
this.pipeline = new HarvestPipeline({
|
|
2784
|
+
apiKeys: config.apiKeys,
|
|
2785
|
+
allowEnvFallback: config.allowEnvFallback,
|
|
2786
|
+
crawl4aiHost: config.crawl4aiHost,
|
|
2787
|
+
llm: config.llm,
|
|
2788
|
+
customSearchAdapter: config.customSearchAdapter,
|
|
2789
|
+
customCrawlerAdapter: config.customCrawlerAdapter,
|
|
2790
|
+
customAcademicSearchAdapter: config.customAcademicSearchAdapter,
|
|
2791
|
+
customSynthesizerAdapter: config.customSynthesizerAdapter
|
|
2792
|
+
});
|
|
2793
|
+
}
|
|
2794
|
+
planner = new CampaignPlanner();
|
|
2795
|
+
pipeline;
|
|
2796
|
+
async planCampaign(topic, options) {
|
|
2797
|
+
const ctx = options?.context ?? this.config.defaultCourseContext;
|
|
2798
|
+
const domain = options?.domain ?? classifyTechnicalDomain(topic, void 0, ctx).domain;
|
|
2799
|
+
return this.planner.createPlanWithLlm(topic, domain, {
|
|
2800
|
+
isDeep: options?.isDeep,
|
|
2801
|
+
maxRounds: options?.maxRounds,
|
|
2802
|
+
context: ctx,
|
|
2803
|
+
llm: this.config.llm
|
|
2804
|
+
});
|
|
2805
|
+
}
|
|
2806
|
+
async runFullHarvest(topic, overrides) {
|
|
2807
|
+
return this.pipeline.run(topic, {
|
|
2808
|
+
apiKeys: this.config.apiKeys,
|
|
2809
|
+
allowEnvFallback: this.config.allowEnvFallback,
|
|
2810
|
+
crawl4aiHost: this.config.crawl4aiHost,
|
|
2811
|
+
llm: this.config.llm,
|
|
2812
|
+
context: this.config.defaultCourseContext,
|
|
2813
|
+
customSearchAdapter: this.config.customSearchAdapter,
|
|
2814
|
+
customCrawlerAdapter: this.config.customCrawlerAdapter,
|
|
2815
|
+
customAcademicSearchAdapter: this.config.customAcademicSearchAdapter,
|
|
2816
|
+
customSynthesizerAdapter: this.config.customSynthesizerAdapter,
|
|
2817
|
+
...overrides
|
|
2818
|
+
});
|
|
2819
|
+
}
|
|
2820
|
+
};
|
|
2821
|
+
var ResearchKitClient = class {
|
|
2822
|
+
constructor(config = {}) {
|
|
2823
|
+
this.config = config;
|
|
2824
|
+
this.discovery = new KnowledgeDiscoveryService(config);
|
|
2825
|
+
this.terminology = new TerminologyGraphService(config);
|
|
2826
|
+
this.academic = new AcademicStandardsService(config);
|
|
2827
|
+
this.verification = new EvidenceVerificationService(config);
|
|
2828
|
+
this.orchestrator = new PedagogicalSynthesisService(config);
|
|
2829
|
+
}
|
|
2830
|
+
/** Service 1: Multi-Engine Web Search, Scraper & YouTube Transcript Ingestion */
|
|
2831
|
+
discovery;
|
|
2832
|
+
/** Service 2: Bilingual Pedagogical Term Extraction & Prerequisite Concept Graph (DAG) */
|
|
2833
|
+
terminology;
|
|
2834
|
+
/** Service 3: Academic Literature Search, BibTeX Citations & CS2023/CSTA K-12 Alignment */
|
|
2835
|
+
academic;
|
|
2836
|
+
/** Service 4: Health Audit, Evidence Scoring, Claim Verification & Provenance Tracking */
|
|
2837
|
+
verification;
|
|
2838
|
+
/** Service 5: Campaign Planning, Semantic Synthesis & Canonical Reference Pack Orchestration */
|
|
2839
|
+
orchestrator;
|
|
1109
2840
|
};
|
|
1110
2841
|
|
|
1111
2842
|
exports.AcademicPaperSchema = AcademicPaperSchema;
|
|
1112
2843
|
exports.AcademicSearchEngine = AcademicSearchEngine;
|
|
2844
|
+
exports.AcademicStandardsService = AcademicStandardsService;
|
|
1113
2845
|
exports.ArXivSearchEngine = ArXivSearchEngine;
|
|
1114
2846
|
exports.CS2023_KNOWLEDGE_AREAS = CS2023_KNOWLEDGE_AREAS;
|
|
2847
|
+
exports.CSTA_K12_STRANDS = CSTA_K12_STRANDS;
|
|
1115
2848
|
exports.CampaignPlanner = CampaignPlanner;
|
|
1116
2849
|
exports.CatalogBuilder = CatalogBuilder;
|
|
1117
2850
|
exports.CitationManager = CitationManager;
|
|
1118
2851
|
exports.ClaimVerifier = ClaimVerifier;
|
|
1119
2852
|
exports.DEFAULT_RESEARCH_CONFIG = DEFAULT_RESEARCH_CONFIG;
|
|
1120
2853
|
exports.EvidenceScorer = EvidenceScorer;
|
|
2854
|
+
exports.EvidenceVerificationService = EvidenceVerificationService;
|
|
1121
2855
|
exports.HarvestPipeline = HarvestPipeline;
|
|
1122
2856
|
exports.HealthAuditor = HealthAuditor;
|
|
2857
|
+
exports.KnowledgeDiscoveryService = KnowledgeDiscoveryService;
|
|
1123
2858
|
exports.LiteratureReviewSynthesizer = LiteratureReviewSynthesizer;
|
|
1124
2859
|
exports.MultiSourceSearchEngine = MultiSourceSearchEngine;
|
|
2860
|
+
exports.PedagogicalSynthesisService = PedagogicalSynthesisService;
|
|
2861
|
+
exports.PedagogicalTermExtractor = PedagogicalTermExtractor;
|
|
1125
2862
|
exports.ProvenanceTracker = ProvenanceTracker;
|
|
1126
2863
|
exports.ReferencePackBuilder = ReferencePackBuilder;
|
|
1127
2864
|
exports.ResearchConfigSchema = ResearchConfigSchema;
|
|
2865
|
+
exports.ResearchKitClient = ResearchKitClient;
|
|
2866
|
+
exports.TerminologyGraphService = TerminologyGraphService;
|
|
1128
2867
|
exports.WebScraper = WebScraper;
|
|
2868
|
+
exports.YouTubeIngestionService = YouTubeIngestionService;
|
|
1129
2869
|
exports.classifyTechnicalDomain = classifyTechnicalDomain;
|
|
2870
|
+
exports.extractSubstantiveExcerpts = extractSubstantiveExcerpts;
|
|
2871
|
+
exports.extractTopSentences = extractTopSentences;
|
|
2872
|
+
exports.extractYouTubeVideoId = extractYouTubeVideoId;
|
|
2873
|
+
exports.formatTimestamp = formatTimestamp;
|
|
2874
|
+
exports.inferPrimaryCodeLanguage = inferPrimaryCodeLanguage;
|
|
2875
|
+
exports.isYouTubeUrl = isYouTubeUrl;
|
|
1130
2876
|
exports.mapTopicToCS2023 = mapTopicToCS2023;
|
|
2877
|
+
exports.normalizeUrlForDedup = normalizeUrlForDedup;
|
|
2878
|
+
exports.reconstructOpenAlexAbstract = reconstructOpenAlexAbstract;
|
|
2879
|
+
exports.stitchCaptionSegments = stitchCaptionSegments;
|
|
1131
2880
|
//# sourceMappingURL=index.cjs.map
|
|
1132
2881
|
//# sourceMappingURL=index.cjs.map
|