@thanh01.pmt/research-kit 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -14
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/catalogBuilder-2i5KASUF.d.ts +82 -0
- package/dist/catalogBuilder-nrGYEY4b.d.cts +82 -0
- package/dist/chunk-LSMSIPTZ.js +3 -0
- package/dist/{chunk-2JEVBCIF.js.map → chunk-LSMSIPTZ.js.map} +1 -1
- package/dist/{chunk-KAFQVI2R.js → chunk-ON6VFCJO.js} +3 -3
- package/dist/{chunk-KAFQVI2R.js.map → chunk-ON6VFCJO.js.map} +1 -1
- package/dist/{chunk-GKF3NDEX.js → chunk-QFYM2NV4.js} +3 -3
- package/dist/{chunk-GKF3NDEX.js.map → chunk-QFYM2NV4.js.map} +1 -1
- package/dist/{chunk-XIF5YNI7.js → chunk-QK3LVGI7.js} +244 -8
- package/dist/chunk-QK3LVGI7.js.map +1 -0
- package/dist/{chunk-J7XKDMGL.js → chunk-X3X26W2I.js} +2 -2
- package/dist/{chunk-J7XKDMGL.js.map → chunk-X3X26W2I.js.map} +1 -1
- package/dist/{chunk-AUGVVZTL.js → chunk-XDDTHWJW.js} +173 -6
- package/dist/chunk-XDDTHWJW.js.map +1 -0
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -2
- package/dist/harvester/index.cjs +248 -6
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +63 -49
- package/dist/harvester/index.d.ts +63 -49
- package/dist/harvester/index.js +3 -3
- package/dist/{index-t8pahlJ4.d.ts → index-DlngvZBr.d.cts} +15 -1
- package/dist/{index-t8pahlJ4.d.cts → index-DlngvZBr.d.ts} +15 -1
- package/dist/index.cjs +443 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -13
- package/dist/index.d.ts +28 -13
- package/dist/index.js +37 -11
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-BTPFe7eb.d.cts → literatureReview-C7hsjrn4.d.cts} +1 -1
- package/dist/{literatureReview-hGbfrBmw.d.ts → literatureReview-DWl13r_3.d.ts} +1 -1
- package/dist/pipeline/index.cjs +361 -6
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -5
- package/dist/pipeline/index.d.ts +49 -5
- package/dist/pipeline/index.js +4 -4
- package/dist/{webScraper-BhiQ8s9x.d.ts → webScraper-DBFUIqoO.d.ts} +1 -1
- package/dist/{webScraper-D9Ldus9p.d.cts → webScraper-yO9P-oLb.d.cts} +1 -1
- package/package.json +9 -8
- package/LICENSE +0 -21
- package/dist/catalogBuilder-CmmQzUnf.d.ts +0 -38
- package/dist/catalogBuilder-DgXYxL2F.d.cts +0 -38
- package/dist/chunk-2JEVBCIF.js +0 -3
- package/dist/chunk-AUGVVZTL.js.map +0 -1
- package/dist/chunk-XIF5YNI7.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -1141,7 +1141,7 @@ var WebScraper = class {
|
|
|
1141
1141
|
}
|
|
1142
1142
|
const res = await fetch(jinaUrl, {
|
|
1143
1143
|
headers,
|
|
1144
|
-
signal: AbortSignal.timeout(
|
|
1144
|
+
signal: AbortSignal.timeout(8e3)
|
|
1145
1145
|
});
|
|
1146
1146
|
if (res.ok) {
|
|
1147
1147
|
const markdown = await res.text();
|
|
@@ -1226,6 +1226,214 @@ ${p.replace(/<[^>]+>/g, " ").trim()}
|
|
|
1226
1226
|
}
|
|
1227
1227
|
};
|
|
1228
1228
|
|
|
1229
|
+
// src/harvester/noiseFilter.ts
|
|
1230
|
+
var ECOMMERCE_DOMAINS = /* @__PURE__ */ new Set([
|
|
1231
|
+
"shopee.vn",
|
|
1232
|
+
"shopee.com",
|
|
1233
|
+
"lazada.vn",
|
|
1234
|
+
"lazada.com",
|
|
1235
|
+
"tiki.vn",
|
|
1236
|
+
"sendo.vn",
|
|
1237
|
+
"aliexpress.com",
|
|
1238
|
+
"taobao.com",
|
|
1239
|
+
"amazon.com",
|
|
1240
|
+
"ebay.com",
|
|
1241
|
+
"banggood.com"
|
|
1242
|
+
]);
|
|
1243
|
+
var ECOMMERCE_AND_COMMERCIAL_PATTERNS = [
|
|
1244
|
+
/\b(shopee|lazada|tiki|sendo|taobao|aliexpress)\b/i,
|
|
1245
|
+
/\b(mua ngay|giá tốt|khuyến mãi|freeship|bán chạy|nhập khẩu chính hãng|cửa hàng|bảng giá|đặt hàng)\b/i,
|
|
1246
|
+
/\b(buy now|add to cart|free shipping|discount|best price|on sale|shop now|in stock|order now)\b/i,
|
|
1247
|
+
/\b(\d+[kKđ]|₫|\$\d+|\d+\s*(?:VNĐ|VND|usd))\b/i
|
|
1248
|
+
];
|
|
1249
|
+
function isCommercialOrNoise(input) {
|
|
1250
|
+
if (input.url) {
|
|
1251
|
+
try {
|
|
1252
|
+
const hostname = new URL(input.url).hostname.toLowerCase().replace(/^www\./, "");
|
|
1253
|
+
if (ECOMMERCE_DOMAINS.has(hostname) || [...ECOMMERCE_DOMAINS].some((d) => hostname.endsWith(`.${d}`))) {
|
|
1254
|
+
return true;
|
|
1255
|
+
}
|
|
1256
|
+
} catch {
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
const combined = `${input.title || ""} ${input.snippet || ""}`;
|
|
1260
|
+
if (!combined.trim()) return false;
|
|
1261
|
+
for (const pattern of ECOMMERCE_AND_COMMERCIAL_PATTERNS) {
|
|
1262
|
+
if (pattern.test(combined)) {
|
|
1263
|
+
return true;
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
return false;
|
|
1267
|
+
}
|
|
1268
|
+
function evaluateTechnicalDensity(markdown) {
|
|
1269
|
+
if (!markdown || markdown.trim().length < 80) {
|
|
1270
|
+
return {
|
|
1271
|
+
score: 1,
|
|
1272
|
+
isSufficientQuality: false,
|
|
1273
|
+
codeDensity: 0,
|
|
1274
|
+
technicalTermDensity: 0,
|
|
1275
|
+
hasStructure: false,
|
|
1276
|
+
reasons: ["Content is too short or empty."]
|
|
1277
|
+
};
|
|
1278
|
+
}
|
|
1279
|
+
const reasons = [];
|
|
1280
|
+
const totalLength = markdown.length;
|
|
1281
|
+
const codeBlocks = markdown.match(/```[\s\S]*?```/g) || [];
|
|
1282
|
+
const inlineCode = markdown.match(/`[^`\n]+`/g) || [];
|
|
1283
|
+
const totalCodeChars = codeBlocks.reduce((sum, b) => sum + b.length, 0) + inlineCode.reduce((sum, c) => sum + c.length, 0);
|
|
1284
|
+
const codeDensity = Math.min(1, totalCodeChars / Math.max(1, totalLength));
|
|
1285
|
+
const headings = markdown.match(/^#{1,4}\s+.+$/gm) || [];
|
|
1286
|
+
const hasStructure = headings.length >= 2;
|
|
1287
|
+
const techKeywords = markdown.match(
|
|
1288
|
+
/\b(function|class|method|struct|interface|api|hardware|sensor|pin|circuit|logic|loop|state|algorithm|test|debug|schema|protocol|variable)\b/gi
|
|
1289
|
+
) || [];
|
|
1290
|
+
const technicalTermDensity = Math.min(1, techKeywords.length * 10 / Math.max(100, markdown.split(/\s+/).length));
|
|
1291
|
+
let score = 5;
|
|
1292
|
+
if (codeDensity > 0.15) {
|
|
1293
|
+
score += 2;
|
|
1294
|
+
} else if (codeDensity > 0.05) {
|
|
1295
|
+
score += 1;
|
|
1296
|
+
} else {
|
|
1297
|
+
reasons.push("Low code or API snippet density.");
|
|
1298
|
+
}
|
|
1299
|
+
if (hasStructure) {
|
|
1300
|
+
score += 1.5;
|
|
1301
|
+
} else {
|
|
1302
|
+
reasons.push("Lacks clear hierarchical markdown headings.");
|
|
1303
|
+
}
|
|
1304
|
+
if (technicalTermDensity > 0.08) {
|
|
1305
|
+
score += 1.5;
|
|
1306
|
+
}
|
|
1307
|
+
const finalScore = Math.min(10, Math.max(1, Math.round(score * 10) / 10));
|
|
1308
|
+
return {
|
|
1309
|
+
score: finalScore,
|
|
1310
|
+
isSufficientQuality: finalScore >= 6,
|
|
1311
|
+
codeDensity,
|
|
1312
|
+
technicalTermDensity,
|
|
1313
|
+
hasStructure,
|
|
1314
|
+
reasons
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
// src/harvester/graphUtils.ts
|
|
1319
|
+
function transitiveReduction(edges) {
|
|
1320
|
+
const uniqueEdges = [];
|
|
1321
|
+
const edgeSet = /* @__PURE__ */ new Set();
|
|
1322
|
+
for (const e of edges) {
|
|
1323
|
+
if (!e.source || !e.target || e.source === e.target) continue;
|
|
1324
|
+
const key = `${e.source}-->${e.target}`;
|
|
1325
|
+
if (!edgeSet.has(key)) {
|
|
1326
|
+
edgeSet.add(key);
|
|
1327
|
+
uniqueEdges.push(e);
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
const adj = /* @__PURE__ */ new Map();
|
|
1331
|
+
for (const e of uniqueEdges) {
|
|
1332
|
+
if (!adj.has(e.source)) adj.set(e.source, /* @__PURE__ */ new Set());
|
|
1333
|
+
adj.get(e.source).add(e.target);
|
|
1334
|
+
}
|
|
1335
|
+
const reducedEdges = [];
|
|
1336
|
+
const removedTransitiveEdges = [];
|
|
1337
|
+
const hasAlternatePath = (start, target) => {
|
|
1338
|
+
const directNeighbors = adj.get(start);
|
|
1339
|
+
if (!directNeighbors) return false;
|
|
1340
|
+
const queue = [];
|
|
1341
|
+
const visited = /* @__PURE__ */ new Set([start]);
|
|
1342
|
+
for (const nxt of directNeighbors) {
|
|
1343
|
+
if (nxt !== target) {
|
|
1344
|
+
queue.push(nxt);
|
|
1345
|
+
visited.add(nxt);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
while (queue.length > 0) {
|
|
1349
|
+
const curr = queue.shift();
|
|
1350
|
+
if (curr === target) return true;
|
|
1351
|
+
const nextSet = adj.get(curr);
|
|
1352
|
+
if (!nextSet) continue;
|
|
1353
|
+
for (const nxt of nextSet) {
|
|
1354
|
+
if (nxt === target) return true;
|
|
1355
|
+
if (!visited.has(nxt)) {
|
|
1356
|
+
visited.add(nxt);
|
|
1357
|
+
queue.push(nxt);
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
return false;
|
|
1362
|
+
};
|
|
1363
|
+
for (const e of uniqueEdges) {
|
|
1364
|
+
if (hasAlternatePath(e.source, e.target) && !hasAlternatePath(e.target, e.source)) {
|
|
1365
|
+
removedTransitiveEdges.push(e);
|
|
1366
|
+
} else {
|
|
1367
|
+
reducedEdges.push(e);
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
return { reducedEdges, removedTransitiveEdges };
|
|
1371
|
+
}
|
|
1372
|
+
function kahnTopologicalSort(nodes, edges) {
|
|
1373
|
+
const nodeSet = new Set(nodes);
|
|
1374
|
+
const validEdges = edges.filter((e) => nodeSet.has(e.source) && nodeSet.has(e.target) && e.source !== e.target);
|
|
1375
|
+
const inDegree = /* @__PURE__ */ new Map();
|
|
1376
|
+
const adj = /* @__PURE__ */ new Map();
|
|
1377
|
+
for (const n of nodes) {
|
|
1378
|
+
inDegree.set(n, 0);
|
|
1379
|
+
adj.set(n, /* @__PURE__ */ new Set());
|
|
1380
|
+
}
|
|
1381
|
+
for (const e of validEdges) {
|
|
1382
|
+
adj.get(e.source).add(e.target);
|
|
1383
|
+
inDegree.set(e.target, (inDegree.get(e.target) ?? 0) + 1);
|
|
1384
|
+
}
|
|
1385
|
+
const queue = [];
|
|
1386
|
+
for (const [node, deg] of inDegree.entries()) {
|
|
1387
|
+
if (deg === 0) queue.push(node);
|
|
1388
|
+
}
|
|
1389
|
+
const order = [];
|
|
1390
|
+
const cyclesBroken = [];
|
|
1391
|
+
const droppedEdges = [];
|
|
1392
|
+
while (order.length < nodes.length) {
|
|
1393
|
+
if (queue.length > 0) {
|
|
1394
|
+
queue.sort((a, b) => a.localeCompare(b));
|
|
1395
|
+
const curr = queue.shift();
|
|
1396
|
+
order.push(curr);
|
|
1397
|
+
const neighbors = adj.get(curr);
|
|
1398
|
+
if (neighbors) {
|
|
1399
|
+
for (const nxt of neighbors) {
|
|
1400
|
+
const nextDeg = (inDegree.get(nxt) ?? 1) - 1;
|
|
1401
|
+
inDegree.set(nxt, nextDeg);
|
|
1402
|
+
if (nextDeg === 0) queue.push(nxt);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
} else {
|
|
1406
|
+
const remaining = nodes.filter((n) => !order.includes(n));
|
|
1407
|
+
if (remaining.length === 0) break;
|
|
1408
|
+
remaining.sort((a, b) => (inDegree.get(a) ?? 0) - (inDegree.get(b) ?? 0) || a.localeCompare(b));
|
|
1409
|
+
const candidate = remaining[0];
|
|
1410
|
+
let dropped = false;
|
|
1411
|
+
for (const [src, targets] of adj.entries()) {
|
|
1412
|
+
if (remaining.includes(src) && targets.has(candidate)) {
|
|
1413
|
+
targets.delete(candidate);
|
|
1414
|
+
droppedEdges.push({ source: src, target: candidate });
|
|
1415
|
+
cyclesBroken.push(`${src} -> ${candidate}`);
|
|
1416
|
+
inDegree.set(candidate, Math.max(0, (inDegree.get(candidate) ?? 1) - 1));
|
|
1417
|
+
if (inDegree.get(candidate) === 0) {
|
|
1418
|
+
queue.push(candidate);
|
|
1419
|
+
}
|
|
1420
|
+
dropped = true;
|
|
1421
|
+
break;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
if (!dropped) {
|
|
1425
|
+
order.push(candidate);
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
return {
|
|
1430
|
+
isDag: cyclesBroken.length === 0,
|
|
1431
|
+
order,
|
|
1432
|
+
cyclesBroken,
|
|
1433
|
+
droppedEdges
|
|
1434
|
+
};
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1229
1437
|
// src/harvester/termExtractor.ts
|
|
1230
1438
|
var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
1231
1439
|
"l\xE0",
|
|
@@ -1439,6 +1647,7 @@ var PedagogicalTermExtractor = class {
|
|
|
1439
1647
|
}
|
|
1440
1648
|
};
|
|
1441
1649
|
for (const { resourceId, doc } of docs) {
|
|
1650
|
+
if (isCommercialOrNoise({ url: doc.url, title: doc.title })) continue;
|
|
1442
1651
|
const md = doc.markdownContent || "";
|
|
1443
1652
|
const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
|
|
1444
1653
|
const bilingualMatches = mdWithoutCode.matchAll(
|
|
@@ -1542,7 +1751,7 @@ ${corpusPreview}`,
|
|
|
1542
1751
|
`
|
|
1543
1752
|
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
1544
1753
|
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
1545
|
-
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
|
|
1754
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."]}`
|
|
1546
1755
|
].filter(Boolean).join("\n");
|
|
1547
1756
|
const rawResponse = await options.llm(prompt, {
|
|
1548
1757
|
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
@@ -1554,9 +1763,26 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
1554
1763
|
const parsed = JSON.parse(jsonMatch[0]);
|
|
1555
1764
|
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
1556
1765
|
const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
|
|
1766
|
+
const rawEdges = [];
|
|
1767
|
+
for (const item of parsed) {
|
|
1768
|
+
const targetId = String(item.id || "");
|
|
1769
|
+
if (!validIds.has(targetId)) continue;
|
|
1770
|
+
for (const p of item.prerequisites || []) {
|
|
1771
|
+
const srcId = String(p);
|
|
1772
|
+
if (validIds.has(srcId) && srcId !== targetId) {
|
|
1773
|
+
rawEdges.push({ source: srcId, target: targetId });
|
|
1774
|
+
}
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
const { reducedEdges } = transitiveReduction(rawEdges);
|
|
1778
|
+
const reducedPrereqMap = /* @__PURE__ */ new Map();
|
|
1779
|
+
for (const e of reducedEdges) {
|
|
1780
|
+
if (!reducedPrereqMap.has(e.target)) reducedPrereqMap.set(e.target, []);
|
|
1781
|
+
reducedPrereqMap.get(e.target).push(e.source);
|
|
1782
|
+
}
|
|
1557
1783
|
const concepts = parsed.map((item, idx) => {
|
|
1558
1784
|
const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
|
|
1559
|
-
const prereqs =
|
|
1785
|
+
const prereqs = reducedPrereqMap.get(id) || [];
|
|
1560
1786
|
return {
|
|
1561
1787
|
id,
|
|
1562
1788
|
term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
|
|
@@ -1568,12 +1794,22 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
1568
1794
|
salienceScore: Number(item.salienceScore) || 8,
|
|
1569
1795
|
definition: String(item.definition || ""),
|
|
1570
1796
|
prerequisites: prereqs,
|
|
1571
|
-
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
|
|
1797
|
+
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"],
|
|
1798
|
+
facets: item.facets && typeof item.facets === "object" ? {
|
|
1799
|
+
computationalPrinciple: item.facets.computationalPrinciple ? String(item.facets.computationalPrinciple) : void 0,
|
|
1800
|
+
mentalModel: item.facets.mentalModel ? String(item.facets.mentalModel) : void 0,
|
|
1801
|
+
syntaxAndTools: Array.isArray(item.facets.syntaxAndTools) ? item.facets.syntaxAndTools.map(String) : void 0
|
|
1802
|
+
} : void 0,
|
|
1803
|
+
commonMisconceptions: Array.isArray(item.commonMisconceptions) ? item.commonMisconceptions.map(String) : void 0,
|
|
1804
|
+
troubleshootingHints: Array.isArray(item.troubleshootingHints) ? item.troubleshootingHints.map(String) : void 0
|
|
1572
1805
|
};
|
|
1573
1806
|
});
|
|
1807
|
+
const topoResult = kahnTopologicalSort(concepts.map((c) => c.id), reducedEdges);
|
|
1808
|
+
const conceptMap = new Map(concepts.map((c) => [c.id, c]));
|
|
1809
|
+
const sortedConcepts = topoResult.order.map((id) => conceptMap.get(id)).filter(Boolean);
|
|
1574
1810
|
return {
|
|
1575
|
-
concepts,
|
|
1576
|
-
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
1811
|
+
concepts: sortedConcepts.length > 0 ? sortedConcepts : concepts,
|
|
1812
|
+
conceptDagTsv: this.formatConceptDagTsv(sortedConcepts.length > 0 ? sortedConcepts : concepts),
|
|
1577
1813
|
isLlmRefined: true,
|
|
1578
1814
|
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
1579
1815
|
};
|
|
@@ -2619,6 +2855,173 @@ Return ONLY a valid JSON object with this structure:`,
|
|
|
2619
2855
|
}
|
|
2620
2856
|
};
|
|
2621
2857
|
|
|
2858
|
+
// src/pipeline/groundingEngine.ts
|
|
2859
|
+
var TechnicalGroundingEngine = class {
|
|
2860
|
+
constructor(searchEngine = new MultiSourceSearchEngine(), scraper = new WebScraper()) {
|
|
2861
|
+
this.searchEngine = searchEngine;
|
|
2862
|
+
this.scraper = scraper;
|
|
2863
|
+
}
|
|
2864
|
+
async buildFactsheet(topic, domain, context, llm) {
|
|
2865
|
+
const hardware = context?.hardwarePlatform || "";
|
|
2866
|
+
const software = context?.softwareEnv || "";
|
|
2867
|
+
const ageBand = context?.ageBand || [12, 18];
|
|
2868
|
+
const queries = [
|
|
2869
|
+
`${hardware} ${software} ${topic} compatibility setup official guide`.trim(),
|
|
2870
|
+
`${hardware || software || topic} beginner student project tutorial`.trim(),
|
|
2871
|
+
`${hardware || software || topic} common errors debugging mistakes pitfalls`.trim()
|
|
2872
|
+
];
|
|
2873
|
+
const rawHits = [];
|
|
2874
|
+
for (const q of queries) {
|
|
2875
|
+
try {
|
|
2876
|
+
const hits = await this.searchEngine.search(q, { limit: 3 });
|
|
2877
|
+
for (const h of hits) {
|
|
2878
|
+
if (!isCommercialOrNoise({ url: h.url, title: h.title, snippet: h.content })) {
|
|
2879
|
+
if (!rawHits.some((existing) => existing.url === h.url)) {
|
|
2880
|
+
rawHits.push(h);
|
|
2881
|
+
}
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
} catch {
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
const hitsToScrape = rawHits.slice(0, 3);
|
|
2888
|
+
const scrapeResults = await Promise.allSettled(
|
|
2889
|
+
hitsToScrape.map(async (h) => {
|
|
2890
|
+
try {
|
|
2891
|
+
const doc = await this.scraper.scrapeUrl(h.url);
|
|
2892
|
+
if (doc && doc.markdownContent) {
|
|
2893
|
+
return {
|
|
2894
|
+
title: h.title,
|
|
2895
|
+
url: h.url,
|
|
2896
|
+
excerpt: doc.markdownContent.slice(0, 1500)
|
|
2897
|
+
};
|
|
2898
|
+
}
|
|
2899
|
+
} catch {
|
|
2900
|
+
}
|
|
2901
|
+
return { title: h.title, url: h.url, excerpt: h.content.slice(0, 500) };
|
|
2902
|
+
})
|
|
2903
|
+
);
|
|
2904
|
+
const scrapedSnippets = scrapeResults.map(
|
|
2905
|
+
(res, idx) => res.status === "fulfilled" ? res.value : { title: hitsToScrape[idx].title, url: hitsToScrape[idx].url, excerpt: hitsToScrape[idx].content.slice(0, 500) }
|
|
2906
|
+
);
|
|
2907
|
+
const defaultBottlenecks = [
|
|
2908
|
+
"Power supply insufficiency when driving motors or actuators simultaneously with microcontrollers.",
|
|
2909
|
+
"Baud rate mismatch or driver incompatibility across host OS versions.",
|
|
2910
|
+
"Syntax confusion between blocking loops (delay/sleep) and non-blocking event handling."
|
|
2911
|
+
];
|
|
2912
|
+
const defaultExemplars = [
|
|
2913
|
+
{
|
|
2914
|
+
title: `${topic} Hands-on Foundation Project`,
|
|
2915
|
+
description: `Verified core implementation integrating ${hardware || "hardware"} and ${software || "software logic"}.`,
|
|
2916
|
+
targetAudience: `Students aged ${ageBand[0]}-${ageBand[1]}`,
|
|
2917
|
+
sourceUrl: scrapedSnippets[0]?.url,
|
|
2918
|
+
hardwareRequirements: hardware ? [hardware] : void 0,
|
|
2919
|
+
softwareRequirements: software ? [software] : void 0
|
|
2920
|
+
}
|
|
2921
|
+
];
|
|
2922
|
+
let factsheet = {
|
|
2923
|
+
topic,
|
|
2924
|
+
domain,
|
|
2925
|
+
compatibilityMatrix: {
|
|
2926
|
+
hardwarePlatform: hardware || void 0,
|
|
2927
|
+
softwareEnv: software || void 0,
|
|
2928
|
+
verifiedCompatible: true,
|
|
2929
|
+
notes: [`Targeting ${hardware || "General Platform"} with ${software || "Standard SDK"}.`]
|
|
2930
|
+
},
|
|
2931
|
+
ageFeasibility: {
|
|
2932
|
+
recommendedMinAge: ageBand[0],
|
|
2933
|
+
recommendedMaxAge: ageBand[1],
|
|
2934
|
+
cognitiveReadinessVerdict: "feasible",
|
|
2935
|
+
potentialBarriers: ["Understanding abstract signal flows and state-based logic."]
|
|
2936
|
+
},
|
|
2937
|
+
exemplarProjects: defaultExemplars,
|
|
2938
|
+
technicalBottlenecks: defaultBottlenecks,
|
|
2939
|
+
groundingSources: scrapedSnippets.map((s) => ({ title: s.title, url: s.url })),
|
|
2940
|
+
markdownReport: ""
|
|
2941
|
+
};
|
|
2942
|
+
if (llm && scrapedSnippets.length > 0) {
|
|
2943
|
+
try {
|
|
2944
|
+
const prompt = [
|
|
2945
|
+
`You are a Lead STEM Curriculum Engineer. Analyze these research excerpts for topic: "${topic}".`,
|
|
2946
|
+
`Hardware: ${hardware || "N/A"}, Software/Env: ${software || "N/A"}, Student Age Band: ${ageBand[0]}-${ageBand[1]} years old.`,
|
|
2947
|
+
`
|
|
2948
|
+
Harvested Sources:
|
|
2949
|
+
${scrapedSnippets.map((s, i) => `[Source ${i + 1}: ${s.title} (${s.url})]
|
|
2950
|
+
${s.excerpt}`).join("\n\n")}`,
|
|
2951
|
+
`
|
|
2952
|
+
Synthesize a strictly valid JSON object with:`,
|
|
2953
|
+
`{`,
|
|
2954
|
+
` "compatibilityVerdict": boolean,`,
|
|
2955
|
+
` "compatibilityNotes": string[],`,
|
|
2956
|
+
` "cognitiveReadiness": "feasible" | "challenging" | "excessive",`,
|
|
2957
|
+
` "potentialBarriers": string[],`,
|
|
2958
|
+
` "exemplars": [{"title": string, "description": string, "targetAudience": string, "hardwareRequirements": string[], "softwareRequirements": string[]}],`,
|
|
2959
|
+
` "technicalBottlenecks": string[]`,
|
|
2960
|
+
`}`
|
|
2961
|
+
].join("\n");
|
|
2962
|
+
const raw = await llm(prompt, {
|
|
2963
|
+
system: "Output strictly valid JSON object without markdown commentary.",
|
|
2964
|
+
temperature: 0.2,
|
|
2965
|
+
jsonMode: true
|
|
2966
|
+
});
|
|
2967
|
+
const jsonMatch = raw.match(/\{[\s\S]*\}/);
|
|
2968
|
+
if (jsonMatch) {
|
|
2969
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
2970
|
+
factsheet.compatibilityMatrix.verifiedCompatible = Boolean(parsed.compatibilityVerdict ?? true);
|
|
2971
|
+
if (Array.isArray(parsed.compatibilityNotes)) factsheet.compatibilityMatrix.notes = parsed.compatibilityNotes.map(String);
|
|
2972
|
+
if (["feasible", "challenging", "excessive"].includes(parsed.cognitiveReadiness)) {
|
|
2973
|
+
factsheet.ageFeasibility.cognitiveReadinessVerdict = parsed.cognitiveReadiness;
|
|
2974
|
+
}
|
|
2975
|
+
if (Array.isArray(parsed.potentialBarriers)) factsheet.ageFeasibility.potentialBarriers = parsed.potentialBarriers.map(String);
|
|
2976
|
+
if (Array.isArray(parsed.exemplars) && parsed.exemplars.length > 0) {
|
|
2977
|
+
factsheet.exemplarProjects = parsed.exemplars.map((ex) => ({
|
|
2978
|
+
title: String(ex.title || "Verified Project"),
|
|
2979
|
+
description: String(ex.description || ""),
|
|
2980
|
+
targetAudience: String(ex.targetAudience || `Ages ${ageBand[0]}-${ageBand[1]}`),
|
|
2981
|
+
hardwareRequirements: Array.isArray(ex.hardwareRequirements) ? ex.hardwareRequirements.map(String) : void 0,
|
|
2982
|
+
softwareRequirements: Array.isArray(ex.softwareRequirements) ? ex.softwareRequirements.map(String) : void 0,
|
|
2983
|
+
sourceUrl: scrapedSnippets[0]?.url
|
|
2984
|
+
}));
|
|
2985
|
+
}
|
|
2986
|
+
if (Array.isArray(parsed.technicalBottlenecks)) factsheet.technicalBottlenecks = parsed.technicalBottlenecks.map(String);
|
|
2987
|
+
}
|
|
2988
|
+
} catch (err) {
|
|
2989
|
+
console.warn("[TechnicalGroundingEngine] LLM synthesis fallback to baseline:", err?.message || err);
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
factsheet.markdownReport = this.renderMarkdownReport(factsheet);
|
|
2993
|
+
return factsheet;
|
|
2994
|
+
}
|
|
2995
|
+
renderMarkdownReport(fs) {
|
|
2996
|
+
const lines = [
|
|
2997
|
+
`# \u{1F6E0}\uFE0F Technical Grounding & Empirical Factsheet: ${fs.topic}`,
|
|
2998
|
+
`> **Domain:** \`${fs.domain.toUpperCase()}\` | **Feasibility:** \`${fs.ageFeasibility.cognitiveReadinessVerdict.toUpperCase()}\``,
|
|
2999
|
+
`
|
|
3000
|
+
## 1. Compatibility & Tooling Verification`,
|
|
3001
|
+
`- **Hardware:** ${fs.compatibilityMatrix.hardwarePlatform || "N/A"}`,
|
|
3002
|
+
`- **Software / SDK:** ${fs.compatibilityMatrix.softwareEnv || "N/A"}`,
|
|
3003
|
+
`- **Verdict:** ${fs.compatibilityMatrix.verifiedCompatible ? "\u2705 Verified Compatible" : "\u26A0\uFE0F Compatibility Warning"}`,
|
|
3004
|
+
...fs.compatibilityMatrix.notes.map((n) => ` - ${n}`),
|
|
3005
|
+
`
|
|
3006
|
+
## 2. Age Feasibility & Learner Barriers (Ages ${fs.ageFeasibility.recommendedMinAge}\u2013${fs.ageFeasibility.recommendedMaxAge})`,
|
|
3007
|
+
...fs.ageFeasibility.potentialBarriers.map((b) => `- \u26A0\uFE0F ${b}`),
|
|
3008
|
+
`
|
|
3009
|
+
## 3. Real-World Exemplar Projects`,
|
|
3010
|
+
...fs.exemplarProjects.map((ex, idx) => `### Exemplar ${idx + 1}: ${ex.title}
|
|
3011
|
+
- **Summary:** ${ex.description}
|
|
3012
|
+
- **Audience:** ${ex.targetAudience}${ex.sourceUrl ? `
|
|
3013
|
+
- **Reference:** [Link](${ex.sourceUrl})` : ""}`),
|
|
3014
|
+
`
|
|
3015
|
+
## 4. Technical Bottlenecks & Pitfalls to Avoid`,
|
|
3016
|
+
...fs.technicalBottlenecks.map((bn) => `- \u{1F6D1} ${bn}`),
|
|
3017
|
+
`
|
|
3018
|
+
## 5. Grounding Citations`,
|
|
3019
|
+
...fs.groundingSources.map((s, idx) => `${idx + 1}. [${s.title}](${s.url})`)
|
|
3020
|
+
];
|
|
3021
|
+
return lines.join("\n");
|
|
3022
|
+
}
|
|
3023
|
+
};
|
|
3024
|
+
|
|
2622
3025
|
// src/services/researchKitClient.ts
|
|
2623
3026
|
var KnowledgeDiscoveryService = class {
|
|
2624
3027
|
constructor(config) {
|
|
@@ -2818,6 +3221,29 @@ var PedagogicalSynthesisService = class {
|
|
|
2818
3221
|
});
|
|
2819
3222
|
}
|
|
2820
3223
|
};
|
|
3224
|
+
var TechnicalGroundingService = class {
|
|
3225
|
+
constructor(config) {
|
|
3226
|
+
this.config = config;
|
|
3227
|
+
const searchEngine = new MultiSourceSearchEngine({
|
|
3228
|
+
apiKeys: config.apiKeys,
|
|
3229
|
+
allowEnvFallback: config.allowEnvFallback,
|
|
3230
|
+
customAdapter: config.customSearchAdapter
|
|
3231
|
+
});
|
|
3232
|
+
const scraper = new WebScraper({
|
|
3233
|
+
crawl4aiHost: config.crawl4aiHost,
|
|
3234
|
+
jinaKey: config.apiKeys?.jina,
|
|
3235
|
+
allowEnvFallback: config.allowEnvFallback,
|
|
3236
|
+
customAdapter: config.customCrawlerAdapter
|
|
3237
|
+
});
|
|
3238
|
+
this.engine = new TechnicalGroundingEngine(searchEngine, scraper);
|
|
3239
|
+
}
|
|
3240
|
+
engine;
|
|
3241
|
+
async buildFactsheet(topic, options) {
|
|
3242
|
+
const ctx = options?.context ?? this.config.defaultCourseContext;
|
|
3243
|
+
const domain = options?.domain ?? classifyTechnicalDomain(topic, void 0, ctx).domain;
|
|
3244
|
+
return this.engine.buildFactsheet(topic, domain, ctx, this.config.llm);
|
|
3245
|
+
}
|
|
3246
|
+
};
|
|
2821
3247
|
var ResearchKitClient = class {
|
|
2822
3248
|
constructor(config = {}) {
|
|
2823
3249
|
this.config = config;
|
|
@@ -2826,6 +3252,7 @@ var ResearchKitClient = class {
|
|
|
2826
3252
|
this.academic = new AcademicStandardsService(config);
|
|
2827
3253
|
this.verification = new EvidenceVerificationService(config);
|
|
2828
3254
|
this.orchestrator = new PedagogicalSynthesisService(config);
|
|
3255
|
+
this.grounding = new TechnicalGroundingService(config);
|
|
2829
3256
|
}
|
|
2830
3257
|
/** Service 1: Multi-Engine Web Search, Scraper & YouTube Transcript Ingestion */
|
|
2831
3258
|
discovery;
|
|
@@ -2837,6 +3264,8 @@ var ResearchKitClient = class {
|
|
|
2837
3264
|
verification;
|
|
2838
3265
|
/** Service 5: Campaign Planning, Semantic Synthesis & Canonical Reference Pack Orchestration */
|
|
2839
3266
|
orchestrator;
|
|
3267
|
+
/** Service 6: Technical Grounding, Hardware/Software Compatibility & Empirical Exemplars */
|
|
3268
|
+
grounding;
|
|
2840
3269
|
};
|
|
2841
3270
|
|
|
2842
3271
|
exports.AcademicPaperSchema = AcademicPaperSchema;
|
|
@@ -2850,6 +3279,8 @@ exports.CatalogBuilder = CatalogBuilder;
|
|
|
2850
3279
|
exports.CitationManager = CitationManager;
|
|
2851
3280
|
exports.ClaimVerifier = ClaimVerifier;
|
|
2852
3281
|
exports.DEFAULT_RESEARCH_CONFIG = DEFAULT_RESEARCH_CONFIG;
|
|
3282
|
+
exports.ECOMMERCE_AND_COMMERCIAL_PATTERNS = ECOMMERCE_AND_COMMERCIAL_PATTERNS;
|
|
3283
|
+
exports.ECOMMERCE_DOMAINS = ECOMMERCE_DOMAINS;
|
|
2853
3284
|
exports.EvidenceScorer = EvidenceScorer;
|
|
2854
3285
|
exports.EvidenceVerificationService = EvidenceVerificationService;
|
|
2855
3286
|
exports.HarvestPipeline = HarvestPipeline;
|
|
@@ -2863,19 +3294,25 @@ exports.ProvenanceTracker = ProvenanceTracker;
|
|
|
2863
3294
|
exports.ReferencePackBuilder = ReferencePackBuilder;
|
|
2864
3295
|
exports.ResearchConfigSchema = ResearchConfigSchema;
|
|
2865
3296
|
exports.ResearchKitClient = ResearchKitClient;
|
|
3297
|
+
exports.TechnicalGroundingEngine = TechnicalGroundingEngine;
|
|
3298
|
+
exports.TechnicalGroundingService = TechnicalGroundingService;
|
|
2866
3299
|
exports.TerminologyGraphService = TerminologyGraphService;
|
|
2867
3300
|
exports.WebScraper = WebScraper;
|
|
2868
3301
|
exports.YouTubeIngestionService = YouTubeIngestionService;
|
|
2869
3302
|
exports.classifyTechnicalDomain = classifyTechnicalDomain;
|
|
3303
|
+
exports.evaluateTechnicalDensity = evaluateTechnicalDensity;
|
|
2870
3304
|
exports.extractSubstantiveExcerpts = extractSubstantiveExcerpts;
|
|
2871
3305
|
exports.extractTopSentences = extractTopSentences;
|
|
2872
3306
|
exports.extractYouTubeVideoId = extractYouTubeVideoId;
|
|
2873
3307
|
exports.formatTimestamp = formatTimestamp;
|
|
2874
3308
|
exports.inferPrimaryCodeLanguage = inferPrimaryCodeLanguage;
|
|
3309
|
+
exports.isCommercialOrNoise = isCommercialOrNoise;
|
|
2875
3310
|
exports.isYouTubeUrl = isYouTubeUrl;
|
|
3311
|
+
exports.kahnTopologicalSort = kahnTopologicalSort;
|
|
2876
3312
|
exports.mapTopicToCS2023 = mapTopicToCS2023;
|
|
2877
3313
|
exports.normalizeUrlForDedup = normalizeUrlForDedup;
|
|
2878
3314
|
exports.reconstructOpenAlexAbstract = reconstructOpenAlexAbstract;
|
|
2879
3315
|
exports.stitchCaptionSegments = stitchCaptionSegments;
|
|
3316
|
+
exports.transitiveReduction = transitiveReduction;
|
|
2880
3317
|
//# sourceMappingURL=index.cjs.map
|
|
2881
3318
|
//# sourceMappingURL=index.cjs.map
|