@thanh01.pmt/research-kit 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -14
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +2 -1
- package/dist/catalogBuilder-BaVRvg-9.d.cts +90 -0
- package/dist/catalogBuilder-DHV6y1gD.d.ts +90 -0
- package/dist/chunk-AKEI7H2C.js +223 -0
- package/dist/chunk-AKEI7H2C.js.map +1 -0
- package/dist/{chunk-XIF5YNI7.js → chunk-HZNSZDSM.js} +318 -8
- package/dist/chunk-HZNSZDSM.js.map +1 -0
- package/dist/chunk-LSMSIPTZ.js +3 -0
- package/dist/{chunk-2JEVBCIF.js.map → chunk-LSMSIPTZ.js.map} +1 -1
- package/dist/{chunk-KAFQVI2R.js → chunk-ON6VFCJO.js} +3 -3
- package/dist/{chunk-KAFQVI2R.js.map → chunk-ON6VFCJO.js.map} +1 -1
- package/dist/{chunk-GKF3NDEX.js → chunk-QFYM2NV4.js} +3 -3
- package/dist/{chunk-GKF3NDEX.js.map → chunk-QFYM2NV4.js.map} +1 -1
- package/dist/{chunk-AUGVVZTL.js → chunk-RW7VS7NI.js} +175 -7
- package/dist/chunk-RW7VS7NI.js.map +1 -0
- package/dist/{chunk-DOXSPRSP.js → chunk-UKGR45VZ.js} +3 -223
- package/dist/chunk-UKGR45VZ.js.map +1 -0
- package/dist/{chunk-J7XKDMGL.js → chunk-X3X26W2I.js} +2 -2
- package/dist/{chunk-J7XKDMGL.js.map → chunk-X3X26W2I.js.map} +1 -1
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -2
- package/dist/harvester/index.cjs +542 -6
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +63 -49
- package/dist/harvester/index.d.ts +63 -49
- package/dist/harvester/index.js +4 -3
- package/dist/{index-t8pahlJ4.d.ts → index-DlngvZBr.d.cts} +15 -1
- package/dist/{index-t8pahlJ4.d.cts → index-DlngvZBr.d.ts} +15 -1
- package/dist/index.cjs +515 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -13
- package/dist/index.d.ts +28 -13
- package/dist/index.js +41 -13
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-BTPFe7eb.d.cts → literatureReview-C7hsjrn4.d.cts} +1 -1
- package/dist/{literatureReview-hGbfrBmw.d.ts → literatureReview-DWl13r_3.d.ts} +1 -1
- package/dist/pipeline/index.cjs +658 -231
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -5
- package/dist/pipeline/index.d.ts +49 -5
- package/dist/pipeline/index.js +6 -5
- package/dist/{webScraper-BhiQ8s9x.d.ts → webScraper-DBFUIqoO.d.ts} +1 -1
- package/dist/{webScraper-D9Ldus9p.d.cts → webScraper-yO9P-oLb.d.cts} +1 -1
- package/package.json +1 -1
- package/dist/catalogBuilder-CmmQzUnf.d.ts +0 -38
- package/dist/catalogBuilder-DgXYxL2F.d.cts +0 -38
- package/dist/chunk-2JEVBCIF.js +0 -3
- package/dist/chunk-AUGVVZTL.js.map +0 -1
- package/dist/chunk-DOXSPRSP.js.map +0 -1
- package/dist/chunk-XIF5YNI7.js.map +0 -1
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { AcademicSearchEngine } from './chunk-AKEI7H2C.js';
|
|
2
|
+
|
|
1
3
|
// src/harvester/domainClassifier.ts
|
|
2
4
|
var HARDWARE_PATTERNS = [
|
|
3
5
|
/\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
|
|
@@ -109,14 +111,25 @@ var MultiSourceSearchEngine = class {
|
|
|
109
111
|
tavilyKey;
|
|
110
112
|
exaKey;
|
|
111
113
|
serperKey;
|
|
114
|
+
searxngHost;
|
|
115
|
+
searxngHeaders;
|
|
112
116
|
allowEnvFallback;
|
|
113
117
|
customAdapter;
|
|
118
|
+
academicEngine;
|
|
114
119
|
constructor(options = {}) {
|
|
115
120
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
116
121
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
117
122
|
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
118
123
|
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
124
|
+
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
125
|
+
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
126
|
+
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
127
|
+
this.searxngHeaders = {
|
|
128
|
+
...options.searxngHeaders || {},
|
|
129
|
+
...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
|
|
130
|
+
};
|
|
119
131
|
this.customAdapter = options.customAdapter;
|
|
132
|
+
this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
|
|
120
133
|
}
|
|
121
134
|
setCustomAdapter(adapter) {
|
|
122
135
|
this.customAdapter = adapter;
|
|
@@ -126,6 +139,12 @@ var MultiSourceSearchEngine = class {
|
|
|
126
139
|
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
127
140
|
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
128
141
|
}
|
|
142
|
+
setSearxng(host, headers) {
|
|
143
|
+
this.searxngHost = host;
|
|
144
|
+
if (headers) {
|
|
145
|
+
this.searxngHeaders = { ...this.searxngHeaders, ...headers };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
129
148
|
async search(query, options = {}) {
|
|
130
149
|
const limit = options.limit ?? 5;
|
|
131
150
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -146,9 +165,30 @@ var MultiSourceSearchEngine = class {
|
|
|
146
165
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
147
166
|
}
|
|
148
167
|
}
|
|
168
|
+
if (preferred === "academic") {
|
|
169
|
+
try {
|
|
170
|
+
const papers = await this.academicEngine.searchPapers(query, { limit });
|
|
171
|
+
if (papers.length > 0) {
|
|
172
|
+
return papers.map((p) => ({
|
|
173
|
+
title: p.title,
|
|
174
|
+
url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
|
|
175
|
+
content: p.tldr || p.abstract || "",
|
|
176
|
+
score: p.citationCount,
|
|
177
|
+
engine: "academic"
|
|
178
|
+
}));
|
|
179
|
+
}
|
|
180
|
+
} catch (e) {
|
|
181
|
+
console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
149
184
|
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
150
185
|
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
151
186
|
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
187
|
+
const searxngHost = options.searxngHost || this.searxngHost;
|
|
188
|
+
if (preferred === "searxng" && searxngHost) {
|
|
189
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
190
|
+
if (searxHits.length > 0) return searxHits;
|
|
191
|
+
}
|
|
152
192
|
if (preferred === "exa" && exaKey) {
|
|
153
193
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
154
194
|
if (exaHits.length > 0) return exaHits;
|
|
@@ -203,10 +243,44 @@ var MultiSourceSearchEngine = class {
|
|
|
203
243
|
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
204
244
|
if (serperHits.length > 0) return serperHits;
|
|
205
245
|
}
|
|
246
|
+
if (searxngHost) {
|
|
247
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
248
|
+
if (searxHits.length > 0) return searxHits;
|
|
249
|
+
}
|
|
206
250
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
207
251
|
if (ddgHits.length > 0) return ddgHits;
|
|
208
252
|
return [];
|
|
209
253
|
}
|
|
254
|
+
async querySearxng(query, searxngHost, limit, customHeaders) {
|
|
255
|
+
try {
|
|
256
|
+
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
257
|
+
const searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
258
|
+
const headers = {
|
|
259
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
260
|
+
...this.searxngHeaders || {},
|
|
261
|
+
...customHeaders || {}
|
|
262
|
+
};
|
|
263
|
+
const res = await fetch(searchUrl, {
|
|
264
|
+
headers,
|
|
265
|
+
signal: AbortSignal.timeout(1e4)
|
|
266
|
+
});
|
|
267
|
+
if (res.ok) {
|
|
268
|
+
const data = await res.json();
|
|
269
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
270
|
+
return data.results.slice(0, limit).map((r) => ({
|
|
271
|
+
title: r.title || "Untitled",
|
|
272
|
+
url: r.url,
|
|
273
|
+
content: r.content || r.snippet || "",
|
|
274
|
+
score: r.score,
|
|
275
|
+
engine: "searxng"
|
|
276
|
+
}));
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
} catch (e) {
|
|
280
|
+
console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
|
|
281
|
+
}
|
|
282
|
+
return [];
|
|
283
|
+
}
|
|
210
284
|
async queryExa(query, exaKey, limit, domainFilter) {
|
|
211
285
|
try {
|
|
212
286
|
const body = {
|
|
@@ -305,6 +379,214 @@ var MultiSourceSearchEngine = class {
|
|
|
305
379
|
}
|
|
306
380
|
};
|
|
307
381
|
|
|
382
|
+
// src/harvester/noiseFilter.ts
|
|
383
|
+
var ECOMMERCE_DOMAINS = /* @__PURE__ */ new Set([
|
|
384
|
+
"shopee.vn",
|
|
385
|
+
"shopee.com",
|
|
386
|
+
"lazada.vn",
|
|
387
|
+
"lazada.com",
|
|
388
|
+
"tiki.vn",
|
|
389
|
+
"sendo.vn",
|
|
390
|
+
"aliexpress.com",
|
|
391
|
+
"taobao.com",
|
|
392
|
+
"amazon.com",
|
|
393
|
+
"ebay.com",
|
|
394
|
+
"banggood.com"
|
|
395
|
+
]);
|
|
396
|
+
var ECOMMERCE_AND_COMMERCIAL_PATTERNS = [
|
|
397
|
+
/\b(shopee|lazada|tiki|sendo|taobao|aliexpress)\b/i,
|
|
398
|
+
/\b(mua ngay|giá tốt|khuyến mãi|freeship|bán chạy|nhập khẩu chính hãng|cửa hàng|bảng giá|đặt hàng)\b/i,
|
|
399
|
+
/\b(buy now|add to cart|free shipping|discount|best price|on sale|shop now|in stock|order now)\b/i,
|
|
400
|
+
/\b(\d+[kKđ]|₫|\$\d+|\d+\s*(?:VNĐ|VND|usd))\b/i
|
|
401
|
+
];
|
|
402
|
+
function isCommercialOrNoise(input) {
|
|
403
|
+
if (input.url) {
|
|
404
|
+
try {
|
|
405
|
+
const hostname = new URL(input.url).hostname.toLowerCase().replace(/^www\./, "");
|
|
406
|
+
if (ECOMMERCE_DOMAINS.has(hostname) || [...ECOMMERCE_DOMAINS].some((d) => hostname.endsWith(`.${d}`))) {
|
|
407
|
+
return true;
|
|
408
|
+
}
|
|
409
|
+
} catch {
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
const combined = `${input.title || ""} ${input.snippet || ""}`;
|
|
413
|
+
if (!combined.trim()) return false;
|
|
414
|
+
for (const pattern of ECOMMERCE_AND_COMMERCIAL_PATTERNS) {
|
|
415
|
+
if (pattern.test(combined)) {
|
|
416
|
+
return true;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
function evaluateTechnicalDensity(markdown) {
|
|
422
|
+
if (!markdown || markdown.trim().length < 80) {
|
|
423
|
+
return {
|
|
424
|
+
score: 1,
|
|
425
|
+
isSufficientQuality: false,
|
|
426
|
+
codeDensity: 0,
|
|
427
|
+
technicalTermDensity: 0,
|
|
428
|
+
hasStructure: false,
|
|
429
|
+
reasons: ["Content is too short or empty."]
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
const reasons = [];
|
|
433
|
+
const totalLength = markdown.length;
|
|
434
|
+
const codeBlocks = markdown.match(/```[\s\S]*?```/g) || [];
|
|
435
|
+
const inlineCode = markdown.match(/`[^`\n]+`/g) || [];
|
|
436
|
+
const totalCodeChars = codeBlocks.reduce((sum, b) => sum + b.length, 0) + inlineCode.reduce((sum, c) => sum + c.length, 0);
|
|
437
|
+
const codeDensity = Math.min(1, totalCodeChars / Math.max(1, totalLength));
|
|
438
|
+
const headings = markdown.match(/^#{1,4}\s+.+$/gm) || [];
|
|
439
|
+
const hasStructure = headings.length >= 2;
|
|
440
|
+
const techKeywords = markdown.match(
|
|
441
|
+
/\b(function|class|method|struct|interface|api|hardware|sensor|pin|circuit|logic|loop|state|algorithm|test|debug|schema|protocol|variable)\b/gi
|
|
442
|
+
) || [];
|
|
443
|
+
const technicalTermDensity = Math.min(1, techKeywords.length * 10 / Math.max(100, markdown.split(/\s+/).length));
|
|
444
|
+
let score = 5;
|
|
445
|
+
if (codeDensity > 0.15) {
|
|
446
|
+
score += 2;
|
|
447
|
+
} else if (codeDensity > 0.05) {
|
|
448
|
+
score += 1;
|
|
449
|
+
} else {
|
|
450
|
+
reasons.push("Low code or API snippet density.");
|
|
451
|
+
}
|
|
452
|
+
if (hasStructure) {
|
|
453
|
+
score += 1.5;
|
|
454
|
+
} else {
|
|
455
|
+
reasons.push("Lacks clear hierarchical markdown headings.");
|
|
456
|
+
}
|
|
457
|
+
if (technicalTermDensity > 0.08) {
|
|
458
|
+
score += 1.5;
|
|
459
|
+
}
|
|
460
|
+
const finalScore = Math.min(10, Math.max(1, Math.round(score * 10) / 10));
|
|
461
|
+
return {
|
|
462
|
+
score: finalScore,
|
|
463
|
+
isSufficientQuality: finalScore >= 6,
|
|
464
|
+
codeDensity,
|
|
465
|
+
technicalTermDensity,
|
|
466
|
+
hasStructure,
|
|
467
|
+
reasons
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// src/harvester/graphUtils.ts
|
|
472
|
+
function transitiveReduction(edges) {
|
|
473
|
+
const uniqueEdges = [];
|
|
474
|
+
const edgeSet = /* @__PURE__ */ new Set();
|
|
475
|
+
for (const e of edges) {
|
|
476
|
+
if (!e.source || !e.target || e.source === e.target) continue;
|
|
477
|
+
const key = `${e.source}-->${e.target}`;
|
|
478
|
+
if (!edgeSet.has(key)) {
|
|
479
|
+
edgeSet.add(key);
|
|
480
|
+
uniqueEdges.push(e);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
const adj = /* @__PURE__ */ new Map();
|
|
484
|
+
for (const e of uniqueEdges) {
|
|
485
|
+
if (!adj.has(e.source)) adj.set(e.source, /* @__PURE__ */ new Set());
|
|
486
|
+
adj.get(e.source).add(e.target);
|
|
487
|
+
}
|
|
488
|
+
const reducedEdges = [];
|
|
489
|
+
const removedTransitiveEdges = [];
|
|
490
|
+
const hasAlternatePath = (start, target) => {
|
|
491
|
+
const directNeighbors = adj.get(start);
|
|
492
|
+
if (!directNeighbors) return false;
|
|
493
|
+
const queue = [];
|
|
494
|
+
const visited = /* @__PURE__ */ new Set([start]);
|
|
495
|
+
for (const nxt of directNeighbors) {
|
|
496
|
+
if (nxt !== target) {
|
|
497
|
+
queue.push(nxt);
|
|
498
|
+
visited.add(nxt);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
while (queue.length > 0) {
|
|
502
|
+
const curr = queue.shift();
|
|
503
|
+
if (curr === target) return true;
|
|
504
|
+
const nextSet = adj.get(curr);
|
|
505
|
+
if (!nextSet) continue;
|
|
506
|
+
for (const nxt of nextSet) {
|
|
507
|
+
if (nxt === target) return true;
|
|
508
|
+
if (!visited.has(nxt)) {
|
|
509
|
+
visited.add(nxt);
|
|
510
|
+
queue.push(nxt);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return false;
|
|
515
|
+
};
|
|
516
|
+
for (const e of uniqueEdges) {
|
|
517
|
+
if (hasAlternatePath(e.source, e.target) && !hasAlternatePath(e.target, e.source)) {
|
|
518
|
+
removedTransitiveEdges.push(e);
|
|
519
|
+
} else {
|
|
520
|
+
reducedEdges.push(e);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
return { reducedEdges, removedTransitiveEdges };
|
|
524
|
+
}
|
|
525
|
+
function kahnTopologicalSort(nodes, edges) {
|
|
526
|
+
const nodeSet = new Set(nodes);
|
|
527
|
+
const validEdges = edges.filter((e) => nodeSet.has(e.source) && nodeSet.has(e.target) && e.source !== e.target);
|
|
528
|
+
const inDegree = /* @__PURE__ */ new Map();
|
|
529
|
+
const adj = /* @__PURE__ */ new Map();
|
|
530
|
+
for (const n of nodes) {
|
|
531
|
+
inDegree.set(n, 0);
|
|
532
|
+
adj.set(n, /* @__PURE__ */ new Set());
|
|
533
|
+
}
|
|
534
|
+
for (const e of validEdges) {
|
|
535
|
+
adj.get(e.source).add(e.target);
|
|
536
|
+
inDegree.set(e.target, (inDegree.get(e.target) ?? 0) + 1);
|
|
537
|
+
}
|
|
538
|
+
const queue = [];
|
|
539
|
+
for (const [node, deg] of inDegree.entries()) {
|
|
540
|
+
if (deg === 0) queue.push(node);
|
|
541
|
+
}
|
|
542
|
+
const order = [];
|
|
543
|
+
const cyclesBroken = [];
|
|
544
|
+
const droppedEdges = [];
|
|
545
|
+
while (order.length < nodes.length) {
|
|
546
|
+
if (queue.length > 0) {
|
|
547
|
+
queue.sort((a, b) => a.localeCompare(b));
|
|
548
|
+
const curr = queue.shift();
|
|
549
|
+
order.push(curr);
|
|
550
|
+
const neighbors = adj.get(curr);
|
|
551
|
+
if (neighbors) {
|
|
552
|
+
for (const nxt of neighbors) {
|
|
553
|
+
const nextDeg = (inDegree.get(nxt) ?? 1) - 1;
|
|
554
|
+
inDegree.set(nxt, nextDeg);
|
|
555
|
+
if (nextDeg === 0) queue.push(nxt);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
} else {
|
|
559
|
+
const remaining = nodes.filter((n) => !order.includes(n));
|
|
560
|
+
if (remaining.length === 0) break;
|
|
561
|
+
remaining.sort((a, b) => (inDegree.get(a) ?? 0) - (inDegree.get(b) ?? 0) || a.localeCompare(b));
|
|
562
|
+
const candidate = remaining[0];
|
|
563
|
+
let dropped = false;
|
|
564
|
+
for (const [src, targets] of adj.entries()) {
|
|
565
|
+
if (remaining.includes(src) && targets.has(candidate)) {
|
|
566
|
+
targets.delete(candidate);
|
|
567
|
+
droppedEdges.push({ source: src, target: candidate });
|
|
568
|
+
cyclesBroken.push(`${src} -> ${candidate}`);
|
|
569
|
+
inDegree.set(candidate, Math.max(0, (inDegree.get(candidate) ?? 1) - 1));
|
|
570
|
+
if (inDegree.get(candidate) === 0) {
|
|
571
|
+
queue.push(candidate);
|
|
572
|
+
}
|
|
573
|
+
dropped = true;
|
|
574
|
+
break;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
if (!dropped) {
|
|
578
|
+
order.push(candidate);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
return {
|
|
583
|
+
isDag: cyclesBroken.length === 0,
|
|
584
|
+
order,
|
|
585
|
+
cyclesBroken,
|
|
586
|
+
droppedEdges
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
|
|
308
590
|
// src/harvester/termExtractor.ts
|
|
309
591
|
var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
310
592
|
"l\xE0",
|
|
@@ -518,6 +800,7 @@ var PedagogicalTermExtractor = class {
|
|
|
518
800
|
}
|
|
519
801
|
};
|
|
520
802
|
for (const { resourceId, doc } of docs) {
|
|
803
|
+
if (isCommercialOrNoise({ url: doc.url, title: doc.title })) continue;
|
|
521
804
|
const md = doc.markdownContent || "";
|
|
522
805
|
const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
|
|
523
806
|
const bilingualMatches = mdWithoutCode.matchAll(
|
|
@@ -621,7 +904,7 @@ ${corpusPreview}`,
|
|
|
621
904
|
`
|
|
622
905
|
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
623
906
|
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
624
|
-
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
|
|
907
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."]}`
|
|
625
908
|
].filter(Boolean).join("\n");
|
|
626
909
|
const rawResponse = await options.llm(prompt, {
|
|
627
910
|
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
@@ -633,9 +916,26 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
633
916
|
const parsed = JSON.parse(jsonMatch[0]);
|
|
634
917
|
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
635
918
|
const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
|
|
919
|
+
const rawEdges = [];
|
|
920
|
+
for (const item of parsed) {
|
|
921
|
+
const targetId = String(item.id || "");
|
|
922
|
+
if (!validIds.has(targetId)) continue;
|
|
923
|
+
for (const p of item.prerequisites || []) {
|
|
924
|
+
const srcId = String(p);
|
|
925
|
+
if (validIds.has(srcId) && srcId !== targetId) {
|
|
926
|
+
rawEdges.push({ source: srcId, target: targetId });
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
const { reducedEdges } = transitiveReduction(rawEdges);
|
|
931
|
+
const reducedPrereqMap = /* @__PURE__ */ new Map();
|
|
932
|
+
for (const e of reducedEdges) {
|
|
933
|
+
if (!reducedPrereqMap.has(e.target)) reducedPrereqMap.set(e.target, []);
|
|
934
|
+
reducedPrereqMap.get(e.target).push(e.source);
|
|
935
|
+
}
|
|
636
936
|
const concepts = parsed.map((item, idx) => {
|
|
637
937
|
const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
|
|
638
|
-
const prereqs =
|
|
938
|
+
const prereqs = reducedPrereqMap.get(id) || [];
|
|
639
939
|
return {
|
|
640
940
|
id,
|
|
641
941
|
term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
|
|
@@ -647,12 +947,22 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
647
947
|
salienceScore: Number(item.salienceScore) || 8,
|
|
648
948
|
definition: String(item.definition || ""),
|
|
649
949
|
prerequisites: prereqs,
|
|
650
|
-
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
|
|
950
|
+
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"],
|
|
951
|
+
facets: item.facets && typeof item.facets === "object" ? {
|
|
952
|
+
computationalPrinciple: item.facets.computationalPrinciple ? String(item.facets.computationalPrinciple) : void 0,
|
|
953
|
+
mentalModel: item.facets.mentalModel ? String(item.facets.mentalModel) : void 0,
|
|
954
|
+
syntaxAndTools: Array.isArray(item.facets.syntaxAndTools) ? item.facets.syntaxAndTools.map(String) : void 0
|
|
955
|
+
} : void 0,
|
|
956
|
+
commonMisconceptions: Array.isArray(item.commonMisconceptions) ? item.commonMisconceptions.map(String) : void 0,
|
|
957
|
+
troubleshootingHints: Array.isArray(item.troubleshootingHints) ? item.troubleshootingHints.map(String) : void 0
|
|
651
958
|
};
|
|
652
959
|
});
|
|
960
|
+
const topoResult = kahnTopologicalSort(concepts.map((c) => c.id), reducedEdges);
|
|
961
|
+
const conceptMap = new Map(concepts.map((c) => [c.id, c]));
|
|
962
|
+
const sortedConcepts = topoResult.order.map((id) => conceptMap.get(id)).filter(Boolean);
|
|
653
963
|
return {
|
|
654
|
-
concepts,
|
|
655
|
-
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
964
|
+
concepts: sortedConcepts.length > 0 ? sortedConcepts : concepts,
|
|
965
|
+
conceptDagTsv: this.formatConceptDagTsv(sortedConcepts.length > 0 ? sortedConcepts : concepts),
|
|
656
966
|
isLlmRefined: true,
|
|
657
967
|
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
658
968
|
};
|
|
@@ -794,6 +1104,6 @@ var HealthAuditor = class {
|
|
|
794
1104
|
}
|
|
795
1105
|
};
|
|
796
1106
|
|
|
797
|
-
export { CatalogBuilder, HealthAuditor, MultiSourceSearchEngine, PedagogicalTermExtractor, classifyTechnicalDomain, inferPrimaryCodeLanguage };
|
|
798
|
-
//# sourceMappingURL=chunk-
|
|
799
|
-
//# sourceMappingURL=chunk-
|
|
1107
|
+
export { CatalogBuilder, ECOMMERCE_AND_COMMERCIAL_PATTERNS, ECOMMERCE_DOMAINS, HealthAuditor, MultiSourceSearchEngine, PedagogicalTermExtractor, classifyTechnicalDomain, evaluateTechnicalDensity, inferPrimaryCodeLanguage, isCommercialOrNoise, kahnTopologicalSort, transitiveReduction };
|
|
1108
|
+
//# sourceMappingURL=chunk-HZNSZDSM.js.map
|
|
1109
|
+
//# sourceMappingURL=chunk-HZNSZDSM.js.map
|