@thanh01.pmt/research-kit 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -14
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +2 -1
- package/dist/catalogBuilder-BaVRvg-9.d.cts +90 -0
- package/dist/catalogBuilder-DHV6y1gD.d.ts +90 -0
- package/dist/chunk-AKEI7H2C.js +223 -0
- package/dist/chunk-AKEI7H2C.js.map +1 -0
- package/dist/{chunk-XIF5YNI7.js → chunk-HZNSZDSM.js} +318 -8
- package/dist/chunk-HZNSZDSM.js.map +1 -0
- package/dist/chunk-LSMSIPTZ.js +3 -0
- package/dist/{chunk-2JEVBCIF.js.map → chunk-LSMSIPTZ.js.map} +1 -1
- package/dist/{chunk-KAFQVI2R.js → chunk-ON6VFCJO.js} +3 -3
- package/dist/{chunk-KAFQVI2R.js.map → chunk-ON6VFCJO.js.map} +1 -1
- package/dist/{chunk-GKF3NDEX.js → chunk-QFYM2NV4.js} +3 -3
- package/dist/{chunk-GKF3NDEX.js.map → chunk-QFYM2NV4.js.map} +1 -1
- package/dist/{chunk-AUGVVZTL.js → chunk-RW7VS7NI.js} +175 -7
- package/dist/chunk-RW7VS7NI.js.map +1 -0
- package/dist/{chunk-DOXSPRSP.js → chunk-UKGR45VZ.js} +3 -223
- package/dist/chunk-UKGR45VZ.js.map +1 -0
- package/dist/{chunk-J7XKDMGL.js → chunk-X3X26W2I.js} +2 -2
- package/dist/{chunk-J7XKDMGL.js.map → chunk-X3X26W2I.js.map} +1 -1
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -2
- package/dist/harvester/index.cjs +542 -6
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +63 -49
- package/dist/harvester/index.d.ts +63 -49
- package/dist/harvester/index.js +4 -3
- package/dist/{index-t8pahlJ4.d.ts → index-DlngvZBr.d.cts} +15 -1
- package/dist/{index-t8pahlJ4.d.cts → index-DlngvZBr.d.ts} +15 -1
- package/dist/index.cjs +515 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -13
- package/dist/index.d.ts +28 -13
- package/dist/index.js +41 -13
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-BTPFe7eb.d.cts → literatureReview-C7hsjrn4.d.cts} +1 -1
- package/dist/{literatureReview-hGbfrBmw.d.ts → literatureReview-DWl13r_3.d.ts} +1 -1
- package/dist/pipeline/index.cjs +658 -231
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -5
- package/dist/pipeline/index.d.ts +49 -5
- package/dist/pipeline/index.js +6 -5
- package/dist/{webScraper-BhiQ8s9x.d.ts → webScraper-DBFUIqoO.d.ts} +1 -1
- package/dist/{webScraper-D9Ldus9p.d.cts → webScraper-yO9P-oLb.d.cts} +1 -1
- package/package.json +1 -1
- package/dist/catalogBuilder-CmmQzUnf.d.ts +0 -38
- package/dist/catalogBuilder-DgXYxL2F.d.cts +0 -38
- package/dist/chunk-2JEVBCIF.js +0 -3
- package/dist/chunk-AUGVVZTL.js.map +0 -1
- package/dist/chunk-DOXSPRSP.js.map +0 -1
- package/dist/chunk-XIF5YNI7.js.map +0 -1
package/dist/pipeline/index.cjs
CHANGED
|
@@ -329,19 +329,250 @@ ${options.synthesizedInsights.executiveSummary}`);
|
|
|
329
329
|
}
|
|
330
330
|
};
|
|
331
331
|
|
|
332
|
+
// src/academic/arxivClient.ts
|
|
333
|
+
var ArXivSearchEngine = class {
|
|
334
|
+
/**
|
|
335
|
+
* Search ArXiv repository using the public atom XML API
|
|
336
|
+
*/
|
|
337
|
+
async search(query, maxResults = 5) {
|
|
338
|
+
const papers = [];
|
|
339
|
+
try {
|
|
340
|
+
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
341
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
342
|
+
if (res.ok) {
|
|
343
|
+
const xmlText = await res.text();
|
|
344
|
+
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
345
|
+
for (const entry of entryMatches) {
|
|
346
|
+
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
347
|
+
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
348
|
+
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
349
|
+
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
350
|
+
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
351
|
+
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
352
|
+
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
353
|
+
if (titleMatch) {
|
|
354
|
+
papers.push({
|
|
355
|
+
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
356
|
+
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
357
|
+
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
358
|
+
authors: authorMatches,
|
|
359
|
+
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
360
|
+
citationCount: 0,
|
|
361
|
+
venue: "arXiv preprint",
|
|
362
|
+
url: arxivUrl,
|
|
363
|
+
pdfUrl,
|
|
364
|
+
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
} catch (e) {
|
|
370
|
+
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
371
|
+
}
|
|
372
|
+
return papers;
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
// src/academic/semanticScholar.ts
|
|
377
|
+
zod.z.object({
|
|
378
|
+
id: zod.z.string(),
|
|
379
|
+
title: zod.z.string(),
|
|
380
|
+
abstract: zod.z.string().optional(),
|
|
381
|
+
authors: zod.z.array(zod.z.string()).default([]),
|
|
382
|
+
year: zod.z.number().optional(),
|
|
383
|
+
citationCount: zod.z.number().default(0),
|
|
384
|
+
venue: zod.z.string().optional(),
|
|
385
|
+
doi: zod.z.string().optional(),
|
|
386
|
+
url: zod.z.string().optional(),
|
|
387
|
+
pdfUrl: zod.z.string().optional(),
|
|
388
|
+
tldr: zod.z.string().optional(),
|
|
389
|
+
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
390
|
+
});
|
|
391
|
+
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
392
|
+
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
393
|
+
const entries = Object.entries(invertedIndex);
|
|
394
|
+
if (entries.length === 0) return void 0;
|
|
395
|
+
let maxIndex = 0;
|
|
396
|
+
for (const [, positions] of entries) {
|
|
397
|
+
if (Array.isArray(positions)) {
|
|
398
|
+
for (const p of positions) {
|
|
399
|
+
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
404
|
+
const words = new Array(maxIndex + 1);
|
|
405
|
+
for (const [word, positions] of entries) {
|
|
406
|
+
if (Array.isArray(positions)) {
|
|
407
|
+
for (const p of positions) {
|
|
408
|
+
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
409
|
+
words[p] = word;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
415
|
+
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
416
|
+
}
|
|
417
|
+
var AcademicSearchEngine = class {
|
|
418
|
+
apiKey;
|
|
419
|
+
allowEnvFallback;
|
|
420
|
+
customAdapter;
|
|
421
|
+
arxivEngine;
|
|
422
|
+
constructor(options = {}) {
|
|
423
|
+
this.arxivEngine = new ArXivSearchEngine();
|
|
424
|
+
if (typeof options === "string") {
|
|
425
|
+
this.apiKey = options;
|
|
426
|
+
this.allowEnvFallback = true;
|
|
427
|
+
} else {
|
|
428
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
429
|
+
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
430
|
+
this.customAdapter = options.customAdapter;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
setCustomAdapter(adapter) {
|
|
434
|
+
this.customAdapter = adapter;
|
|
435
|
+
}
|
|
436
|
+
setApiKey(apiKey) {
|
|
437
|
+
this.apiKey = apiKey;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
441
|
+
*/
|
|
442
|
+
async searchPapers(query, options = {}) {
|
|
443
|
+
const adapter = options.customAdapter || this.customAdapter;
|
|
444
|
+
if (adapter) {
|
|
445
|
+
try {
|
|
446
|
+
const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
|
|
447
|
+
if (Array.isArray(customPapers) && customPapers.length > 0) {
|
|
448
|
+
return customPapers;
|
|
449
|
+
}
|
|
450
|
+
} catch (e) {
|
|
451
|
+
console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
const limit = options.limit ?? 5;
|
|
455
|
+
const papers = [];
|
|
456
|
+
const seenTitles = /* @__PURE__ */ new Set();
|
|
457
|
+
const pushUnique = (p) => {
|
|
458
|
+
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
459
|
+
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
460
|
+
seenTitles.add(normTitle);
|
|
461
|
+
papers.push(p);
|
|
462
|
+
};
|
|
463
|
+
try {
|
|
464
|
+
const apiKey = options.apiKey || this.apiKey;
|
|
465
|
+
const headers = { "Content-Type": "application/json" };
|
|
466
|
+
if (apiKey) {
|
|
467
|
+
headers["x-api-key"] = apiKey;
|
|
468
|
+
}
|
|
469
|
+
const params = new URLSearchParams({
|
|
470
|
+
query,
|
|
471
|
+
limit: limit.toString(),
|
|
472
|
+
fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
|
|
473
|
+
});
|
|
474
|
+
if (options.yearRange) {
|
|
475
|
+
params.append("year", options.yearRange);
|
|
476
|
+
}
|
|
477
|
+
const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
|
|
478
|
+
headers,
|
|
479
|
+
signal: AbortSignal.timeout(6e3)
|
|
480
|
+
});
|
|
481
|
+
if (res.ok) {
|
|
482
|
+
const data = await res.json();
|
|
483
|
+
if (Array.isArray(data?.data)) {
|
|
484
|
+
for (const item of data.data) {
|
|
485
|
+
pushUnique({
|
|
486
|
+
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
487
|
+
title: item.title || "Untitled",
|
|
488
|
+
abstract: item.abstract || void 0,
|
|
489
|
+
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
490
|
+
year: item.year,
|
|
491
|
+
citationCount: item.citationCount || 0,
|
|
492
|
+
venue: item.venue || void 0,
|
|
493
|
+
doi: item.externalIds?.DOI,
|
|
494
|
+
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
495
|
+
pdfUrl: item.openAccessPdf?.url,
|
|
496
|
+
tldr: item.tldr?.text,
|
|
497
|
+
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
if (papers.length >= limit) return papers.slice(0, limit);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
} catch (e) {
|
|
504
|
+
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
505
|
+
}
|
|
506
|
+
if (papers.length < limit) {
|
|
507
|
+
try {
|
|
508
|
+
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
509
|
+
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
510
|
+
signal: AbortSignal.timeout(6e3)
|
|
511
|
+
});
|
|
512
|
+
if (res.ok) {
|
|
513
|
+
const data = await res.json();
|
|
514
|
+
if (Array.isArray(data?.results)) {
|
|
515
|
+
for (const item of data.results) {
|
|
516
|
+
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
517
|
+
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
518
|
+
pushUnique({
|
|
519
|
+
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
520
|
+
title: item.title || "Untitled",
|
|
521
|
+
abstract: abstractText,
|
|
522
|
+
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
523
|
+
year: item.publication_year,
|
|
524
|
+
citationCount: item.cited_by_count || 0,
|
|
525
|
+
venue: item.primary_location?.source?.display_name,
|
|
526
|
+
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
527
|
+
url: item.doi || item.id,
|
|
528
|
+
pdfUrl: item.open_access?.oa_url,
|
|
529
|
+
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
} catch (e) {
|
|
535
|
+
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
539
|
+
try {
|
|
540
|
+
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
541
|
+
for (const p of arxivPapers) {
|
|
542
|
+
pushUnique(p);
|
|
543
|
+
}
|
|
544
|
+
} catch (e) {
|
|
545
|
+
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return papers.slice(0, limit);
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
|
|
332
552
|
// src/harvester/multiSourceSearch.ts
|
|
333
553
|
var MultiSourceSearchEngine = class {
|
|
334
554
|
tavilyKey;
|
|
335
555
|
exaKey;
|
|
336
556
|
serperKey;
|
|
557
|
+
searxngHost;
|
|
558
|
+
searxngHeaders;
|
|
337
559
|
allowEnvFallback;
|
|
338
560
|
customAdapter;
|
|
561
|
+
academicEngine;
|
|
339
562
|
constructor(options = {}) {
|
|
340
563
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
341
564
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
342
565
|
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
343
566
|
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
567
|
+
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
568
|
+
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
569
|
+
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
570
|
+
this.searxngHeaders = {
|
|
571
|
+
...options.searxngHeaders || {},
|
|
572
|
+
...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
|
|
573
|
+
};
|
|
344
574
|
this.customAdapter = options.customAdapter;
|
|
575
|
+
this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
|
|
345
576
|
}
|
|
346
577
|
setCustomAdapter(adapter) {
|
|
347
578
|
this.customAdapter = adapter;
|
|
@@ -351,6 +582,12 @@ var MultiSourceSearchEngine = class {
|
|
|
351
582
|
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
352
583
|
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
353
584
|
}
|
|
585
|
+
setSearxng(host, headers) {
|
|
586
|
+
this.searxngHost = host;
|
|
587
|
+
if (headers) {
|
|
588
|
+
this.searxngHeaders = { ...this.searxngHeaders, ...headers };
|
|
589
|
+
}
|
|
590
|
+
}
|
|
354
591
|
async search(query, options = {}) {
|
|
355
592
|
const limit = options.limit ?? 5;
|
|
356
593
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -371,9 +608,30 @@ var MultiSourceSearchEngine = class {
|
|
|
371
608
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
372
609
|
}
|
|
373
610
|
}
|
|
611
|
+
if (preferred === "academic") {
|
|
612
|
+
try {
|
|
613
|
+
const papers = await this.academicEngine.searchPapers(query, { limit });
|
|
614
|
+
if (papers.length > 0) {
|
|
615
|
+
return papers.map((p) => ({
|
|
616
|
+
title: p.title,
|
|
617
|
+
url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
|
|
618
|
+
content: p.tldr || p.abstract || "",
|
|
619
|
+
score: p.citationCount,
|
|
620
|
+
engine: "academic"
|
|
621
|
+
}));
|
|
622
|
+
}
|
|
623
|
+
} catch (e) {
|
|
624
|
+
console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
374
627
|
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
375
628
|
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
376
629
|
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
630
|
+
const searxngHost = options.searxngHost || this.searxngHost;
|
|
631
|
+
if (preferred === "searxng" && searxngHost) {
|
|
632
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
633
|
+
if (searxHits.length > 0) return searxHits;
|
|
634
|
+
}
|
|
377
635
|
if (preferred === "exa" && exaKey) {
|
|
378
636
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
379
637
|
if (exaHits.length > 0) return exaHits;
|
|
@@ -428,10 +686,44 @@ var MultiSourceSearchEngine = class {
|
|
|
428
686
|
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
429
687
|
if (serperHits.length > 0) return serperHits;
|
|
430
688
|
}
|
|
689
|
+
if (searxngHost) {
|
|
690
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
691
|
+
if (searxHits.length > 0) return searxHits;
|
|
692
|
+
}
|
|
431
693
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
432
694
|
if (ddgHits.length > 0) return ddgHits;
|
|
433
695
|
return [];
|
|
434
696
|
}
|
|
697
|
+
async querySearxng(query, searxngHost, limit, customHeaders) {
|
|
698
|
+
try {
|
|
699
|
+
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
700
|
+
const searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
701
|
+
const headers = {
|
|
702
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
703
|
+
...this.searxngHeaders || {},
|
|
704
|
+
...customHeaders || {}
|
|
705
|
+
};
|
|
706
|
+
const res = await fetch(searchUrl, {
|
|
707
|
+
headers,
|
|
708
|
+
signal: AbortSignal.timeout(1e4)
|
|
709
|
+
});
|
|
710
|
+
if (res.ok) {
|
|
711
|
+
const data = await res.json();
|
|
712
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
713
|
+
return data.results.slice(0, limit).map((r) => ({
|
|
714
|
+
title: r.title || "Untitled",
|
|
715
|
+
url: r.url,
|
|
716
|
+
content: r.content || r.snippet || "",
|
|
717
|
+
score: r.score,
|
|
718
|
+
engine: "searxng"
|
|
719
|
+
}));
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
} catch (e) {
|
|
723
|
+
console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
|
|
724
|
+
}
|
|
725
|
+
return [];
|
|
726
|
+
}
|
|
435
727
|
async queryExa(query, exaKey, limit, domainFilter) {
|
|
436
728
|
try {
|
|
437
729
|
const body = {
|
|
@@ -775,7 +1067,7 @@ var WebScraper = class {
|
|
|
775
1067
|
}
|
|
776
1068
|
const res = await fetch(jinaUrl, {
|
|
777
1069
|
headers,
|
|
778
|
-
signal: AbortSignal.timeout(
|
|
1070
|
+
signal: AbortSignal.timeout(8e3)
|
|
779
1071
|
});
|
|
780
1072
|
if (res.ok) {
|
|
781
1073
|
const markdown = await res.text();
|
|
@@ -971,11 +1263,170 @@ var HealthAuditor = class {
|
|
|
971
1263
|
}
|
|
972
1264
|
};
|
|
973
1265
|
|
|
974
|
-
// src/harvester/
|
|
975
|
-
var
|
|
976
|
-
"
|
|
977
|
-
"
|
|
978
|
-
"
|
|
1266
|
+
// src/harvester/noiseFilter.ts
|
|
1267
|
+
var ECOMMERCE_DOMAINS = /* @__PURE__ */ new Set([
|
|
1268
|
+
"shopee.vn",
|
|
1269
|
+
"shopee.com",
|
|
1270
|
+
"lazada.vn",
|
|
1271
|
+
"lazada.com",
|
|
1272
|
+
"tiki.vn",
|
|
1273
|
+
"sendo.vn",
|
|
1274
|
+
"aliexpress.com",
|
|
1275
|
+
"taobao.com",
|
|
1276
|
+
"amazon.com",
|
|
1277
|
+
"ebay.com",
|
|
1278
|
+
"banggood.com"
|
|
1279
|
+
]);
|
|
1280
|
+
var ECOMMERCE_AND_COMMERCIAL_PATTERNS = [
|
|
1281
|
+
/\b(shopee|lazada|tiki|sendo|taobao|aliexpress)\b/i,
|
|
1282
|
+
/\b(mua ngay|giá tốt|khuyến mãi|freeship|bán chạy|nhập khẩu chính hãng|cửa hàng|bảng giá|đặt hàng)\b/i,
|
|
1283
|
+
/\b(buy now|add to cart|free shipping|discount|best price|on sale|shop now|in stock|order now)\b/i,
|
|
1284
|
+
/\b(\d+[kKđ]|₫|\$\d+|\d+\s*(?:VNĐ|VND|usd))\b/i
|
|
1285
|
+
];
|
|
1286
|
+
function isCommercialOrNoise(input) {
|
|
1287
|
+
if (input.url) {
|
|
1288
|
+
try {
|
|
1289
|
+
const hostname = new URL(input.url).hostname.toLowerCase().replace(/^www\./, "");
|
|
1290
|
+
if (ECOMMERCE_DOMAINS.has(hostname) || [...ECOMMERCE_DOMAINS].some((d) => hostname.endsWith(`.${d}`))) {
|
|
1291
|
+
return true;
|
|
1292
|
+
}
|
|
1293
|
+
} catch {
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
const combined = `${input.title || ""} ${input.snippet || ""}`;
|
|
1297
|
+
if (!combined.trim()) return false;
|
|
1298
|
+
for (const pattern of ECOMMERCE_AND_COMMERCIAL_PATTERNS) {
|
|
1299
|
+
if (pattern.test(combined)) {
|
|
1300
|
+
return true;
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
return false;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
// src/harvester/graphUtils.ts
|
|
1307
|
+
function transitiveReduction(edges) {
|
|
1308
|
+
const uniqueEdges = [];
|
|
1309
|
+
const edgeSet = /* @__PURE__ */ new Set();
|
|
1310
|
+
for (const e of edges) {
|
|
1311
|
+
if (!e.source || !e.target || e.source === e.target) continue;
|
|
1312
|
+
const key = `${e.source}-->${e.target}`;
|
|
1313
|
+
if (!edgeSet.has(key)) {
|
|
1314
|
+
edgeSet.add(key);
|
|
1315
|
+
uniqueEdges.push(e);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
const adj = /* @__PURE__ */ new Map();
|
|
1319
|
+
for (const e of uniqueEdges) {
|
|
1320
|
+
if (!adj.has(e.source)) adj.set(e.source, /* @__PURE__ */ new Set());
|
|
1321
|
+
adj.get(e.source).add(e.target);
|
|
1322
|
+
}
|
|
1323
|
+
const reducedEdges = [];
|
|
1324
|
+
const removedTransitiveEdges = [];
|
|
1325
|
+
const hasAlternatePath = (start, target) => {
|
|
1326
|
+
const directNeighbors = adj.get(start);
|
|
1327
|
+
if (!directNeighbors) return false;
|
|
1328
|
+
const queue = [];
|
|
1329
|
+
const visited = /* @__PURE__ */ new Set([start]);
|
|
1330
|
+
for (const nxt of directNeighbors) {
|
|
1331
|
+
if (nxt !== target) {
|
|
1332
|
+
queue.push(nxt);
|
|
1333
|
+
visited.add(nxt);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
while (queue.length > 0) {
|
|
1337
|
+
const curr = queue.shift();
|
|
1338
|
+
if (curr === target) return true;
|
|
1339
|
+
const nextSet = adj.get(curr);
|
|
1340
|
+
if (!nextSet) continue;
|
|
1341
|
+
for (const nxt of nextSet) {
|
|
1342
|
+
if (nxt === target) return true;
|
|
1343
|
+
if (!visited.has(nxt)) {
|
|
1344
|
+
visited.add(nxt);
|
|
1345
|
+
queue.push(nxt);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
return false;
|
|
1350
|
+
};
|
|
1351
|
+
for (const e of uniqueEdges) {
|
|
1352
|
+
if (hasAlternatePath(e.source, e.target) && !hasAlternatePath(e.target, e.source)) {
|
|
1353
|
+
removedTransitiveEdges.push(e);
|
|
1354
|
+
} else {
|
|
1355
|
+
reducedEdges.push(e);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
return { reducedEdges, removedTransitiveEdges };
|
|
1359
|
+
}
|
|
1360
|
+
function kahnTopologicalSort(nodes, edges) {
|
|
1361
|
+
const nodeSet = new Set(nodes);
|
|
1362
|
+
const validEdges = edges.filter((e) => nodeSet.has(e.source) && nodeSet.has(e.target) && e.source !== e.target);
|
|
1363
|
+
const inDegree = /* @__PURE__ */ new Map();
|
|
1364
|
+
const adj = /* @__PURE__ */ new Map();
|
|
1365
|
+
for (const n of nodes) {
|
|
1366
|
+
inDegree.set(n, 0);
|
|
1367
|
+
adj.set(n, /* @__PURE__ */ new Set());
|
|
1368
|
+
}
|
|
1369
|
+
for (const e of validEdges) {
|
|
1370
|
+
adj.get(e.source).add(e.target);
|
|
1371
|
+
inDegree.set(e.target, (inDegree.get(e.target) ?? 0) + 1);
|
|
1372
|
+
}
|
|
1373
|
+
const queue = [];
|
|
1374
|
+
for (const [node, deg] of inDegree.entries()) {
|
|
1375
|
+
if (deg === 0) queue.push(node);
|
|
1376
|
+
}
|
|
1377
|
+
const order = [];
|
|
1378
|
+
const cyclesBroken = [];
|
|
1379
|
+
const droppedEdges = [];
|
|
1380
|
+
while (order.length < nodes.length) {
|
|
1381
|
+
if (queue.length > 0) {
|
|
1382
|
+
queue.sort((a, b) => a.localeCompare(b));
|
|
1383
|
+
const curr = queue.shift();
|
|
1384
|
+
order.push(curr);
|
|
1385
|
+
const neighbors = adj.get(curr);
|
|
1386
|
+
if (neighbors) {
|
|
1387
|
+
for (const nxt of neighbors) {
|
|
1388
|
+
const nextDeg = (inDegree.get(nxt) ?? 1) - 1;
|
|
1389
|
+
inDegree.set(nxt, nextDeg);
|
|
1390
|
+
if (nextDeg === 0) queue.push(nxt);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
} else {
|
|
1394
|
+
const remaining = nodes.filter((n) => !order.includes(n));
|
|
1395
|
+
if (remaining.length === 0) break;
|
|
1396
|
+
remaining.sort((a, b) => (inDegree.get(a) ?? 0) - (inDegree.get(b) ?? 0) || a.localeCompare(b));
|
|
1397
|
+
const candidate = remaining[0];
|
|
1398
|
+
let dropped = false;
|
|
1399
|
+
for (const [src, targets] of adj.entries()) {
|
|
1400
|
+
if (remaining.includes(src) && targets.has(candidate)) {
|
|
1401
|
+
targets.delete(candidate);
|
|
1402
|
+
droppedEdges.push({ source: src, target: candidate });
|
|
1403
|
+
cyclesBroken.push(`${src} -> ${candidate}`);
|
|
1404
|
+
inDegree.set(candidate, Math.max(0, (inDegree.get(candidate) ?? 1) - 1));
|
|
1405
|
+
if (inDegree.get(candidate) === 0) {
|
|
1406
|
+
queue.push(candidate);
|
|
1407
|
+
}
|
|
1408
|
+
dropped = true;
|
|
1409
|
+
break;
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
if (!dropped) {
|
|
1413
|
+
order.push(candidate);
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
return {
|
|
1418
|
+
isDag: cyclesBroken.length === 0,
|
|
1419
|
+
order,
|
|
1420
|
+
cyclesBroken,
|
|
1421
|
+
droppedEdges
|
|
1422
|
+
};
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
// src/harvester/termExtractor.ts
|
|
1426
|
+
var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
1427
|
+
"l\xE0",
|
|
1428
|
+
"c\u1EE7a",
|
|
1429
|
+
"trong",
|
|
979
1430
|
"\u0111\u01B0\u1EE3c",
|
|
980
1431
|
"nh\u1EEFng",
|
|
981
1432
|
"c\xE1c",
|
|
@@ -1184,6 +1635,7 @@ var PedagogicalTermExtractor = class {
|
|
|
1184
1635
|
}
|
|
1185
1636
|
};
|
|
1186
1637
|
for (const { resourceId, doc } of docs) {
|
|
1638
|
+
if (isCommercialOrNoise({ url: doc.url, title: doc.title })) continue;
|
|
1187
1639
|
const md = doc.markdownContent || "";
|
|
1188
1640
|
const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
|
|
1189
1641
|
const bilingualMatches = mdWithoutCode.matchAll(
|
|
@@ -1287,7 +1739,7 @@ ${corpusPreview}`,
|
|
|
1287
1739
|
`
|
|
1288
1740
|
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
1289
1741
|
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
1290
|
-
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
|
|
1742
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."]}`
|
|
1291
1743
|
].filter(Boolean).join("\n");
|
|
1292
1744
|
const rawResponse = await options.llm(prompt, {
|
|
1293
1745
|
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
@@ -1299,9 +1751,26 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
1299
1751
|
const parsed = JSON.parse(jsonMatch[0]);
|
|
1300
1752
|
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
1301
1753
|
const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
|
|
1754
|
+
const rawEdges = [];
|
|
1755
|
+
for (const item of parsed) {
|
|
1756
|
+
const targetId = String(item.id || "");
|
|
1757
|
+
if (!validIds.has(targetId)) continue;
|
|
1758
|
+
for (const p of item.prerequisites || []) {
|
|
1759
|
+
const srcId = String(p);
|
|
1760
|
+
if (validIds.has(srcId) && srcId !== targetId) {
|
|
1761
|
+
rawEdges.push({ source: srcId, target: targetId });
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
const { reducedEdges } = transitiveReduction(rawEdges);
|
|
1766
|
+
const reducedPrereqMap = /* @__PURE__ */ new Map();
|
|
1767
|
+
for (const e of reducedEdges) {
|
|
1768
|
+
if (!reducedPrereqMap.has(e.target)) reducedPrereqMap.set(e.target, []);
|
|
1769
|
+
reducedPrereqMap.get(e.target).push(e.source);
|
|
1770
|
+
}
|
|
1302
1771
|
const concepts = parsed.map((item, idx) => {
|
|
1303
1772
|
const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
|
|
1304
|
-
const prereqs =
|
|
1773
|
+
const prereqs = reducedPrereqMap.get(id) || [];
|
|
1305
1774
|
return {
|
|
1306
1775
|
id,
|
|
1307
1776
|
term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
|
|
@@ -1313,12 +1782,22 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
1313
1782
|
salienceScore: Number(item.salienceScore) || 8,
|
|
1314
1783
|
definition: String(item.definition || ""),
|
|
1315
1784
|
prerequisites: prereqs,
|
|
1316
|
-
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
|
|
1785
|
+
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"],
|
|
1786
|
+
facets: item.facets && typeof item.facets === "object" ? {
|
|
1787
|
+
computationalPrinciple: item.facets.computationalPrinciple ? String(item.facets.computationalPrinciple) : void 0,
|
|
1788
|
+
mentalModel: item.facets.mentalModel ? String(item.facets.mentalModel) : void 0,
|
|
1789
|
+
syntaxAndTools: Array.isArray(item.facets.syntaxAndTools) ? item.facets.syntaxAndTools.map(String) : void 0
|
|
1790
|
+
} : void 0,
|
|
1791
|
+
commonMisconceptions: Array.isArray(item.commonMisconceptions) ? item.commonMisconceptions.map(String) : void 0,
|
|
1792
|
+
troubleshootingHints: Array.isArray(item.troubleshootingHints) ? item.troubleshootingHints.map(String) : void 0
|
|
1317
1793
|
};
|
|
1318
1794
|
});
|
|
1795
|
+
const topoResult = kahnTopologicalSort(concepts.map((c) => c.id), reducedEdges);
|
|
1796
|
+
const conceptMap = new Map(concepts.map((c) => [c.id, c]));
|
|
1797
|
+
const sortedConcepts = topoResult.order.map((id) => conceptMap.get(id)).filter(Boolean);
|
|
1319
1798
|
return {
|
|
1320
|
-
concepts,
|
|
1321
|
-
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
1799
|
+
concepts: sortedConcepts.length > 0 ? sortedConcepts : concepts,
|
|
1800
|
+
conceptDagTsv: this.formatConceptDagTsv(sortedConcepts.length > 0 ? sortedConcepts : concepts),
|
|
1322
1801
|
isLlmRefined: true,
|
|
1323
1802
|
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
1324
1803
|
};
|
|
@@ -1506,226 +1985,6 @@ var ProvenanceTracker = class {
|
|
|
1506
1985
|
}
|
|
1507
1986
|
};
|
|
1508
1987
|
|
|
1509
|
-
// src/academic/arxivClient.ts
|
|
1510
|
-
var ArXivSearchEngine = class {
|
|
1511
|
-
/**
|
|
1512
|
-
* Search ArXiv repository using the public atom XML API
|
|
1513
|
-
*/
|
|
1514
|
-
async search(query, maxResults = 5) {
|
|
1515
|
-
const papers = [];
|
|
1516
|
-
try {
|
|
1517
|
-
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
1518
|
-
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
1519
|
-
if (res.ok) {
|
|
1520
|
-
const xmlText = await res.text();
|
|
1521
|
-
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
1522
|
-
for (const entry of entryMatches) {
|
|
1523
|
-
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
1524
|
-
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
1525
|
-
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
1526
|
-
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
1527
|
-
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
1528
|
-
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
1529
|
-
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
1530
|
-
if (titleMatch) {
|
|
1531
|
-
papers.push({
|
|
1532
|
-
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
1533
|
-
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
1534
|
-
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
1535
|
-
authors: authorMatches,
|
|
1536
|
-
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
1537
|
-
citationCount: 0,
|
|
1538
|
-
venue: "arXiv preprint",
|
|
1539
|
-
url: arxivUrl,
|
|
1540
|
-
pdfUrl,
|
|
1541
|
-
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
1542
|
-
});
|
|
1543
|
-
}
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1546
|
-
} catch (e) {
|
|
1547
|
-
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
1548
|
-
}
|
|
1549
|
-
return papers;
|
|
1550
|
-
}
|
|
1551
|
-
};
|
|
1552
|
-
|
|
1553
|
-
// src/academic/semanticScholar.ts
|
|
1554
|
-
zod.z.object({
|
|
1555
|
-
id: zod.z.string(),
|
|
1556
|
-
title: zod.z.string(),
|
|
1557
|
-
abstract: zod.z.string().optional(),
|
|
1558
|
-
authors: zod.z.array(zod.z.string()).default([]),
|
|
1559
|
-
year: zod.z.number().optional(),
|
|
1560
|
-
citationCount: zod.z.number().default(0),
|
|
1561
|
-
venue: zod.z.string().optional(),
|
|
1562
|
-
doi: zod.z.string().optional(),
|
|
1563
|
-
url: zod.z.string().optional(),
|
|
1564
|
-
pdfUrl: zod.z.string().optional(),
|
|
1565
|
-
tldr: zod.z.string().optional(),
|
|
1566
|
-
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
1567
|
-
});
|
|
1568
|
-
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
1569
|
-
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
1570
|
-
const entries = Object.entries(invertedIndex);
|
|
1571
|
-
if (entries.length === 0) return void 0;
|
|
1572
|
-
let maxIndex = 0;
|
|
1573
|
-
for (const [, positions] of entries) {
|
|
1574
|
-
if (Array.isArray(positions)) {
|
|
1575
|
-
for (const p of positions) {
|
|
1576
|
-
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
1577
|
-
}
|
|
1578
|
-
}
|
|
1579
|
-
}
|
|
1580
|
-
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
1581
|
-
const words = new Array(maxIndex + 1);
|
|
1582
|
-
for (const [word, positions] of entries) {
|
|
1583
|
-
if (Array.isArray(positions)) {
|
|
1584
|
-
for (const p of positions) {
|
|
1585
|
-
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
1586
|
-
words[p] = word;
|
|
1587
|
-
}
|
|
1588
|
-
}
|
|
1589
|
-
}
|
|
1590
|
-
}
|
|
1591
|
-
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
1592
|
-
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
1593
|
-
}
|
|
1594
|
-
var AcademicSearchEngine = class {
|
|
1595
|
-
apiKey;
|
|
1596
|
-
allowEnvFallback;
|
|
1597
|
-
customAdapter;
|
|
1598
|
-
arxivEngine;
|
|
1599
|
-
constructor(options = {}) {
|
|
1600
|
-
this.arxivEngine = new ArXivSearchEngine();
|
|
1601
|
-
if (typeof options === "string") {
|
|
1602
|
-
this.apiKey = options;
|
|
1603
|
-
this.allowEnvFallback = true;
|
|
1604
|
-
} else {
|
|
1605
|
-
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
1606
|
-
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
1607
|
-
this.customAdapter = options.customAdapter;
|
|
1608
|
-
}
|
|
1609
|
-
}
|
|
1610
|
-
setCustomAdapter(adapter) {
|
|
1611
|
-
this.customAdapter = adapter;
|
|
1612
|
-
}
|
|
1613
|
-
setApiKey(apiKey) {
|
|
1614
|
-
this.apiKey = apiKey;
|
|
1615
|
-
}
|
|
1616
|
-
/**
|
|
1617
|
-
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
1618
|
-
*/
|
|
1619
|
-
async searchPapers(query, options = {}) {
|
|
1620
|
-
const adapter = options.customAdapter || this.customAdapter;
|
|
1621
|
-
if (adapter) {
|
|
1622
|
-
try {
|
|
1623
|
-
const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
|
|
1624
|
-
if (Array.isArray(customPapers) && customPapers.length > 0) {
|
|
1625
|
-
return customPapers;
|
|
1626
|
-
}
|
|
1627
|
-
} catch (e) {
|
|
1628
|
-
console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
const limit = options.limit ?? 5;
|
|
1632
|
-
const papers = [];
|
|
1633
|
-
const seenTitles = /* @__PURE__ */ new Set();
|
|
1634
|
-
const pushUnique = (p) => {
|
|
1635
|
-
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
1636
|
-
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
1637
|
-
seenTitles.add(normTitle);
|
|
1638
|
-
papers.push(p);
|
|
1639
|
-
};
|
|
1640
|
-
try {
|
|
1641
|
-
const apiKey = options.apiKey || this.apiKey;
|
|
1642
|
-
const headers = { "Content-Type": "application/json" };
|
|
1643
|
-
if (apiKey) {
|
|
1644
|
-
headers["x-api-key"] = apiKey;
|
|
1645
|
-
}
|
|
1646
|
-
const params = new URLSearchParams({
|
|
1647
|
-
query,
|
|
1648
|
-
limit: limit.toString(),
|
|
1649
|
-
fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
|
|
1650
|
-
});
|
|
1651
|
-
if (options.yearRange) {
|
|
1652
|
-
params.append("year", options.yearRange);
|
|
1653
|
-
}
|
|
1654
|
-
const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
|
|
1655
|
-
headers,
|
|
1656
|
-
signal: AbortSignal.timeout(6e3)
|
|
1657
|
-
});
|
|
1658
|
-
if (res.ok) {
|
|
1659
|
-
const data = await res.json();
|
|
1660
|
-
if (Array.isArray(data?.data)) {
|
|
1661
|
-
for (const item of data.data) {
|
|
1662
|
-
pushUnique({
|
|
1663
|
-
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
1664
|
-
title: item.title || "Untitled",
|
|
1665
|
-
abstract: item.abstract || void 0,
|
|
1666
|
-
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
1667
|
-
year: item.year,
|
|
1668
|
-
citationCount: item.citationCount || 0,
|
|
1669
|
-
venue: item.venue || void 0,
|
|
1670
|
-
doi: item.externalIds?.DOI,
|
|
1671
|
-
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
1672
|
-
pdfUrl: item.openAccessPdf?.url,
|
|
1673
|
-
tldr: item.tldr?.text,
|
|
1674
|
-
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
1675
|
-
});
|
|
1676
|
-
}
|
|
1677
|
-
if (papers.length >= limit) return papers.slice(0, limit);
|
|
1678
|
-
}
|
|
1679
|
-
}
|
|
1680
|
-
} catch (e) {
|
|
1681
|
-
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
1682
|
-
}
|
|
1683
|
-
if (papers.length < limit) {
|
|
1684
|
-
try {
|
|
1685
|
-
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
1686
|
-
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
1687
|
-
signal: AbortSignal.timeout(6e3)
|
|
1688
|
-
});
|
|
1689
|
-
if (res.ok) {
|
|
1690
|
-
const data = await res.json();
|
|
1691
|
-
if (Array.isArray(data?.results)) {
|
|
1692
|
-
for (const item of data.results) {
|
|
1693
|
-
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
1694
|
-
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
1695
|
-
pushUnique({
|
|
1696
|
-
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
1697
|
-
title: item.title || "Untitled",
|
|
1698
|
-
abstract: abstractText,
|
|
1699
|
-
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
1700
|
-
year: item.publication_year,
|
|
1701
|
-
citationCount: item.cited_by_count || 0,
|
|
1702
|
-
venue: item.primary_location?.source?.display_name,
|
|
1703
|
-
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
1704
|
-
url: item.doi || item.id,
|
|
1705
|
-
pdfUrl: item.open_access?.oa_url,
|
|
1706
|
-
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
1707
|
-
});
|
|
1708
|
-
}
|
|
1709
|
-
}
|
|
1710
|
-
}
|
|
1711
|
-
} catch (e) {
|
|
1712
|
-
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
1713
|
-
}
|
|
1714
|
-
}
|
|
1715
|
-
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
1716
|
-
try {
|
|
1717
|
-
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
1718
|
-
for (const p of arxivPapers) {
|
|
1719
|
-
pushUnique(p);
|
|
1720
|
-
}
|
|
1721
|
-
} catch (e) {
|
|
1722
|
-
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
1723
|
-
}
|
|
1724
|
-
}
|
|
1725
|
-
return papers.slice(0, limit);
|
|
1726
|
-
}
|
|
1727
|
-
};
|
|
1728
|
-
|
|
1729
1988
|
// src/academic/cs2023Taxonomy.ts
|
|
1730
1989
|
var CSTA_K12_STRANDS = {
|
|
1731
1990
|
CS: {
|
|
@@ -2320,9 +2579,177 @@ Return ONLY a valid JSON object with this structure:`,
|
|
|
2320
2579
|
}
|
|
2321
2580
|
};
|
|
2322
2581
|
|
|
2582
|
+
// src/pipeline/groundingEngine.ts
|
|
2583
|
+
var TechnicalGroundingEngine = class {
|
|
2584
|
+
constructor(searchEngine = new MultiSourceSearchEngine(), scraper = new WebScraper()) {
|
|
2585
|
+
this.searchEngine = searchEngine;
|
|
2586
|
+
this.scraper = scraper;
|
|
2587
|
+
}
|
|
2588
|
+
async buildFactsheet(topic, domain, context, llm) {
|
|
2589
|
+
const hardware = context?.hardwarePlatform || "";
|
|
2590
|
+
const software = context?.softwareEnv || "";
|
|
2591
|
+
const ageBand = context?.ageBand || [12, 18];
|
|
2592
|
+
const queries = [
|
|
2593
|
+
`${hardware} ${software} ${topic} compatibility setup official guide`.trim(),
|
|
2594
|
+
`${hardware || software || topic} beginner student project tutorial`.trim(),
|
|
2595
|
+
`${hardware || software || topic} common errors debugging mistakes pitfalls`.trim()
|
|
2596
|
+
];
|
|
2597
|
+
const rawHits = [];
|
|
2598
|
+
for (const q of queries) {
|
|
2599
|
+
try {
|
|
2600
|
+
const hits = await this.searchEngine.search(q, { limit: 3 });
|
|
2601
|
+
for (const h of hits) {
|
|
2602
|
+
if (!isCommercialOrNoise({ url: h.url, title: h.title, snippet: h.content })) {
|
|
2603
|
+
if (!rawHits.some((existing) => existing.url === h.url)) {
|
|
2604
|
+
rawHits.push(h);
|
|
2605
|
+
}
|
|
2606
|
+
}
|
|
2607
|
+
}
|
|
2608
|
+
} catch {
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
const hitsToScrape = rawHits.slice(0, 3);
|
|
2612
|
+
const scrapeResults = await Promise.allSettled(
|
|
2613
|
+
hitsToScrape.map(async (h) => {
|
|
2614
|
+
try {
|
|
2615
|
+
const doc = await this.scraper.scrapeUrl(h.url);
|
|
2616
|
+
if (doc && doc.markdownContent) {
|
|
2617
|
+
return {
|
|
2618
|
+
title: h.title,
|
|
2619
|
+
url: h.url,
|
|
2620
|
+
excerpt: doc.markdownContent.slice(0, 1500)
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
} catch {
|
|
2624
|
+
}
|
|
2625
|
+
return { title: h.title, url: h.url, excerpt: h.content.slice(0, 500) };
|
|
2626
|
+
})
|
|
2627
|
+
);
|
|
2628
|
+
const scrapedSnippets = scrapeResults.map(
|
|
2629
|
+
(res, idx) => res.status === "fulfilled" ? res.value : { title: hitsToScrape[idx].title, url: hitsToScrape[idx].url, excerpt: hitsToScrape[idx].content.slice(0, 500) }
|
|
2630
|
+
);
|
|
2631
|
+
const defaultBottlenecks = [
|
|
2632
|
+
"Power supply insufficiency when driving motors or actuators simultaneously with microcontrollers.",
|
|
2633
|
+
"Baud rate mismatch or driver incompatibility across host OS versions.",
|
|
2634
|
+
"Syntax confusion between blocking loops (delay/sleep) and non-blocking event handling."
|
|
2635
|
+
];
|
|
2636
|
+
const defaultExemplars = [
|
|
2637
|
+
{
|
|
2638
|
+
title: `${topic} Hands-on Foundation Project`,
|
|
2639
|
+
description: `Verified core implementation integrating ${hardware || "hardware"} and ${software || "software logic"}.`,
|
|
2640
|
+
targetAudience: `Students aged ${ageBand[0]}-${ageBand[1]}`,
|
|
2641
|
+
sourceUrl: scrapedSnippets[0]?.url,
|
|
2642
|
+
hardwareRequirements: hardware ? [hardware] : void 0,
|
|
2643
|
+
softwareRequirements: software ? [software] : void 0
|
|
2644
|
+
}
|
|
2645
|
+
];
|
|
2646
|
+
let factsheet = {
|
|
2647
|
+
topic,
|
|
2648
|
+
domain,
|
|
2649
|
+
compatibilityMatrix: {
|
|
2650
|
+
hardwarePlatform: hardware || void 0,
|
|
2651
|
+
softwareEnv: software || void 0,
|
|
2652
|
+
verifiedCompatible: true,
|
|
2653
|
+
notes: [`Targeting ${hardware || "General Platform"} with ${software || "Standard SDK"}.`]
|
|
2654
|
+
},
|
|
2655
|
+
ageFeasibility: {
|
|
2656
|
+
recommendedMinAge: ageBand[0],
|
|
2657
|
+
recommendedMaxAge: ageBand[1],
|
|
2658
|
+
cognitiveReadinessVerdict: "feasible",
|
|
2659
|
+
potentialBarriers: ["Understanding abstract signal flows and state-based logic."]
|
|
2660
|
+
},
|
|
2661
|
+
exemplarProjects: defaultExemplars,
|
|
2662
|
+
technicalBottlenecks: defaultBottlenecks,
|
|
2663
|
+
groundingSources: scrapedSnippets.map((s) => ({ title: s.title, url: s.url })),
|
|
2664
|
+
markdownReport: ""
|
|
2665
|
+
};
|
|
2666
|
+
if (llm && scrapedSnippets.length > 0) {
|
|
2667
|
+
try {
|
|
2668
|
+
const prompt = [
|
|
2669
|
+
`You are a Lead STEM Curriculum Engineer. Analyze these research excerpts for topic: "${topic}".`,
|
|
2670
|
+
`Hardware: ${hardware || "N/A"}, Software/Env: ${software || "N/A"}, Student Age Band: ${ageBand[0]}-${ageBand[1]} years old.`,
|
|
2671
|
+
`
|
|
2672
|
+
Harvested Sources:
|
|
2673
|
+
${scrapedSnippets.map((s, i) => `[Source ${i + 1}: ${s.title} (${s.url})]
|
|
2674
|
+
${s.excerpt}`).join("\n\n")}`,
|
|
2675
|
+
`
|
|
2676
|
+
Synthesize a strictly valid JSON object with:`,
|
|
2677
|
+
`{`,
|
|
2678
|
+
` "compatibilityVerdict": boolean,`,
|
|
2679
|
+
` "compatibilityNotes": string[],`,
|
|
2680
|
+
` "cognitiveReadiness": "feasible" | "challenging" | "excessive",`,
|
|
2681
|
+
` "potentialBarriers": string[],`,
|
|
2682
|
+
` "exemplars": [{"title": string, "description": string, "targetAudience": string, "hardwareRequirements": string[], "softwareRequirements": string[]}],`,
|
|
2683
|
+
` "technicalBottlenecks": string[]`,
|
|
2684
|
+
`}`
|
|
2685
|
+
].join("\n");
|
|
2686
|
+
const raw = await llm(prompt, {
|
|
2687
|
+
system: "Output strictly valid JSON object without markdown commentary.",
|
|
2688
|
+
temperature: 0.2,
|
|
2689
|
+
jsonMode: true
|
|
2690
|
+
});
|
|
2691
|
+
const jsonMatch = raw.match(/\{[\s\S]*\}/);
|
|
2692
|
+
if (jsonMatch) {
|
|
2693
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
2694
|
+
factsheet.compatibilityMatrix.verifiedCompatible = Boolean(parsed.compatibilityVerdict ?? true);
|
|
2695
|
+
if (Array.isArray(parsed.compatibilityNotes)) factsheet.compatibilityMatrix.notes = parsed.compatibilityNotes.map(String);
|
|
2696
|
+
if (["feasible", "challenging", "excessive"].includes(parsed.cognitiveReadiness)) {
|
|
2697
|
+
factsheet.ageFeasibility.cognitiveReadinessVerdict = parsed.cognitiveReadiness;
|
|
2698
|
+
}
|
|
2699
|
+
if (Array.isArray(parsed.potentialBarriers)) factsheet.ageFeasibility.potentialBarriers = parsed.potentialBarriers.map(String);
|
|
2700
|
+
if (Array.isArray(parsed.exemplars) && parsed.exemplars.length > 0) {
|
|
2701
|
+
factsheet.exemplarProjects = parsed.exemplars.map((ex) => ({
|
|
2702
|
+
title: String(ex.title || "Verified Project"),
|
|
2703
|
+
description: String(ex.description || ""),
|
|
2704
|
+
targetAudience: String(ex.targetAudience || `Ages ${ageBand[0]}-${ageBand[1]}`),
|
|
2705
|
+
hardwareRequirements: Array.isArray(ex.hardwareRequirements) ? ex.hardwareRequirements.map(String) : void 0,
|
|
2706
|
+
softwareRequirements: Array.isArray(ex.softwareRequirements) ? ex.softwareRequirements.map(String) : void 0,
|
|
2707
|
+
sourceUrl: scrapedSnippets[0]?.url
|
|
2708
|
+
}));
|
|
2709
|
+
}
|
|
2710
|
+
if (Array.isArray(parsed.technicalBottlenecks)) factsheet.technicalBottlenecks = parsed.technicalBottlenecks.map(String);
|
|
2711
|
+
}
|
|
2712
|
+
} catch (err) {
|
|
2713
|
+
console.warn("[TechnicalGroundingEngine] LLM synthesis fallback to baseline:", err?.message || err);
|
|
2714
|
+
}
|
|
2715
|
+
}
|
|
2716
|
+
factsheet.markdownReport = this.renderMarkdownReport(factsheet);
|
|
2717
|
+
return factsheet;
|
|
2718
|
+
}
|
|
2719
|
+
renderMarkdownReport(fs) {
|
|
2720
|
+
const lines = [
|
|
2721
|
+
`# \u{1F6E0}\uFE0F Technical Grounding & Empirical Factsheet: ${fs.topic}`,
|
|
2722
|
+
`> **Domain:** \`${fs.domain.toUpperCase()}\` | **Feasibility:** \`${fs.ageFeasibility.cognitiveReadinessVerdict.toUpperCase()}\``,
|
|
2723
|
+
`
|
|
2724
|
+
## 1. Compatibility & Tooling Verification`,
|
|
2725
|
+
`- **Hardware:** ${fs.compatibilityMatrix.hardwarePlatform || "N/A"}`,
|
|
2726
|
+
`- **Software / SDK:** ${fs.compatibilityMatrix.softwareEnv || "N/A"}`,
|
|
2727
|
+
`- **Verdict:** ${fs.compatibilityMatrix.verifiedCompatible ? "\u2705 Verified Compatible" : "\u26A0\uFE0F Compatibility Warning"}`,
|
|
2728
|
+
...fs.compatibilityMatrix.notes.map((n) => ` - ${n}`),
|
|
2729
|
+
`
|
|
2730
|
+
## 2. Age Feasibility & Learner Barriers (Ages ${fs.ageFeasibility.recommendedMinAge}\u2013${fs.ageFeasibility.recommendedMaxAge})`,
|
|
2731
|
+
...fs.ageFeasibility.potentialBarriers.map((b) => `- \u26A0\uFE0F ${b}`),
|
|
2732
|
+
`
|
|
2733
|
+
## 3. Real-World Exemplar Projects`,
|
|
2734
|
+
...fs.exemplarProjects.map((ex, idx) => `### Exemplar ${idx + 1}: ${ex.title}
|
|
2735
|
+
- **Summary:** ${ex.description}
|
|
2736
|
+
- **Audience:** ${ex.targetAudience}${ex.sourceUrl ? `
|
|
2737
|
+
- **Reference:** [Link](${ex.sourceUrl})` : ""}`),
|
|
2738
|
+
`
|
|
2739
|
+
## 4. Technical Bottlenecks & Pitfalls to Avoid`,
|
|
2740
|
+
...fs.technicalBottlenecks.map((bn) => `- \u{1F6D1} ${bn}`),
|
|
2741
|
+
`
|
|
2742
|
+
## 5. Grounding Citations`,
|
|
2743
|
+
...fs.groundingSources.map((s, idx) => `${idx + 1}. [${s.title}](${s.url})`)
|
|
2744
|
+
];
|
|
2745
|
+
return lines.join("\n");
|
|
2746
|
+
}
|
|
2747
|
+
};
|
|
2748
|
+
|
|
2323
2749
|
exports.CampaignPlanner = CampaignPlanner;
|
|
2324
2750
|
exports.HarvestPipeline = HarvestPipeline;
|
|
2325
2751
|
exports.ReferencePackBuilder = ReferencePackBuilder;
|
|
2752
|
+
exports.TechnicalGroundingEngine = TechnicalGroundingEngine;
|
|
2326
2753
|
exports.normalizeUrlForDedup = normalizeUrlForDedup;
|
|
2327
2754
|
//# sourceMappingURL=index.cjs.map
|
|
2328
2755
|
//# sourceMappingURL=index.cjs.map
|