@thanh01.pmt/research-kit 1.2.0 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/academic/index.cjs +11 -2
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +2 -1
- package/dist/{catalogBuilder-nrGYEY4b.d.cts → catalogBuilder-B4eiLS_S.d.cts} +27 -5
- package/dist/{catalogBuilder-2i5KASUF.d.ts → catalogBuilder-BNB_gaWP.d.ts} +27 -5
- package/dist/chunk-AKEI7H2C.js +223 -0
- package/dist/chunk-AKEI7H2C.js.map +1 -0
- package/dist/{chunk-X3X26W2I.js → chunk-HESXWQEF.js} +12 -7
- package/dist/chunk-HESXWQEF.js.map +1 -0
- package/dist/{chunk-QFYM2NV4.js → chunk-KXJDQBSJ.js} +3 -3
- package/dist/{chunk-QFYM2NV4.js.map → chunk-KXJDQBSJ.js.map} +1 -1
- package/dist/{chunk-XDDTHWJW.js → chunk-P6RHBJY2.js} +89 -25
- package/dist/chunk-P6RHBJY2.js.map +1 -0
- package/dist/{chunk-ON6VFCJO.js → chunk-QQZQNBJC.js} +51 -2
- package/dist/chunk-QQZQNBJC.js.map +1 -0
- package/dist/{chunk-QK3LVGI7.js → chunk-SAHBBO3U.js} +170 -8
- package/dist/chunk-SAHBBO3U.js.map +1 -0
- package/dist/{chunk-DOXSPRSP.js → chunk-SDDZFFL6.js} +14 -225
- package/dist/chunk-SDDZFFL6.js.map +1 -0
- package/dist/chunk-YICMTZS5.js +527 -0
- package/dist/chunk-YICMTZS5.js.map +1 -0
- package/dist/config/index.cjs +10 -5
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -2
- package/dist/harvester/index.cjs +965 -6
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +87 -5
- package/dist/harvester/index.d.ts +87 -5
- package/dist/harvester/index.js +4 -3
- package/dist/{index-DlngvZBr.d.ts → index-9rDiFuR4.d.cts} +43 -12
- package/dist/{index-DlngvZBr.d.cts → index-9rDiFuR4.d.ts} +43 -12
- package/dist/index.cjs +852 -32
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -11
- package/dist/index.d.ts +13 -11
- package/dist/index.js +21 -12
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-C7hsjrn4.d.cts → literatureReview-BwidpYJo.d.cts} +2 -1
- package/dist/{literatureReview-DWl13r_3.d.ts → literatureReview-DOzgm9Q2.d.ts} +2 -1
- package/dist/pipeline/index.cjs +545 -249
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +10 -4
- package/dist/pipeline/index.d.ts +10 -4
- package/dist/pipeline/index.js +6 -5
- package/dist/{webScraper-yO9P-oLb.d.cts → webScraper-CAQjE46m.d.cts} +11 -2
- package/dist/{webScraper-DBFUIqoO.d.ts → webScraper-CXIpvyA6.d.ts} +11 -2
- package/package.json +1 -1
- package/dist/chunk-DOXSPRSP.js.map +0 -1
- package/dist/chunk-LSMSIPTZ.js +0 -3
- package/dist/chunk-LSMSIPTZ.js.map +0 -1
- package/dist/chunk-ON6VFCJO.js.map +0 -1
- package/dist/chunk-QK3LVGI7.js.map +0 -1
- package/dist/chunk-X3X26W2I.js.map +0 -1
- package/dist/chunk-XDDTHWJW.js.map +0 -1
package/dist/pipeline/index.cjs
CHANGED
|
@@ -329,19 +329,254 @@ ${options.synthesizedInsights.executiveSummary}`);
|
|
|
329
329
|
}
|
|
330
330
|
};
|
|
331
331
|
|
|
332
|
+
// src/academic/arxivClient.ts
|
|
333
|
+
var ArXivSearchEngine = class {
|
|
334
|
+
/**
|
|
335
|
+
* Search ArXiv repository using the public atom XML API
|
|
336
|
+
*/
|
|
337
|
+
async search(query, maxResults = 5) {
|
|
338
|
+
const papers = [];
|
|
339
|
+
try {
|
|
340
|
+
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
341
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
342
|
+
if (res.ok) {
|
|
343
|
+
const xmlText = await res.text();
|
|
344
|
+
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
345
|
+
for (const entry of entryMatches) {
|
|
346
|
+
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
347
|
+
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
348
|
+
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
349
|
+
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
350
|
+
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
351
|
+
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
352
|
+
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
353
|
+
if (titleMatch) {
|
|
354
|
+
papers.push({
|
|
355
|
+
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
356
|
+
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
357
|
+
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
358
|
+
authors: authorMatches,
|
|
359
|
+
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
360
|
+
citationCount: 0,
|
|
361
|
+
venue: "arXiv preprint",
|
|
362
|
+
url: arxivUrl,
|
|
363
|
+
pdfUrl,
|
|
364
|
+
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
} catch (e) {
|
|
370
|
+
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
371
|
+
}
|
|
372
|
+
return papers;
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
// src/academic/semanticScholar.ts
|
|
377
|
+
zod.z.object({
|
|
378
|
+
id: zod.z.string(),
|
|
379
|
+
title: zod.z.string(),
|
|
380
|
+
abstract: zod.z.string().optional(),
|
|
381
|
+
authors: zod.z.array(zod.z.string()).default([]),
|
|
382
|
+
year: zod.z.number().optional(),
|
|
383
|
+
citationCount: zod.z.number().default(0),
|
|
384
|
+
venue: zod.z.string().optional(),
|
|
385
|
+
doi: zod.z.string().optional(),
|
|
386
|
+
url: zod.z.string().optional(),
|
|
387
|
+
pdfUrl: zod.z.string().optional(),
|
|
388
|
+
tldr: zod.z.string().optional(),
|
|
389
|
+
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
390
|
+
});
|
|
391
|
+
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
392
|
+
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
393
|
+
const entries = Object.entries(invertedIndex);
|
|
394
|
+
if (entries.length === 0) return void 0;
|
|
395
|
+
let maxIndex = 0;
|
|
396
|
+
for (const [, positions] of entries) {
|
|
397
|
+
if (Array.isArray(positions)) {
|
|
398
|
+
for (const p of positions) {
|
|
399
|
+
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
404
|
+
const words = new Array(maxIndex + 1);
|
|
405
|
+
for (const [word, positions] of entries) {
|
|
406
|
+
if (Array.isArray(positions)) {
|
|
407
|
+
for (const p of positions) {
|
|
408
|
+
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
409
|
+
words[p] = word;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
415
|
+
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
416
|
+
}
|
|
417
|
+
var AcademicSearchEngine = class {
|
|
418
|
+
apiKey;
|
|
419
|
+
allowEnvFallback;
|
|
420
|
+
customAdapter;
|
|
421
|
+
arxivEngine;
|
|
422
|
+
constructor(options = {}) {
|
|
423
|
+
this.arxivEngine = new ArXivSearchEngine();
|
|
424
|
+
if (typeof options === "string") {
|
|
425
|
+
this.apiKey = options;
|
|
426
|
+
this.allowEnvFallback = true;
|
|
427
|
+
} else {
|
|
428
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
429
|
+
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
430
|
+
this.customAdapter = options.customAdapter;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
setCustomAdapter(adapter) {
|
|
434
|
+
this.customAdapter = adapter;
|
|
435
|
+
}
|
|
436
|
+
setApiKey(apiKey) {
|
|
437
|
+
this.apiKey = apiKey;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
441
|
+
*/
|
|
442
|
+
async searchPapers(query, options = {}) {
|
|
443
|
+
const adapter = options.customAdapter || this.customAdapter;
|
|
444
|
+
if (adapter) {
|
|
445
|
+
try {
|
|
446
|
+
const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
|
|
447
|
+
if (Array.isArray(customPapers) && customPapers.length > 0) {
|
|
448
|
+
return customPapers;
|
|
449
|
+
}
|
|
450
|
+
} catch (e) {
|
|
451
|
+
console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
const limit = options.limit ?? 5;
|
|
455
|
+
const papers = [];
|
|
456
|
+
const seenTitles = /* @__PURE__ */ new Set();
|
|
457
|
+
const pushUnique = (p) => {
|
|
458
|
+
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
459
|
+
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
460
|
+
seenTitles.add(normTitle);
|
|
461
|
+
papers.push(p);
|
|
462
|
+
};
|
|
463
|
+
try {
|
|
464
|
+
const apiKey = options.apiKey || this.apiKey;
|
|
465
|
+
const headers = { "Content-Type": "application/json" };
|
|
466
|
+
if (apiKey) {
|
|
467
|
+
headers["x-api-key"] = apiKey;
|
|
468
|
+
}
|
|
469
|
+
const params = new URLSearchParams({
|
|
470
|
+
query,
|
|
471
|
+
limit: limit.toString(),
|
|
472
|
+
fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
|
|
473
|
+
});
|
|
474
|
+
if (options.yearRange) {
|
|
475
|
+
params.append("year", options.yearRange);
|
|
476
|
+
}
|
|
477
|
+
const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
|
|
478
|
+
headers,
|
|
479
|
+
signal: AbortSignal.timeout(6e3)
|
|
480
|
+
});
|
|
481
|
+
if (res.ok) {
|
|
482
|
+
const data = await res.json();
|
|
483
|
+
if (Array.isArray(data?.data)) {
|
|
484
|
+
for (const item of data.data) {
|
|
485
|
+
pushUnique({
|
|
486
|
+
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
487
|
+
title: item.title || "Untitled",
|
|
488
|
+
abstract: item.abstract || void 0,
|
|
489
|
+
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
490
|
+
year: item.year,
|
|
491
|
+
citationCount: item.citationCount || 0,
|
|
492
|
+
venue: item.venue || void 0,
|
|
493
|
+
doi: item.externalIds?.DOI,
|
|
494
|
+
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
495
|
+
pdfUrl: item.openAccessPdf?.url,
|
|
496
|
+
tldr: item.tldr?.text,
|
|
497
|
+
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
if (papers.length >= limit) return papers.slice(0, limit);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
} catch (e) {
|
|
504
|
+
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
505
|
+
}
|
|
506
|
+
if (papers.length < limit) {
|
|
507
|
+
try {
|
|
508
|
+
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
509
|
+
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
510
|
+
signal: AbortSignal.timeout(6e3)
|
|
511
|
+
});
|
|
512
|
+
if (res.ok) {
|
|
513
|
+
const data = await res.json();
|
|
514
|
+
if (Array.isArray(data?.results)) {
|
|
515
|
+
for (const item of data.results) {
|
|
516
|
+
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
517
|
+
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
518
|
+
pushUnique({
|
|
519
|
+
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
520
|
+
title: item.title || "Untitled",
|
|
521
|
+
abstract: abstractText,
|
|
522
|
+
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
523
|
+
year: item.publication_year,
|
|
524
|
+
citationCount: item.cited_by_count || 0,
|
|
525
|
+
venue: item.primary_location?.source?.display_name,
|
|
526
|
+
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
527
|
+
url: item.doi || item.id,
|
|
528
|
+
pdfUrl: item.open_access?.oa_url,
|
|
529
|
+
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
} catch (e) {
|
|
535
|
+
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
539
|
+
try {
|
|
540
|
+
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
541
|
+
for (const p of arxivPapers) {
|
|
542
|
+
pushUnique(p);
|
|
543
|
+
}
|
|
544
|
+
} catch (e) {
|
|
545
|
+
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return papers.slice(0, limit);
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
|
|
332
552
|
// src/harvester/multiSourceSearch.ts
|
|
333
553
|
var MultiSourceSearchEngine = class {
|
|
334
554
|
tavilyKey;
|
|
335
555
|
exaKey;
|
|
336
556
|
serperKey;
|
|
557
|
+
brightDataKey;
|
|
558
|
+
brightDataZone;
|
|
559
|
+
searxngHost;
|
|
560
|
+
searxngHeaders;
|
|
337
561
|
allowEnvFallback;
|
|
338
562
|
customAdapter;
|
|
563
|
+
academicEngine;
|
|
339
564
|
constructor(options = {}) {
|
|
340
565
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
341
566
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
342
567
|
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
343
568
|
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
569
|
+
this.brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || (this.allowEnvFallback ? process.env.BRIGHTDATA_API_KEY || process.env.BRIGHT_DATA_KEY : void 0);
|
|
570
|
+
this.brightDataZone = options.brightDataZone || (this.allowEnvFallback ? process.env.BRIGHTDATA_SERP_ZONE || process.env.BRIGHTDATA_UNLOCKER_ZONE : void 0) || "cli_unlocker";
|
|
571
|
+
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
572
|
+
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
573
|
+
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
574
|
+
this.searxngHeaders = {
|
|
575
|
+
...options.searxngHeaders || {},
|
|
576
|
+
...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
|
|
577
|
+
};
|
|
344
578
|
this.customAdapter = options.customAdapter;
|
|
579
|
+
this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
|
|
345
580
|
}
|
|
346
581
|
setCustomAdapter(adapter) {
|
|
347
582
|
this.customAdapter = adapter;
|
|
@@ -350,6 +585,18 @@ var MultiSourceSearchEngine = class {
|
|
|
350
585
|
if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
|
|
351
586
|
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
352
587
|
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
588
|
+
if (keys.brightdata !== void 0) this.brightDataKey = keys.brightdata;
|
|
589
|
+
if (keys.brightData !== void 0) this.brightDataKey = keys.brightData;
|
|
590
|
+
}
|
|
591
|
+
setBrightData(key, zone) {
|
|
592
|
+
if (key !== void 0) this.brightDataKey = key;
|
|
593
|
+
if (zone !== void 0) this.brightDataZone = zone;
|
|
594
|
+
}
|
|
595
|
+
setSearxng(host, headers) {
|
|
596
|
+
this.searxngHost = host;
|
|
597
|
+
if (headers) {
|
|
598
|
+
this.searxngHeaders = { ...this.searxngHeaders, ...headers };
|
|
599
|
+
}
|
|
353
600
|
}
|
|
354
601
|
async search(query, options = {}) {
|
|
355
602
|
const limit = options.limit ?? 5;
|
|
@@ -371,9 +618,39 @@ var MultiSourceSearchEngine = class {
|
|
|
371
618
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
372
619
|
}
|
|
373
620
|
}
|
|
621
|
+
if (preferred === "academic") {
|
|
622
|
+
try {
|
|
623
|
+
const papers = await this.academicEngine.searchPapers(query, { limit });
|
|
624
|
+
if (papers.length > 0) {
|
|
625
|
+
return papers.map((p) => ({
|
|
626
|
+
title: p.title,
|
|
627
|
+
url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
|
|
628
|
+
content: p.tldr || p.abstract || "",
|
|
629
|
+
score: p.citationCount,
|
|
630
|
+
engine: "academic"
|
|
631
|
+
}));
|
|
632
|
+
}
|
|
633
|
+
} catch (e) {
|
|
634
|
+
console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
374
637
|
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
375
638
|
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
376
639
|
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
640
|
+
const brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || this.brightDataKey;
|
|
641
|
+
const brightDataZone = options.brightDataZone || this.brightDataZone;
|
|
642
|
+
const searxngHost = options.searxngHost || this.searxngHost;
|
|
643
|
+
if (preferred === "searxng" && searxngHost) {
|
|
644
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
645
|
+
if (searxHits.length > 0) return searxHits;
|
|
646
|
+
}
|
|
647
|
+
if (preferred === "brightdata" && brightDataKey) {
|
|
648
|
+
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
649
|
+
zone: brightDataZone,
|
|
650
|
+
engine: options.brightDataEngine
|
|
651
|
+
});
|
|
652
|
+
if (bdHits.length > 0) return bdHits;
|
|
653
|
+
}
|
|
377
654
|
if (preferred === "exa" && exaKey) {
|
|
378
655
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
379
656
|
if (exaHits.length > 0) return exaHits;
|
|
@@ -395,6 +672,9 @@ var MultiSourceSearchEngine = class {
|
|
|
395
672
|
include_answer: true,
|
|
396
673
|
max_results: limit
|
|
397
674
|
};
|
|
675
|
+
if (options.timeRange) {
|
|
676
|
+
body.time_range = options.timeRange;
|
|
677
|
+
}
|
|
398
678
|
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
399
679
|
body.include_domains = options.domainFilter;
|
|
400
680
|
}
|
|
@@ -421,18 +701,103 @@ var MultiSourceSearchEngine = class {
|
|
|
421
701
|
}
|
|
422
702
|
}
|
|
423
703
|
if (exaKey) {
|
|
424
|
-
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
704
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
425
705
|
if (exaHits.length > 0) return exaHits;
|
|
426
706
|
}
|
|
427
707
|
if (serperKey) {
|
|
428
|
-
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
708
|
+
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
|
|
429
709
|
if (serperHits.length > 0) return serperHits;
|
|
430
710
|
}
|
|
711
|
+
if (brightDataKey) {
|
|
712
|
+
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
713
|
+
zone: brightDataZone,
|
|
714
|
+
engine: options.brightDataEngine
|
|
715
|
+
});
|
|
716
|
+
if (bdHits.length > 0) return bdHits;
|
|
717
|
+
}
|
|
718
|
+
if (searxngHost) {
|
|
719
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders, options.timeRange);
|
|
720
|
+
if (searxHits.length > 0) return searxHits;
|
|
721
|
+
}
|
|
431
722
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
432
723
|
if (ddgHits.length > 0) return ddgHits;
|
|
433
724
|
return [];
|
|
434
725
|
}
|
|
435
|
-
async
|
|
726
|
+
async queryBrightData(query, key, limit = 5, options = {}) {
|
|
727
|
+
try {
|
|
728
|
+
const zone = options.zone || this.brightDataZone || "cli_unlocker";
|
|
729
|
+
const engine = options.engine || "bing";
|
|
730
|
+
let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
|
|
731
|
+
if (engine === "google") {
|
|
732
|
+
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&brd_json=1&hl=en&gl=us`;
|
|
733
|
+
} else if (engine === "yandex") {
|
|
734
|
+
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
|
|
735
|
+
}
|
|
736
|
+
const res = await fetch("https://api.brightdata.com/request", {
|
|
737
|
+
method: "POST",
|
|
738
|
+
headers: {
|
|
739
|
+
Authorization: `Bearer ${key}`,
|
|
740
|
+
"Content-Type": "application/json"
|
|
741
|
+
},
|
|
742
|
+
body: JSON.stringify({
|
|
743
|
+
zone,
|
|
744
|
+
url: searchUrl,
|
|
745
|
+
format: "raw"
|
|
746
|
+
}),
|
|
747
|
+
signal: AbortSignal.timeout(15e3)
|
|
748
|
+
});
|
|
749
|
+
if (res.ok) {
|
|
750
|
+
const data = await res.json();
|
|
751
|
+
const organic = Array.isArray(data?.organic) ? data.organic : [];
|
|
752
|
+
if (organic.length > 0) {
|
|
753
|
+
return organic.slice(0, limit).map((r, idx) => ({
|
|
754
|
+
title: r.title || "Untitled",
|
|
755
|
+
url: r.link,
|
|
756
|
+
content: r.description || r.snippet || "",
|
|
757
|
+
score: r.rank ? 1 / r.rank : 1 / (idx + 1),
|
|
758
|
+
engine: "brightdata"
|
|
759
|
+
}));
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
} catch (e) {
|
|
763
|
+
console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
|
|
764
|
+
}
|
|
765
|
+
return [];
|
|
766
|
+
}
|
|
767
|
+
async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
|
|
768
|
+
try {
|
|
769
|
+
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
770
|
+
let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
771
|
+
if (timeRange) {
|
|
772
|
+
searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
|
|
773
|
+
}
|
|
774
|
+
const headers = {
|
|
775
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
776
|
+
...this.searxngHeaders || {},
|
|
777
|
+
...customHeaders || {}
|
|
778
|
+
};
|
|
779
|
+
const res = await fetch(searchUrl, {
|
|
780
|
+
headers,
|
|
781
|
+
signal: AbortSignal.timeout(1e4)
|
|
782
|
+
});
|
|
783
|
+
if (res.ok) {
|
|
784
|
+
const data = await res.json();
|
|
785
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
786
|
+
return data.results.slice(0, limit).map((r) => ({
|
|
787
|
+
title: r.title || "Untitled",
|
|
788
|
+
url: r.url,
|
|
789
|
+
content: r.content || r.snippet || "",
|
|
790
|
+
score: r.score,
|
|
791
|
+
engine: "searxng"
|
|
792
|
+
}));
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
} catch (e) {
|
|
796
|
+
console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
|
|
797
|
+
}
|
|
798
|
+
return [];
|
|
799
|
+
}
|
|
800
|
+
async queryExa(query, exaKey, limit, domainFilter, timeRange) {
|
|
436
801
|
try {
|
|
437
802
|
const body = {
|
|
438
803
|
query,
|
|
@@ -442,6 +807,14 @@ var MultiSourceSearchEngine = class {
|
|
|
442
807
|
if (domainFilter && domainFilter.length > 0) {
|
|
443
808
|
body.includeDomains = domainFilter;
|
|
444
809
|
}
|
|
810
|
+
if (timeRange) {
|
|
811
|
+
const now = /* @__PURE__ */ new Date();
|
|
812
|
+
if (timeRange === "day") now.setDate(now.getDate() - 1);
|
|
813
|
+
else if (timeRange === "week") now.setDate(now.getDate() - 7);
|
|
814
|
+
else if (timeRange === "month") now.setMonth(now.getMonth() - 1);
|
|
815
|
+
else if (timeRange === "year") now.setFullYear(now.getFullYear() - 1);
|
|
816
|
+
body.startPublishedDate = now.toISOString();
|
|
817
|
+
}
|
|
445
818
|
const res = await fetch("https://api.exa.ai/search", {
|
|
446
819
|
method: "POST",
|
|
447
820
|
headers: {
|
|
@@ -468,15 +841,22 @@ var MultiSourceSearchEngine = class {
|
|
|
468
841
|
}
|
|
469
842
|
return [];
|
|
470
843
|
}
|
|
471
|
-
async querySerper(query, serperKey, limit) {
|
|
844
|
+
async querySerper(query, serperKey, limit, timeRange) {
|
|
472
845
|
try {
|
|
846
|
+
const body = { q: query, num: limit };
|
|
847
|
+
if (timeRange) {
|
|
848
|
+
const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
|
|
849
|
+
if (tbsMap[timeRange]) {
|
|
850
|
+
body.tbs = tbsMap[timeRange];
|
|
851
|
+
}
|
|
852
|
+
}
|
|
473
853
|
const res = await fetch("https://google.serper.dev/search", {
|
|
474
854
|
method: "POST",
|
|
475
855
|
headers: {
|
|
476
856
|
"X-API-KEY": serperKey,
|
|
477
857
|
"Content-Type": "application/json"
|
|
478
858
|
},
|
|
479
|
-
body: JSON.stringify(
|
|
859
|
+
body: JSON.stringify(body),
|
|
480
860
|
signal: AbortSignal.timeout(1e4)
|
|
481
861
|
});
|
|
482
862
|
if (res.ok) {
|
|
@@ -725,6 +1105,8 @@ function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
|
|
|
725
1105
|
var WebScraper = class {
|
|
726
1106
|
crawl4aiHost;
|
|
727
1107
|
jinaKey;
|
|
1108
|
+
brightDataKey;
|
|
1109
|
+
brightDataZone;
|
|
728
1110
|
allowEnvFallback;
|
|
729
1111
|
customAdapter;
|
|
730
1112
|
youtubeService;
|
|
@@ -732,6 +1114,8 @@ var WebScraper = class {
|
|
|
732
1114
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
733
1115
|
this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
|
|
734
1116
|
this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
|
|
1117
|
+
this.brightDataKey = options.brightDataKey || (this.allowEnvFallback ? process.env.BRIGHTDATA_API_KEY || process.env.BRIGHT_DATA_KEY : void 0);
|
|
1118
|
+
this.brightDataZone = options.brightDataZone || (this.allowEnvFallback ? process.env.BRIGHTDATA_UNLOCKER_ZONE : void 0) || "cli_unlocker";
|
|
735
1119
|
this.customAdapter = options.customAdapter;
|
|
736
1120
|
this.youtubeService = new YouTubeIngestionService();
|
|
737
1121
|
}
|
|
@@ -741,8 +1125,15 @@ var WebScraper = class {
|
|
|
741
1125
|
setJinaKey(key) {
|
|
742
1126
|
this.jinaKey = key;
|
|
743
1127
|
}
|
|
1128
|
+
setBrightData(key, zone) {
|
|
1129
|
+
if (key !== void 0) this.brightDataKey = key;
|
|
1130
|
+
if (zone !== void 0) this.brightDataZone = zone;
|
|
1131
|
+
}
|
|
744
1132
|
async scrapeUrl(targetUrl, options = {}) {
|
|
745
1133
|
const adapter = options.customAdapter || this.customAdapter;
|
|
1134
|
+
const bdKey = options.brightDataKey || this.brightDataKey;
|
|
1135
|
+
const bdZone = options.brightDataZone || this.brightDataZone;
|
|
1136
|
+
const preferred = options.preferredProvider;
|
|
746
1137
|
if (adapter) {
|
|
747
1138
|
try {
|
|
748
1139
|
const result = await adapter.crawl(targetUrl);
|
|
@@ -763,6 +1154,13 @@ var WebScraper = class {
|
|
|
763
1154
|
console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
|
|
764
1155
|
}
|
|
765
1156
|
}
|
|
1157
|
+
if (preferred === "brightdata" && bdKey) {
|
|
1158
|
+
try {
|
|
1159
|
+
return await this.scrapeWithBrightData(targetUrl, bdKey, bdZone);
|
|
1160
|
+
} catch (e) {
|
|
1161
|
+
console.warn(`[WebScraper] Preferred Bright Data failed for ${targetUrl}:`, e?.message || e);
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
766
1164
|
try {
|
|
767
1165
|
const jinaUrl = `https://r.jina.ai/${targetUrl}`;
|
|
768
1166
|
const jinaKey = options.jinaKey || this.jinaKey;
|
|
@@ -807,6 +1205,13 @@ var WebScraper = class {
|
|
|
807
1205
|
} catch {
|
|
808
1206
|
}
|
|
809
1207
|
}
|
|
1208
|
+
if (bdKey) {
|
|
1209
|
+
try {
|
|
1210
|
+
return await this.scrapeWithBrightData(targetUrl, bdKey, bdZone);
|
|
1211
|
+
} catch (e) {
|
|
1212
|
+
console.warn(`[WebScraper] Bright Data fallback failed for ${targetUrl}:`, e?.message || e);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
810
1215
|
try {
|
|
811
1216
|
const res = await fetch(targetUrl, {
|
|
812
1217
|
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
|
|
@@ -831,6 +1236,30 @@ ${p.replace(/<[^>]+>/g, " ").trim()}
|
|
|
831
1236
|
}
|
|
832
1237
|
throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
|
|
833
1238
|
}
|
|
1239
|
+
async scrapeWithBrightData(targetUrl, key, zone) {
|
|
1240
|
+
const reqZone = zone || this.brightDataZone || "cli_unlocker";
|
|
1241
|
+
const res = await fetch("https://api.brightdata.com/request", {
|
|
1242
|
+
method: "POST",
|
|
1243
|
+
headers: {
|
|
1244
|
+
Authorization: `Bearer ${key}`,
|
|
1245
|
+
"Content-Type": "application/json"
|
|
1246
|
+
},
|
|
1247
|
+
body: JSON.stringify({
|
|
1248
|
+
zone: reqZone,
|
|
1249
|
+
url: targetUrl,
|
|
1250
|
+
format: "raw",
|
|
1251
|
+
data_format: "markdown"
|
|
1252
|
+
}),
|
|
1253
|
+
signal: AbortSignal.timeout(2e4)
|
|
1254
|
+
});
|
|
1255
|
+
if (res.ok) {
|
|
1256
|
+
const markdown = await res.text();
|
|
1257
|
+
if (markdown && markdown.length > 30) {
|
|
1258
|
+
return this.packageDocument(targetUrl, markdown, "brightdata");
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
throw new Error(`Bright Data Web Unlocker failed with HTTP ${res.status}`);
|
|
1262
|
+
}
|
|
834
1263
|
packageDocument(url, markdown, provider, explicitTitle) {
|
|
835
1264
|
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
|
|
836
1265
|
const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
|
|
@@ -1447,7 +1876,7 @@ ${corpusPreview}`,
|
|
|
1447
1876
|
`
|
|
1448
1877
|
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
1449
1878
|
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
1450
|
-
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."]}`
|
|
1879
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."], "troubleshootingHints": ["2-minute quick fix for common student pitfall"]}`
|
|
1451
1880
|
].filter(Boolean).join("\n");
|
|
1452
1881
|
const rawResponse = await options.llm(prompt, {
|
|
1453
1882
|
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
@@ -1693,226 +2122,6 @@ var ProvenanceTracker = class {
|
|
|
1693
2122
|
}
|
|
1694
2123
|
};
|
|
1695
2124
|
|
|
1696
|
-
// src/academic/arxivClient.ts
|
|
1697
|
-
var ArXivSearchEngine = class {
|
|
1698
|
-
/**
|
|
1699
|
-
* Search ArXiv repository using the public atom XML API
|
|
1700
|
-
*/
|
|
1701
|
-
async search(query, maxResults = 5) {
|
|
1702
|
-
const papers = [];
|
|
1703
|
-
try {
|
|
1704
|
-
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
1705
|
-
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
1706
|
-
if (res.ok) {
|
|
1707
|
-
const xmlText = await res.text();
|
|
1708
|
-
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
1709
|
-
for (const entry of entryMatches) {
|
|
1710
|
-
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
1711
|
-
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
1712
|
-
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
1713
|
-
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
1714
|
-
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
1715
|
-
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
1716
|
-
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
1717
|
-
if (titleMatch) {
|
|
1718
|
-
papers.push({
|
|
1719
|
-
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
1720
|
-
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
1721
|
-
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
1722
|
-
authors: authorMatches,
|
|
1723
|
-
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
1724
|
-
citationCount: 0,
|
|
1725
|
-
venue: "arXiv preprint",
|
|
1726
|
-
url: arxivUrl,
|
|
1727
|
-
pdfUrl,
|
|
1728
|
-
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
1729
|
-
});
|
|
1730
|
-
}
|
|
1731
|
-
}
|
|
1732
|
-
}
|
|
1733
|
-
} catch (e) {
|
|
1734
|
-
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
1735
|
-
}
|
|
1736
|
-
return papers;
|
|
1737
|
-
}
|
|
1738
|
-
};
|
|
1739
|
-
|
|
1740
|
-
// src/academic/semanticScholar.ts
|
|
1741
|
-
zod.z.object({
|
|
1742
|
-
id: zod.z.string(),
|
|
1743
|
-
title: zod.z.string(),
|
|
1744
|
-
abstract: zod.z.string().optional(),
|
|
1745
|
-
authors: zod.z.array(zod.z.string()).default([]),
|
|
1746
|
-
year: zod.z.number().optional(),
|
|
1747
|
-
citationCount: zod.z.number().default(0),
|
|
1748
|
-
venue: zod.z.string().optional(),
|
|
1749
|
-
doi: zod.z.string().optional(),
|
|
1750
|
-
url: zod.z.string().optional(),
|
|
1751
|
-
pdfUrl: zod.z.string().optional(),
|
|
1752
|
-
tldr: zod.z.string().optional(),
|
|
1753
|
-
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
1754
|
-
});
|
|
1755
|
-
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
1756
|
-
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
1757
|
-
const entries = Object.entries(invertedIndex);
|
|
1758
|
-
if (entries.length === 0) return void 0;
|
|
1759
|
-
let maxIndex = 0;
|
|
1760
|
-
for (const [, positions] of entries) {
|
|
1761
|
-
if (Array.isArray(positions)) {
|
|
1762
|
-
for (const p of positions) {
|
|
1763
|
-
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
1764
|
-
}
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
1768
|
-
const words = new Array(maxIndex + 1);
|
|
1769
|
-
for (const [word, positions] of entries) {
|
|
1770
|
-
if (Array.isArray(positions)) {
|
|
1771
|
-
for (const p of positions) {
|
|
1772
|
-
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
1773
|
-
words[p] = word;
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
}
|
|
1777
|
-
}
|
|
1778
|
-
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
1779
|
-
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
1780
|
-
}
|
|
1781
|
-
var AcademicSearchEngine = class {
|
|
1782
|
-
apiKey;
|
|
1783
|
-
allowEnvFallback;
|
|
1784
|
-
customAdapter;
|
|
1785
|
-
arxivEngine;
|
|
1786
|
-
constructor(options = {}) {
|
|
1787
|
-
this.arxivEngine = new ArXivSearchEngine();
|
|
1788
|
-
if (typeof options === "string") {
|
|
1789
|
-
this.apiKey = options;
|
|
1790
|
-
this.allowEnvFallback = true;
|
|
1791
|
-
} else {
|
|
1792
|
-
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
1793
|
-
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
1794
|
-
this.customAdapter = options.customAdapter;
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
setCustomAdapter(adapter) {
|
|
1798
|
-
this.customAdapter = adapter;
|
|
1799
|
-
}
|
|
1800
|
-
setApiKey(apiKey) {
|
|
1801
|
-
this.apiKey = apiKey;
|
|
1802
|
-
}
|
|
1803
|
-
/**
|
|
1804
|
-
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
1805
|
-
*/
|
|
1806
|
-
async searchPapers(query, options = {}) {
|
|
1807
|
-
const adapter = options.customAdapter || this.customAdapter;
|
|
1808
|
-
if (adapter) {
|
|
1809
|
-
try {
|
|
1810
|
-
const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
|
|
1811
|
-
if (Array.isArray(customPapers) && customPapers.length > 0) {
|
|
1812
|
-
return customPapers;
|
|
1813
|
-
}
|
|
1814
|
-
} catch (e) {
|
|
1815
|
-
console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
|
|
1816
|
-
}
|
|
1817
|
-
}
|
|
1818
|
-
const limit = options.limit ?? 5;
|
|
1819
|
-
const papers = [];
|
|
1820
|
-
const seenTitles = /* @__PURE__ */ new Set();
|
|
1821
|
-
const pushUnique = (p) => {
|
|
1822
|
-
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
1823
|
-
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
1824
|
-
seenTitles.add(normTitle);
|
|
1825
|
-
papers.push(p);
|
|
1826
|
-
};
|
|
1827
|
-
try {
|
|
1828
|
-
const apiKey = options.apiKey || this.apiKey;
|
|
1829
|
-
const headers = { "Content-Type": "application/json" };
|
|
1830
|
-
if (apiKey) {
|
|
1831
|
-
headers["x-api-key"] = apiKey;
|
|
1832
|
-
}
|
|
1833
|
-
const params = new URLSearchParams({
|
|
1834
|
-
query,
|
|
1835
|
-
limit: limit.toString(),
|
|
1836
|
-
fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
|
|
1837
|
-
});
|
|
1838
|
-
if (options.yearRange) {
|
|
1839
|
-
params.append("year", options.yearRange);
|
|
1840
|
-
}
|
|
1841
|
-
const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
|
|
1842
|
-
headers,
|
|
1843
|
-
signal: AbortSignal.timeout(6e3)
|
|
1844
|
-
});
|
|
1845
|
-
if (res.ok) {
|
|
1846
|
-
const data = await res.json();
|
|
1847
|
-
if (Array.isArray(data?.data)) {
|
|
1848
|
-
for (const item of data.data) {
|
|
1849
|
-
pushUnique({
|
|
1850
|
-
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
1851
|
-
title: item.title || "Untitled",
|
|
1852
|
-
abstract: item.abstract || void 0,
|
|
1853
|
-
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
1854
|
-
year: item.year,
|
|
1855
|
-
citationCount: item.citationCount || 0,
|
|
1856
|
-
venue: item.venue || void 0,
|
|
1857
|
-
doi: item.externalIds?.DOI,
|
|
1858
|
-
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
1859
|
-
pdfUrl: item.openAccessPdf?.url,
|
|
1860
|
-
tldr: item.tldr?.text,
|
|
1861
|
-
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
1862
|
-
});
|
|
1863
|
-
}
|
|
1864
|
-
if (papers.length >= limit) return papers.slice(0, limit);
|
|
1865
|
-
}
|
|
1866
|
-
}
|
|
1867
|
-
} catch (e) {
|
|
1868
|
-
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
1869
|
-
}
|
|
1870
|
-
if (papers.length < limit) {
|
|
1871
|
-
try {
|
|
1872
|
-
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
1873
|
-
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
1874
|
-
signal: AbortSignal.timeout(6e3)
|
|
1875
|
-
});
|
|
1876
|
-
if (res.ok) {
|
|
1877
|
-
const data = await res.json();
|
|
1878
|
-
if (Array.isArray(data?.results)) {
|
|
1879
|
-
for (const item of data.results) {
|
|
1880
|
-
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
1881
|
-
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
1882
|
-
pushUnique({
|
|
1883
|
-
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
1884
|
-
title: item.title || "Untitled",
|
|
1885
|
-
abstract: abstractText,
|
|
1886
|
-
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
1887
|
-
year: item.publication_year,
|
|
1888
|
-
citationCount: item.cited_by_count || 0,
|
|
1889
|
-
venue: item.primary_location?.source?.display_name,
|
|
1890
|
-
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
1891
|
-
url: item.doi || item.id,
|
|
1892
|
-
pdfUrl: item.open_access?.oa_url,
|
|
1893
|
-
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
1894
|
-
});
|
|
1895
|
-
}
|
|
1896
|
-
}
|
|
1897
|
-
}
|
|
1898
|
-
} catch (e) {
|
|
1899
|
-
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
1900
|
-
}
|
|
1901
|
-
}
|
|
1902
|
-
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
1903
|
-
try {
|
|
1904
|
-
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
1905
|
-
for (const p of arxivPapers) {
|
|
1906
|
-
pushUnique(p);
|
|
1907
|
-
}
|
|
1908
|
-
} catch (e) {
|
|
1909
|
-
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
1910
|
-
}
|
|
1911
|
-
}
|
|
1912
|
-
return papers.slice(0, limit);
|
|
1913
|
-
}
|
|
1914
|
-
};
|
|
1915
|
-
|
|
1916
2125
|
// src/academic/cs2023Taxonomy.ts
|
|
1917
2126
|
var CSTA_K12_STRANDS = {
|
|
1918
2127
|
CS: {
|
|
@@ -1942,7 +2151,15 @@ var CSTA_K12_STRANDS = {
|
|
|
1942
2151
|
nameVi: "Thu\u1EADt to\xE1n & L\u1EADp tr\xECnh",
|
|
1943
2152
|
description: "Algorithms, variables, control structures, modularity, and iterative program development.",
|
|
1944
2153
|
subconcepts: ["Algorithms", "Variables", "Control", "Modularity", "Program Development"]
|
|
1945
|
-
}
|
|
2154
|
+
},
|
|
2155
|
+
IC: {
|
|
2156
|
+
code: "IC",
|
|
2157
|
+
name: "Impacts of Computing",
|
|
2158
|
+
nameVi: "T\xE1c \u0111\u1ED9ng c\u1EE7a Tin h\u1ECDc & AI",
|
|
2159
|
+
description: "Culture, social interactions, safety, law, and ethics of computing and AI systems.",
|
|
2160
|
+
subconcepts: ["Culture", "Social Interactions", "Safety, Law, & Ethics"]
|
|
2161
|
+
}
|
|
2162
|
+
};
|
|
1946
2163
|
var CS2023_KNOWLEDGE_AREAS = {
|
|
1947
2164
|
AL: {
|
|
1948
2165
|
code: "AL",
|
|
@@ -2009,7 +2226,14 @@ var CS2023_KNOWLEDGE_AREAS = {
|
|
|
2009
2226
|
name: "Security & Cryptography",
|
|
2010
2227
|
description: "Authentication, cryptography, secure coding, threat modeling, and network defense.",
|
|
2011
2228
|
coreTopics: ["Symmetric & Asymmetric Encryption", "OAuth & JWT Auth", "Secure Coding & Vulnerabilities", "Identity & Access Management"]
|
|
2012
|
-
}
|
|
2229
|
+
},
|
|
2230
|
+
SPD: {
|
|
2231
|
+
code: "SPD",
|
|
2232
|
+
name: "Specialized Platform Development",
|
|
2233
|
+
description: "Web, mobile, cloud, IoT, and robotics platform engineering.",
|
|
2234
|
+
coreTopics: ["Mobile & Desktop App Lifecycle", "Web Full-Stack Architecture", "IoT & Edge Device Connectivity", "Platform SDK Integration"]
|
|
2235
|
+
}
|
|
2236
|
+
};
|
|
2013
2237
|
function mapTopicToCS2023(topic, context) {
|
|
2014
2238
|
const combined = [
|
|
2015
2239
|
topic,
|
|
@@ -2047,9 +2271,18 @@ function mapTopicToCS2023(topic, context) {
|
|
|
2047
2271
|
return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88 };
|
|
2048
2272
|
}
|
|
2049
2273
|
if (/\b(algorithm|data structure|tree|graph|dynamic programming|sorting|thuật toán|cấu trúc dữ liệu|đồ thị|quy hoạch động)\b/.test(combined)) {
|
|
2050
|
-
return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
|
|
2274
|
+
return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9, isCsApplicable: true };
|
|
2051
2275
|
}
|
|
2052
|
-
|
|
2276
|
+
const isExplicitNonCoding = /\b(noncoding|non-coding|phi lập trình|không lập trình|applied science|khoa học ứng dụng|general science|khoa học tự nhiên|applied-noncoding)\b/i.test(combined);
|
|
2277
|
+
if (isExplicitNonCoding) {
|
|
2278
|
+
return {
|
|
2279
|
+
area: CS2023_KNOWLEDGE_AREAS.SPD,
|
|
2280
|
+
k12Strand: CSTA_K12_STRANDS.IC,
|
|
2281
|
+
matchConfidence: 0.25,
|
|
2282
|
+
isCsApplicable: false
|
|
2283
|
+
};
|
|
2284
|
+
}
|
|
2285
|
+
return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.45, isCsApplicable: true };
|
|
2053
2286
|
}
|
|
2054
2287
|
|
|
2055
2288
|
// src/academic/citationManager.ts
|
|
@@ -2542,35 +2775,68 @@ var TechnicalGroundingEngine = class {
|
|
|
2542
2775
|
try {
|
|
2543
2776
|
const doc = await this.scraper.scrapeUrl(h.url);
|
|
2544
2777
|
if (doc && doc.markdownContent) {
|
|
2778
|
+
const substantiveExcerpt = doc.keyExcerpts && doc.keyExcerpts.length > 0 ? doc.keyExcerpts.slice(0, 3).join("\n\n") : doc.markdownContent.slice(0, 1500);
|
|
2545
2779
|
return {
|
|
2546
2780
|
title: h.title,
|
|
2547
2781
|
url: h.url,
|
|
2548
|
-
excerpt:
|
|
2782
|
+
excerpt: substantiveExcerpt,
|
|
2783
|
+
codeBlocks: (doc.extractedCodeBlocks || []).slice(0, 2),
|
|
2784
|
+
tables: (doc.extractedTables || []).slice(0, 2),
|
|
2785
|
+
keyExcerpts: (doc.keyExcerpts || []).slice(0, 4)
|
|
2549
2786
|
};
|
|
2550
2787
|
}
|
|
2551
2788
|
} catch {
|
|
2552
2789
|
}
|
|
2553
|
-
return {
|
|
2790
|
+
return {
|
|
2791
|
+
title: h.title,
|
|
2792
|
+
url: h.url,
|
|
2793
|
+
excerpt: h.content.slice(0, 500),
|
|
2794
|
+
codeBlocks: [],
|
|
2795
|
+
tables: [],
|
|
2796
|
+
keyExcerpts: []
|
|
2797
|
+
};
|
|
2554
2798
|
})
|
|
2555
2799
|
);
|
|
2556
2800
|
const scrapedSnippets = scrapeResults.map(
|
|
2557
|
-
(res, idx) => res.status === "fulfilled" ? res.value : {
|
|
2801
|
+
(res, idx) => res.status === "fulfilled" ? res.value : {
|
|
2802
|
+
title: hitsToScrape[idx].title,
|
|
2803
|
+
url: hitsToScrape[idx].url,
|
|
2804
|
+
excerpt: hitsToScrape[idx].content.slice(0, 500),
|
|
2805
|
+
codeBlocks: [],
|
|
2806
|
+
tables: [],
|
|
2807
|
+
keyExcerpts: []
|
|
2808
|
+
}
|
|
2558
2809
|
);
|
|
2559
|
-
const
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2810
|
+
const preservedCodeSnippets = [];
|
|
2811
|
+
const preservedTables = [];
|
|
2812
|
+
const preservedKeyExcerpts = [];
|
|
2813
|
+
for (const s of scrapedSnippets) {
|
|
2814
|
+
if (s.codeBlocks && s.codeBlocks.length > 0) {
|
|
2815
|
+
preservedCodeSnippets.push(...s.codeBlocks);
|
|
2816
|
+
}
|
|
2817
|
+
if (s.tables && s.tables.length > 0) {
|
|
2818
|
+
preservedTables.push(...s.tables);
|
|
2819
|
+
}
|
|
2820
|
+
if (s.keyExcerpts && s.keyExcerpts.length > 0) {
|
|
2821
|
+
preservedKeyExcerpts.push(...s.keyExcerpts);
|
|
2822
|
+
}
|
|
2823
|
+
}
|
|
2824
|
+
const defaultExemplars = scrapedSnippets.length > 0 ? [
|
|
2565
2825
|
{
|
|
2566
|
-
title: `${topic}
|
|
2567
|
-
description:
|
|
2826
|
+
title: scrapedSnippets[0].title || `${topic} Practical Foundation`,
|
|
2827
|
+
description: scrapedSnippets[0].excerpt ? scrapedSnippets[0].excerpt.slice(0, 220) : `Hands-on implementation project for ${topic}.`,
|
|
2568
2828
|
targetAudience: `Students aged ${ageBand[0]}-${ageBand[1]}`,
|
|
2569
|
-
sourceUrl: scrapedSnippets[0]
|
|
2829
|
+
sourceUrl: scrapedSnippets[0].url,
|
|
2570
2830
|
hardwareRequirements: hardware ? [hardware] : void 0,
|
|
2571
2831
|
softwareRequirements: software ? [software] : void 0
|
|
2572
2832
|
}
|
|
2573
|
-
];
|
|
2833
|
+
] : [];
|
|
2834
|
+
const defaultBottlenecks = scrapedSnippets.length > 0 ? domain === "hardware" || Boolean(hardware) ? [
|
|
2835
|
+
"Power supply decoupling and current sufficiency when powering sensors or actuators.",
|
|
2836
|
+
"Logic level shifting and hardware pin mapping conflicts."
|
|
2837
|
+
] : [
|
|
2838
|
+
"SDK/runtime version compatibility and local environment dependencies."
|
|
2839
|
+
] : [];
|
|
2574
2840
|
let factsheet = {
|
|
2575
2841
|
topic,
|
|
2576
2842
|
domain,
|
|
@@ -2578,18 +2844,21 @@ var TechnicalGroundingEngine = class {
|
|
|
2578
2844
|
hardwarePlatform: hardware || void 0,
|
|
2579
2845
|
softwareEnv: software || void 0,
|
|
2580
2846
|
verifiedCompatible: true,
|
|
2581
|
-
notes: [`Targeting ${hardware || "General Platform"} with ${software || "Standard SDK"}.`]
|
|
2847
|
+
notes: hardware || software ? [`Targeting ${hardware || "General Platform"} with ${software || "Standard SDK"}.`] : [`Baseline technical verification for ${topic}.`]
|
|
2582
2848
|
},
|
|
2583
2849
|
ageFeasibility: {
|
|
2584
2850
|
recommendedMinAge: ageBand[0],
|
|
2585
2851
|
recommendedMaxAge: ageBand[1],
|
|
2586
2852
|
cognitiveReadinessVerdict: "feasible",
|
|
2587
|
-
potentialBarriers: [
|
|
2853
|
+
potentialBarriers: []
|
|
2588
2854
|
},
|
|
2589
2855
|
exemplarProjects: defaultExemplars,
|
|
2590
2856
|
technicalBottlenecks: defaultBottlenecks,
|
|
2591
2857
|
groundingSources: scrapedSnippets.map((s) => ({ title: s.title, url: s.url })),
|
|
2592
|
-
markdownReport: ""
|
|
2858
|
+
markdownReport: "",
|
|
2859
|
+
codeSnippets: preservedCodeSnippets.slice(0, 4),
|
|
2860
|
+
pinoutTables: preservedTables.slice(0, 4),
|
|
2861
|
+
keyExcerpts: preservedKeyExcerpts.slice(0, 6)
|
|
2593
2862
|
};
|
|
2594
2863
|
if (llm && scrapedSnippets.length > 0) {
|
|
2595
2864
|
try {
|
|
@@ -2598,8 +2867,19 @@ var TechnicalGroundingEngine = class {
|
|
|
2598
2867
|
`Hardware: ${hardware || "N/A"}, Software/Env: ${software || "N/A"}, Student Age Band: ${ageBand[0]}-${ageBand[1]} years old.`,
|
|
2599
2868
|
`
|
|
2600
2869
|
Harvested Sources:
|
|
2601
|
-
${scrapedSnippets.map((s, i) =>
|
|
2602
|
-
${s.
|
|
2870
|
+
${scrapedSnippets.map((s, i) => {
|
|
2871
|
+
const parts = [`[Source ${i + 1}: ${s.title} (${s.url})]
|
|
2872
|
+
${s.excerpt}`];
|
|
2873
|
+
if (s.codeBlocks && s.codeBlocks.length > 0) {
|
|
2874
|
+
parts.push("Code Snippets:\n" + s.codeBlocks.map((c) => `\`\`\`${c.language || ""}
|
|
2875
|
+
${c.code}
|
|
2876
|
+
\`\`\``).join("\n"));
|
|
2877
|
+
}
|
|
2878
|
+
if (s.tables && s.tables.length > 0) {
|
|
2879
|
+
parts.push("Tables/Pinouts:\n" + s.tables.join("\n\n"));
|
|
2880
|
+
}
|
|
2881
|
+
return parts.join("\n");
|
|
2882
|
+
}).join("\n\n")}`,
|
|
2603
2883
|
`
|
|
2604
2884
|
Synthesize a strictly valid JSON object with:`,
|
|
2605
2885
|
`{`,
|
|
@@ -2638,7 +2918,7 @@ Synthesize a strictly valid JSON object with:`,
|
|
|
2638
2918
|
if (Array.isArray(parsed.technicalBottlenecks)) factsheet.technicalBottlenecks = parsed.technicalBottlenecks.map(String);
|
|
2639
2919
|
}
|
|
2640
2920
|
} catch (err) {
|
|
2641
|
-
console.warn("[TechnicalGroundingEngine] LLM synthesis
|
|
2921
|
+
console.warn("[TechnicalGroundingEngine] LLM synthesis error:", err?.message || err);
|
|
2642
2922
|
}
|
|
2643
2923
|
}
|
|
2644
2924
|
factsheet.markdownReport = this.renderMarkdownReport(factsheet);
|
|
@@ -2670,6 +2950,22 @@ Synthesize a strictly valid JSON object with:`,
|
|
|
2670
2950
|
## 5. Grounding Citations`,
|
|
2671
2951
|
...fs.groundingSources.map((s, idx) => `${idx + 1}. [${s.title}](${s.url})`)
|
|
2672
2952
|
];
|
|
2953
|
+
if (fs.codeSnippets && fs.codeSnippets.length > 0) {
|
|
2954
|
+
lines.push(
|
|
2955
|
+
`
|
|
2956
|
+
## 6. Verified Code Snippets & Syntax Reference`,
|
|
2957
|
+
...fs.codeSnippets.map((c) => `\`\`\`${c.language || "text"}
|
|
2958
|
+
${c.code}
|
|
2959
|
+
\`\`\``)
|
|
2960
|
+
);
|
|
2961
|
+
}
|
|
2962
|
+
if (fs.pinoutTables && fs.pinoutTables.length > 0) {
|
|
2963
|
+
lines.push(
|
|
2964
|
+
`
|
|
2965
|
+
## 7. Pinout & Specification Tables`,
|
|
2966
|
+
...fs.pinoutTables
|
|
2967
|
+
);
|
|
2968
|
+
}
|
|
2673
2969
|
return lines.join("\n");
|
|
2674
2970
|
}
|
|
2675
2971
|
};
|