@thanh01.pmt/research-kit 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/academic/index.js +2 -1
- package/dist/{catalogBuilder-nrGYEY4b.d.cts → catalogBuilder-BaVRvg-9.d.cts} +8 -0
- package/dist/{catalogBuilder-2i5KASUF.d.ts → catalogBuilder-DHV6y1gD.d.ts} +8 -0
- package/dist/chunk-AKEI7H2C.js +223 -0
- package/dist/chunk-AKEI7H2C.js.map +1 -0
- package/dist/{chunk-QK3LVGI7.js → chunk-HZNSZDSM.js} +76 -2
- package/dist/chunk-HZNSZDSM.js.map +1 -0
- package/dist/{chunk-XDDTHWJW.js → chunk-RW7VS7NI.js} +5 -4
- package/dist/chunk-RW7VS7NI.js.map +1 -0
- package/dist/{chunk-DOXSPRSP.js → chunk-UKGR45VZ.js} +3 -223
- package/dist/chunk-UKGR45VZ.js.map +1 -0
- package/dist/harvester/index.cjs +294 -0
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +1 -1
- package/dist/harvester/index.d.ts +1 -1
- package/dist/harvester/index.js +2 -1
- package/dist/index.cjs +72 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +8 -6
- package/dist/index.js.map +1 -1
- package/dist/pipeline/index.cjs +292 -220
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +1 -1
- package/dist/pipeline/index.d.ts +1 -1
- package/dist/pipeline/index.js +4 -3
- package/package.json +8 -9
- package/dist/chunk-DOXSPRSP.js.map +0 -1
- package/dist/chunk-QK3LVGI7.js.map +0 -1
- package/dist/chunk-XDDTHWJW.js.map +0 -1
package/dist/pipeline/index.cjs
CHANGED
|
@@ -329,19 +329,250 @@ ${options.synthesizedInsights.executiveSummary}`);
|
|
|
329
329
|
}
|
|
330
330
|
};
|
|
331
331
|
|
|
332
|
+
// src/academic/arxivClient.ts
|
|
333
|
+
var ArXivSearchEngine = class {
|
|
334
|
+
/**
|
|
335
|
+
* Search ArXiv repository using the public atom XML API
|
|
336
|
+
*/
|
|
337
|
+
async search(query, maxResults = 5) {
|
|
338
|
+
const papers = [];
|
|
339
|
+
try {
|
|
340
|
+
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
341
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
342
|
+
if (res.ok) {
|
|
343
|
+
const xmlText = await res.text();
|
|
344
|
+
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
345
|
+
for (const entry of entryMatches) {
|
|
346
|
+
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
347
|
+
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
348
|
+
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
349
|
+
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
350
|
+
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
351
|
+
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
352
|
+
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
353
|
+
if (titleMatch) {
|
|
354
|
+
papers.push({
|
|
355
|
+
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
356
|
+
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
357
|
+
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
358
|
+
authors: authorMatches,
|
|
359
|
+
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
360
|
+
citationCount: 0,
|
|
361
|
+
venue: "arXiv preprint",
|
|
362
|
+
url: arxivUrl,
|
|
363
|
+
pdfUrl,
|
|
364
|
+
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
} catch (e) {
|
|
370
|
+
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
371
|
+
}
|
|
372
|
+
return papers;
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
// src/academic/semanticScholar.ts
|
|
377
|
+
zod.z.object({
|
|
378
|
+
id: zod.z.string(),
|
|
379
|
+
title: zod.z.string(),
|
|
380
|
+
abstract: zod.z.string().optional(),
|
|
381
|
+
authors: zod.z.array(zod.z.string()).default([]),
|
|
382
|
+
year: zod.z.number().optional(),
|
|
383
|
+
citationCount: zod.z.number().default(0),
|
|
384
|
+
venue: zod.z.string().optional(),
|
|
385
|
+
doi: zod.z.string().optional(),
|
|
386
|
+
url: zod.z.string().optional(),
|
|
387
|
+
pdfUrl: zod.z.string().optional(),
|
|
388
|
+
tldr: zod.z.string().optional(),
|
|
389
|
+
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
390
|
+
});
|
|
391
|
+
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
392
|
+
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
393
|
+
const entries = Object.entries(invertedIndex);
|
|
394
|
+
if (entries.length === 0) return void 0;
|
|
395
|
+
let maxIndex = 0;
|
|
396
|
+
for (const [, positions] of entries) {
|
|
397
|
+
if (Array.isArray(positions)) {
|
|
398
|
+
for (const p of positions) {
|
|
399
|
+
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
404
|
+
const words = new Array(maxIndex + 1);
|
|
405
|
+
for (const [word, positions] of entries) {
|
|
406
|
+
if (Array.isArray(positions)) {
|
|
407
|
+
for (const p of positions) {
|
|
408
|
+
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
409
|
+
words[p] = word;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
415
|
+
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
416
|
+
}
|
|
417
|
+
var AcademicSearchEngine = class {
|
|
418
|
+
apiKey;
|
|
419
|
+
allowEnvFallback;
|
|
420
|
+
customAdapter;
|
|
421
|
+
arxivEngine;
|
|
422
|
+
constructor(options = {}) {
|
|
423
|
+
this.arxivEngine = new ArXivSearchEngine();
|
|
424
|
+
if (typeof options === "string") {
|
|
425
|
+
this.apiKey = options;
|
|
426
|
+
this.allowEnvFallback = true;
|
|
427
|
+
} else {
|
|
428
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
429
|
+
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
430
|
+
this.customAdapter = options.customAdapter;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
setCustomAdapter(adapter) {
|
|
434
|
+
this.customAdapter = adapter;
|
|
435
|
+
}
|
|
436
|
+
setApiKey(apiKey) {
|
|
437
|
+
this.apiKey = apiKey;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
441
|
+
*/
|
|
442
|
+
async searchPapers(query, options = {}) {
|
|
443
|
+
const adapter = options.customAdapter || this.customAdapter;
|
|
444
|
+
if (adapter) {
|
|
445
|
+
try {
|
|
446
|
+
const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
|
|
447
|
+
if (Array.isArray(customPapers) && customPapers.length > 0) {
|
|
448
|
+
return customPapers;
|
|
449
|
+
}
|
|
450
|
+
} catch (e) {
|
|
451
|
+
console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
const limit = options.limit ?? 5;
|
|
455
|
+
const papers = [];
|
|
456
|
+
const seenTitles = /* @__PURE__ */ new Set();
|
|
457
|
+
const pushUnique = (p) => {
|
|
458
|
+
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
459
|
+
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
460
|
+
seenTitles.add(normTitle);
|
|
461
|
+
papers.push(p);
|
|
462
|
+
};
|
|
463
|
+
try {
|
|
464
|
+
const apiKey = options.apiKey || this.apiKey;
|
|
465
|
+
const headers = { "Content-Type": "application/json" };
|
|
466
|
+
if (apiKey) {
|
|
467
|
+
headers["x-api-key"] = apiKey;
|
|
468
|
+
}
|
|
469
|
+
const params = new URLSearchParams({
|
|
470
|
+
query,
|
|
471
|
+
limit: limit.toString(),
|
|
472
|
+
fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
|
|
473
|
+
});
|
|
474
|
+
if (options.yearRange) {
|
|
475
|
+
params.append("year", options.yearRange);
|
|
476
|
+
}
|
|
477
|
+
const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
|
|
478
|
+
headers,
|
|
479
|
+
signal: AbortSignal.timeout(6e3)
|
|
480
|
+
});
|
|
481
|
+
if (res.ok) {
|
|
482
|
+
const data = await res.json();
|
|
483
|
+
if (Array.isArray(data?.data)) {
|
|
484
|
+
for (const item of data.data) {
|
|
485
|
+
pushUnique({
|
|
486
|
+
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
487
|
+
title: item.title || "Untitled",
|
|
488
|
+
abstract: item.abstract || void 0,
|
|
489
|
+
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
490
|
+
year: item.year,
|
|
491
|
+
citationCount: item.citationCount || 0,
|
|
492
|
+
venue: item.venue || void 0,
|
|
493
|
+
doi: item.externalIds?.DOI,
|
|
494
|
+
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
495
|
+
pdfUrl: item.openAccessPdf?.url,
|
|
496
|
+
tldr: item.tldr?.text,
|
|
497
|
+
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
if (papers.length >= limit) return papers.slice(0, limit);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
} catch (e) {
|
|
504
|
+
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
505
|
+
}
|
|
506
|
+
if (papers.length < limit) {
|
|
507
|
+
try {
|
|
508
|
+
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
509
|
+
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
510
|
+
signal: AbortSignal.timeout(6e3)
|
|
511
|
+
});
|
|
512
|
+
if (res.ok) {
|
|
513
|
+
const data = await res.json();
|
|
514
|
+
if (Array.isArray(data?.results)) {
|
|
515
|
+
for (const item of data.results) {
|
|
516
|
+
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
517
|
+
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
518
|
+
pushUnique({
|
|
519
|
+
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
520
|
+
title: item.title || "Untitled",
|
|
521
|
+
abstract: abstractText,
|
|
522
|
+
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
523
|
+
year: item.publication_year,
|
|
524
|
+
citationCount: item.cited_by_count || 0,
|
|
525
|
+
venue: item.primary_location?.source?.display_name,
|
|
526
|
+
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
527
|
+
url: item.doi || item.id,
|
|
528
|
+
pdfUrl: item.open_access?.oa_url,
|
|
529
|
+
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
} catch (e) {
|
|
535
|
+
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
539
|
+
try {
|
|
540
|
+
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
541
|
+
for (const p of arxivPapers) {
|
|
542
|
+
pushUnique(p);
|
|
543
|
+
}
|
|
544
|
+
} catch (e) {
|
|
545
|
+
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return papers.slice(0, limit);
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
|
|
332
552
|
// src/harvester/multiSourceSearch.ts
|
|
333
553
|
var MultiSourceSearchEngine = class {
|
|
334
554
|
tavilyKey;
|
|
335
555
|
exaKey;
|
|
336
556
|
serperKey;
|
|
557
|
+
searxngHost;
|
|
558
|
+
searxngHeaders;
|
|
337
559
|
allowEnvFallback;
|
|
338
560
|
customAdapter;
|
|
561
|
+
academicEngine;
|
|
339
562
|
constructor(options = {}) {
|
|
340
563
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
341
564
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
342
565
|
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
343
566
|
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
567
|
+
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
568
|
+
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
569
|
+
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
570
|
+
this.searxngHeaders = {
|
|
571
|
+
...options.searxngHeaders || {},
|
|
572
|
+
...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
|
|
573
|
+
};
|
|
344
574
|
this.customAdapter = options.customAdapter;
|
|
575
|
+
this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
|
|
345
576
|
}
|
|
346
577
|
setCustomAdapter(adapter) {
|
|
347
578
|
this.customAdapter = adapter;
|
|
@@ -351,6 +582,12 @@ var MultiSourceSearchEngine = class {
|
|
|
351
582
|
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
352
583
|
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
353
584
|
}
|
|
585
|
+
setSearxng(host, headers) {
|
|
586
|
+
this.searxngHost = host;
|
|
587
|
+
if (headers) {
|
|
588
|
+
this.searxngHeaders = { ...this.searxngHeaders, ...headers };
|
|
589
|
+
}
|
|
590
|
+
}
|
|
354
591
|
async search(query, options = {}) {
|
|
355
592
|
const limit = options.limit ?? 5;
|
|
356
593
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -371,9 +608,30 @@ var MultiSourceSearchEngine = class {
|
|
|
371
608
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
372
609
|
}
|
|
373
610
|
}
|
|
611
|
+
if (preferred === "academic") {
|
|
612
|
+
try {
|
|
613
|
+
const papers = await this.academicEngine.searchPapers(query, { limit });
|
|
614
|
+
if (papers.length > 0) {
|
|
615
|
+
return papers.map((p) => ({
|
|
616
|
+
title: p.title,
|
|
617
|
+
url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
|
|
618
|
+
content: p.tldr || p.abstract || "",
|
|
619
|
+
score: p.citationCount,
|
|
620
|
+
engine: "academic"
|
|
621
|
+
}));
|
|
622
|
+
}
|
|
623
|
+
} catch (e) {
|
|
624
|
+
console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
374
627
|
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
375
628
|
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
376
629
|
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
630
|
+
const searxngHost = options.searxngHost || this.searxngHost;
|
|
631
|
+
if (preferred === "searxng" && searxngHost) {
|
|
632
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
633
|
+
if (searxHits.length > 0) return searxHits;
|
|
634
|
+
}
|
|
377
635
|
if (preferred === "exa" && exaKey) {
|
|
378
636
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
379
637
|
if (exaHits.length > 0) return exaHits;
|
|
@@ -428,10 +686,44 @@ var MultiSourceSearchEngine = class {
|
|
|
428
686
|
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
429
687
|
if (serperHits.length > 0) return serperHits;
|
|
430
688
|
}
|
|
689
|
+
if (searxngHost) {
|
|
690
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
691
|
+
if (searxHits.length > 0) return searxHits;
|
|
692
|
+
}
|
|
431
693
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
432
694
|
if (ddgHits.length > 0) return ddgHits;
|
|
433
695
|
return [];
|
|
434
696
|
}
|
|
697
|
+
async querySearxng(query, searxngHost, limit, customHeaders) {
|
|
698
|
+
try {
|
|
699
|
+
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
700
|
+
const searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
701
|
+
const headers = {
|
|
702
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
703
|
+
...this.searxngHeaders || {},
|
|
704
|
+
...customHeaders || {}
|
|
705
|
+
};
|
|
706
|
+
const res = await fetch(searchUrl, {
|
|
707
|
+
headers,
|
|
708
|
+
signal: AbortSignal.timeout(1e4)
|
|
709
|
+
});
|
|
710
|
+
if (res.ok) {
|
|
711
|
+
const data = await res.json();
|
|
712
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
713
|
+
return data.results.slice(0, limit).map((r) => ({
|
|
714
|
+
title: r.title || "Untitled",
|
|
715
|
+
url: r.url,
|
|
716
|
+
content: r.content || r.snippet || "",
|
|
717
|
+
score: r.score,
|
|
718
|
+
engine: "searxng"
|
|
719
|
+
}));
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
} catch (e) {
|
|
723
|
+
console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
|
|
724
|
+
}
|
|
725
|
+
return [];
|
|
726
|
+
}
|
|
435
727
|
async queryExa(query, exaKey, limit, domainFilter) {
|
|
436
728
|
try {
|
|
437
729
|
const body = {
|
|
@@ -1693,226 +1985,6 @@ var ProvenanceTracker = class {
|
|
|
1693
1985
|
}
|
|
1694
1986
|
};
|
|
1695
1987
|
|
|
1696
|
-
// src/academic/arxivClient.ts
|
|
1697
|
-
var ArXivSearchEngine = class {
|
|
1698
|
-
/**
|
|
1699
|
-
* Search ArXiv repository using the public atom XML API
|
|
1700
|
-
*/
|
|
1701
|
-
async search(query, maxResults = 5) {
|
|
1702
|
-
const papers = [];
|
|
1703
|
-
try {
|
|
1704
|
-
const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
|
|
1705
|
-
const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
|
|
1706
|
-
if (res.ok) {
|
|
1707
|
-
const xmlText = await res.text();
|
|
1708
|
-
const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
|
1709
|
-
for (const entry of entryMatches) {
|
|
1710
|
-
const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
|
|
1711
|
-
const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
|
|
1712
|
-
const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
|
|
1713
|
-
const publishedMatch = entry.match(/<published>(\d{4})/);
|
|
1714
|
-
const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
|
|
1715
|
-
const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
|
|
1716
|
-
const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
|
|
1717
|
-
if (titleMatch) {
|
|
1718
|
-
papers.push({
|
|
1719
|
-
id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
|
|
1720
|
-
title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
|
|
1721
|
-
abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
|
|
1722
|
-
authors: authorMatches,
|
|
1723
|
-
year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
|
|
1724
|
-
citationCount: 0,
|
|
1725
|
-
venue: "arXiv preprint",
|
|
1726
|
-
url: arxivUrl,
|
|
1727
|
-
pdfUrl,
|
|
1728
|
-
fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
|
|
1729
|
-
});
|
|
1730
|
-
}
|
|
1731
|
-
}
|
|
1732
|
-
}
|
|
1733
|
-
} catch (e) {
|
|
1734
|
-
console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
|
|
1735
|
-
}
|
|
1736
|
-
return papers;
|
|
1737
|
-
}
|
|
1738
|
-
};
|
|
1739
|
-
|
|
1740
|
-
// src/academic/semanticScholar.ts
|
|
1741
|
-
zod.z.object({
|
|
1742
|
-
id: zod.z.string(),
|
|
1743
|
-
title: zod.z.string(),
|
|
1744
|
-
abstract: zod.z.string().optional(),
|
|
1745
|
-
authors: zod.z.array(zod.z.string()).default([]),
|
|
1746
|
-
year: zod.z.number().optional(),
|
|
1747
|
-
citationCount: zod.z.number().default(0),
|
|
1748
|
-
venue: zod.z.string().optional(),
|
|
1749
|
-
doi: zod.z.string().optional(),
|
|
1750
|
-
url: zod.z.string().optional(),
|
|
1751
|
-
pdfUrl: zod.z.string().optional(),
|
|
1752
|
-
tldr: zod.z.string().optional(),
|
|
1753
|
-
fieldsOfStudy: zod.z.array(zod.z.string()).default([])
|
|
1754
|
-
});
|
|
1755
|
-
function reconstructOpenAlexAbstract(invertedIndex) {
|
|
1756
|
-
if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
|
|
1757
|
-
const entries = Object.entries(invertedIndex);
|
|
1758
|
-
if (entries.length === 0) return void 0;
|
|
1759
|
-
let maxIndex = 0;
|
|
1760
|
-
for (const [, positions] of entries) {
|
|
1761
|
-
if (Array.isArray(positions)) {
|
|
1762
|
-
for (const p of positions) {
|
|
1763
|
-
if (typeof p === "number" && p > maxIndex) maxIndex = p;
|
|
1764
|
-
}
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
if (maxIndex > 5e3) maxIndex = 5e3;
|
|
1768
|
-
const words = new Array(maxIndex + 1);
|
|
1769
|
-
for (const [word, positions] of entries) {
|
|
1770
|
-
if (Array.isArray(positions)) {
|
|
1771
|
-
for (const p of positions) {
|
|
1772
|
-
if (typeof p === "number" && p >= 0 && p <= maxIndex) {
|
|
1773
|
-
words[p] = word;
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
}
|
|
1777
|
-
}
|
|
1778
|
-
const reconstructed = words.filter(Boolean).join(" ").trim();
|
|
1779
|
-
return reconstructed.length > 0 ? reconstructed : void 0;
|
|
1780
|
-
}
|
|
1781
|
-
var AcademicSearchEngine = class {
|
|
1782
|
-
apiKey;
|
|
1783
|
-
allowEnvFallback;
|
|
1784
|
-
customAdapter;
|
|
1785
|
-
arxivEngine;
|
|
1786
|
-
constructor(options = {}) {
|
|
1787
|
-
this.arxivEngine = new ArXivSearchEngine();
|
|
1788
|
-
if (typeof options === "string") {
|
|
1789
|
-
this.apiKey = options;
|
|
1790
|
-
this.allowEnvFallback = true;
|
|
1791
|
-
} else {
|
|
1792
|
-
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
1793
|
-
this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
|
|
1794
|
-
this.customAdapter = options.customAdapter;
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
setCustomAdapter(adapter) {
|
|
1798
|
-
this.customAdapter = adapter;
|
|
1799
|
-
}
|
|
1800
|
-
setApiKey(apiKey) {
|
|
1801
|
-
this.apiKey = apiKey;
|
|
1802
|
-
}
|
|
1803
|
-
/**
|
|
1804
|
-
* Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
|
|
1805
|
-
*/
|
|
1806
|
-
async searchPapers(query, options = {}) {
|
|
1807
|
-
const adapter = options.customAdapter || this.customAdapter;
|
|
1808
|
-
if (adapter) {
|
|
1809
|
-
try {
|
|
1810
|
-
const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
|
|
1811
|
-
if (Array.isArray(customPapers) && customPapers.length > 0) {
|
|
1812
|
-
return customPapers;
|
|
1813
|
-
}
|
|
1814
|
-
} catch (e) {
|
|
1815
|
-
console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
|
|
1816
|
-
}
|
|
1817
|
-
}
|
|
1818
|
-
const limit = options.limit ?? 5;
|
|
1819
|
-
const papers = [];
|
|
1820
|
-
const seenTitles = /* @__PURE__ */ new Set();
|
|
1821
|
-
const pushUnique = (p) => {
|
|
1822
|
-
const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
|
|
1823
|
-
if (!normTitle || seenTitles.has(normTitle)) return;
|
|
1824
|
-
seenTitles.add(normTitle);
|
|
1825
|
-
papers.push(p);
|
|
1826
|
-
};
|
|
1827
|
-
try {
|
|
1828
|
-
const apiKey = options.apiKey || this.apiKey;
|
|
1829
|
-
const headers = { "Content-Type": "application/json" };
|
|
1830
|
-
if (apiKey) {
|
|
1831
|
-
headers["x-api-key"] = apiKey;
|
|
1832
|
-
}
|
|
1833
|
-
const params = new URLSearchParams({
|
|
1834
|
-
query,
|
|
1835
|
-
limit: limit.toString(),
|
|
1836
|
-
fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
|
|
1837
|
-
});
|
|
1838
|
-
if (options.yearRange) {
|
|
1839
|
-
params.append("year", options.yearRange);
|
|
1840
|
-
}
|
|
1841
|
-
const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
|
|
1842
|
-
headers,
|
|
1843
|
-
signal: AbortSignal.timeout(6e3)
|
|
1844
|
-
});
|
|
1845
|
-
if (res.ok) {
|
|
1846
|
-
const data = await res.json();
|
|
1847
|
-
if (Array.isArray(data?.data)) {
|
|
1848
|
-
for (const item of data.data) {
|
|
1849
|
-
pushUnique({
|
|
1850
|
-
id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
|
|
1851
|
-
title: item.title || "Untitled",
|
|
1852
|
-
abstract: item.abstract || void 0,
|
|
1853
|
-
authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
|
|
1854
|
-
year: item.year,
|
|
1855
|
-
citationCount: item.citationCount || 0,
|
|
1856
|
-
venue: item.venue || void 0,
|
|
1857
|
-
doi: item.externalIds?.DOI,
|
|
1858
|
-
url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
|
|
1859
|
-
pdfUrl: item.openAccessPdf?.url,
|
|
1860
|
-
tldr: item.tldr?.text,
|
|
1861
|
-
fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
|
|
1862
|
-
});
|
|
1863
|
-
}
|
|
1864
|
-
if (papers.length >= limit) return papers.slice(0, limit);
|
|
1865
|
-
}
|
|
1866
|
-
}
|
|
1867
|
-
} catch (e) {
|
|
1868
|
-
console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
|
|
1869
|
-
}
|
|
1870
|
-
if (papers.length < limit) {
|
|
1871
|
-
try {
|
|
1872
|
-
const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
|
|
1873
|
-
headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
|
|
1874
|
-
signal: AbortSignal.timeout(6e3)
|
|
1875
|
-
});
|
|
1876
|
-
if (res.ok) {
|
|
1877
|
-
const data = await res.json();
|
|
1878
|
-
if (Array.isArray(data?.results)) {
|
|
1879
|
-
for (const item of data.results) {
|
|
1880
|
-
const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
|
|
1881
|
-
const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
|
|
1882
|
-
pushUnique({
|
|
1883
|
-
id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
|
|
1884
|
-
title: item.title || "Untitled",
|
|
1885
|
-
abstract: abstractText,
|
|
1886
|
-
authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
|
|
1887
|
-
year: item.publication_year,
|
|
1888
|
-
citationCount: item.cited_by_count || 0,
|
|
1889
|
-
venue: item.primary_location?.source?.display_name,
|
|
1890
|
-
doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
|
|
1891
|
-
url: item.doi || item.id,
|
|
1892
|
-
pdfUrl: item.open_access?.oa_url,
|
|
1893
|
-
fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
|
|
1894
|
-
});
|
|
1895
|
-
}
|
|
1896
|
-
}
|
|
1897
|
-
}
|
|
1898
|
-
} catch (e) {
|
|
1899
|
-
console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
|
|
1900
|
-
}
|
|
1901
|
-
}
|
|
1902
|
-
if (papers.length < limit && options.includeArxivFallback !== false) {
|
|
1903
|
-
try {
|
|
1904
|
-
const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
|
|
1905
|
-
for (const p of arxivPapers) {
|
|
1906
|
-
pushUnique(p);
|
|
1907
|
-
}
|
|
1908
|
-
} catch (e) {
|
|
1909
|
-
console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
|
|
1910
|
-
}
|
|
1911
|
-
}
|
|
1912
|
-
return papers.slice(0, limit);
|
|
1913
|
-
}
|
|
1914
|
-
};
|
|
1915
|
-
|
|
1916
1988
|
// src/academic/cs2023Taxonomy.ts
|
|
1917
1989
|
var CSTA_K12_STRANDS = {
|
|
1918
1990
|
CS: {
|