@thanh01.pmt/research-kit 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/academic/index.js +2 -1
- package/dist/{catalogBuilder-nrGYEY4b.d.cts → catalogBuilder-BaVRvg-9.d.cts} +8 -0
- package/dist/{catalogBuilder-2i5KASUF.d.ts → catalogBuilder-DHV6y1gD.d.ts} +8 -0
- package/dist/chunk-AKEI7H2C.js +223 -0
- package/dist/chunk-AKEI7H2C.js.map +1 -0
- package/dist/{chunk-QK3LVGI7.js → chunk-HZNSZDSM.js} +76 -2
- package/dist/chunk-HZNSZDSM.js.map +1 -0
- package/dist/{chunk-XDDTHWJW.js → chunk-RW7VS7NI.js} +5 -4
- package/dist/chunk-RW7VS7NI.js.map +1 -0
- package/dist/{chunk-DOXSPRSP.js → chunk-UKGR45VZ.js} +3 -223
- package/dist/chunk-UKGR45VZ.js.map +1 -0
- package/dist/harvester/index.cjs +294 -0
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +1 -1
- package/dist/harvester/index.d.ts +1 -1
- package/dist/harvester/index.js +2 -1
- package/dist/index.cjs +72 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +8 -6
- package/dist/index.js.map +1 -1
- package/dist/pipeline/index.cjs +292 -220
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +1 -1
- package/dist/pipeline/index.d.ts +1 -1
- package/dist/pipeline/index.js +4 -3
- package/package.json +8 -9
- package/dist/chunk-DOXSPRSP.js.map +0 -1
- package/dist/chunk-QK3LVGI7.js.map +0 -1
- package/dist/chunk-XDDTHWJW.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { T as TechnicalDomain, S as ScrapedDocument } from '../webScraper-yO9P-oLb.cjs';
|
|
2
2
|
export { D as DomainClassificationResult, E as ExtractedCodeBlock, a as WebScraper, W as WebScraperOptions, c as classifyTechnicalDomain, e as extractSubstantiveExcerpts, i as inferPrimaryCodeLanguage } from '../webScraper-yO9P-oLb.cjs';
|
|
3
|
-
export { C as CatalogBuilder, H as HarvestedResourceItem, M as MultiSearchOptions, a as MultiSourceSearchEngine, R as ResourceCatalogBundle, S as SearchHit } from '../catalogBuilder-
|
|
3
|
+
export { C as CatalogBuilder, H as HarvestedResourceItem, M as MultiSearchOptions, a as MultiSourceSearchEngine, R as ResourceCatalogBundle, S as SearchHit } from '../catalogBuilder-BaVRvg-9.cjs';
|
|
4
4
|
import { E as ExtractedConceptNode, C as CourseResearchContext, L as LlmTextFn } from '../index-DlngvZBr.cjs';
|
|
5
5
|
import 'zod';
|
|
6
6
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { T as TechnicalDomain, S as ScrapedDocument } from '../webScraper-DBFUIqoO.js';
|
|
2
2
|
export { D as DomainClassificationResult, E as ExtractedCodeBlock, a as WebScraper, W as WebScraperOptions, c as classifyTechnicalDomain, e as extractSubstantiveExcerpts, i as inferPrimaryCodeLanguage } from '../webScraper-DBFUIqoO.js';
|
|
3
|
-
export { C as CatalogBuilder, H as HarvestedResourceItem, M as MultiSearchOptions, a as MultiSourceSearchEngine, R as ResourceCatalogBundle, S as SearchHit } from '../catalogBuilder-
|
|
3
|
+
export { C as CatalogBuilder, H as HarvestedResourceItem, M as MultiSearchOptions, a as MultiSourceSearchEngine, R as ResourceCatalogBundle, S as SearchHit } from '../catalogBuilder-DHV6y1gD.js';
|
|
4
4
|
import { E as ExtractedConceptNode, C as CourseResearchContext, L as LlmTextFn } from '../index-DlngvZBr.js';
|
|
5
5
|
import 'zod';
|
|
6
6
|
|
package/dist/harvester/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import '../chunk-LSMSIPTZ.js';
|
|
2
|
-
export { CatalogBuilder, ECOMMERCE_AND_COMMERCIAL_PATTERNS, ECOMMERCE_DOMAINS, HealthAuditor, MultiSourceSearchEngine, PedagogicalTermExtractor, classifyTechnicalDomain, evaluateTechnicalDensity, inferPrimaryCodeLanguage, isCommercialOrNoise, kahnTopologicalSort, transitiveReduction } from '../chunk-
|
|
2
|
+
export { CatalogBuilder, ECOMMERCE_AND_COMMERCIAL_PATTERNS, ECOMMERCE_DOMAINS, HealthAuditor, MultiSourceSearchEngine, PedagogicalTermExtractor, classifyTechnicalDomain, evaluateTechnicalDensity, inferPrimaryCodeLanguage, isCommercialOrNoise, kahnTopologicalSort, transitiveReduction } from '../chunk-HZNSZDSM.js';
|
|
3
|
+
import '../chunk-AKEI7H2C.js';
|
|
3
4
|
export { WebScraper, YouTubeIngestionService, extractSubstantiveExcerpts, extractYouTubeVideoId, formatTimestamp, isYouTubeUrl, stitchCaptionSegments } from '../chunk-ON6VFCJO.js';
|
|
4
5
|
//# sourceMappingURL=index.js.map
|
|
5
6
|
//# sourceMappingURL=index.js.map
|
package/dist/index.cjs
CHANGED
|
@@ -700,14 +700,25 @@ var MultiSourceSearchEngine = class {
|
|
|
700
700
|
tavilyKey;
|
|
701
701
|
exaKey;
|
|
702
702
|
serperKey;
|
|
703
|
+
searxngHost;
|
|
704
|
+
searxngHeaders;
|
|
703
705
|
allowEnvFallback;
|
|
704
706
|
customAdapter;
|
|
707
|
+
academicEngine;
|
|
705
708
|
constructor(options = {}) {
|
|
706
709
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
707
710
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
708
711
|
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
709
712
|
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
713
|
+
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
714
|
+
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
715
|
+
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
716
|
+
this.searxngHeaders = {
|
|
717
|
+
...options.searxngHeaders || {},
|
|
718
|
+
...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
|
|
719
|
+
};
|
|
710
720
|
this.customAdapter = options.customAdapter;
|
|
721
|
+
this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
|
|
711
722
|
}
|
|
712
723
|
setCustomAdapter(adapter) {
|
|
713
724
|
this.customAdapter = adapter;
|
|
@@ -717,6 +728,12 @@ var MultiSourceSearchEngine = class {
|
|
|
717
728
|
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
718
729
|
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
719
730
|
}
|
|
731
|
+
setSearxng(host, headers) {
|
|
732
|
+
this.searxngHost = host;
|
|
733
|
+
if (headers) {
|
|
734
|
+
this.searxngHeaders = { ...this.searxngHeaders, ...headers };
|
|
735
|
+
}
|
|
736
|
+
}
|
|
720
737
|
async search(query, options = {}) {
|
|
721
738
|
const limit = options.limit ?? 5;
|
|
722
739
|
const adapter = options.customAdapter || this.customAdapter;
|
|
@@ -737,9 +754,30 @@ var MultiSourceSearchEngine = class {
|
|
|
737
754
|
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
738
755
|
}
|
|
739
756
|
}
|
|
757
|
+
if (preferred === "academic") {
|
|
758
|
+
try {
|
|
759
|
+
const papers = await this.academicEngine.searchPapers(query, { limit });
|
|
760
|
+
if (papers.length > 0) {
|
|
761
|
+
return papers.map((p) => ({
|
|
762
|
+
title: p.title,
|
|
763
|
+
url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
|
|
764
|
+
content: p.tldr || p.abstract || "",
|
|
765
|
+
score: p.citationCount,
|
|
766
|
+
engine: "academic"
|
|
767
|
+
}));
|
|
768
|
+
}
|
|
769
|
+
} catch (e) {
|
|
770
|
+
console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
740
773
|
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
741
774
|
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
742
775
|
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
776
|
+
const searxngHost = options.searxngHost || this.searxngHost;
|
|
777
|
+
if (preferred === "searxng" && searxngHost) {
|
|
778
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
779
|
+
if (searxHits.length > 0) return searxHits;
|
|
780
|
+
}
|
|
743
781
|
if (preferred === "exa" && exaKey) {
|
|
744
782
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
745
783
|
if (exaHits.length > 0) return exaHits;
|
|
@@ -794,10 +832,44 @@ var MultiSourceSearchEngine = class {
|
|
|
794
832
|
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
795
833
|
if (serperHits.length > 0) return serperHits;
|
|
796
834
|
}
|
|
835
|
+
if (searxngHost) {
|
|
836
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
837
|
+
if (searxHits.length > 0) return searxHits;
|
|
838
|
+
}
|
|
797
839
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
798
840
|
if (ddgHits.length > 0) return ddgHits;
|
|
799
841
|
return [];
|
|
800
842
|
}
|
|
843
|
+
async querySearxng(query, searxngHost, limit, customHeaders) {
|
|
844
|
+
try {
|
|
845
|
+
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
846
|
+
const searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
847
|
+
const headers = {
|
|
848
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
849
|
+
...this.searxngHeaders || {},
|
|
850
|
+
...customHeaders || {}
|
|
851
|
+
};
|
|
852
|
+
const res = await fetch(searchUrl, {
|
|
853
|
+
headers,
|
|
854
|
+
signal: AbortSignal.timeout(1e4)
|
|
855
|
+
});
|
|
856
|
+
if (res.ok) {
|
|
857
|
+
const data = await res.json();
|
|
858
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
859
|
+
return data.results.slice(0, limit).map((r) => ({
|
|
860
|
+
title: r.title || "Untitled",
|
|
861
|
+
url: r.url,
|
|
862
|
+
content: r.content || r.snippet || "",
|
|
863
|
+
score: r.score,
|
|
864
|
+
engine: "searxng"
|
|
865
|
+
}));
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
} catch (e) {
|
|
869
|
+
console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
|
|
870
|
+
}
|
|
871
|
+
return [];
|
|
872
|
+
}
|
|
801
873
|
async queryExa(query, exaKey, limit, domainFilter) {
|
|
802
874
|
try {
|
|
803
875
|
const body = {
|