@thanh01.pmt/research-kit 1.2.4 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/academic/index.cjs +122 -0
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +27 -4
- package/dist/academic/index.d.ts +27 -4
- package/dist/academic/index.js +2 -1
- package/dist/agent/index.cjs +3441 -0
- package/dist/agent/index.cjs.map +1 -0
- package/dist/agent/index.d.cts +10 -0
- package/dist/agent/index.d.ts +10 -0
- package/dist/agent/index.js +10 -0
- package/dist/agent/index.js.map +1 -0
- package/dist/{catalogBuilder-B4eiLS_S.d.cts → catalogBuilder-BBCa9jqI.d.cts} +2 -2
- package/dist/{catalogBuilder-BNB_gaWP.d.ts → catalogBuilder-D-UiM37y.d.ts} +2 -2
- package/dist/chunk-2GBXVPOQ.js +3 -0
- package/dist/{chunk-J3VCGH7Z.js.map → chunk-2GBXVPOQ.js.map} +1 -1
- package/dist/chunk-2RMBLKP5.js +581 -0
- package/dist/chunk-2RMBLKP5.js.map +1 -0
- package/dist/{chunk-2XIEWCUE.js → chunk-32B5PVTQ.js} +2 -2
- package/dist/{chunk-2XIEWCUE.js.map → chunk-32B5PVTQ.js.map} +1 -1
- package/dist/chunk-LDCZ33AC.js +3 -0
- package/dist/chunk-LDCZ33AC.js.map +1 -0
- package/dist/chunk-NSY6KU4G.js +125 -0
- package/dist/chunk-NSY6KU4G.js.map +1 -0
- package/dist/{chunk-SAHBBO3U.js → chunk-RYLZYZRF.js} +4 -580
- package/dist/chunk-RYLZYZRF.js.map +1 -0
- package/dist/{chunk-GK5KI5PB.js → chunk-S3JUCDKD.js} +62 -43
- package/dist/chunk-S3JUCDKD.js.map +1 -0
- package/dist/chunk-TYQEC24G.js +3 -0
- package/dist/chunk-TYQEC24G.js.map +1 -0
- package/dist/chunk-WWQ45RQK.js +1114 -0
- package/dist/chunk-WWQ45RQK.js.map +1 -0
- package/dist/{chunk-YICMTZS5.js → chunk-ZNGU4ZYX.js} +171 -3
- package/dist/chunk-ZNGU4ZYX.js.map +1 -0
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -1
- package/dist/harvester/index.cjs +168 -1
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +5 -161
- package/dist/harvester/index.d.ts +5 -161
- package/dist/harvester/index.js +4 -2
- package/dist/index-CN8gei7k.d.ts +518 -0
- package/dist/{index-9rDiFuR4.d.ts → index-CxoD81QN.d.cts} +1 -1
- package/dist/{index-9rDiFuR4.d.cts → index-CxoD81QN.d.ts} +1 -1
- package/dist/index-onQDKDNo.d.cts +518 -0
- package/dist/index.cjs +1463 -39
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +11 -174
- package/dist/index.d.ts +11 -174
- package/dist/index.js +25 -10
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-DOzgm9Q2.d.ts → literatureReview-Bn2iEGRO.d.ts} +1 -1
- package/dist/{literatureReview-BwidpYJo.d.cts → literatureReview-Bvlqk4DB.d.cts} +1 -1
- package/dist/pipeline/index.cjs +59 -41
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +5 -4
- package/dist/pipeline/index.d.ts +5 -4
- package/dist/pipeline/index.js +3 -2
- package/dist/socialPulseHarvester-CN4MwrpY.d.cts +204 -0
- package/dist/socialPulseHarvester-IbdT3zD1.d.ts +204 -0
- package/dist/{webScraper-CAQjE46m.d.cts → webScraper-BVvNW4yj.d.cts} +1 -1
- package/dist/{webScraper-CXIpvyA6.d.ts → webScraper-C9pB5Rgd.d.ts} +1 -1
- package/package.json +6 -1
- package/dist/chunk-GK5KI5PB.js.map +0 -1
- package/dist/chunk-J3VCGH7Z.js +0 -3
- package/dist/chunk-SAHBBO3U.js.map +0 -1
- package/dist/chunk-YICMTZS5.js.map +0 -1
|
@@ -1,560 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
// src/harvester/domainClassifier.ts
|
|
4
|
-
var HARDWARE_PATTERNS = [
|
|
5
|
-
/\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
|
|
6
|
-
/\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
|
|
7
|
-
/(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
|
|
8
|
-
];
|
|
9
|
-
var PLATFORM_PATTERNS = [
|
|
10
|
-
/\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
|
|
11
|
-
/(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
|
|
12
|
-
];
|
|
13
|
-
var ACADEMIC_PATTERNS = [
|
|
14
|
-
/\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
|
|
15
|
-
/(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
|
|
16
|
-
];
|
|
17
|
-
function inferPrimaryCodeLanguage(topic, context, domain) {
|
|
18
|
-
const combined = [
|
|
19
|
-
topic,
|
|
20
|
-
context?.courseTitle,
|
|
21
|
-
context?.hardwarePlatform,
|
|
22
|
-
context?.softwareEnv,
|
|
23
|
-
...context?.inScopeTopics || []
|
|
24
|
-
].filter(Boolean).join(" ").toLowerCase();
|
|
25
|
-
if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
|
|
26
|
-
if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
|
|
27
|
-
if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
|
|
28
|
-
if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
|
|
29
|
-
if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
|
|
30
|
-
if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
|
|
31
|
-
if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
|
|
32
|
-
if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
|
|
33
|
-
if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
|
|
34
|
-
if (/\b(rust|cargo)\b/.test(combined)) return "rust";
|
|
35
|
-
if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
|
|
36
|
-
if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
|
|
37
|
-
if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
|
|
38
|
-
if (domain === "hardware") return "cpp";
|
|
39
|
-
return "text";
|
|
40
|
-
}
|
|
41
|
-
function classifyTechnicalDomain(topic, explicitDomain, context) {
|
|
42
|
-
if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
|
|
43
|
-
const domain = explicitDomain.toLowerCase();
|
|
44
|
-
return {
|
|
45
|
-
domain,
|
|
46
|
-
confidence: 1,
|
|
47
|
-
rationale: "Explicitly configured by user flag.",
|
|
48
|
-
recommendedExtractors: getExtractorsForDomain(domain),
|
|
49
|
-
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
const combined = [
|
|
53
|
-
topic,
|
|
54
|
-
context?.courseTitle,
|
|
55
|
-
context?.positioning,
|
|
56
|
-
context?.hardwarePlatform,
|
|
57
|
-
context?.softwareEnv,
|
|
58
|
-
...context?.inScopeTopics || []
|
|
59
|
-
].filter(Boolean).join(" ");
|
|
60
|
-
if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
|
|
61
|
-
return {
|
|
62
|
-
domain: "hardware",
|
|
63
|
-
confidence: 0.95,
|
|
64
|
-
rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
|
|
65
|
-
recommendedExtractors: getExtractorsForDomain("hardware"),
|
|
66
|
-
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
|
|
70
|
-
return {
|
|
71
|
-
domain: "platform",
|
|
72
|
-
confidence: 0.9,
|
|
73
|
-
rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
|
|
74
|
-
recommendedExtractors: getExtractorsForDomain("platform"),
|
|
75
|
-
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
|
|
79
|
-
return {
|
|
80
|
-
domain: "academic",
|
|
81
|
-
confidence: 0.9,
|
|
82
|
-
rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
|
|
83
|
-
recommendedExtractors: getExtractorsForDomain("academic"),
|
|
84
|
-
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
return {
|
|
88
|
-
domain: "software",
|
|
89
|
-
confidence: 0.85,
|
|
90
|
-
rationale: "Topic is a programming language, game engine, client/server library, or software application.",
|
|
91
|
-
recommendedExtractors: getExtractorsForDomain("software"),
|
|
92
|
-
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
function getExtractorsForDomain(domain) {
|
|
96
|
-
switch (domain) {
|
|
97
|
-
case "hardware":
|
|
98
|
-
return ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"];
|
|
99
|
-
case "platform":
|
|
100
|
-
return ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"];
|
|
101
|
-
case "academic":
|
|
102
|
-
return ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"];
|
|
103
|
-
case "software":
|
|
104
|
-
default:
|
|
105
|
-
return ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"];
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
// src/harvester/multiSourceSearch.ts
|
|
110
|
-
var MultiSourceSearchEngine = class {
|
|
111
|
-
tavilyKey;
|
|
112
|
-
exaKey;
|
|
113
|
-
serperKey;
|
|
114
|
-
brightDataKey;
|
|
115
|
-
brightDataZone;
|
|
116
|
-
searxngHost;
|
|
117
|
-
searxngHeaders;
|
|
118
|
-
allowEnvFallback;
|
|
119
|
-
customAdapter;
|
|
120
|
-
academicEngine;
|
|
121
|
-
constructor(options = {}) {
|
|
122
|
-
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
123
|
-
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
124
|
-
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
125
|
-
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
126
|
-
this.brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || (this.allowEnvFallback ? process.env.BRIGHTDATA_API_KEY || process.env.BRIGHT_DATA_KEY : void 0);
|
|
127
|
-
this.brightDataZone = options.brightDataZone || (this.allowEnvFallback ? process.env.BRIGHTDATA_SERP_ZONE || process.env.BRIGHTDATA_UNLOCKER_ZONE : void 0) || "cli_unlocker";
|
|
128
|
-
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
129
|
-
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
130
|
-
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
131
|
-
this.searxngHeaders = {
|
|
132
|
-
...options.searxngHeaders || {},
|
|
133
|
-
...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
|
|
134
|
-
};
|
|
135
|
-
this.customAdapter = options.customAdapter;
|
|
136
|
-
this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
|
|
137
|
-
}
|
|
138
|
-
setCustomAdapter(adapter) {
|
|
139
|
-
this.customAdapter = adapter;
|
|
140
|
-
}
|
|
141
|
-
setApiKeys(keys) {
|
|
142
|
-
if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
|
|
143
|
-
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
144
|
-
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
145
|
-
if (keys.brightdata !== void 0) this.brightDataKey = keys.brightdata;
|
|
146
|
-
if (keys.brightData !== void 0) this.brightDataKey = keys.brightData;
|
|
147
|
-
}
|
|
148
|
-
setBrightData(key, zone) {
|
|
149
|
-
if (key !== void 0) this.brightDataKey = key;
|
|
150
|
-
if (zone !== void 0) this.brightDataZone = zone;
|
|
151
|
-
}
|
|
152
|
-
setSearxng(host, headers) {
|
|
153
|
-
this.searxngHost = host;
|
|
154
|
-
if (headers) {
|
|
155
|
-
this.searxngHeaders = { ...this.searxngHeaders, ...headers };
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
async search(query, options = {}) {
|
|
159
|
-
const limit = options.limit ?? 5;
|
|
160
|
-
const adapter = options.customAdapter || this.customAdapter;
|
|
161
|
-
const preferred = options.preferredEngine;
|
|
162
|
-
if (adapter && (!preferred || preferred === "custom")) {
|
|
163
|
-
try {
|
|
164
|
-
const customResults = await adapter.search(query, { limit, domainFilter: options.domainFilter });
|
|
165
|
-
if (Array.isArray(customResults) && customResults.length > 0) {
|
|
166
|
-
return customResults.map((r) => ({
|
|
167
|
-
title: r.title,
|
|
168
|
-
url: r.url,
|
|
169
|
-
content: r.content,
|
|
170
|
-
score: r.score,
|
|
171
|
-
engine: "custom"
|
|
172
|
-
}));
|
|
173
|
-
}
|
|
174
|
-
} catch (e) {
|
|
175
|
-
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
if (preferred === "academic") {
|
|
179
|
-
try {
|
|
180
|
-
const papers = await this.academicEngine.searchPapers(query, { limit });
|
|
181
|
-
if (papers.length > 0) {
|
|
182
|
-
return papers.map((p) => ({
|
|
183
|
-
title: p.title,
|
|
184
|
-
url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
|
|
185
|
-
content: p.tldr || p.abstract || "",
|
|
186
|
-
score: p.citationCount,
|
|
187
|
-
engine: "academic"
|
|
188
|
-
}));
|
|
189
|
-
}
|
|
190
|
-
} catch (e) {
|
|
191
|
-
console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
195
|
-
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
196
|
-
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
197
|
-
const brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || this.brightDataKey;
|
|
198
|
-
const brightDataZone = options.brightDataZone || this.brightDataZone;
|
|
199
|
-
const searxngHost = options.searxngHost || this.searxngHost;
|
|
200
|
-
if (preferred === "searxng" && searxngHost) {
|
|
201
|
-
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
202
|
-
if (searxHits.length > 0) return searxHits;
|
|
203
|
-
}
|
|
204
|
-
if (preferred === "brightdata" && brightDataKey) {
|
|
205
|
-
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
206
|
-
zone: brightDataZone,
|
|
207
|
-
engine: options.brightDataEngine
|
|
208
|
-
});
|
|
209
|
-
if (bdHits.length > 0) return bdHits;
|
|
210
|
-
}
|
|
211
|
-
if (preferred === "exa" && exaKey) {
|
|
212
|
-
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
213
|
-
if (exaHits.length > 0) return exaHits;
|
|
214
|
-
}
|
|
215
|
-
if (preferred === "serper" && serperKey) {
|
|
216
|
-
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
217
|
-
if (serperHits.length > 0) return serperHits;
|
|
218
|
-
}
|
|
219
|
-
if (preferred === "duckduckgo") {
|
|
220
|
-
const ddgHits2 = await this.queryDuckDuckGo(query, limit);
|
|
221
|
-
if (ddgHits2.length > 0) return ddgHits2;
|
|
222
|
-
}
|
|
223
|
-
if (tavilyKey && (!preferred || preferred === "tavily")) {
|
|
224
|
-
try {
|
|
225
|
-
const body = {
|
|
226
|
-
api_key: tavilyKey,
|
|
227
|
-
query,
|
|
228
|
-
search_depth: "advanced",
|
|
229
|
-
include_answer: true,
|
|
230
|
-
max_results: limit
|
|
231
|
-
};
|
|
232
|
-
if (options.timeRange) {
|
|
233
|
-
body.time_range = options.timeRange;
|
|
234
|
-
}
|
|
235
|
-
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
236
|
-
body.include_domains = options.domainFilter;
|
|
237
|
-
}
|
|
238
|
-
const res = await fetch("https://api.tavily.com/search", {
|
|
239
|
-
method: "POST",
|
|
240
|
-
headers: { "Content-Type": "application/json" },
|
|
241
|
-
body: JSON.stringify(body),
|
|
242
|
-
signal: AbortSignal.timeout(12e3)
|
|
243
|
-
});
|
|
244
|
-
if (res.ok) {
|
|
245
|
-
const data = await res.json();
|
|
246
|
-
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
247
|
-
return data.results.map((r) => ({
|
|
248
|
-
title: r.title || "Untitled",
|
|
249
|
-
url: r.url,
|
|
250
|
-
content: r.content || "",
|
|
251
|
-
score: r.score,
|
|
252
|
-
engine: "tavily"
|
|
253
|
-
}));
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
} catch (e) {
|
|
257
|
-
console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
if (exaKey) {
|
|
261
|
-
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
262
|
-
if (exaHits.length > 0) return exaHits;
|
|
263
|
-
}
|
|
264
|
-
if (serperKey) {
|
|
265
|
-
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
|
|
266
|
-
if (serperHits.length > 0) return serperHits;
|
|
267
|
-
}
|
|
268
|
-
if (brightDataKey) {
|
|
269
|
-
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
270
|
-
zone: brightDataZone,
|
|
271
|
-
engine: options.brightDataEngine
|
|
272
|
-
});
|
|
273
|
-
if (bdHits.length > 0) return bdHits;
|
|
274
|
-
}
|
|
275
|
-
if (searxngHost) {
|
|
276
|
-
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders, options.timeRange);
|
|
277
|
-
if (searxHits.length > 0) return searxHits;
|
|
278
|
-
}
|
|
279
|
-
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
280
|
-
if (ddgHits.length > 0) return ddgHits;
|
|
281
|
-
return [];
|
|
282
|
-
}
|
|
283
|
-
async queryBrightData(query, key, limit = 5, options = {}) {
|
|
284
|
-
try {
|
|
285
|
-
const zone = options.zone || this.brightDataZone || "cli_unlocker";
|
|
286
|
-
const engine = options.engine || "bing";
|
|
287
|
-
let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
|
|
288
|
-
if (engine === "google") {
|
|
289
|
-
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&brd_json=1&hl=en&gl=us`;
|
|
290
|
-
} else if (engine === "yandex") {
|
|
291
|
-
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
|
|
292
|
-
}
|
|
293
|
-
const res = await fetch("https://api.brightdata.com/request", {
|
|
294
|
-
method: "POST",
|
|
295
|
-
headers: {
|
|
296
|
-
Authorization: `Bearer ${key}`,
|
|
297
|
-
"Content-Type": "application/json"
|
|
298
|
-
},
|
|
299
|
-
body: JSON.stringify({
|
|
300
|
-
zone,
|
|
301
|
-
url: searchUrl,
|
|
302
|
-
format: "raw"
|
|
303
|
-
}),
|
|
304
|
-
signal: AbortSignal.timeout(15e3)
|
|
305
|
-
});
|
|
306
|
-
if (res.ok) {
|
|
307
|
-
const data = await res.json();
|
|
308
|
-
const organic = Array.isArray(data?.organic) ? data.organic : [];
|
|
309
|
-
if (organic.length > 0) {
|
|
310
|
-
return organic.slice(0, limit).map((r, idx) => ({
|
|
311
|
-
title: r.title || "Untitled",
|
|
312
|
-
url: r.link,
|
|
313
|
-
content: r.description || r.snippet || "",
|
|
314
|
-
score: r.rank ? 1 / r.rank : 1 / (idx + 1),
|
|
315
|
-
engine: "brightdata"
|
|
316
|
-
}));
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
} catch (e) {
|
|
320
|
-
console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
|
|
321
|
-
}
|
|
322
|
-
return [];
|
|
323
|
-
}
|
|
324
|
-
async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
|
|
325
|
-
try {
|
|
326
|
-
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
327
|
-
let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
328
|
-
if (timeRange) {
|
|
329
|
-
searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
|
|
330
|
-
}
|
|
331
|
-
const headers = {
|
|
332
|
-
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
333
|
-
...this.searxngHeaders || {},
|
|
334
|
-
...customHeaders || {}
|
|
335
|
-
};
|
|
336
|
-
const res = await fetch(searchUrl, {
|
|
337
|
-
headers,
|
|
338
|
-
signal: AbortSignal.timeout(1e4)
|
|
339
|
-
});
|
|
340
|
-
if (res.ok) {
|
|
341
|
-
const data = await res.json();
|
|
342
|
-
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
343
|
-
return data.results.slice(0, limit).map((r) => ({
|
|
344
|
-
title: r.title || "Untitled",
|
|
345
|
-
url: r.url,
|
|
346
|
-
content: r.content || r.snippet || "",
|
|
347
|
-
score: r.score,
|
|
348
|
-
engine: "searxng"
|
|
349
|
-
}));
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
} catch (e) {
|
|
353
|
-
console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
|
|
354
|
-
}
|
|
355
|
-
return [];
|
|
356
|
-
}
|
|
357
|
-
async queryExa(query, exaKey, limit, domainFilter, timeRange) {
|
|
358
|
-
try {
|
|
359
|
-
const body = {
|
|
360
|
-
query,
|
|
361
|
-
numResults: limit,
|
|
362
|
-
useAutoprompt: true
|
|
363
|
-
};
|
|
364
|
-
if (domainFilter && domainFilter.length > 0) {
|
|
365
|
-
body.includeDomains = domainFilter;
|
|
366
|
-
}
|
|
367
|
-
if (timeRange) {
|
|
368
|
-
const now = /* @__PURE__ */ new Date();
|
|
369
|
-
if (timeRange === "day") now.setDate(now.getDate() - 1);
|
|
370
|
-
else if (timeRange === "week") now.setDate(now.getDate() - 7);
|
|
371
|
-
else if (timeRange === "month") now.setMonth(now.getMonth() - 1);
|
|
372
|
-
else if (timeRange === "year") now.setFullYear(now.getFullYear() - 1);
|
|
373
|
-
body.startPublishedDate = now.toISOString();
|
|
374
|
-
}
|
|
375
|
-
const res = await fetch("https://api.exa.ai/search", {
|
|
376
|
-
method: "POST",
|
|
377
|
-
headers: {
|
|
378
|
-
"x-api-key": exaKey,
|
|
379
|
-
"Content-Type": "application/json"
|
|
380
|
-
},
|
|
381
|
-
body: JSON.stringify(body),
|
|
382
|
-
signal: AbortSignal.timeout(1e4)
|
|
383
|
-
});
|
|
384
|
-
if (res.ok) {
|
|
385
|
-
const data = await res.json();
|
|
386
|
-
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
387
|
-
return data.results.map((r) => ({
|
|
388
|
-
title: r.title || "Untitled",
|
|
389
|
-
url: r.url,
|
|
390
|
-
content: r.text || r.snippet || "",
|
|
391
|
-
score: r.score,
|
|
392
|
-
engine: "exa"
|
|
393
|
-
}));
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
} catch (e) {
|
|
397
|
-
console.warn("[MultiSourceSearchEngine] Exa attempt failed:", e?.message || e);
|
|
398
|
-
}
|
|
399
|
-
return [];
|
|
400
|
-
}
|
|
401
|
-
async querySerper(query, serperKey, limit, timeRange) {
|
|
402
|
-
try {
|
|
403
|
-
const body = { q: query, num: limit };
|
|
404
|
-
if (timeRange) {
|
|
405
|
-
const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
|
|
406
|
-
if (tbsMap[timeRange]) {
|
|
407
|
-
body.tbs = tbsMap[timeRange];
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
const res = await fetch("https://google.serper.dev/search", {
|
|
411
|
-
method: "POST",
|
|
412
|
-
headers: {
|
|
413
|
-
"X-API-KEY": serperKey,
|
|
414
|
-
"Content-Type": "application/json"
|
|
415
|
-
},
|
|
416
|
-
body: JSON.stringify(body),
|
|
417
|
-
signal: AbortSignal.timeout(1e4)
|
|
418
|
-
});
|
|
419
|
-
if (res.ok) {
|
|
420
|
-
const data = await res.json();
|
|
421
|
-
if (Array.isArray(data?.organic) && data.organic.length > 0) {
|
|
422
|
-
return data.organic.slice(0, limit).map((r) => ({
|
|
423
|
-
title: r.title || "Untitled",
|
|
424
|
-
url: r.link,
|
|
425
|
-
content: r.snippet || "",
|
|
426
|
-
engine: "serper"
|
|
427
|
-
}));
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
} catch (e) {
|
|
431
|
-
console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
|
|
432
|
-
}
|
|
433
|
-
return [];
|
|
434
|
-
}
|
|
435
|
-
async queryDuckDuckGo(query, limit) {
|
|
436
|
-
try {
|
|
437
|
-
const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
438
|
-
const res = await fetch(ddgUrl, {
|
|
439
|
-
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
|
|
440
|
-
signal: AbortSignal.timeout(8e3)
|
|
441
|
-
});
|
|
442
|
-
if (res.ok) {
|
|
443
|
-
const html = await res.text();
|
|
444
|
-
const hits = [];
|
|
445
|
-
const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
446
|
-
let match;
|
|
447
|
-
while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
|
|
448
|
-
const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
449
|
-
const rawUrl = match[2]?.trim();
|
|
450
|
-
const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
|
|
451
|
-
const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
|
|
452
|
-
if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
|
|
453
|
-
hits.push({
|
|
454
|
-
title: rawTitle,
|
|
455
|
-
url: cleanUrl,
|
|
456
|
-
content: snippet,
|
|
457
|
-
engine: "duckduckgo"
|
|
458
|
-
});
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
if (hits.length > 0) return hits;
|
|
462
|
-
}
|
|
463
|
-
} catch (e) {
|
|
464
|
-
console.warn("[MultiSourceSearchEngine] DuckDuckGo attempt failed:", e?.message || e);
|
|
465
|
-
}
|
|
466
|
-
return [];
|
|
467
|
-
}
|
|
468
|
-
};
|
|
469
|
-
|
|
470
|
-
// src/harvester/noiseFilter.ts
|
|
471
|
-
var ECOMMERCE_DOMAINS = /* @__PURE__ */ new Set([
|
|
472
|
-
"shopee.vn",
|
|
473
|
-
"shopee.com",
|
|
474
|
-
"lazada.vn",
|
|
475
|
-
"lazada.com",
|
|
476
|
-
"tiki.vn",
|
|
477
|
-
"sendo.vn",
|
|
478
|
-
"aliexpress.com",
|
|
479
|
-
"taobao.com",
|
|
480
|
-
"amazon.com",
|
|
481
|
-
"ebay.com",
|
|
482
|
-
"banggood.com"
|
|
483
|
-
]);
|
|
484
|
-
var ECOMMERCE_AND_COMMERCIAL_PATTERNS = [
|
|
485
|
-
/\b(shopee|lazada|tiki|sendo|taobao|aliexpress)\b/i,
|
|
486
|
-
/\b(mua ngay|giá tốt|khuyến mãi|freeship|bán chạy|nhập khẩu chính hãng|cửa hàng|bảng giá|đặt hàng)\b/i,
|
|
487
|
-
/\b(buy now|add to cart|free shipping|discount|best price|on sale|shop now|in stock|order now)\b/i,
|
|
488
|
-
/\b(\d+[kKđ]|₫|\$\d+|\d+\s*(?:VNĐ|VND|usd))\b/i
|
|
489
|
-
];
|
|
490
|
-
function isCommercialOrNoise(input) {
|
|
491
|
-
if (input.url) {
|
|
492
|
-
try {
|
|
493
|
-
const hostname = new URL(input.url).hostname.toLowerCase().replace(/^www\./, "");
|
|
494
|
-
if (ECOMMERCE_DOMAINS.has(hostname) || [...ECOMMERCE_DOMAINS].some((d) => hostname.endsWith(`.${d}`))) {
|
|
495
|
-
return true;
|
|
496
|
-
}
|
|
497
|
-
} catch {
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
const combined = `${input.title || ""} ${input.snippet || ""}`;
|
|
501
|
-
if (!combined.trim()) return false;
|
|
502
|
-
for (const pattern of ECOMMERCE_AND_COMMERCIAL_PATTERNS) {
|
|
503
|
-
if (pattern.test(combined)) {
|
|
504
|
-
return true;
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
return false;
|
|
508
|
-
}
|
|
509
|
-
function evaluateTechnicalDensity(markdown) {
|
|
510
|
-
if (!markdown || markdown.trim().length < 80) {
|
|
511
|
-
return {
|
|
512
|
-
score: 1,
|
|
513
|
-
isSufficientQuality: false,
|
|
514
|
-
codeDensity: 0,
|
|
515
|
-
technicalTermDensity: 0,
|
|
516
|
-
hasStructure: false,
|
|
517
|
-
reasons: ["Content is too short or empty."]
|
|
518
|
-
};
|
|
519
|
-
}
|
|
520
|
-
const reasons = [];
|
|
521
|
-
const totalLength = markdown.length;
|
|
522
|
-
const codeBlocks = markdown.match(/```[\s\S]*?```/g) || [];
|
|
523
|
-
const inlineCode = markdown.match(/`[^`\n]+`/g) || [];
|
|
524
|
-
const totalCodeChars = codeBlocks.reduce((sum, b) => sum + b.length, 0) + inlineCode.reduce((sum, c) => sum + c.length, 0);
|
|
525
|
-
const codeDensity = Math.min(1, totalCodeChars / Math.max(1, totalLength));
|
|
526
|
-
const headings = markdown.match(/^#{1,4}\s+.+$/gm) || [];
|
|
527
|
-
const hasStructure = headings.length >= 2;
|
|
528
|
-
const techKeywords = markdown.match(
|
|
529
|
-
/\b(function|class|method|struct|interface|api|hardware|sensor|pin|circuit|logic|loop|state|algorithm|test|debug|schema|protocol|variable)\b/gi
|
|
530
|
-
) || [];
|
|
531
|
-
const technicalTermDensity = Math.min(1, techKeywords.length * 10 / Math.max(100, markdown.split(/\s+/).length));
|
|
532
|
-
let score = 5;
|
|
533
|
-
if (codeDensity > 0.15) {
|
|
534
|
-
score += 2;
|
|
535
|
-
} else if (codeDensity > 0.05) {
|
|
536
|
-
score += 1;
|
|
537
|
-
} else {
|
|
538
|
-
reasons.push("Low code or API snippet density.");
|
|
539
|
-
}
|
|
540
|
-
if (hasStructure) {
|
|
541
|
-
score += 1.5;
|
|
542
|
-
} else {
|
|
543
|
-
reasons.push("Lacks clear hierarchical markdown headings.");
|
|
544
|
-
}
|
|
545
|
-
if (technicalTermDensity > 0.08) {
|
|
546
|
-
score += 1.5;
|
|
547
|
-
}
|
|
548
|
-
const finalScore = Math.min(10, Math.max(1, Math.round(score * 10) / 10));
|
|
549
|
-
return {
|
|
550
|
-
score: finalScore,
|
|
551
|
-
isSufficientQuality: finalScore >= 6,
|
|
552
|
-
codeDensity,
|
|
553
|
-
technicalTermDensity,
|
|
554
|
-
hasStructure,
|
|
555
|
-
reasons
|
|
556
|
-
};
|
|
557
|
-
}
|
|
1
|
+
import { isCommercialOrNoise, inferPrimaryCodeLanguage } from './chunk-2RMBLKP5.js';
|
|
558
2
|
|
|
559
3
|
// src/harvester/graphUtils.ts
|
|
560
4
|
function transitiveReduction(edges) {
|
|
@@ -1172,26 +616,6 @@ ${c}
|
|
|
1172
616
|
}
|
|
1173
617
|
};
|
|
1174
618
|
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
const issues = [];
|
|
1179
|
-
const warnings = [];
|
|
1180
|
-
if (!doc.markdownContent || doc.markdownContent.length < 80) {
|
|
1181
|
-
issues.push("Document content is too short or empty (< 80 characters).");
|
|
1182
|
-
}
|
|
1183
|
-
if (doc.markdownContent.includes("403 Forbidden") || doc.markdownContent.includes("Cloudflare Ray ID") || doc.markdownContent.includes("Access Denied")) {
|
|
1184
|
-
issues.push("Document blocked by firewall / Cloudflare / WAF.");
|
|
1185
|
-
}
|
|
1186
|
-
if (doc.extractedCodeSnippets.length === 0 && !doc.markdownContent.includes("|")) {
|
|
1187
|
-
warnings.push("Document does not contain any code snippets or reference tables.");
|
|
1188
|
-
}
|
|
1189
|
-
const passed = issues.length === 0;
|
|
1190
|
-
const score = passed ? warnings.length > 0 ? 80 : 100 : 20;
|
|
1191
|
-
return { passed, score, issues, warnings };
|
|
1192
|
-
}
|
|
1193
|
-
};
|
|
1194
|
-
|
|
1195
|
-
export { CatalogBuilder, ECOMMERCE_AND_COMMERCIAL_PATTERNS, ECOMMERCE_DOMAINS, HealthAuditor, MultiSourceSearchEngine, PedagogicalTermExtractor, classifyTechnicalDomain, evaluateTechnicalDensity, inferPrimaryCodeLanguage, isCommercialOrNoise, kahnTopologicalSort, transitiveReduction };
|
|
1196
|
-
//# sourceMappingURL=chunk-SAHBBO3U.js.map
|
|
1197
|
-
//# sourceMappingURL=chunk-SAHBBO3U.js.map
|
|
619
|
+
export { CatalogBuilder, PedagogicalTermExtractor, kahnTopologicalSort, transitiveReduction };
|
|
620
|
+
//# sourceMappingURL=chunk-RYLZYZRF.js.map
|
|
621
|
+
//# sourceMappingURL=chunk-RYLZYZRF.js.map
|