@thanh01.pmt/research-kit 1.1.1 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +27 -14
  2. package/dist/academic/index.d.cts +3 -3
  3. package/dist/academic/index.d.ts +3 -3
  4. package/dist/academic/index.js +2 -1
  5. package/dist/catalogBuilder-BaVRvg-9.d.cts +90 -0
  6. package/dist/catalogBuilder-DHV6y1gD.d.ts +90 -0
  7. package/dist/chunk-AKEI7H2C.js +223 -0
  8. package/dist/chunk-AKEI7H2C.js.map +1 -0
  9. package/dist/{chunk-XIF5YNI7.js → chunk-HZNSZDSM.js} +318 -8
  10. package/dist/chunk-HZNSZDSM.js.map +1 -0
  11. package/dist/chunk-LSMSIPTZ.js +3 -0
  12. package/dist/{chunk-2JEVBCIF.js.map → chunk-LSMSIPTZ.js.map} +1 -1
  13. package/dist/{chunk-KAFQVI2R.js → chunk-ON6VFCJO.js} +3 -3
  14. package/dist/{chunk-KAFQVI2R.js.map → chunk-ON6VFCJO.js.map} +1 -1
  15. package/dist/{chunk-GKF3NDEX.js → chunk-QFYM2NV4.js} +3 -3
  16. package/dist/{chunk-GKF3NDEX.js.map → chunk-QFYM2NV4.js.map} +1 -1
  17. package/dist/{chunk-AUGVVZTL.js → chunk-RW7VS7NI.js} +175 -7
  18. package/dist/chunk-RW7VS7NI.js.map +1 -0
  19. package/dist/{chunk-DOXSPRSP.js → chunk-UKGR45VZ.js} +3 -223
  20. package/dist/chunk-UKGR45VZ.js.map +1 -0
  21. package/dist/{chunk-J7XKDMGL.js → chunk-X3X26W2I.js} +2 -2
  22. package/dist/{chunk-J7XKDMGL.js.map → chunk-X3X26W2I.js.map} +1 -1
  23. package/dist/config/index.cjs.map +1 -1
  24. package/dist/config/index.d.cts +1 -1
  25. package/dist/config/index.d.ts +1 -1
  26. package/dist/config/index.js +1 -1
  27. package/dist/evidence/index.cjs.map +1 -1
  28. package/dist/evidence/index.d.cts +2 -2
  29. package/dist/evidence/index.d.ts +2 -2
  30. package/dist/evidence/index.js +2 -2
  31. package/dist/harvester/index.cjs +542 -6
  32. package/dist/harvester/index.cjs.map +1 -1
  33. package/dist/harvester/index.d.cts +63 -49
  34. package/dist/harvester/index.d.ts +63 -49
  35. package/dist/harvester/index.js +4 -3
  36. package/dist/{index-t8pahlJ4.d.ts → index-DlngvZBr.d.cts} +15 -1
  37. package/dist/{index-t8pahlJ4.d.cts → index-DlngvZBr.d.ts} +15 -1
  38. package/dist/index.cjs +515 -6
  39. package/dist/index.cjs.map +1 -1
  40. package/dist/index.d.cts +28 -13
  41. package/dist/index.d.ts +28 -13
  42. package/dist/index.js +41 -13
  43. package/dist/index.js.map +1 -1
  44. package/dist/{literatureReview-BTPFe7eb.d.cts → literatureReview-C7hsjrn4.d.cts} +1 -1
  45. package/dist/{literatureReview-hGbfrBmw.d.ts → literatureReview-DWl13r_3.d.ts} +1 -1
  46. package/dist/pipeline/index.cjs +658 -231
  47. package/dist/pipeline/index.cjs.map +1 -1
  48. package/dist/pipeline/index.d.cts +49 -5
  49. package/dist/pipeline/index.d.ts +49 -5
  50. package/dist/pipeline/index.js +6 -5
  51. package/dist/{webScraper-BhiQ8s9x.d.ts → webScraper-DBFUIqoO.d.ts} +1 -1
  52. package/dist/{webScraper-D9Ldus9p.d.cts → webScraper-yO9P-oLb.d.cts} +1 -1
  53. package/package.json +1 -1
  54. package/dist/catalogBuilder-CmmQzUnf.d.ts +0 -38
  55. package/dist/catalogBuilder-DgXYxL2F.d.cts +0 -38
  56. package/dist/chunk-2JEVBCIF.js +0 -3
  57. package/dist/chunk-AUGVVZTL.js.map +0 -1
  58. package/dist/chunk-DOXSPRSP.js.map +0 -1
  59. package/dist/chunk-XIF5YNI7.js.map +0 -1
@@ -1,5 +1,7 @@
1
1
  'use strict';
2
2
 
3
+ var zod = require('zod');
4
+
3
5
  // src/harvester/domainClassifier.ts
4
6
  var HARDWARE_PATTERNS = [
5
7
  /\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
@@ -106,19 +108,250 @@ function getExtractorsForDomain(domain) {
106
108
  }
107
109
  }
108
110
 
111
+ // src/academic/arxivClient.ts
112
+ var ArXivSearchEngine = class {
113
+ /**
114
+ * Search ArXiv repository using the public atom XML API
115
+ */
116
+ async search(query, maxResults = 5) {
117
+ const papers = [];
118
+ try {
119
+ const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${maxResults}&sortBy=relevance&sortOrder=descending`;
120
+ const res = await fetch(url, { signal: AbortSignal.timeout(1e4) });
121
+ if (res.ok) {
122
+ const xmlText = await res.text();
123
+ const entryMatches = xmlText.match(/<entry>[\s\S]*?<\/entry>/g) || [];
124
+ for (const entry of entryMatches) {
125
+ const idMatch = entry.match(/<id>([\s\S]*?)<\/id>/);
126
+ const titleMatch = entry.match(/<title>([\s\S]*?)<\/title>/);
127
+ const summaryMatch = entry.match(/<summary>([\s\S]*?)<\/summary>/);
128
+ const publishedMatch = entry.match(/<published>(\d{4})/);
129
+ const authorMatches = [...entry.matchAll(/<author>\s*<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim()).filter(Boolean);
130
+ const arxivUrl = idMatch ? idMatch[1]?.trim() : "";
131
+ const pdfUrl = arxivUrl ? arxivUrl.replace("/abs/", "/pdf/") + ".pdf" : void 0;
132
+ if (titleMatch) {
133
+ papers.push({
134
+ id: arxivUrl || `arxiv-${Math.random().toString(36).substring(7)}`,
135
+ title: titleMatch[1]?.trim().replace(/\s+/g, " ") || "Untitled ArXiv Paper",
136
+ abstract: summaryMatch ? summaryMatch[1]?.trim().replace(/\s+/g, " ") : "",
137
+ authors: authorMatches,
138
+ year: publishedMatch ? parseInt(publishedMatch[1], 10) : (/* @__PURE__ */ new Date()).getFullYear(),
139
+ citationCount: 0,
140
+ venue: "arXiv preprint",
141
+ url: arxivUrl,
142
+ pdfUrl,
143
+ fieldsOfStudy: ["Computer Science", "AI & Machine Learning"]
144
+ });
145
+ }
146
+ }
147
+ }
148
+ } catch (e) {
149
+ console.warn("[ArXivSearchEngine] ArXiv search failed:", e?.message || e);
150
+ }
151
+ return papers;
152
+ }
153
+ };
154
+
155
+ // src/academic/semanticScholar.ts
156
+ zod.z.object({
157
+ id: zod.z.string(),
158
+ title: zod.z.string(),
159
+ abstract: zod.z.string().optional(),
160
+ authors: zod.z.array(zod.z.string()).default([]),
161
+ year: zod.z.number().optional(),
162
+ citationCount: zod.z.number().default(0),
163
+ venue: zod.z.string().optional(),
164
+ doi: zod.z.string().optional(),
165
+ url: zod.z.string().optional(),
166
+ pdfUrl: zod.z.string().optional(),
167
+ tldr: zod.z.string().optional(),
168
+ fieldsOfStudy: zod.z.array(zod.z.string()).default([])
169
+ });
170
+ function reconstructOpenAlexAbstract(invertedIndex) {
171
+ if (!invertedIndex || typeof invertedIndex !== "object") return void 0;
172
+ const entries = Object.entries(invertedIndex);
173
+ if (entries.length === 0) return void 0;
174
+ let maxIndex = 0;
175
+ for (const [, positions] of entries) {
176
+ if (Array.isArray(positions)) {
177
+ for (const p of positions) {
178
+ if (typeof p === "number" && p > maxIndex) maxIndex = p;
179
+ }
180
+ }
181
+ }
182
+ if (maxIndex > 5e3) maxIndex = 5e3;
183
+ const words = new Array(maxIndex + 1);
184
+ for (const [word, positions] of entries) {
185
+ if (Array.isArray(positions)) {
186
+ for (const p of positions) {
187
+ if (typeof p === "number" && p >= 0 && p <= maxIndex) {
188
+ words[p] = word;
189
+ }
190
+ }
191
+ }
192
+ }
193
+ const reconstructed = words.filter(Boolean).join(" ").trim();
194
+ return reconstructed.length > 0 ? reconstructed : void 0;
195
+ }
196
+ var AcademicSearchEngine = class {
197
+ apiKey;
198
+ allowEnvFallback;
199
+ customAdapter;
200
+ arxivEngine;
201
+ constructor(options = {}) {
202
+ this.arxivEngine = new ArXivSearchEngine();
203
+ if (typeof options === "string") {
204
+ this.apiKey = options;
205
+ this.allowEnvFallback = true;
206
+ } else {
207
+ this.allowEnvFallback = options.allowEnvFallback ?? true;
208
+ this.apiKey = options.apiKey || (this.allowEnvFallback ? process.env.SEMANTIC_SCHOLAR_API_KEY : void 0);
209
+ this.customAdapter = options.customAdapter;
210
+ }
211
+ }
212
+ setCustomAdapter(adapter) {
213
+ this.customAdapter = adapter;
214
+ }
215
+ setApiKey(apiKey) {
216
+ this.apiKey = apiKey;
217
+ }
218
+ /**
219
+ * Search academic papers across Semantic Scholar with OpenAlex and ArXiv fallback or Custom Adapter
220
+ */
221
+ async searchPapers(query, options = {}) {
222
+ const adapter = options.customAdapter || this.customAdapter;
223
+ if (adapter) {
224
+ try {
225
+ const customPapers = await adapter.searchPapers(query, { limit: options.limit, yearRange: options.yearRange });
226
+ if (Array.isArray(customPapers) && customPapers.length > 0) {
227
+ return customPapers;
228
+ }
229
+ } catch (e) {
230
+ console.warn("[AcademicSearchEngine] Custom academic adapter failed:", e?.message || e);
231
+ }
232
+ }
233
+ const limit = options.limit ?? 5;
234
+ const papers = [];
235
+ const seenTitles = /* @__PURE__ */ new Set();
236
+ const pushUnique = (p) => {
237
+ const normTitle = p.title.toLowerCase().replace(/[^a-z0-9\u00C0-\u1EF9]+/g, "");
238
+ if (!normTitle || seenTitles.has(normTitle)) return;
239
+ seenTitles.add(normTitle);
240
+ papers.push(p);
241
+ };
242
+ try {
243
+ const apiKey = options.apiKey || this.apiKey;
244
+ const headers = { "Content-Type": "application/json" };
245
+ if (apiKey) {
246
+ headers["x-api-key"] = apiKey;
247
+ }
248
+ const params = new URLSearchParams({
249
+ query,
250
+ limit: limit.toString(),
251
+ fields: "paperId,title,abstract,authors,year,citationCount,venue,externalIds,url,openAccessPdf,tldr,fieldsOfStudy"
252
+ });
253
+ if (options.yearRange) {
254
+ params.append("year", options.yearRange);
255
+ }
256
+ const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
257
+ headers,
258
+ signal: AbortSignal.timeout(6e3)
259
+ });
260
+ if (res.ok) {
261
+ const data = await res.json();
262
+ if (Array.isArray(data?.data)) {
263
+ for (const item of data.data) {
264
+ pushUnique({
265
+ id: item.paperId || `s2-${Math.random().toString(36).slice(2, 8)}`,
266
+ title: item.title || "Untitled",
267
+ abstract: item.abstract || void 0,
268
+ authors: Array.isArray(item.authors) ? item.authors.map((a) => a.name).filter(Boolean) : [],
269
+ year: item.year,
270
+ citationCount: item.citationCount || 0,
271
+ venue: item.venue || void 0,
272
+ doi: item.externalIds?.DOI,
273
+ url: item.url || (item.externalIds?.DOI ? `https://doi.org/${item.externalIds.DOI}` : void 0),
274
+ pdfUrl: item.openAccessPdf?.url,
275
+ tldr: item.tldr?.text,
276
+ fieldsOfStudy: Array.isArray(item.fieldsOfStudy) && item.fieldsOfStudy.length > 0 ? item.fieldsOfStudy : ["Computer Science"]
277
+ });
278
+ }
279
+ if (papers.length >= limit) return papers.slice(0, limit);
280
+ }
281
+ }
282
+ } catch (e) {
283
+ console.warn("[AcademicSearchEngine] Semantic Scholar failed:", e?.message || e);
284
+ }
285
+ if (papers.length < limit) {
286
+ try {
287
+ const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
288
+ headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
289
+ signal: AbortSignal.timeout(6e3)
290
+ });
291
+ if (res.ok) {
292
+ const data = await res.json();
293
+ if (Array.isArray(data?.results)) {
294
+ for (const item of data.results) {
295
+ const abstractText = reconstructOpenAlexAbstract(item.abstract_inverted_index);
296
+ const concepts = Array.isArray(item.concepts) ? item.concepts.slice(0, 3).map((c) => c.display_name).filter(Boolean) : ["Computer Science"];
297
+ pushUnique({
298
+ id: item.id || `oa-${Math.random().toString(36).slice(2, 8)}`,
299
+ title: item.title || "Untitled",
300
+ abstract: abstractText,
301
+ authors: Array.isArray(item.authorships) ? item.authorships.map((a) => a.author?.display_name).filter(Boolean) : [],
302
+ year: item.publication_year,
303
+ citationCount: item.cited_by_count || 0,
304
+ venue: item.primary_location?.source?.display_name,
305
+ doi: item.doi ? String(item.doi).replace(/^https?:\/\/doi\.org\//i, "") : void 0,
306
+ url: item.doi || item.id,
307
+ pdfUrl: item.open_access?.oa_url,
308
+ fieldsOfStudy: concepts.length > 0 ? concepts : ["Computer Science"]
309
+ });
310
+ }
311
+ }
312
+ }
313
+ } catch (e) {
314
+ console.warn("[AcademicSearchEngine] OpenAlex failed:", e?.message || e);
315
+ }
316
+ }
317
+ if (papers.length < limit && options.includeArxivFallback !== false) {
318
+ try {
319
+ const arxivPapers = await this.arxivEngine.search(query, limit - papers.length);
320
+ for (const p of arxivPapers) {
321
+ pushUnique(p);
322
+ }
323
+ } catch (e) {
324
+ console.warn("[AcademicSearchEngine] ArXiv fallback failed:", e?.message || e);
325
+ }
326
+ }
327
+ return papers.slice(0, limit);
328
+ }
329
+ };
330
+
109
331
  // src/harvester/multiSourceSearch.ts
110
332
  var MultiSourceSearchEngine = class {
111
333
  tavilyKey;
112
334
  exaKey;
113
335
  serperKey;
336
+ searxngHost;
337
+ searxngHeaders;
114
338
  allowEnvFallback;
115
339
  customAdapter;
340
+ academicEngine;
116
341
  constructor(options = {}) {
117
342
  this.allowEnvFallback = options.allowEnvFallback ?? true;
118
343
  this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
119
344
  this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
120
345
  this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
346
+ this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
347
+ const cfId = process.env.CF_ACCESS_CLIENT_ID;
348
+ const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
349
+ this.searxngHeaders = {
350
+ ...options.searxngHeaders || {},
351
+ ...cfId && cfSecret ? { "CF-Access-Client-Id": cfId, "CF-Access-Client-Secret": cfSecret } : {}
352
+ };
121
353
  this.customAdapter = options.customAdapter;
354
+ this.academicEngine = new AcademicSearchEngine({ allowEnvFallback: this.allowEnvFallback });
122
355
  }
123
356
  setCustomAdapter(adapter) {
124
357
  this.customAdapter = adapter;
@@ -128,6 +361,12 @@ var MultiSourceSearchEngine = class {
128
361
  if (keys.exa !== void 0) this.exaKey = keys.exa;
129
362
  if (keys.serper !== void 0) this.serperKey = keys.serper;
130
363
  }
364
+ setSearxng(host, headers) {
365
+ this.searxngHost = host;
366
+ if (headers) {
367
+ this.searxngHeaders = { ...this.searxngHeaders, ...headers };
368
+ }
369
+ }
131
370
  async search(query, options = {}) {
132
371
  const limit = options.limit ?? 5;
133
372
  const adapter = options.customAdapter || this.customAdapter;
@@ -148,9 +387,30 @@ var MultiSourceSearchEngine = class {
148
387
  console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
149
388
  }
150
389
  }
390
+ if (preferred === "academic") {
391
+ try {
392
+ const papers = await this.academicEngine.searchPapers(query, { limit });
393
+ if (papers.length > 0) {
394
+ return papers.map((p) => ({
395
+ title: p.title,
396
+ url: p.url || p.pdfUrl || (p.doi ? `https://doi.org/${p.doi}` : `https://openalex.org/${p.id}`),
397
+ content: p.tldr || p.abstract || "",
398
+ score: p.citationCount,
399
+ engine: "academic"
400
+ }));
401
+ }
402
+ } catch (e) {
403
+ console.warn("[MultiSourceSearchEngine] Academic search failed:", e?.message || e);
404
+ }
405
+ }
151
406
  const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
152
407
  const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
153
408
  const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
409
+ const searxngHost = options.searxngHost || this.searxngHost;
410
+ if (preferred === "searxng" && searxngHost) {
411
+ const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
412
+ if (searxHits.length > 0) return searxHits;
413
+ }
154
414
  if (preferred === "exa" && exaKey) {
155
415
  const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
156
416
  if (exaHits.length > 0) return exaHits;
@@ -205,10 +465,44 @@ var MultiSourceSearchEngine = class {
205
465
  const serperHits = await this.querySerper(query, serperKey, limit);
206
466
  if (serperHits.length > 0) return serperHits;
207
467
  }
468
+ if (searxngHost) {
469
+ const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
470
+ if (searxHits.length > 0) return searxHits;
471
+ }
208
472
  const ddgHits = await this.queryDuckDuckGo(query, limit);
209
473
  if (ddgHits.length > 0) return ddgHits;
210
474
  return [];
211
475
  }
476
+ async querySearxng(query, searxngHost, limit, customHeaders) {
477
+ try {
478
+ const cleanHost = searxngHost.replace(/\/$/, "");
479
+ const searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
480
+ const headers = {
481
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
482
+ ...this.searxngHeaders || {},
483
+ ...customHeaders || {}
484
+ };
485
+ const res = await fetch(searchUrl, {
486
+ headers,
487
+ signal: AbortSignal.timeout(1e4)
488
+ });
489
+ if (res.ok) {
490
+ const data = await res.json();
491
+ if (Array.isArray(data?.results) && data.results.length > 0) {
492
+ return data.results.slice(0, limit).map((r) => ({
493
+ title: r.title || "Untitled",
494
+ url: r.url,
495
+ content: r.content || r.snippet || "",
496
+ score: r.score,
497
+ engine: "searxng"
498
+ }));
499
+ }
500
+ }
501
+ } catch (e) {
502
+ console.warn("[MultiSourceSearchEngine] SearXNG attempt failed:", e?.message || e);
503
+ }
504
+ return [];
505
+ }
212
506
  async queryExa(query, exaKey, limit, domainFilter) {
213
507
  try {
214
508
  const body = {
@@ -552,7 +846,7 @@ var WebScraper = class {
552
846
  }
553
847
  const res = await fetch(jinaUrl, {
554
848
  headers,
555
- signal: AbortSignal.timeout(15e3)
849
+ signal: AbortSignal.timeout(8e3)
556
850
  });
557
851
  if (res.ok) {
558
852
  const markdown = await res.text();
@@ -637,6 +931,214 @@ ${p.replace(/<[^>]+>/g, " ").trim()}
637
931
  }
638
932
  };
639
933
 
934
+ // src/harvester/noiseFilter.ts
935
+ var ECOMMERCE_DOMAINS = /* @__PURE__ */ new Set([
936
+ "shopee.vn",
937
+ "shopee.com",
938
+ "lazada.vn",
939
+ "lazada.com",
940
+ "tiki.vn",
941
+ "sendo.vn",
942
+ "aliexpress.com",
943
+ "taobao.com",
944
+ "amazon.com",
945
+ "ebay.com",
946
+ "banggood.com"
947
+ ]);
948
+ var ECOMMERCE_AND_COMMERCIAL_PATTERNS = [
949
+ /\b(shopee|lazada|tiki|sendo|taobao|aliexpress)\b/i,
950
+ /\b(mua ngay|giá tốt|khuyến mãi|freeship|bán chạy|nhập khẩu chính hãng|cửa hàng|bảng giá|đặt hàng)\b/i,
951
+ /\b(buy now|add to cart|free shipping|discount|best price|on sale|shop now|in stock|order now)\b/i,
952
+ /\b(\d+[kKđ]|₫|\$\d+|\d+\s*(?:VNĐ|VND|usd))\b/i
953
+ ];
954
+ function isCommercialOrNoise(input) {
955
+ if (input.url) {
956
+ try {
957
+ const hostname = new URL(input.url).hostname.toLowerCase().replace(/^www\./, "");
958
+ if (ECOMMERCE_DOMAINS.has(hostname) || [...ECOMMERCE_DOMAINS].some((d) => hostname.endsWith(`.${d}`))) {
959
+ return true;
960
+ }
961
+ } catch {
962
+ }
963
+ }
964
+ const combined = `${input.title || ""} ${input.snippet || ""}`;
965
+ if (!combined.trim()) return false;
966
+ for (const pattern of ECOMMERCE_AND_COMMERCIAL_PATTERNS) {
967
+ if (pattern.test(combined)) {
968
+ return true;
969
+ }
970
+ }
971
+ return false;
972
+ }
973
+ function evaluateTechnicalDensity(markdown) {
974
+ if (!markdown || markdown.trim().length < 80) {
975
+ return {
976
+ score: 1,
977
+ isSufficientQuality: false,
978
+ codeDensity: 0,
979
+ technicalTermDensity: 0,
980
+ hasStructure: false,
981
+ reasons: ["Content is too short or empty."]
982
+ };
983
+ }
984
+ const reasons = [];
985
+ const totalLength = markdown.length;
986
+ const codeBlocks = markdown.match(/```[\s\S]*?```/g) || [];
987
+ const inlineCode = markdown.match(/`[^`\n]+`/g) || [];
988
+ const totalCodeChars = codeBlocks.reduce((sum, b) => sum + b.length, 0) + inlineCode.reduce((sum, c) => sum + c.length, 0);
989
+ const codeDensity = Math.min(1, totalCodeChars / Math.max(1, totalLength));
990
+ const headings = markdown.match(/^#{1,4}\s+.+$/gm) || [];
991
+ const hasStructure = headings.length >= 2;
992
+ const techKeywords = markdown.match(
993
+ /\b(function|class|method|struct|interface|api|hardware|sensor|pin|circuit|logic|loop|state|algorithm|test|debug|schema|protocol|variable)\b/gi
994
+ ) || [];
995
+ const technicalTermDensity = Math.min(1, techKeywords.length * 10 / Math.max(100, markdown.split(/\s+/).length));
996
+ let score = 5;
997
+ if (codeDensity > 0.15) {
998
+ score += 2;
999
+ } else if (codeDensity > 0.05) {
1000
+ score += 1;
1001
+ } else {
1002
+ reasons.push("Low code or API snippet density.");
1003
+ }
1004
+ if (hasStructure) {
1005
+ score += 1.5;
1006
+ } else {
1007
+ reasons.push("Lacks clear hierarchical markdown headings.");
1008
+ }
1009
+ if (technicalTermDensity > 0.08) {
1010
+ score += 1.5;
1011
+ }
1012
+ const finalScore = Math.min(10, Math.max(1, Math.round(score * 10) / 10));
1013
+ return {
1014
+ score: finalScore,
1015
+ isSufficientQuality: finalScore >= 6,
1016
+ codeDensity,
1017
+ technicalTermDensity,
1018
+ hasStructure,
1019
+ reasons
1020
+ };
1021
+ }
1022
+
1023
+ // src/harvester/graphUtils.ts
1024
+ function transitiveReduction(edges) {
1025
+ const uniqueEdges = [];
1026
+ const edgeSet = /* @__PURE__ */ new Set();
1027
+ for (const e of edges) {
1028
+ if (!e.source || !e.target || e.source === e.target) continue;
1029
+ const key = `${e.source}-->${e.target}`;
1030
+ if (!edgeSet.has(key)) {
1031
+ edgeSet.add(key);
1032
+ uniqueEdges.push(e);
1033
+ }
1034
+ }
1035
+ const adj = /* @__PURE__ */ new Map();
1036
+ for (const e of uniqueEdges) {
1037
+ if (!adj.has(e.source)) adj.set(e.source, /* @__PURE__ */ new Set());
1038
+ adj.get(e.source).add(e.target);
1039
+ }
1040
+ const reducedEdges = [];
1041
+ const removedTransitiveEdges = [];
1042
+ const hasAlternatePath = (start, target) => {
1043
+ const directNeighbors = adj.get(start);
1044
+ if (!directNeighbors) return false;
1045
+ const queue = [];
1046
+ const visited = /* @__PURE__ */ new Set([start]);
1047
+ for (const nxt of directNeighbors) {
1048
+ if (nxt !== target) {
1049
+ queue.push(nxt);
1050
+ visited.add(nxt);
1051
+ }
1052
+ }
1053
+ while (queue.length > 0) {
1054
+ const curr = queue.shift();
1055
+ if (curr === target) return true;
1056
+ const nextSet = adj.get(curr);
1057
+ if (!nextSet) continue;
1058
+ for (const nxt of nextSet) {
1059
+ if (nxt === target) return true;
1060
+ if (!visited.has(nxt)) {
1061
+ visited.add(nxt);
1062
+ queue.push(nxt);
1063
+ }
1064
+ }
1065
+ }
1066
+ return false;
1067
+ };
1068
+ for (const e of uniqueEdges) {
1069
+ if (hasAlternatePath(e.source, e.target) && !hasAlternatePath(e.target, e.source)) {
1070
+ removedTransitiveEdges.push(e);
1071
+ } else {
1072
+ reducedEdges.push(e);
1073
+ }
1074
+ }
1075
+ return { reducedEdges, removedTransitiveEdges };
1076
+ }
1077
+ function kahnTopologicalSort(nodes, edges) {
1078
+ const nodeSet = new Set(nodes);
1079
+ const validEdges = edges.filter((e) => nodeSet.has(e.source) && nodeSet.has(e.target) && e.source !== e.target);
1080
+ const inDegree = /* @__PURE__ */ new Map();
1081
+ const adj = /* @__PURE__ */ new Map();
1082
+ for (const n of nodes) {
1083
+ inDegree.set(n, 0);
1084
+ adj.set(n, /* @__PURE__ */ new Set());
1085
+ }
1086
+ for (const e of validEdges) {
1087
+ adj.get(e.source).add(e.target);
1088
+ inDegree.set(e.target, (inDegree.get(e.target) ?? 0) + 1);
1089
+ }
1090
+ const queue = [];
1091
+ for (const [node, deg] of inDegree.entries()) {
1092
+ if (deg === 0) queue.push(node);
1093
+ }
1094
+ const order = [];
1095
+ const cyclesBroken = [];
1096
+ const droppedEdges = [];
1097
+ while (order.length < nodes.length) {
1098
+ if (queue.length > 0) {
1099
+ queue.sort((a, b) => a.localeCompare(b));
1100
+ const curr = queue.shift();
1101
+ order.push(curr);
1102
+ const neighbors = adj.get(curr);
1103
+ if (neighbors) {
1104
+ for (const nxt of neighbors) {
1105
+ const nextDeg = (inDegree.get(nxt) ?? 1) - 1;
1106
+ inDegree.set(nxt, nextDeg);
1107
+ if (nextDeg === 0) queue.push(nxt);
1108
+ }
1109
+ }
1110
+ } else {
1111
+ const remaining = nodes.filter((n) => !order.includes(n));
1112
+ if (remaining.length === 0) break;
1113
+ remaining.sort((a, b) => (inDegree.get(a) ?? 0) - (inDegree.get(b) ?? 0) || a.localeCompare(b));
1114
+ const candidate = remaining[0];
1115
+ let dropped = false;
1116
+ for (const [src, targets] of adj.entries()) {
1117
+ if (remaining.includes(src) && targets.has(candidate)) {
1118
+ targets.delete(candidate);
1119
+ droppedEdges.push({ source: src, target: candidate });
1120
+ cyclesBroken.push(`${src} -> ${candidate}`);
1121
+ inDegree.set(candidate, Math.max(0, (inDegree.get(candidate) ?? 1) - 1));
1122
+ if (inDegree.get(candidate) === 0) {
1123
+ queue.push(candidate);
1124
+ }
1125
+ dropped = true;
1126
+ break;
1127
+ }
1128
+ }
1129
+ if (!dropped) {
1130
+ order.push(candidate);
1131
+ }
1132
+ }
1133
+ }
1134
+ return {
1135
+ isDag: cyclesBroken.length === 0,
1136
+ order,
1137
+ cyclesBroken,
1138
+ droppedEdges
1139
+ };
1140
+ }
1141
+
640
1142
  // src/harvester/termExtractor.ts
641
1143
  var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
642
1144
  "l\xE0",
@@ -850,6 +1352,7 @@ var PedagogicalTermExtractor = class {
850
1352
  }
851
1353
  };
852
1354
  for (const { resourceId, doc } of docs) {
1355
+ if (isCommercialOrNoise({ url: doc.url, title: doc.title })) continue;
853
1356
  const md = doc.markdownContent || "";
854
1357
  const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
855
1358
  const bilingualMatches = mdWithoutCode.matchAll(
@@ -953,7 +1456,7 @@ ${corpusPreview}`,
953
1456
  `
954
1457
  Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
955
1458
  `IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
956
- `Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
1459
+ `Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."]}`
957
1460
  ].filter(Boolean).join("\n");
958
1461
  const rawResponse = await options.llm(prompt, {
959
1462
  system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
@@ -965,9 +1468,26 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
965
1468
  const parsed = JSON.parse(jsonMatch[0]);
966
1469
  if (Array.isArray(parsed) && parsed.length > 0) {
967
1470
  const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
1471
+ const rawEdges = [];
1472
+ for (const item of parsed) {
1473
+ const targetId = String(item.id || "");
1474
+ if (!validIds.has(targetId)) continue;
1475
+ for (const p of item.prerequisites || []) {
1476
+ const srcId = String(p);
1477
+ if (validIds.has(srcId) && srcId !== targetId) {
1478
+ rawEdges.push({ source: srcId, target: targetId });
1479
+ }
1480
+ }
1481
+ }
1482
+ const { reducedEdges } = transitiveReduction(rawEdges);
1483
+ const reducedPrereqMap = /* @__PURE__ */ new Map();
1484
+ for (const e of reducedEdges) {
1485
+ if (!reducedPrereqMap.has(e.target)) reducedPrereqMap.set(e.target, []);
1486
+ reducedPrereqMap.get(e.target).push(e.source);
1487
+ }
968
1488
  const concepts = parsed.map((item, idx) => {
969
1489
  const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
970
- const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
1490
+ const prereqs = reducedPrereqMap.get(id) || [];
971
1491
  return {
972
1492
  id,
973
1493
  term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
@@ -979,12 +1499,22 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
979
1499
  salienceScore: Number(item.salienceScore) || 8,
980
1500
  definition: String(item.definition || ""),
981
1501
  prerequisites: prereqs,
982
- sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
1502
+ sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"],
1503
+ facets: item.facets && typeof item.facets === "object" ? {
1504
+ computationalPrinciple: item.facets.computationalPrinciple ? String(item.facets.computationalPrinciple) : void 0,
1505
+ mentalModel: item.facets.mentalModel ? String(item.facets.mentalModel) : void 0,
1506
+ syntaxAndTools: Array.isArray(item.facets.syntaxAndTools) ? item.facets.syntaxAndTools.map(String) : void 0
1507
+ } : void 0,
1508
+ commonMisconceptions: Array.isArray(item.commonMisconceptions) ? item.commonMisconceptions.map(String) : void 0,
1509
+ troubleshootingHints: Array.isArray(item.troubleshootingHints) ? item.troubleshootingHints.map(String) : void 0
983
1510
  };
984
1511
  });
1512
+ const topoResult = kahnTopologicalSort(concepts.map((c) => c.id), reducedEdges);
1513
+ const conceptMap = new Map(concepts.map((c) => [c.id, c]));
1514
+ const sortedConcepts = topoResult.order.map((id) => conceptMap.get(id)).filter(Boolean);
985
1515
  return {
986
- concepts,
987
- conceptDagTsv: this.formatConceptDagTsv(concepts),
1516
+ concepts: sortedConcepts.length > 0 ? sortedConcepts : concepts,
1517
+ conceptDagTsv: this.formatConceptDagTsv(sortedConcepts.length > 0 ? sortedConcepts : concepts),
988
1518
  isLlmRefined: true,
989
1519
  unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
990
1520
  };
@@ -1127,17 +1657,23 @@ var HealthAuditor = class {
1127
1657
  };
1128
1658
 
1129
1659
  exports.CatalogBuilder = CatalogBuilder;
1660
+ exports.ECOMMERCE_AND_COMMERCIAL_PATTERNS = ECOMMERCE_AND_COMMERCIAL_PATTERNS;
1661
+ exports.ECOMMERCE_DOMAINS = ECOMMERCE_DOMAINS;
1130
1662
  exports.HealthAuditor = HealthAuditor;
1131
1663
  exports.MultiSourceSearchEngine = MultiSourceSearchEngine;
1132
1664
  exports.PedagogicalTermExtractor = PedagogicalTermExtractor;
1133
1665
  exports.WebScraper = WebScraper;
1134
1666
  exports.YouTubeIngestionService = YouTubeIngestionService;
1135
1667
  exports.classifyTechnicalDomain = classifyTechnicalDomain;
1668
+ exports.evaluateTechnicalDensity = evaluateTechnicalDensity;
1136
1669
  exports.extractSubstantiveExcerpts = extractSubstantiveExcerpts;
1137
1670
  exports.extractYouTubeVideoId = extractYouTubeVideoId;
1138
1671
  exports.formatTimestamp = formatTimestamp;
1139
1672
  exports.inferPrimaryCodeLanguage = inferPrimaryCodeLanguage;
1673
+ exports.isCommercialOrNoise = isCommercialOrNoise;
1140
1674
  exports.isYouTubeUrl = isYouTubeUrl;
1675
+ exports.kahnTopologicalSort = kahnTopologicalSort;
1141
1676
  exports.stitchCaptionSegments = stitchCaptionSegments;
1677
+ exports.transitiveReduction = transitiveReduction;
1142
1678
  //# sourceMappingURL=index.cjs.map
1143
1679
  //# sourceMappingURL=index.cjs.map