@thanh01.pmt/research-kit 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -14
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/catalogBuilder-2i5KASUF.d.ts +82 -0
- package/dist/catalogBuilder-nrGYEY4b.d.cts +82 -0
- package/dist/chunk-LSMSIPTZ.js +3 -0
- package/dist/{chunk-2JEVBCIF.js.map → chunk-LSMSIPTZ.js.map} +1 -1
- package/dist/{chunk-KAFQVI2R.js → chunk-ON6VFCJO.js} +3 -3
- package/dist/{chunk-KAFQVI2R.js.map → chunk-ON6VFCJO.js.map} +1 -1
- package/dist/{chunk-GKF3NDEX.js → chunk-QFYM2NV4.js} +3 -3
- package/dist/{chunk-GKF3NDEX.js.map → chunk-QFYM2NV4.js.map} +1 -1
- package/dist/{chunk-XIF5YNI7.js → chunk-QK3LVGI7.js} +244 -8
- package/dist/chunk-QK3LVGI7.js.map +1 -0
- package/dist/{chunk-J7XKDMGL.js → chunk-X3X26W2I.js} +2 -2
- package/dist/{chunk-J7XKDMGL.js.map → chunk-X3X26W2I.js.map} +1 -1
- package/dist/{chunk-AUGVVZTL.js → chunk-XDDTHWJW.js} +173 -6
- package/dist/chunk-XDDTHWJW.js.map +1 -0
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -2
- package/dist/harvester/index.cjs +248 -6
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +63 -49
- package/dist/harvester/index.d.ts +63 -49
- package/dist/harvester/index.js +3 -3
- package/dist/{index-t8pahlJ4.d.ts → index-DlngvZBr.d.cts} +15 -1
- package/dist/{index-t8pahlJ4.d.cts → index-DlngvZBr.d.ts} +15 -1
- package/dist/index.cjs +443 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -13
- package/dist/index.d.ts +28 -13
- package/dist/index.js +37 -11
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-BTPFe7eb.d.cts → literatureReview-C7hsjrn4.d.cts} +1 -1
- package/dist/{literatureReview-hGbfrBmw.d.ts → literatureReview-DWl13r_3.d.ts} +1 -1
- package/dist/pipeline/index.cjs +361 -6
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -5
- package/dist/pipeline/index.d.ts +49 -5
- package/dist/pipeline/index.js +4 -4
- package/dist/{webScraper-BhiQ8s9x.d.ts → webScraper-DBFUIqoO.d.ts} +1 -1
- package/dist/{webScraper-D9Ldus9p.d.cts → webScraper-yO9P-oLb.d.cts} +1 -1
- package/package.json +9 -8
- package/LICENSE +0 -21
- package/dist/catalogBuilder-CmmQzUnf.d.ts +0 -38
- package/dist/catalogBuilder-DgXYxL2F.d.cts +0 -38
- package/dist/chunk-2JEVBCIF.js +0 -3
- package/dist/chunk-AUGVVZTL.js.map +0 -1
- package/dist/chunk-XIF5YNI7.js.map +0 -1
package/README.md
CHANGED
|
@@ -24,31 +24,39 @@ It solves the fundamental problems of hallucinations, obsolete technical syntax,
|
|
|
24
24
|
|
|
25
25
|
---
|
|
26
26
|
|
|
27
|
-
## 📦 The
|
|
27
|
+
## 📦 The 6 Core Services
|
|
28
28
|
|
|
29
29
|
`@thanh01.pmt/research-kit` exposes a unified facade (`ResearchKitClient`) or standalone service classes:
|
|
30
30
|
|
|
31
31
|
```
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
▼
|
|
38
|
-
|
|
39
|
-
│
|
|
40
|
-
│
|
|
41
|
-
│
|
|
42
|
-
│
|
|
43
|
-
|
|
32
|
+
┌───────────────────────────┐
|
|
33
|
+
│ ResearchKitClient │
|
|
34
|
+
└─────────────┬─────────────┘
|
|
35
|
+
│
|
|
36
|
+
┌───────────────┬───────────────┬───────────┼───────────┬───────────────┬───────────────┐
|
|
37
|
+
▼ ▼ ▼ ▼ ▼ ▼ ▼
|
|
38
|
+
┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────────┐ ┌───────────────┐
|
|
39
|
+
│ Service 1 │ │ Service 2 │ │ Service 3 │ │ Service 4 │ │ Service 5 │ │ Service 6 │
|
|
40
|
+
│ Discovery │ │ Term & │ │ Academic │ │ Evidence │ │ Pedagogical │ │ Technical │
|
|
41
|
+
│ & Scrape │ │ Clean DAG │ │ Standards │ │ & NLI │ │ Synthesis │ │ Grounding │
|
|
42
|
+
│ Ingestion │ │ Extraction│ │ Alignment │ │ Auditor │ │ Orchestrator │ │ Factsheet │
|
|
43
|
+
└───────────┘ └───────────┘ └───────────┘ └───────────┘ └───────────────┘ └───────────────┘
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
### 1. `KnowledgeDiscoveryService` (Discovery & Multimodal Ingestion)
|
|
47
47
|
* **Multi-Engine Search**: Auto-routes queries across Tavily, Exa, Serper, SearXNG, DuckDuckGo, and custom adapters.
|
|
48
|
+
* **Commercial Noise Filter & Quality Auditor**: Automatically identifies and rejects e-commerce clutter (Shopee, Lazada, Tiki, AliExpress, eBay, Amazon product listings) and calculates substantive technical density (code blocks, technical headings, tables).
|
|
48
49
|
* **High-Fidelity Scraper**: Jina Reader API / Crawl4AI / Direct Fetch extracting clean Markdown, formatted code snippets with correct language tags (`lua`, `cpp`, `python`, `swift`), and tabular structures.
|
|
49
50
|
* **YouTube Transcript Ingestion**: Parses video transcripts, stitches caption fragments into timestamped paragraphs (every ~30–45s), and filters out sponsor segments and subscribe filler noise.
|
|
50
51
|
|
|
51
|
-
### 2. `TerminologyGraphService` (
|
|
52
|
+
### 2. `TerminologyGraphService` (Clean Prerequisite DAG & Concept Facets)
|
|
53
|
+
* **Deterministic DAG & Prerequisite Engine**: Applies transitive reduction to strip redundant shortcut edges ($A \to B \to C \implies$ remove $A \to C$) and runs Kahn's topological sort with cycle detection to guarantee clean, acyclic dependency graphs.
|
|
54
|
+
* **Multi-dimensional Concept Facets**: Concept extraction captures cognitive facets across Marr's tripartite layers:
|
|
55
|
+
* `facets.ulo`: Conceptual purpose, real-world rationale & scientific nature.
|
|
56
|
+
* `facets.cio`: Algorithmic mental model, data flow & internal mechanics.
|
|
57
|
+
* `facets.sio`: Syntactic implementation, pinout connections & code drill.
|
|
58
|
+
* `commonMisconceptions`: Empirical student pitfalls and misconceptions.
|
|
59
|
+
* `troubleshootingHints`: Practical debugging and setup fixes.
|
|
52
60
|
* **Tier 1 (Deterministic Candidate Harvester)**: Rapidly mines candidate terms from markdown headings, inline code blocks, bold markers, bilingual parenthetical pairs `Thuật ngữ (English)`, and Vietnamese technical head nouns.
|
|
53
61
|
* **Tier 2 (LLM Semantic Curator & Ontologist)**: Deduplicates synonyms (e.g., subsumes *LDR*, *Quang trở*, *Cảm biến ánh sáng* into `C_PHOTORESISTOR`), assigns Marr cognitive layers (`ULO`/`CIO`/`SIO`) and Bloom taxonomy levels, and constructs genuine pedagogical prerequisite DAG edges (`CONCEPT_DAG.tsv`).
|
|
54
62
|
|
|
@@ -68,6 +76,11 @@ It solves the fundamental problems of hallucinations, obsolete technical syntax,
|
|
|
68
76
|
* **URL Normalization & Deduplication**: Strips tracking parameters (`utm_*`, `fbclid`, `gclid`, `#hash`) to eliminate redundant crawling.
|
|
69
77
|
* **Canonical Reference Pack**: Bundles all verified evidence into `REFERENCE_PACK.md`, `RESOURCE_CATALOG.md`, `KNOWLEDGE_GRAPH.md`, and `RESOURCE_INDEX.json`.
|
|
70
78
|
|
|
79
|
+
### 6. `TechnicalGroundingService` (Hardware/Software Compatibility & Empirical Exemplars)
|
|
80
|
+
* **Compatibility Matrix**: Validates firmware, OS versions, libraries, and pinout requirements.
|
|
81
|
+
* **Age Feasibility Constraints**: Verifies if cognitive and motor skills match target age groups (e.g., fine soldering vs. breadboard dupont wires).
|
|
82
|
+
* **Empirical Exemplar Projects**: Extracts verified hands-on student projects with concrete hardware lists, target outcomes, and troubleshooting gotchas into `TECHNICAL_FACTSHEET.md`.
|
|
83
|
+
|
|
71
84
|
---
|
|
72
85
|
|
|
73
86
|
## 🚀 Quick Start
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { A as AcademicPaper } from '../index-
|
|
2
|
-
export {
|
|
3
|
-
export { C as CS2023KnowledgeArea, c as CS2023_KNOWLEDGE_AREAS, a as CSTAK12Strand, b as CSTA_K12_STRANDS, L as LiteratureReviewReport, d as LiteratureReviewSynthesizer, S as SotaComparisonRow, T as TaxonomyMappingResult, m as mapTopicToCS2023 } from '../literatureReview-
|
|
1
|
+
import { A as AcademicPaper } from '../index-DlngvZBr.cjs';
|
|
2
|
+
export { j as AcademicPaperSchema, l as AcademicSearchEngine, k as AcademicSearchOptions, r as reconstructOpenAlexAbstract } from '../index-DlngvZBr.cjs';
|
|
3
|
+
export { C as CS2023KnowledgeArea, c as CS2023_KNOWLEDGE_AREAS, a as CSTAK12Strand, b as CSTA_K12_STRANDS, L as LiteratureReviewReport, d as LiteratureReviewSynthesizer, S as SotaComparisonRow, T as TaxonomyMappingResult, m as mapTopicToCS2023 } from '../literatureReview-C7hsjrn4.cjs';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
6
6
|
/**
|
package/dist/academic/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { A as AcademicPaper } from '../index-
|
|
2
|
-
export {
|
|
3
|
-
export { C as CS2023KnowledgeArea, c as CS2023_KNOWLEDGE_AREAS, a as CSTAK12Strand, b as CSTA_K12_STRANDS, L as LiteratureReviewReport, d as LiteratureReviewSynthesizer, S as SotaComparisonRow, T as TaxonomyMappingResult, m as mapTopicToCS2023 } from '../literatureReview-
|
|
1
|
+
import { A as AcademicPaper } from '../index-DlngvZBr.js';
|
|
2
|
+
export { j as AcademicPaperSchema, l as AcademicSearchEngine, k as AcademicSearchOptions, r as reconstructOpenAlexAbstract } from '../index-DlngvZBr.js';
|
|
3
|
+
export { C as CS2023KnowledgeArea, c as CS2023_KNOWLEDGE_AREAS, a as CSTAK12Strand, b as CSTA_K12_STRANDS, L as LiteratureReviewReport, d as LiteratureReviewSynthesizer, S as SotaComparisonRow, T as TaxonomyMappingResult, m as mapTopicToCS2023 } from '../literatureReview-DWl13r_3.js';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { R as ResearchApiKeys, a as CustomSearchAdapter, C as CourseResearchContext, E as ExtractedConceptNode } from './index-DlngvZBr.js';
|
|
2
|
+
import { T as TechnicalDomain, S as ScrapedDocument } from './webScraper-DBFUIqoO.js';
|
|
3
|
+
|
|
4
|
+
interface SearchHit {
|
|
5
|
+
title: string;
|
|
6
|
+
url: string;
|
|
7
|
+
content: string;
|
|
8
|
+
score?: number;
|
|
9
|
+
engine: 'tavily' | 'exa' | 'serper' | 'duckduckgo' | 'searxng' | 'academic' | 'custom';
|
|
10
|
+
}
|
|
11
|
+
interface MultiSearchOptions {
|
|
12
|
+
limit?: number;
|
|
13
|
+
tavilyKey?: string;
|
|
14
|
+
exaKey?: string;
|
|
15
|
+
serperKey?: string;
|
|
16
|
+
apiKeys?: ResearchApiKeys;
|
|
17
|
+
allowEnvFallback?: boolean;
|
|
18
|
+
searxngHost?: string;
|
|
19
|
+
domainFilter?: string[];
|
|
20
|
+
preferredEngine?: 'tavily' | 'exa' | 'serper' | 'duckduckgo' | 'searxng' | 'academic' | 'custom';
|
|
21
|
+
customAdapter?: CustomSearchAdapter;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Unified multi-engine search engine with host-injected API keys, automatic provider failover, and custom adapter support
|
|
25
|
+
*/
|
|
26
|
+
declare class MultiSourceSearchEngine {
|
|
27
|
+
private tavilyKey?;
|
|
28
|
+
private exaKey?;
|
|
29
|
+
private serperKey?;
|
|
30
|
+
private allowEnvFallback;
|
|
31
|
+
private customAdapter?;
|
|
32
|
+
constructor(options?: {
|
|
33
|
+
tavilyKey?: string;
|
|
34
|
+
exaKey?: string;
|
|
35
|
+
serperKey?: string;
|
|
36
|
+
apiKeys?: ResearchApiKeys;
|
|
37
|
+
allowEnvFallback?: boolean;
|
|
38
|
+
customAdapter?: CustomSearchAdapter;
|
|
39
|
+
});
|
|
40
|
+
setCustomAdapter(adapter: CustomSearchAdapter): void;
|
|
41
|
+
setApiKeys(keys: ResearchApiKeys): void;
|
|
42
|
+
search(query: string, options?: MultiSearchOptions): Promise<SearchHit[]>;
|
|
43
|
+
private queryExa;
|
|
44
|
+
private querySerper;
|
|
45
|
+
private queryDuckDuckGo;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface HarvestedResourceItem {
|
|
49
|
+
id: string;
|
|
50
|
+
title: string;
|
|
51
|
+
url: string;
|
|
52
|
+
domain: TechnicalDomain;
|
|
53
|
+
relevanceScore: number;
|
|
54
|
+
byteSize: number;
|
|
55
|
+
tablesCount: number;
|
|
56
|
+
codeSnippetsCount: number;
|
|
57
|
+
summary: string;
|
|
58
|
+
keyTakeaways?: string[];
|
|
59
|
+
}
|
|
60
|
+
interface ResourceCatalogBundle {
|
|
61
|
+
resourceIndexJson: string;
|
|
62
|
+
resourceCatalogMd: string;
|
|
63
|
+
knowledgeGraphMd: string;
|
|
64
|
+
conceptDagTsv?: string;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Builds the 3-Layer Catalog & Distilled Knowledge Graph from real harvested content and extracted concepts
|
|
68
|
+
*/
|
|
69
|
+
declare class CatalogBuilder {
|
|
70
|
+
buildBundle(topic: string, domain: TechnicalDomain, items: {
|
|
71
|
+
doc: ScrapedDocument;
|
|
72
|
+
score: number;
|
|
73
|
+
summary: string;
|
|
74
|
+
keyTakeaways?: string[];
|
|
75
|
+
}[], options?: {
|
|
76
|
+
context?: CourseResearchContext;
|
|
77
|
+
concepts?: ExtractedConceptNode[];
|
|
78
|
+
conceptDagTsv?: string;
|
|
79
|
+
}): ResourceCatalogBundle;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export { CatalogBuilder as C, type HarvestedResourceItem as H, type MultiSearchOptions as M, type ResourceCatalogBundle as R, type SearchHit as S, MultiSourceSearchEngine as a };
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { R as ResearchApiKeys, a as CustomSearchAdapter, C as CourseResearchContext, E as ExtractedConceptNode } from './index-DlngvZBr.cjs';
|
|
2
|
+
import { T as TechnicalDomain, S as ScrapedDocument } from './webScraper-yO9P-oLb.cjs';
|
|
3
|
+
|
|
4
|
+
interface SearchHit {
|
|
5
|
+
title: string;
|
|
6
|
+
url: string;
|
|
7
|
+
content: string;
|
|
8
|
+
score?: number;
|
|
9
|
+
engine: 'tavily' | 'exa' | 'serper' | 'duckduckgo' | 'searxng' | 'academic' | 'custom';
|
|
10
|
+
}
|
|
11
|
+
interface MultiSearchOptions {
|
|
12
|
+
limit?: number;
|
|
13
|
+
tavilyKey?: string;
|
|
14
|
+
exaKey?: string;
|
|
15
|
+
serperKey?: string;
|
|
16
|
+
apiKeys?: ResearchApiKeys;
|
|
17
|
+
allowEnvFallback?: boolean;
|
|
18
|
+
searxngHost?: string;
|
|
19
|
+
domainFilter?: string[];
|
|
20
|
+
preferredEngine?: 'tavily' | 'exa' | 'serper' | 'duckduckgo' | 'searxng' | 'academic' | 'custom';
|
|
21
|
+
customAdapter?: CustomSearchAdapter;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Unified multi-engine search engine with host-injected API keys, automatic provider failover, and custom adapter support
|
|
25
|
+
*/
|
|
26
|
+
declare class MultiSourceSearchEngine {
|
|
27
|
+
private tavilyKey?;
|
|
28
|
+
private exaKey?;
|
|
29
|
+
private serperKey?;
|
|
30
|
+
private allowEnvFallback;
|
|
31
|
+
private customAdapter?;
|
|
32
|
+
constructor(options?: {
|
|
33
|
+
tavilyKey?: string;
|
|
34
|
+
exaKey?: string;
|
|
35
|
+
serperKey?: string;
|
|
36
|
+
apiKeys?: ResearchApiKeys;
|
|
37
|
+
allowEnvFallback?: boolean;
|
|
38
|
+
customAdapter?: CustomSearchAdapter;
|
|
39
|
+
});
|
|
40
|
+
setCustomAdapter(adapter: CustomSearchAdapter): void;
|
|
41
|
+
setApiKeys(keys: ResearchApiKeys): void;
|
|
42
|
+
search(query: string, options?: MultiSearchOptions): Promise<SearchHit[]>;
|
|
43
|
+
private queryExa;
|
|
44
|
+
private querySerper;
|
|
45
|
+
private queryDuckDuckGo;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface HarvestedResourceItem {
|
|
49
|
+
id: string;
|
|
50
|
+
title: string;
|
|
51
|
+
url: string;
|
|
52
|
+
domain: TechnicalDomain;
|
|
53
|
+
relevanceScore: number;
|
|
54
|
+
byteSize: number;
|
|
55
|
+
tablesCount: number;
|
|
56
|
+
codeSnippetsCount: number;
|
|
57
|
+
summary: string;
|
|
58
|
+
keyTakeaways?: string[];
|
|
59
|
+
}
|
|
60
|
+
interface ResourceCatalogBundle {
|
|
61
|
+
resourceIndexJson: string;
|
|
62
|
+
resourceCatalogMd: string;
|
|
63
|
+
knowledgeGraphMd: string;
|
|
64
|
+
conceptDagTsv?: string;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Builds the 3-Layer Catalog & Distilled Knowledge Graph from real harvested content and extracted concepts
|
|
68
|
+
*/
|
|
69
|
+
declare class CatalogBuilder {
|
|
70
|
+
buildBundle(topic: string, domain: TechnicalDomain, items: {
|
|
71
|
+
doc: ScrapedDocument;
|
|
72
|
+
score: number;
|
|
73
|
+
summary: string;
|
|
74
|
+
keyTakeaways?: string[];
|
|
75
|
+
}[], options?: {
|
|
76
|
+
context?: CourseResearchContext;
|
|
77
|
+
concepts?: ExtractedConceptNode[];
|
|
78
|
+
conceptDagTsv?: string;
|
|
79
|
+
}): ResourceCatalogBundle;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export { CatalogBuilder as C, type HarvestedResourceItem as H, type MultiSearchOptions as M, type ResourceCatalogBundle as R, type SearchHit as S, MultiSourceSearchEngine as a };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":[],"names":[],"mappings":"","file":"chunk-
|
|
1
|
+
{"version":3,"sources":[],"names":[],"mappings":"","file":"chunk-LSMSIPTZ.js"}
|
|
@@ -243,7 +243,7 @@ var WebScraper = class {
|
|
|
243
243
|
}
|
|
244
244
|
const res = await fetch(jinaUrl, {
|
|
245
245
|
headers,
|
|
246
|
-
signal: AbortSignal.timeout(
|
|
246
|
+
signal: AbortSignal.timeout(8e3)
|
|
247
247
|
});
|
|
248
248
|
if (res.ok) {
|
|
249
249
|
const markdown = await res.text();
|
|
@@ -329,5 +329,5 @@ ${p.replace(/<[^>]+>/g, " ").trim()}
|
|
|
329
329
|
};
|
|
330
330
|
|
|
331
331
|
export { WebScraper, YouTubeIngestionService, extractSubstantiveExcerpts, extractYouTubeVideoId, formatTimestamp, isYouTubeUrl, stitchCaptionSegments };
|
|
332
|
-
//# sourceMappingURL=chunk-
|
|
333
|
-
//# sourceMappingURL=chunk-
|
|
332
|
+
//# sourceMappingURL=chunk-ON6VFCJO.js.map
|
|
333
|
+
//# sourceMappingURL=chunk-ON6VFCJO.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/harvester/youtubeTranscript.ts","../src/harvester/webScraper.ts"],"names":[],"mappings":";AAkBA,IAAM,sBAAA,GAAyB;AAAA,EAC7B,wJAAA;AAAA,EACA;AACF,CAAA;AAEO,SAAS,sBAAsB,OAAA,EAAgC;AACpE,EAAA,MAAM,OAAA,GAAU,QAAQ,IAAA,EAAK;AAC7B,EAAA,IAAI,qBAAA,CAAsB,IAAA,CAAK,OAAO,CAAA,EAAG,OAAO,OAAA;AAEhD,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,IAAI,GAAA,CAAI,OAAO,CAAA;AACzB,IAAA,MAAM,OAAO,CAAA,CAAE,QAAA,CAAS,QAAQ,QAAA,EAAU,EAAE,EAAE,WAAA,EAAY;AAC1D,IAAA,IAAI,SAAS,UAAA,EAAY;AACvB,MAAA,MAAM,EAAA,GAAK,EAAE,QAAA,CAAS,KAAA,CAAM,GAAG,CAAA,CAAE,MAAA,CAAO,OAAO,CAAA,CAAE,CAAC,CAAA;AAClD,MAAA,OAAO,EAAA,IAAM,qBAAA,CAAsB,IAAA,CAAK,EAAE,IAAI,EAAA,GAAK,IAAA;AAAA,IACrD;AACA,IAAA,IAAI,IAAA,KAAS,aAAA,IAAiB,IAAA,KAAS,eAAA,IAAmB,SAAS,sBAAA,EAAwB;AACzF,MAAA,MAAM,CAAA,GAAI,CAAA,CAAE,YAAA,CAAa,GAAA,CAAI,GAAG,CAAA;AAChC,MAAA,IAAI,CAAA,IAAK,qBAAA,CAAsB,IAAA,CAAK,CAAC,GAAG,OAAO,CAAA;AAC/C,MAAA,MAAM,QAAQ,CAAA,CAAE,QAAA,CAAS,MAAM,GAAG,CAAA,CAAE,OAAO,OAAO,CAAA;AAClD,MAAA,IAAA,CAAK,KAAA,CAAM,CAAC,CAAA,KAAM,OAAA,IAAW,MAAM,CAAC,CAAA,KAAM,QAAA,IAAY,KAAA,CAAM,CAAC,CAAA,KAAM,GAAA,KAAQ,KAAA,CAAM,CAAC,CAAA,EAAG;AACnF,QAAA,OAAO,qBAAA,CAAsB,KAAK,KAAA,CAAM,CAAC,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,IAAA;AAAA,MAC3D;AAAA,IACF;AAAA,EACF,CAAA,CAAA,MAAQ;AAAA,EAER;AACA,EAAA,OAAO,IAAA;AACT;AAEO,SAAS,aAAa,GAAA,EAAsB;AACjD,EAAA,OAAO,qBAAA,CAAsB,GAAG,CAAA,KAAM,IAAA;AACxC;AAEO,SAAS,gBAAgB,OAAA,EAAyB;AACvD,EAAA,MAAM,QAAQ,IAAA,CAAK,GAAA,CAAI,GAAG,IAAA,CAAK,KAAA,CAAM,OAAO,CAAC,CAAA;AAC7C,EAAA,MAAM,GAAA,GAAM,IAAA,CAAK,KAAA,CAAM,KAAA,GAAQ,IAAI,CAAA;AACnC,EAAA,MAAM,IAAA,GAAO,IAAA,CAAK,KAAA,CAAO,KAAA,GAAQ,OAAQ,EAAE,CAAA;AAC3C,EAAA,MAAM,OAAO,KAAA,GAAQ,EAAA;AACrB,EAAA,IAAI,MAAM,CAAA,EAAG;AACX,IAAA,OAAO,CAAA,EAAG,OAAO,GAAG,CAAA,CAAE,SAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA,EAAI,MAAA,CAAO,IAAI,EAAE,QAAA,CAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA,EAAI,MAAA,CAAO,IAAI,CAAA,CAAE,QAAA,CAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,EAC1G;AACA,EAAA,OAAO,CAAA,EAAG,MAAA,CAAO,IAAI,CAAA,CAAE,SAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA,EAAI,OAAO,IAAI,CAAA,CAAE,QAAA,CAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAC1E;AAEA,SAAS,mBAAmB,GAAA,EAAqB;AAC/C,EAAA,OAAO,GAAA,CACJ,OAAA,CAAQ,QAAA,EAAU,GAAG,EACrB,OAAA,CAAQ,OAAA,EAAS,GAAG,CAAA,CACpB,OAAA,CAAQ,OAAA,EAAS,GAAG,CAAA,CACpB,QAAQ,SAAA,EAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,QAAA,EAAU,GAAG,CAAA,CACrB,OAAA,CAAQ,WAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,SAAA,EAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EACnB,IAAA,EAAK;AACV;AAMO,SAAS,sBAAsB,QAAA,EAA+F;AACnI,EAAA,MAAM,OAAA,GAAU,QAAA,CACb,GAAA,CAAI,CAAC,CAAA,MAAO;AAAA,IACX,UAAU,CAAA,CAAE,QAAA;AAAA,IACZ,aAAa,CAAA,CAAE,WAAA;AAAA,IACf,IAAA,EAAM,kBAAA,CAAmB,CAAA,CAAE,IAAI;AAAA,IAC/B,CAAA,CACD,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,KAAK,MAAA,GAAS,CAAA,IAAK,CAAC,sBAAA,CAAuB,IAAA,CAAK,CAAC,EAAA,KAAO,EAAA,CAAG,KAAK,CAAA,CAAE,IAAI,CAAC,CAAC,CAAA;AAE3F,EAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AAElC,EAAA,MAAM,WAAgC,EAAC;AACvC,EAAA,IAAI,WAAA,GAAc,OAAA,CAAQ,CAAC,CAAA,CAAG,QAAA;AAC9B,EAAA,IAAI,YAAY,OAAA,CAAQ,CAAC,EAAG,QAAA,GAAW,OAAA,CAAQ,CAAC,CAAA,CAAG,WAAA;AACnD,EAAA,IAAI,cAAwB,EAAC;AAE7B,EAAA,KAAA,MAAW,OAAO,OAAA,EAAS;AACzB,IAAA,WAAA,CAAY,IAAA,CAAK,IAAI,IAAI,CAAA;AACzB,IAAA,SAAA,GAAY,KAAK,GAAA,CAAI,SAAA,EAAW,GAAA,CAAI,QAAA,GAAW,IAAI,WAAW,CAAA;AAC9D,IAAA,MAAM,MAAA,GAAS,WAAA,CAAY,IAAA,CAAK,GAAG,CAAA;AACnC,IAAA,MAAM,OAAA,GAAU,IAAI,QAAA,GAAW,WAAA;AAC/B,IAAA,MAAM,YAAA,GAAe,SAAA,CAAU,IAAA,CAAK,GAAA,CAAI,IAAI,CAAA;AAE5C,IAAA,IAAK,WAAW,EAAA,IAAM,YAAA,IAAiB,WAAW,EAAA,IAAM,MAAA,CAAO,UAAU,GAAA,EAAK;AAC5E,MAAA,QAAA,CAAS,IAAA,CAAK;AAAA,QACZ,QAAA,EAAU,WAAA;AAAA,QACV,WAAA,EAAa,KAAK,GAAA,CAAI,CAAA,EAAG,KAAK,KAAA,CAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAAA,QAC5D,SAAA,EAAW,gBAAgB,WAAW,CAAA;AAAA,QACtC,IAAA,EAAM;AAAA,OACP,CAAA;AACD,MAAA,WAAA,GAAc,EAAC;AACf,MAAA,WAAA,GAAc,GAAA,CAAI,WAAW,GAAA,CAAI,WAAA;AAAA,IACnC;AAAA,EACF;AAEA,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,EAAG;AAC1B,IAAA,QAAA,CAAS,IAAA,CAAK;AAAA,MACZ,QAAA,EAAU,WAAA;AAAA,MACV,WAAA,EAAa,KAAK,GAAA,CAAI,CAAA,EAAG,KAAK,KAAA,CAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAAA,MAC5D,SAAA,EAAW,gBAAgB,WAAW,CAAA;AAAA,MACtC,IAAA,EAAM,WAAA,CAAY,IAAA,CAAK,GAAG;AAAA,KAC3B,CAAA;AAAA,EACH;AAEA,EAAA,OAAO,QAAA;AACT;AAKO,IAAM,0BAAN,MAA8B;AAAA,EACnC,MAAM,eAAA,CAAgB,OAAA,EAAiB,qBAA+B,CAAC,IAAA,EAAM,IAAI,CAAA,EAAqC;AACpH,IAAA,MAAM,OAAA,GAAU,sBAAsB,OAAO,CAAA;AAC7C,IAAA,IAAI,CAAC,OAAA,EAAS;AACZ,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iCAAA,EAAoC,OAAO,CAAA,CAAE,CAAA;AAAA,IAC/D;AAEA,IAAA,MAAM,YAAA,GAAe,mCAAmC,OAAO,CAAA,CAAA;AAC/D,IAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,YAAA,EAAc;AAAA,MACpC,OAAA,EAAS;AAAA,QACP,YAAA,EAAc,uHAAA;AAAA,QACd,iBAAA,EAAmB;AAAA,OACrB;AAAA,MACA,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,IAAK;AAAA,KAClC,CAAA;AAED,IAAA,IAAI,CAAC,IAAI,EAAA,EAAI;AACX,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,oCAAA,EAAuC,GAAA,CAAI,MAAM,CAAA,CAAA,CAAG,CAAA;AAAA,IACtE;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,GAAA,CAAI,IAAA,EAAK;AAC5B,IAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,+EAA+E,CAAA;AAC9G,IAAA,IAAI,KAAA,GAAQ,kBAAkB,OAAO,CAAA,CAAA,CAAA;AACrC,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI,gBAAA,GAAmB,EAAA;AACvB,IAAA,IAAI,cAA8E,EAAC;AACnF,IAAA,IAAI,YAAA,GAAe,IAAA;AACnB,IAAA,IAAI,eAAA,GAAkB,KAAA;AAEtB,IAAA,IAAI,WAAA,GAAc,CAAC,CAAA,EAAG;AACpB,MAAA,IAAI;AACF,QAAA,MAAM,UAAA,GAAa,IAAA,CAAK,KAAA,CAAM,WAAA,CAAY,CAAC,CAAC,CAAA;AAC5C,QAAA,KAAA,GAAQ,UAAA,EAAY,cAAc,KAAA,IAAS,KAAA;AAC3C,QAAA,MAAA,GAAS,YAAY,YAAA,EAAc,MAAA;AACnC,QAAA,gBAAA,GAAmB,UAAA,EAAY,cAAc,gBAAA,IAAoB,EAAA;AAEjE,QAAA,MAAM,MAAA,GACJ,UAAA,EAAY,QAAA,EAAU,+BAAA,EAAiC,iBAAiB,EAAC;AAE3E,QAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AAErB,UAAA,IAAI,cACF,MAAA,CAAO,IAAA;AAAA,YACL,CAAC,MAAM,kBAAA,CAAmB,QAAA,CAAS,EAAE,YAAY,CAAA,IAAK,EAAE,IAAA,KAAS;AAAA,WACnE,IACA,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAAM,kBAAA,CAAmB,QAAA,CAAS,CAAA,CAAE,YAAY,CAAC,CAAA,IAC9D,MAAA,CAAO,CAAC,CAAA;AAEV,UAAA,IAAI,aAAa,OAAA,EAAS;AACxB,YAAA,YAAA,GAAe,YAAY,YAAA,IAAgB,IAAA;AAC3C,YAAA,eAAA,GAAkB,YAAY,IAAA,KAAS,KAAA;AACvC,YAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,WAAA,CAAY,OAAA,EAAS;AAAA,cAChD,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,GAAK;AAAA,aAClC,CAAA;AACD,YAAA,IAAI,SAAS,EAAA,EAAI;AACf,cAAA,MAAM,GAAA,GAAM,MAAM,QAAA,CAAS,IAAA,EAAK;AAChC,cAAA,MAAM,WAAA,GAAc;AAAA,gBAClB,GAAG,GAAA,CAAI,QAAA,CAAS,yEAAyE;AAAA,eAC3F;AACA,cAAA,KAAA,MAAW,KAAK,WAAA,EAAa;AAC3B,gBAAA,WAAA,CAAY,IAAA,CAAK;AAAA,kBACf,QAAA,EAAU,UAAA,CAAW,CAAA,CAAE,CAAC,KAAK,GAAG,CAAA;AAAA,kBAChC,WAAA,EAAa,UAAA,CAAW,CAAA,CAAE,CAAC,KAAK,GAAG,CAAA;AAAA,kBACnC,OAAO,CAAA,CAAE,CAAC,KAAK,EAAA,EAAI,OAAA,CAAQ,YAAY,GAAG;AAAA,iBAC3C,CAAA;AAAA,cACH;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAA,CAAA,MAAQ;AAAA,MAER;AAAA,IACF;AAEA,IAAA,IAAI,KAAA,KAAU,CAAA,eAAA,EAAkB,OAAO,CAAA,CAAA,CAAA,EAAK;AAC1C,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,KAAA,CAAM,6BAA6B,CAAA;AAC1D,MAAA,IAAI,SAAA,GAAY,CAAC,CAAA,EAAG;AAClB,QAAA,KAAA,GAAQ,mBAAmB,SAAA,CAAU,CAAC,EAAE,OAAA,CAAQ,kBAAA,EAAoB,EAAE,CAAC,CAAA;AAAA,MACzE;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,sBAAsB,WAAW,CAAA;AAClD,IAAA,MAAM,OAAA,GAAoB;AAAA,MACxB,KAAK,KAAK,CAAA,CAAA;AAAA,MACV,CAAA,cAAA,EAAiB,YAAY,CAAA,EAAG,MAAA,GAAS,CAAA,gBAAA,EAAmB,MAAM,CAAA,CAAA,GAAK,EAAE,CAAA,mBAAA,EAAsB,YAAY,CAAA,EAAA,EAAK,eAAA,GAAkB,oBAAoB,EAAE,CAAA;AAAA,KAC1J;AAEA,IAAA,IAAI,gBAAA,CAAiB,MAAK,EAAG;AAC3B,MAAA,OAAA,CAAQ,IAAA,CAAK;AAAA;AAAA,EAAsC,iBAAiB,IAAA,EAAK,CAAE,MAAM,CAAA,EAAG,IAAI,CAAC,CAAA,CAAE,CAAA;AAAA,IAC7F;AAEA,IAAA,IAAI,QAAA,CAAS,SAAS,CAAA,EAAG;AACvB,MAAA,OAAA,CAAQ,IAAA,CAAK;AAAA,yBAAA,CAA6B,CAAA;AAC1C,MAAA,KAAA,MAAW,OAAO,QAAA,EAAU;AAC1B,QAAA,OAAA,CAAQ,KAAK,CAAA,KAAA,EAAQ,GAAA,CAAI,SAAS,CAAA,IAAA,EAAO,GAAA,CAAI,IAAI,CAAA,CAAE,CAAA;AAAA,MACrD;AAAA,IACF;AAEA,IAAA,OAAO;AAAA,MACL,OAAA;AAAA,MACA,GAAA,EAAK,YAAA;AAAA,MACL,KAAA;AAAA,MACA,MAAA;AAAA,MACA,QAAA,EAAU,YAAA;AAAA,MACV,eAAA;AAAA,MACA,QAAA,EAAU,QAAA;AAAA,MACV,gBAAA,EAAkB,OAAA,CAAQ,IAAA,CAAK,IAAI;AAAA,KACrC;AAAA,EACF;AACF;;;AC/MO,SAAS,0BAAA,CAA2B,QAAA,EAAkB,WAAA,GAAc,CAAA,EAAa;AACtF,EAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,iBAAA,EAAmB,IAAI,CAAA;AAC5D,EAAA,MAAM,SAAS,WAAA,CACZ,KAAA,CAAM,QAAQ,CAAA,CACd,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,MAAA,EAAQ,GAAG,CAAA,CAAE,IAAA,EAAM,CAAA,CACxC,MAAA,CAAO,CAAC,CAAA,KAAM;AACb,IAAA,IAAI,EAAE,MAAA,GAAS,EAAA,IAAM,CAAA,CAAE,MAAA,GAAS,KAAK,OAAO,KAAA;AAC5C,IAAA,IAAI,CAAA,CAAE,WAAW,GAAG,CAAA,IAAK,EAAE,UAAA,CAAW,GAAG,GAAG,OAAO,KAAA;AACnD,IAAA,MAAM,SAAA,GAAA,CAAa,CAAA,CAAE,KAAA,CAAM,sBAAsB,CAAA,IAAK,EAAC,EAAG,IAAA,CAAK,EAAE,CAAA,CAAE,MAAA,GAAS,CAAA,CAAE,MAAA;AAC9E,IAAA,IAAI,SAAA,GAAY,MAAM,OAAO,KAAA;AAC7B,IAAA,IAAI,qHAAA,CAAsH,IAAA,CAAK,CAAC,CAAA,EAAG;AACjI,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,OAAO,IAAA;AAAA,EACT,CAAC,CAAA;AAEH,EAAA,OAAO,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,WAAW,CAAA;AACpC;AAKO,IAAM,aAAN,MAAiB;AAAA,EACd,YAAA;AAAA,EACA,OAAA;AAAA,EACA,gBAAA;AAAA,EACA,aAAA;AAAA,EACA,cAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAA6B,EAAC,EAAG;AAC3C,IAAA,IAAA,CAAK,gBAAA,GAAmB,QAAQ,gBAAA,IAAoB,IAAA;AACpD,IAAA,IAAA,CAAK,YAAA,GACH,QAAQ,YAAA,KAAiB,IAAA,CAAK,mBAAmB,OAAA,CAAQ,GAAA,CAAI,gBAAgB,MAAA,CAAA,IAAc,wBAAA;AAC7F,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA,KAAY,KAAK,gBAAA,GAAmB,OAAA,CAAQ,IAAI,YAAA,GAAe,MAAA,CAAA;AACtF,IAAA,IAAA,CAAK,gBAAgB,OAAA,CAAQ,aAAA;AAC7B,IAAA,IAAA,CAAK,cAAA,GAAiB,IAAI,uBAAA,EAAwB;AAAA,EACpD;AAAA,EAEA,iBAAiB,OAAA,EAA+B;AAC9C,IAAA,IAAA,CAAK,aAAA,GAAgB,OAAA;AAAA,EACvB;AAAA,EAEA,WAAW,GAAA,EAAc;AACvB,IAAA,IAAA,CAAK,OAAA,GAAU,GAAA;AAAA,EACjB;AAAA,EAEA,MAAM,SAAA,CACJ,SAAA,EACA,OAAA,GAAqG,EAAC,EAC5E;AAC1B,IAAA,MAAM,OAAA,GAAU,OAAA,CAAQ,aAAA,IAAiB,IAAA,CAAK,aAAA;AAG9C,IAAA,IAAI,OAAA,EAAS;AACX,MAAA,IAAI;AACF,QAAA,MAAM,MAAA,GAAS,MAAM,OAAA,CAAQ,KAAA,CAAM,SAAS,CAAA;AAC5C,QAAA,IAAI,MAAA,IAAU,OAAO,eAAA,EAAiB;AACpC,UAAA,OAAO,KAAK,eAAA,CAAgB,SAAA,EAAW,OAAO,eAAA,EAAiB,QAAA,EAAU,OAAO,KAAK,CAAA;AAAA,QACvF;AAAA,MACF,SAAS,CAAA,EAAQ;AACf,QAAA,OAAA,CAAQ,KAAK,CAAA,+CAAA,EAAkD,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,MAC9F;AAAA,IACF;AAGA,IAAA,IAAI,YAAA,CAAa,SAAS,CAAA,EAAG;AAC3B,MAAA,IAAI;AACF,QAAA,MAAM,KAAK,MAAM,IAAA,CAAK,eAAe,eAAA,CAAgB,SAAA,EAAW,QAAQ,kBAAkB,CAAA;AAC1F,QAAA,IAAI,EAAA,CAAG,gBAAA,IAAoB,EAAA,CAAG,gBAAA,CAAiB,SAAS,EAAA,EAAI;AAC1D,UAAA,OAAO,IAAA,CAAK,gBAAgB,EAAA,CAAG,GAAA,EAAK,GAAG,gBAAA,EAAkB,SAAA,EAAW,GAAG,KAAK,CAAA;AAAA,QAC9E;AAAA,MACF,SAAS,CAAA,EAAQ;AACf,QAAA,OAAA,CAAQ,KAAK,CAAA,sDAAA,EAAyD,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,MACrG;AAAA,IACF;AAGA,IAAA,IAAI;AACF,MAAA,MAAM,OAAA,GAAU,qBAAqB,SAAS,CAAA,CAAA;AAC9C,MAAA,MAAM,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,IAAA,CAAK,OAAA;AACxC,MAAA,MAAM,OAAA,GAAkC;AAAA,QACtC,YAAA,EAAc,sBAAA;AAAA,QACd,iBAAA,EAAmB;AAAA,OACrB;AACA,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,OAAA,CAAQ,eAAe,CAAA,GAAI,CAAA,OAAA,EAAU,OAAO,CAAA,CAAA;AAAA,MAC9C;AAEA,MAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,OAAA,EAAS;AAAA,QAC/B,OAAA;AAAA,QACA,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,IAAK;AAAA,OAClC,CAAA;AAED,MAAA,IAAI,IAAI,EAAA,EAAI;AACV,QAAA,MAAM,QAAA,GAAW,MAAM,GAAA,CAAI,IAAA,EAAK;AAChC,QAAA,IAAI,QAAA,IAAY,QAAA,CAAS,MAAA,GAAS,GAAA,EAAK;AACrC,UAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,SAAA,EAAW,QAAA,EAAU,MAAM,CAAA;AAAA,QACzD;AAAA,MACF;AAAA,IACF,SAAS,CAAA,EAAQ;AACf,MAAA,OAAA,CAAQ,KAAK,CAAA,oCAAA,EAAuC,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,IACnF;AAGA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAI;AACF,QAAA,MAAM,MAAM,MAAM,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,YAAY,CAAA,MAAA,CAAA,EAAU;AAAA,UACpD,MAAA,EAAQ,MAAA;AAAA,UACR,OAAA,EAAS,EAAE,cAAA,EAAgB,kBAAA,EAAmB;AAAA,UAC9C,IAAA,EAAM,KAAK,SAAA,CAAU;AAAA,YACnB,IAAA,EAAM,SAAA;AAAA,YACN,QAAA,EAAU;AAAA,WACX,CAAA;AAAA,UACD,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,IAAK;AAAA,SAClC,CAAA;AAED,QAAA,IAAI,IAAI,EAAA,EAAI;AACV,UAAA,MAAM,IAAA,GAAO,MAAM,GAAA,CAAI,IAAA,EAAK;AAC5B,UAAA,MAAM,WAAW,IAAA,EAAM,OAAA,GAAU,CAAC,CAAA,EAAG,YAAY,IAAA,EAAM,QAAA;AACvD,UAAA,IAAI,QAAA,EAAU;AACZ,YAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,SAAA,EAAW,QAAA,EAAU,UAAU,CAAA;AAAA,UAC7D;AAAA,QACF;AAAA,MACF,CAAA,CAAA,MAAQ;AAAA,MAER;AAAA,IACF;AAGA,IAAA,IAAI;AACF,MAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,SAAA,EAAW;AAAA,QACjC,OAAA,EAAS,EAAE,YAAA,EAAc,iDAAA,EAAkD;AAAA,QAC3E,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,GAAK;AAAA,OAClC,CAAA;AACD,MAAA,IAAI,IAAI,EAAA,EAAI;AACV,QAAA,MAAM,OAAA,GAAU,MAAM,GAAA,CAAI,IAAA,EAAK;AAC/B,QAAA,MAAM,WAAW,OAAA,CAAQ,KAAA,CAAM,kCAAkC,CAAA,GAAI,CAAC,GAAG,IAAA,EAAK;AAC9E,QAAA,MAAM,KAAA,GAAQ,QACX,OAAA,CAAQ,qDAAA,EAAuD,EAAE,CAAA,CACjE,OAAA,CAAQ,kDAAA,EAAoD,EAAE,CAAA,CAC9D,OAAA,CAAQ,8CAA8C,EAAE,CAAA,CACxD,OAAA,CAAQ,qDAAA,EAAuD,EAAE,CAAA,CACjE,QAAQ,oCAAA,EAAsC,CAAC,CAAA,EAAG,GAAA,EAAK,GAAA,KAAQ;AAAA,EAAK,GAAA,CAAI,MAAA,CAAO,MAAA,CAAO,GAAG,CAAC,CAAC,CAAA,CAAA,EAAI,GAAA,CAAI,OAAA,CAAQ,UAAA,EAAY,EAAE,CAAA,CAAE,MAAM;AAAA,CAAI,CAAA,CACrI,OAAA,CAAQ,kDAAA,EAAoD,CAAC,GAAG,IAAA,KAAS;AAAA;AAAA,EAAa,KAAK,OAAA,CAAQ,UAAA,EAAY,EAAE,CAAA,CAAE,MAAM;AAAA;AAAA,CAAY,CAAA,CACrI,OAAA,CAAQ,2BAAA,EAA6B,CAAC,GAAG,CAAA,KAAM;AAAA,EAAK,EAAE,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA,CAAE,MAAM;AAAA,CAAI,CAAA,CACzF,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA,CACvB,OAAA,CAAQ,SAAA,EAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,SAAA,EAAW,MAAM,EACzB,IAAA,EAAK;AAER,QAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,SAAA,EAAW,KAAA,EAAO,UAAU,QAAQ,CAAA;AAAA,MAClE;AAAA,IACF,SAAS,CAAA,EAAQ;AACf,MAAA,OAAA,CAAQ,KAAK,CAAA,qCAAA,EAAwC,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,IACpF;AAEA,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,kCAAA,EAAqC,SAAS,CAAA,CAAE,CAAA;AAAA,EAClE;AAAA,EAEQ,eAAA,CACN,GAAA,EACA,QAAA,EACA,QAAA,EACA,aAAA,EACiB;AACjB,IAAA,MAAM,aAAa,QAAA,CAAS,KAAA,CAAM,aAAa,CAAA,IAAK,QAAA,CAAS,MAAM,mBAAmB,CAAA;AACtF,IAAA,MAAM,KAAA,GAAA,CAAS,kBAAkB,UAAA,GAAa,UAAA,CAAW,CAAC,CAAA,CAAG,IAAA,KAAS,GAAA,CAAI,KAAA,CAAM,GAAG,CAAA,CAAE,MAAA,CAAO,OAAO,CAAA,CAAE,GAAA,MAAS,UAAA,CAAA,EAAa,KAAA,CAAM,GAAG,GAAG,CAAA;AAGvI,IAAA,MAAM,mBAAmB,CAAC,GAAG,QAAA,CAAS,QAAA,CAAS,sCAAsC,CAAC,CAAA;AACtF,IAAA,MAAM,mBAAA,GAA4C,gBAAA,CAC/C,GAAA,CAAI,CAAC,CAAA,MAAO;AAAA,MACX,WAAW,CAAA,CAAE,CAAC,KAAK,EAAA,EAAI,IAAA,GAAO,WAAA,EAAY;AAAA,MAC1C,IAAA,EAAA,CAAO,CAAA,CAAE,CAAC,CAAA,IAAK,IAAI,IAAA;AAAK,KAC1B,CAAE,EACD,MAAA,CAAO,CAAC,MAAM,CAAA,CAAE,IAAA,CAAK,SAAS,CAAC,CAAA;AAClC,IAAA,MAAM,wBAAwB,mBAAA,CAAoB,GAAA,CAAI,CAAC,CAAA,KAAM,EAAE,IAAI,CAAA;AAGnE,IAAA,MAAM,WAAA,GACJ,SAAS,KAAA,CAAM,wDAAwD,KACvE,QAAA,CAAS,KAAA,CAAM,yBAAyB,CAAA,IACxC,EAAC;AACH,IAAA,MAAM,eAAA,GAAkB,WAAA,CAAY,GAAA,CAAI,CAAC,MAAM,CAAA,CAAE,IAAA,EAAM,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,SAAS,EAAE,CAAA;AAGpF,IAAA,MAAM,iBAAA,GAAoB,CAAC,GAAG,QAAA,CAAS,QAAA,CAAS,mBAAmB,CAAC,CAAA,CACjE,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,CAAC,CAAA,CAAG,IAAA,EAAM,CAAA,CACvB,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,MAAA,GAAS,CAAA,IAAK,CAAA,CAAE,MAAA,GAAS,GAAG,CAAA;AAE/C,IAAA,MAAM,WAAA,GAAc,0BAAA,CAA2B,QAAA,EAAU,CAAC,CAAA;AAE1D,IAAA,OAAO;AAAA,MACL,GAAA;AAAA,MACA,KAAA;AAAA,MACA,eAAA,EAAiB,QAAA;AAAA,MACjB,QAAA,EAAU,MAAA,CAAO,UAAA,CAAW,QAAA,EAAU,OAAO,CAAA;AAAA,MAC7C,qBAAA;AAAA,MACA,mBAAA;AAAA,MACA,eAAA;AAAA,MACA,iBAAA;AAAA,MACA,WAAA;AAAA,MACA,aAAa,eAAA,CAAgB,MAAA;AAAA,MAC7B,aAAA,EAAe;AAAA,KACjB;AAAA,EACF;AACF","file":"chunk-KAFQVI2R.js","sourcesContent":["export interface TranscriptSegment {\n startSec: number;\n durationSec: number;\n timestamp: string;\n text: string;\n}\n\nexport interface YouTubeTranscriptResult {\n videoId: string;\n url: string;\n title: string;\n author?: string;\n language: string;\n isAutoGenerated: boolean;\n segments: TranscriptSegment[];\n stitchedMarkdown: string;\n}\n\nconst SPONSOR_NOISE_PATTERNS = [\n /\\b(sponsored by|this video is brought to you by|use (?:promo )?code\\b|don't forget to like and subscribe|hit the bell icon|link in the description)\\b/i,\n /(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\\s+này)?\\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i,\n];\n\nexport function extractYouTubeVideoId(urlOrId: string): string | null {\n const trimmed = urlOrId.trim();\n if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;\n\n try {\n const u = new URL(trimmed);\n const host = u.hostname.replace(/^www\\./, '').toLowerCase();\n if (host === 'youtu.be') {\n const id = u.pathname.split('/').filter(Boolean)[0];\n return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;\n }\n if (host === 'youtube.com' || host === 'm.youtube.com' || host === 'youtube-nocookie.com') {\n const v = u.searchParams.get('v');\n if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;\n const parts = u.pathname.split('/').filter(Boolean);\n if ((parts[0] === 'embed' || parts[0] === 'shorts' || parts[0] === 'v') && parts[1]) {\n return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;\n }\n }\n } catch {\n // not a valid URL\n }\n return null;\n}\n\nexport function isYouTubeUrl(url: string): boolean {\n return extractYouTubeVideoId(url) !== null;\n}\n\nexport function formatTimestamp(seconds: number): string {\n const total = Math.max(0, Math.floor(seconds));\n const hrs = Math.floor(total / 3600);\n const mins = Math.floor((total % 3600) / 60);\n const secs = total % 60;\n if (hrs > 0) {\n return `${String(hrs).padStart(2, '0')}:${String(mins).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;\n }\n return `${String(mins).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;\n}\n\nfunction decodeHtmlEntities(str: string): string {\n return str\n .replace(/&/g, '&')\n .replace(/</g, '<')\n .replace(/>/g, '>')\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/'/g, \"'\")\n .replace(/ /g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\n/**\n * Stitches short caption fragments into coherent timestamped paragraphs (every ~30-45 seconds or sentence boundary)\n * and strips sponsor/subscribe filler noise.\n */\nexport function stitchCaptionSegments(segments: Array<{ startSec: number; durationSec: number; text: string }>): TranscriptSegment[] {\n const cleaned = segments\n .map((s) => ({\n startSec: s.startSec,\n durationSec: s.durationSec,\n text: decodeHtmlEntities(s.text),\n }))\n .filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));\n\n if (cleaned.length === 0) return [];\n\n const stitched: TranscriptSegment[] = [];\n let bucketStart = cleaned[0]!.startSec;\n let bucketEnd = cleaned[0]!.startSec + cleaned[0]!.durationSec;\n let bucketTexts: string[] = [];\n\n for (const seg of cleaned) {\n bucketTexts.push(seg.text);\n bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);\n const joined = bucketTexts.join(' ');\n const elapsed = seg.startSec - bucketStart;\n const endsSentence = /[.!?…]$/.test(seg.text);\n\n if ((elapsed >= 30 && endsSentence) || elapsed >= 50 || joined.length >= 360) {\n stitched.push({\n startSec: bucketStart,\n durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),\n timestamp: formatTimestamp(bucketStart),\n text: joined,\n });\n bucketTexts = [];\n bucketStart = seg.startSec + seg.durationSec;\n }\n }\n\n if (bucketTexts.length > 0) {\n stitched.push({\n startSec: bucketStart,\n durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),\n timestamp: formatTimestamp(bucketStart),\n text: bucketTexts.join(' '),\n });\n }\n\n return stitched;\n}\n\n/**\n * YouTube Transcript Ingestion Service (Idea §3.1)\n */\nexport class YouTubeIngestionService {\n async fetchTranscript(urlOrId: string, preferredLanguages: string[] = ['vi', 'en']): Promise<YouTubeTranscriptResult> {\n const videoId = extractYouTubeVideoId(urlOrId);\n if (!videoId) {\n throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);\n }\n\n const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;\n const res = await fetch(canonicalUrl, {\n headers: {\n 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',\n 'Accept-Language': 'vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7',\n },\n signal: AbortSignal.timeout(12000),\n });\n\n if (!res.ok) {\n throw new Error(`Failed to fetch YouTube watch page (${res.status})`);\n }\n\n const html = await res.text();\n const playerMatch = html.match(/ytInitialPlayerResponse\\s*=\\s*(\\{[\\s\\S]*?\\});\\s*(?:var\\s|window\\.|<\\/script>)/);\n let title = `YouTube Video (${videoId})`;\n let author: string | undefined;\n let shortDescription = '';\n let rawSegments: Array<{ startSec: number; durationSec: number; text: string }> = [];\n let selectedLang = 'en';\n let isAutoGenerated = false;\n\n if (playerMatch?.[1]) {\n try {\n const playerData = JSON.parse(playerMatch[1]);\n title = playerData?.videoDetails?.title || title;\n author = playerData?.videoDetails?.author;\n shortDescription = playerData?.videoDetails?.shortDescription || '';\n\n const tracks: any[] =\n playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];\n\n if (tracks.length > 0) {\n // Prioritize manual tracks in preferredLanguages, then auto-generated in preferredLanguages, then first track\n let chosenTrack =\n tracks.find(\n (t) => preferredLanguages.includes(t.languageCode) && t.kind !== 'asr'\n ) ||\n tracks.find((t) => preferredLanguages.includes(t.languageCode)) ||\n tracks[0];\n\n if (chosenTrack?.baseUrl) {\n selectedLang = chosenTrack.languageCode || 'en';\n isAutoGenerated = chosenTrack.kind === 'asr';\n const trackRes = await fetch(chosenTrack.baseUrl, {\n signal: AbortSignal.timeout(10000),\n });\n if (trackRes.ok) {\n const xml = await trackRes.text();\n const textMatches = [\n ...xml.matchAll(/<text\\s+start=\"([\\d.]+)\"(?:\\s+dur=\"([\\d.]+)\")?[^>]*>([\\s\\S]*?)<\\/text>/g),\n ];\n for (const m of textMatches) {\n rawSegments.push({\n startSec: parseFloat(m[1] || '0'),\n durationSec: parseFloat(m[2] || '2'),\n text: (m[3] || '').replace(/<[^>]+>/g, ' '),\n });\n }\n }\n }\n }\n } catch {\n // Fallback to meta tags below\n }\n }\n\n if (title === `YouTube Video (${videoId})`) {\n const titleMeta = html.match(/<title>([\\s\\S]*?)<\\/title>/i);\n if (titleMeta?.[1]) {\n title = decodeHtmlEntities(titleMeta[1].replace(/\\s*-\\s*YouTube$/i, ''));\n }\n }\n\n const stitched = stitchCaptionSegments(rawSegments);\n const mdLines: string[] = [\n `# ${title}`,\n `> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ''} | **Language:** \\`${selectedLang}\\`${isAutoGenerated ? ' (Auto-caption)' : ''}`,\n ];\n\n if (shortDescription.trim()) {\n mdLines.push(`\\n## Video Overview & Description\\n${shortDescription.trim().slice(0, 1500)}`);\n }\n\n if (stitched.length > 0) {\n mdLines.push(`\\n## Timestamped Transcript`);\n for (const seg of stitched) {\n mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);\n }\n }\n\n return {\n videoId,\n url: canonicalUrl,\n title,\n author,\n language: selectedLang,\n isAutoGenerated,\n segments: stitched,\n stitchedMarkdown: mdLines.join('\\n'),\n };\n }\n}\n","import { CustomCrawlerAdapter } from '../config/researchConfig';\nimport { isYouTubeUrl, YouTubeIngestionService } from './youtubeTranscript';\n\nexport interface ExtractedCodeBlock {\n language: string;\n code: string;\n}\n\nexport interface ScrapedDocument {\n url: string;\n title: string;\n markdownContent: string;\n byteSize: number;\n extractedCodeSnippets: string[];\n extractedCodeBlocks?: ExtractedCodeBlock[];\n extractedTables?: string[];\n extractedHeadings?: string[];\n keyExcerpts?: string[];\n tablesCount: number;\n fetchProvider: 'jina' | 'crawl4ai' | 'youtube' | 'direct' | 'custom';\n}\n\nexport interface WebScraperOptions {\n crawl4aiHost?: string;\n jinaKey?: string;\n allowEnvFallback?: boolean;\n customAdapter?: CustomCrawlerAdapter;\n}\n\n/**\n * Extracts top substantive paragraphs from Markdown (stripping navigation, links-only lines, and cookie banners).\n */\nexport function extractSubstantiveExcerpts(markdown: string, maxExcerpts = 6): string[] {\n const withoutCode = markdown.replace(/```[\\s\\S]*?```/g, '\\n');\n const blocks = withoutCode\n .split(/\\n{2,}/)\n .map((b) => b.replace(/\\s+/g, ' ').trim())\n .filter((b) => {\n if (b.length < 60 || b.length > 900) return false;\n if (b.startsWith('#') || b.startsWith('|')) return false;\n const linkRatio = (b.match(/\\[[^\\]]+\\]\\([^)]+\\)/g) || []).join('').length / b.length;\n if (linkRatio > 0.45) return false;\n if (/\\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\\b/i.test(b)) {\n return false;\n }\n return true;\n });\n\n return blocks.slice(0, maxExcerpts);\n}\n\n/**\n * High-fidelity web content & YouTube transcript extractor with Custom Crawler & Jina Auth support\n */\nexport class WebScraper {\n private crawl4aiHost?: string;\n private jinaKey?: string;\n private allowEnvFallback: boolean;\n private customAdapter?: CustomCrawlerAdapter;\n private youtubeService: YouTubeIngestionService;\n\n constructor(options: WebScraperOptions = {}) {\n this.allowEnvFallback = options.allowEnvFallback ?? true;\n this.crawl4aiHost =\n options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : undefined) || 'http://localhost:11235';\n this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : undefined);\n this.customAdapter = options.customAdapter;\n this.youtubeService = new YouTubeIngestionService();\n }\n\n setCustomAdapter(adapter: CustomCrawlerAdapter) {\n this.customAdapter = adapter;\n }\n\n setJinaKey(key?: string) {\n this.jinaKey = key;\n }\n\n async scrapeUrl(\n targetUrl: string,\n options: { customAdapter?: CustomCrawlerAdapter; jinaKey?: string; preferredLanguages?: string[] } = {}\n ): Promise<ScrapedDocument> {\n const adapter = options.customAdapter || this.customAdapter;\n\n // 0. Custom Crawler Adapter (e.g. MCP bridge, Playwright, Headless Chrome)\n if (adapter) {\n try {\n const result = await adapter.crawl(targetUrl);\n if (result && result.markdownContent) {\n return this.packageDocument(targetUrl, result.markdownContent, 'custom', result.title);\n }\n } catch (e: any) {\n console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);\n }\n }\n\n // 0.5. YouTube Video URL -> Route to YouTubeIngestionService (Idea §3.1)\n if (isYouTubeUrl(targetUrl)) {\n try {\n const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);\n if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {\n return this.packageDocument(yt.url, yt.stitchedMarkdown, 'youtube', yt.title);\n }\n } catch (e: any) {\n console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);\n }\n }\n\n // 1. Try Jina Reader API (with optional Bearer auth from host app or env)\n try {\n const jinaUrl = `https://r.jina.ai/${targetUrl}`;\n const jinaKey = options.jinaKey || this.jinaKey;\n const headers: Record<string, string> = {\n 'User-Agent': 'OrchableComposer/1.0',\n 'X-Return-Format': 'markdown',\n };\n if (jinaKey) {\n headers['Authorization'] = `Bearer ${jinaKey}`;\n }\n\n const res = await fetch(jinaUrl, {\n headers,\n signal: AbortSignal.timeout(15000),\n });\n\n if (res.ok) {\n const markdown = await res.text();\n if (markdown && markdown.length > 100) {\n return this.packageDocument(targetUrl, markdown, 'jina');\n }\n }\n } catch (e: any) {\n console.warn(`[WebScraper] Jina Reader failed for ${targetUrl}:`, e?.message || e);\n }\n\n // 2. Try Crawl4AI Local Server if reachable\n if (this.crawl4aiHost) {\n try {\n const res = await fetch(`${this.crawl4aiHost}/crawl`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({\n urls: targetUrl,\n priority: 10,\n }),\n signal: AbortSignal.timeout(15000),\n });\n\n if (res.ok) {\n const data = await res.json();\n const markdown = data?.results?.[0]?.markdown || data?.markdown;\n if (markdown) {\n return this.packageDocument(targetUrl, markdown, 'crawl4ai');\n }\n }\n } catch {\n // ignore if local crawl4ai is not running\n }\n }\n\n // 3. Fallback: Direct Fetch & Structured HTML-to-Markdown Conversion\n try {\n const res = await fetch(targetUrl, {\n headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' },\n signal: AbortSignal.timeout(10000),\n });\n if (res.ok) {\n const rawHtml = await res.text();\n const titleTag = rawHtml.match(/<title[^>]*>([\\s\\S]*?)<\\/title>/i)?.[1]?.trim();\n const clean = rawHtml\n .replace(/<script\\b[^<]*(?:(?!<\\/script>)<[^<]*)*<\\/script>/gi, '')\n .replace(/<style\\b[^<]*(?:(?!<\\/style>)<[^<]*)*<\\/style>/gi, '')\n .replace(/<nav\\b[^<]*(?:(?!<\\/nav>)<[^<]*)*<\\/nav>/gi, '')\n .replace(/<footer\\b[^<]*(?:(?!<\\/footer>)<[^<]*)*<\\/footer>/gi, '')\n .replace(/<h([1-3])[^>]*>([\\s\\S]*?)<\\/h\\1>/gi, (_, lvl, txt) => `\\n${'#'.repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, '').trim()}\\n`)\n .replace(/<pre[^>]*><code[^>]*>([\\s\\S]*?)<\\/code><\\/pre>/gi, (_, code) => `\\n\\`\\`\\`\\n${code.replace(/<[^>]+>/g, '').trim()}\\n\\`\\`\\`\\n`)\n .replace(/<p[^>]*>([\\s\\S]*?)<\\/p>/gi, (_, p) => `\\n${p.replace(/<[^>]+>/g, ' ').trim()}\\n`)\n .replace(/<[^>]+>/g, ' ')\n .replace(/[ \\t]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim();\n\n return this.packageDocument(targetUrl, clean, 'direct', titleTag);\n }\n } catch (e: any) {\n console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);\n }\n\n throw new Error(`Unable to fetch content from URL: ${targetUrl}`);\n }\n\n private packageDocument(\n url: string,\n markdown: string,\n provider: 'jina' | 'crawl4ai' | 'youtube' | 'direct' | 'custom',\n explicitTitle?: string\n ): ScrapedDocument {\n const titleMatch = markdown.match(/^#\\s+(.+)$/m) || markdown.match(/^Title:\\s*(.+)$/im);\n const title = (explicitTitle || (titleMatch ? titleMatch[1]!.trim() : url.split('/').filter(Boolean).pop() || 'Document')).slice(0, 180);\n\n // Extract code blocks with language tags\n const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\\n([\\s\\S]*?)```/g)];\n const extractedCodeBlocks: ExtractedCodeBlock[] = codeBlockMatches\n .map((m) => ({\n language: (m[1] || '').trim().toLowerCase(),\n code: (m[2] || '').trim(),\n }))\n .filter((b) => b.code.length > 0);\n const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);\n\n // Extract markdown tables (contiguous pipe lines with separator row)\n const tableBlocks =\n markdown.match(/(?:^\\|.+\\|\\s*\\n)+^\\|[-:| ]+\\|\\s*\\n(?:^\\|.+\\|\\s*\\n?)+/gm) ||\n markdown.match(/\\|[\\s\\S]*?\\|[\\s\\S]*?\\|/g) ||\n [];\n const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);\n\n // Extract section headings\n const extractedHeadings = [...markdown.matchAll(/^#{1,3}\\s+(.+)$/gm)]\n .map((m) => m[1]!.trim())\n .filter((h) => h.length > 2 && h.length < 120);\n\n const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);\n\n return {\n url,\n title,\n markdownContent: markdown,\n byteSize: Buffer.byteLength(markdown, 'utf-8'),\n extractedCodeSnippets,\n extractedCodeBlocks,\n extractedTables,\n extractedHeadings,\n keyExcerpts,\n tablesCount: extractedTables.length,\n fetchProvider: provider,\n };\n }\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../src/harvester/youtubeTranscript.ts","../src/harvester/webScraper.ts"],"names":[],"mappings":";AAkBA,IAAM,sBAAA,GAAyB;AAAA,EAC7B,wJAAA;AAAA,EACA;AACF,CAAA;AAEO,SAAS,sBAAsB,OAAA,EAAgC;AACpE,EAAA,MAAM,OAAA,GAAU,QAAQ,IAAA,EAAK;AAC7B,EAAA,IAAI,qBAAA,CAAsB,IAAA,CAAK,OAAO,CAAA,EAAG,OAAO,OAAA;AAEhD,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,IAAI,GAAA,CAAI,OAAO,CAAA;AACzB,IAAA,MAAM,OAAO,CAAA,CAAE,QAAA,CAAS,QAAQ,QAAA,EAAU,EAAE,EAAE,WAAA,EAAY;AAC1D,IAAA,IAAI,SAAS,UAAA,EAAY;AACvB,MAAA,MAAM,EAAA,GAAK,EAAE,QAAA,CAAS,KAAA,CAAM,GAAG,CAAA,CAAE,MAAA,CAAO,OAAO,CAAA,CAAE,CAAC,CAAA;AAClD,MAAA,OAAO,EAAA,IAAM,qBAAA,CAAsB,IAAA,CAAK,EAAE,IAAI,EAAA,GAAK,IAAA;AAAA,IACrD;AACA,IAAA,IAAI,IAAA,KAAS,aAAA,IAAiB,IAAA,KAAS,eAAA,IAAmB,SAAS,sBAAA,EAAwB;AACzF,MAAA,MAAM,CAAA,GAAI,CAAA,CAAE,YAAA,CAAa,GAAA,CAAI,GAAG,CAAA;AAChC,MAAA,IAAI,CAAA,IAAK,qBAAA,CAAsB,IAAA,CAAK,CAAC,GAAG,OAAO,CAAA;AAC/C,MAAA,MAAM,QAAQ,CAAA,CAAE,QAAA,CAAS,MAAM,GAAG,CAAA,CAAE,OAAO,OAAO,CAAA;AAClD,MAAA,IAAA,CAAK,KAAA,CAAM,CAAC,CAAA,KAAM,OAAA,IAAW,MAAM,CAAC,CAAA,KAAM,QAAA,IAAY,KAAA,CAAM,CAAC,CAAA,KAAM,GAAA,KAAQ,KAAA,CAAM,CAAC,CAAA,EAAG;AACnF,QAAA,OAAO,qBAAA,CAAsB,KAAK,KAAA,CAAM,CAAC,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,IAAA;AAAA,MAC3D;AAAA,IACF;AAAA,EACF,CAAA,CAAA,MAAQ;AAAA,EAER;AACA,EAAA,OAAO,IAAA;AACT;AAEO,SAAS,aAAa,GAAA,EAAsB;AACjD,EAAA,OAAO,qBAAA,CAAsB,GAAG,CAAA,KAAM,IAAA;AACxC;AAEO,SAAS,gBAAgB,OAAA,EAAyB;AACvD,EAAA,MAAM,QAAQ,IAAA,CAAK,GAAA,CAAI,GAAG,IAAA,CAAK,KAAA,CAAM,OAAO,CAAC,CAAA;AAC7C,EAAA,MAAM,GAAA,GAAM,IAAA,CAAK,KAAA,CAAM,KAAA,GAAQ,IAAI,CAAA;AACnC,EAAA,MAAM,IAAA,GAAO,IAAA,CAAK,KAAA,CAAO,KAAA,GAAQ,OAAQ,EAAE,CAAA;AAC3C,EAAA,MAAM,OAAO,KAAA,GAAQ,EAAA;AACrB,EAAA,IAAI,MAAM,CAAA,EAAG;AACX,IAAA,OAAO,CAAA,EAAG,OAAO,GAAG,CAAA,CAAE,SAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA,EAAI,MAAA,CAAO,IAAI,EAAE,QAAA,CAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA,EAAI,MAAA,CAAO,IAAI,CAAA,CAAE,QAAA,CAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,EAC1G;AACA,EAAA,OAAO,CAAA,EAAG,MAAA,CAAO,IAAI,CAAA,CAAE,SAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA,EAAI,OAAO,IAAI,CAAA,CAAE,QAAA,CAAS,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAC1E;AAEA,SAAS,mBAAmB,GAAA,EAAqB;AAC/C,EAAA,OAAO,GAAA,CACJ,OAAA,CAAQ,QAAA,EAAU,GAAG,EACrB,OAAA,CAAQ,OAAA,EAAS,GAAG,CAAA,CACpB,OAAA,CAAQ,OAAA,EAAS,GAAG,CAAA,CACpB,QAAQ,SAAA,EAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,QAAA,EAAU,GAAG,CAAA,CACrB,OAAA,CAAQ,WAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,SAAA,EAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EACnB,IAAA,EAAK;AACV;AAMO,SAAS,sBAAsB,QAAA,EAA+F;AACnI,EAAA,MAAM,OAAA,GAAU,QAAA,CACb,GAAA,CAAI,CAAC,CAAA,MAAO;AAAA,IACX,UAAU,CAAA,CAAE,QAAA;AAAA,IACZ,aAAa,CAAA,CAAE,WAAA;AAAA,IACf,IAAA,EAAM,kBAAA,CAAmB,CAAA,CAAE,IAAI;AAAA,IAC/B,CAAA,CACD,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,KAAK,MAAA,GAAS,CAAA,IAAK,CAAC,sBAAA,CAAuB,IAAA,CAAK,CAAC,EAAA,KAAO,EAAA,CAAG,KAAK,CAAA,CAAE,IAAI,CAAC,CAAC,CAAA;AAE3F,EAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AAElC,EAAA,MAAM,WAAgC,EAAC;AACvC,EAAA,IAAI,WAAA,GAAc,OAAA,CAAQ,CAAC,CAAA,CAAG,QAAA;AAC9B,EAAA,IAAI,YAAY,OAAA,CAAQ,CAAC,EAAG,QAAA,GAAW,OAAA,CAAQ,CAAC,CAAA,CAAG,WAAA;AACnD,EAAA,IAAI,cAAwB,EAAC;AAE7B,EAAA,KAAA,MAAW,OAAO,OAAA,EAAS;AACzB,IAAA,WAAA,CAAY,IAAA,CAAK,IAAI,IAAI,CAAA;AACzB,IAAA,SAAA,GAAY,KAAK,GAAA,CAAI,SAAA,EAAW,GAAA,CAAI,QAAA,GAAW,IAAI,WAAW,CAAA;AAC9D,IAAA,MAAM,MAAA,GAAS,WAAA,CAAY,IAAA,CAAK,GAAG,CAAA;AACnC,IAAA,MAAM,OAAA,GAAU,IAAI,QAAA,GAAW,WAAA;AAC/B,IAAA,MAAM,YAAA,GAAe,SAAA,CAAU,IAAA,CAAK,GAAA,CAAI,IAAI,CAAA;AAE5C,IAAA,IAAK,WAAW,EAAA,IAAM,YAAA,IAAiB,WAAW,EAAA,IAAM,MAAA,CAAO,UAAU,GAAA,EAAK;AAC5E,MAAA,QAAA,CAAS,IAAA,CAAK;AAAA,QACZ,QAAA,EAAU,WAAA;AAAA,QACV,WAAA,EAAa,KAAK,GAAA,CAAI,CAAA,EAAG,KAAK,KAAA,CAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAAA,QAC5D,SAAA,EAAW,gBAAgB,WAAW,CAAA;AAAA,QACtC,IAAA,EAAM;AAAA,OACP,CAAA;AACD,MAAA,WAAA,GAAc,EAAC;AACf,MAAA,WAAA,GAAc,GAAA,CAAI,WAAW,GAAA,CAAI,WAAA;AAAA,IACnC;AAAA,EACF;AAEA,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,EAAG;AAC1B,IAAA,QAAA,CAAS,IAAA,CAAK;AAAA,MACZ,QAAA,EAAU,WAAA;AAAA,MACV,WAAA,EAAa,KAAK,GAAA,CAAI,CAAA,EAAG,KAAK,KAAA,CAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAAA,MAC5D,SAAA,EAAW,gBAAgB,WAAW,CAAA;AAAA,MACtC,IAAA,EAAM,WAAA,CAAY,IAAA,CAAK,GAAG;AAAA,KAC3B,CAAA;AAAA,EACH;AAEA,EAAA,OAAO,QAAA;AACT;AAKO,IAAM,0BAAN,MAA8B;AAAA,EACnC,MAAM,eAAA,CAAgB,OAAA,EAAiB,qBAA+B,CAAC,IAAA,EAAM,IAAI,CAAA,EAAqC;AACpH,IAAA,MAAM,OAAA,GAAU,sBAAsB,OAAO,CAAA;AAC7C,IAAA,IAAI,CAAC,OAAA,EAAS;AACZ,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iCAAA,EAAoC,OAAO,CAAA,CAAE,CAAA;AAAA,IAC/D;AAEA,IAAA,MAAM,YAAA,GAAe,mCAAmC,OAAO,CAAA,CAAA;AAC/D,IAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,YAAA,EAAc;AAAA,MACpC,OAAA,EAAS;AAAA,QACP,YAAA,EAAc,uHAAA;AAAA,QACd,iBAAA,EAAmB;AAAA,OACrB;AAAA,MACA,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,IAAK;AAAA,KAClC,CAAA;AAED,IAAA,IAAI,CAAC,IAAI,EAAA,EAAI;AACX,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,oCAAA,EAAuC,GAAA,CAAI,MAAM,CAAA,CAAA,CAAG,CAAA;AAAA,IACtE;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,GAAA,CAAI,IAAA,EAAK;AAC5B,IAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,+EAA+E,CAAA;AAC9G,IAAA,IAAI,KAAA,GAAQ,kBAAkB,OAAO,CAAA,CAAA,CAAA;AACrC,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI,gBAAA,GAAmB,EAAA;AACvB,IAAA,IAAI,cAA8E,EAAC;AACnF,IAAA,IAAI,YAAA,GAAe,IAAA;AACnB,IAAA,IAAI,eAAA,GAAkB,KAAA;AAEtB,IAAA,IAAI,WAAA,GAAc,CAAC,CAAA,EAAG;AACpB,MAAA,IAAI;AACF,QAAA,MAAM,UAAA,GAAa,IAAA,CAAK,KAAA,CAAM,WAAA,CAAY,CAAC,CAAC,CAAA;AAC5C,QAAA,KAAA,GAAQ,UAAA,EAAY,cAAc,KAAA,IAAS,KAAA;AAC3C,QAAA,MAAA,GAAS,YAAY,YAAA,EAAc,MAAA;AACnC,QAAA,gBAAA,GAAmB,UAAA,EAAY,cAAc,gBAAA,IAAoB,EAAA;AAEjE,QAAA,MAAM,MAAA,GACJ,UAAA,EAAY,QAAA,EAAU,+BAAA,EAAiC,iBAAiB,EAAC;AAE3E,QAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AAErB,UAAA,IAAI,cACF,MAAA,CAAO,IAAA;AAAA,YACL,CAAC,MAAM,kBAAA,CAAmB,QAAA,CAAS,EAAE,YAAY,CAAA,IAAK,EAAE,IAAA,KAAS;AAAA,WACnE,IACA,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAAM,kBAAA,CAAmB,QAAA,CAAS,CAAA,CAAE,YAAY,CAAC,CAAA,IAC9D,MAAA,CAAO,CAAC,CAAA;AAEV,UAAA,IAAI,aAAa,OAAA,EAAS;AACxB,YAAA,YAAA,GAAe,YAAY,YAAA,IAAgB,IAAA;AAC3C,YAAA,eAAA,GAAkB,YAAY,IAAA,KAAS,KAAA;AACvC,YAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,WAAA,CAAY,OAAA,EAAS;AAAA,cAChD,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,GAAK;AAAA,aAClC,CAAA;AACD,YAAA,IAAI,SAAS,EAAA,EAAI;AACf,cAAA,MAAM,GAAA,GAAM,MAAM,QAAA,CAAS,IAAA,EAAK;AAChC,cAAA,MAAM,WAAA,GAAc;AAAA,gBAClB,GAAG,GAAA,CAAI,QAAA,CAAS,yEAAyE;AAAA,eAC3F;AACA,cAAA,KAAA,MAAW,KAAK,WAAA,EAAa;AAC3B,gBAAA,WAAA,CAAY,IAAA,CAAK;AAAA,kBACf,QAAA,EAAU,UAAA,CAAW,CAAA,CAAE,CAAC,KAAK,GAAG,CAAA;AAAA,kBAChC,WAAA,EAAa,UAAA,CAAW,CAAA,CAAE,CAAC,KAAK,GAAG,CAAA;AAAA,kBACnC,OAAO,CAAA,CAAE,CAAC,KAAK,EAAA,EAAI,OAAA,CAAQ,YAAY,GAAG;AAAA,iBAC3C,CAAA;AAAA,cACH;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAA,CAAA,MAAQ;AAAA,MAER;AAAA,IACF;AAEA,IAAA,IAAI,KAAA,KAAU,CAAA,eAAA,EAAkB,OAAO,CAAA,CAAA,CAAA,EAAK;AAC1C,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,KAAA,CAAM,6BAA6B,CAAA;AAC1D,MAAA,IAAI,SAAA,GAAY,CAAC,CAAA,EAAG;AAClB,QAAA,KAAA,GAAQ,mBAAmB,SAAA,CAAU,CAAC,EAAE,OAAA,CAAQ,kBAAA,EAAoB,EAAE,CAAC,CAAA;AAAA,MACzE;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,sBAAsB,WAAW,CAAA;AAClD,IAAA,MAAM,OAAA,GAAoB;AAAA,MACxB,KAAK,KAAK,CAAA,CAAA;AAAA,MACV,CAAA,cAAA,EAAiB,YAAY,CAAA,EAAG,MAAA,GAAS,CAAA,gBAAA,EAAmB,MAAM,CAAA,CAAA,GAAK,EAAE,CAAA,mBAAA,EAAsB,YAAY,CAAA,EAAA,EAAK,eAAA,GAAkB,oBAAoB,EAAE,CAAA;AAAA,KAC1J;AAEA,IAAA,IAAI,gBAAA,CAAiB,MAAK,EAAG;AAC3B,MAAA,OAAA,CAAQ,IAAA,CAAK;AAAA;AAAA,EAAsC,iBAAiB,IAAA,EAAK,CAAE,MAAM,CAAA,EAAG,IAAI,CAAC,CAAA,CAAE,CAAA;AAAA,IAC7F;AAEA,IAAA,IAAI,QAAA,CAAS,SAAS,CAAA,EAAG;AACvB,MAAA,OAAA,CAAQ,IAAA,CAAK;AAAA,yBAAA,CAA6B,CAAA;AAC1C,MAAA,KAAA,MAAW,OAAO,QAAA,EAAU;AAC1B,QAAA,OAAA,CAAQ,KAAK,CAAA,KAAA,EAAQ,GAAA,CAAI,SAAS,CAAA,IAAA,EAAO,GAAA,CAAI,IAAI,CAAA,CAAE,CAAA;AAAA,MACrD;AAAA,IACF;AAEA,IAAA,OAAO;AAAA,MACL,OAAA;AAAA,MACA,GAAA,EAAK,YAAA;AAAA,MACL,KAAA;AAAA,MACA,MAAA;AAAA,MACA,QAAA,EAAU,YAAA;AAAA,MACV,eAAA;AAAA,MACA,QAAA,EAAU,QAAA;AAAA,MACV,gBAAA,EAAkB,OAAA,CAAQ,IAAA,CAAK,IAAI;AAAA,KACrC;AAAA,EACF;AACF;;;AC/MO,SAAS,0BAAA,CAA2B,QAAA,EAAkB,WAAA,GAAc,CAAA,EAAa;AACtF,EAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,iBAAA,EAAmB,IAAI,CAAA;AAC5D,EAAA,MAAM,SAAS,WAAA,CACZ,KAAA,CAAM,QAAQ,CAAA,CACd,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,MAAA,EAAQ,GAAG,CAAA,CAAE,IAAA,EAAM,CAAA,CACxC,MAAA,CAAO,CAAC,CAAA,KAAM;AACb,IAAA,IAAI,EAAE,MAAA,GAAS,EAAA,IAAM,CAAA,CAAE,MAAA,GAAS,KAAK,OAAO,KAAA;AAC5C,IAAA,IAAI,CAAA,CAAE,WAAW,GAAG,CAAA,IAAK,EAAE,UAAA,CAAW,GAAG,GAAG,OAAO,KAAA;AACnD,IAAA,MAAM,SAAA,GAAA,CAAa,CAAA,CAAE,KAAA,CAAM,sBAAsB,CAAA,IAAK,EAAC,EAAG,IAAA,CAAK,EAAE,CAAA,CAAE,MAAA,GAAS,CAAA,CAAE,MAAA;AAC9E,IAAA,IAAI,SAAA,GAAY,MAAM,OAAO,KAAA;AAC7B,IAAA,IAAI,qHAAA,CAAsH,IAAA,CAAK,CAAC,CAAA,EAAG;AACjI,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,OAAO,IAAA;AAAA,EACT,CAAC,CAAA;AAEH,EAAA,OAAO,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,WAAW,CAAA;AACpC;AAKO,IAAM,aAAN,MAAiB;AAAA,EACd,YAAA;AAAA,EACA,OAAA;AAAA,EACA,gBAAA;AAAA,EACA,aAAA;AAAA,EACA,cAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAA6B,EAAC,EAAG;AAC3C,IAAA,IAAA,CAAK,gBAAA,GAAmB,QAAQ,gBAAA,IAAoB,IAAA;AACpD,IAAA,IAAA,CAAK,YAAA,GACH,QAAQ,YAAA,KAAiB,IAAA,CAAK,mBAAmB,OAAA,CAAQ,GAAA,CAAI,gBAAgB,MAAA,CAAA,IAAc,wBAAA;AAC7F,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA,KAAY,KAAK,gBAAA,GAAmB,OAAA,CAAQ,IAAI,YAAA,GAAe,MAAA,CAAA;AACtF,IAAA,IAAA,CAAK,gBAAgB,OAAA,CAAQ,aAAA;AAC7B,IAAA,IAAA,CAAK,cAAA,GAAiB,IAAI,uBAAA,EAAwB;AAAA,EACpD;AAAA,EAEA,iBAAiB,OAAA,EAA+B;AAC9C,IAAA,IAAA,CAAK,aAAA,GAAgB,OAAA;AAAA,EACvB;AAAA,EAEA,WAAW,GAAA,EAAc;AACvB,IAAA,IAAA,CAAK,OAAA,GAAU,GAAA;AAAA,EACjB;AAAA,EAEA,MAAM,SAAA,CACJ,SAAA,EACA,OAAA,GAAqG,EAAC,EAC5E;AAC1B,IAAA,MAAM,OAAA,GAAU,OAAA,CAAQ,aAAA,IAAiB,IAAA,CAAK,aAAA;AAG9C,IAAA,IAAI,OAAA,EAAS;AACX,MAAA,IAAI;AACF,QAAA,MAAM,MAAA,GAAS,MAAM,OAAA,CAAQ,KAAA,CAAM,SAAS,CAAA;AAC5C,QAAA,IAAI,MAAA,IAAU,OAAO,eAAA,EAAiB;AACpC,UAAA,OAAO,KAAK,eAAA,CAAgB,SAAA,EAAW,OAAO,eAAA,EAAiB,QAAA,EAAU,OAAO,KAAK,CAAA;AAAA,QACvF;AAAA,MACF,SAAS,CAAA,EAAQ;AACf,QAAA,OAAA,CAAQ,KAAK,CAAA,+CAAA,EAAkD,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,MAC9F;AAAA,IACF;AAGA,IAAA,IAAI,YAAA,CAAa,SAAS,CAAA,EAAG;AAC3B,MAAA,IAAI;AACF,QAAA,MAAM,KAAK,MAAM,IAAA,CAAK,eAAe,eAAA,CAAgB,SAAA,EAAW,QAAQ,kBAAkB,CAAA;AAC1F,QAAA,IAAI,EAAA,CAAG,gBAAA,IAAoB,EAAA,CAAG,gBAAA,CAAiB,SAAS,EAAA,EAAI;AAC1D,UAAA,OAAO,IAAA,CAAK,gBAAgB,EAAA,CAAG,GAAA,EAAK,GAAG,gBAAA,EAAkB,SAAA,EAAW,GAAG,KAAK,CAAA;AAAA,QAC9E;AAAA,MACF,SAAS,CAAA,EAAQ;AACf,QAAA,OAAA,CAAQ,KAAK,CAAA,sDAAA,EAAyD,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,MACrG;AAAA,IACF;AAGA,IAAA,IAAI;AACF,MAAA,MAAM,OAAA,GAAU,qBAAqB,SAAS,CAAA,CAAA;AAC9C,MAAA,MAAM,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,IAAA,CAAK,OAAA;AACxC,MAAA,MAAM,OAAA,GAAkC;AAAA,QACtC,YAAA,EAAc,sBAAA;AAAA,QACd,iBAAA,EAAmB;AAAA,OACrB;AACA,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,OAAA,CAAQ,eAAe,CAAA,GAAI,CAAA,OAAA,EAAU,OAAO,CAAA,CAAA;AAAA,MAC9C;AAEA,MAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,OAAA,EAAS;AAAA,QAC/B,OAAA;AAAA,QACA,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,GAAI;AAAA,OACjC,CAAA;AAED,MAAA,IAAI,IAAI,EAAA,EAAI;AACV,QAAA,MAAM,QAAA,GAAW,MAAM,GAAA,CAAI,IAAA,EAAK;AAChC,QAAA,IAAI,QAAA,IAAY,QAAA,CAAS,MAAA,GAAS,GAAA,EAAK;AACrC,UAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,SAAA,EAAW,QAAA,EAAU,MAAM,CAAA;AAAA,QACzD;AAAA,MACF;AAAA,IACF,SAAS,CAAA,EAAQ;AACf,MAAA,OAAA,CAAQ,KAAK,CAAA,oCAAA,EAAuC,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,IACnF;AAGA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAI;AACF,QAAA,MAAM,MAAM,MAAM,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,YAAY,CAAA,MAAA,CAAA,EAAU;AAAA,UACpD,MAAA,EAAQ,MAAA;AAAA,UACR,OAAA,EAAS,EAAE,cAAA,EAAgB,kBAAA,EAAmB;AAAA,UAC9C,IAAA,EAAM,KAAK,SAAA,CAAU;AAAA,YACnB,IAAA,EAAM,SAAA;AAAA,YACN,QAAA,EAAU;AAAA,WACX,CAAA;AAAA,UACD,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,IAAK;AAAA,SAClC,CAAA;AAED,QAAA,IAAI,IAAI,EAAA,EAAI;AACV,UAAA,MAAM,IAAA,GAAO,MAAM,GAAA,CAAI,IAAA,EAAK;AAC5B,UAAA,MAAM,WAAW,IAAA,EAAM,OAAA,GAAU,CAAC,CAAA,EAAG,YAAY,IAAA,EAAM,QAAA;AACvD,UAAA,IAAI,QAAA,EAAU;AACZ,YAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,SAAA,EAAW,QAAA,EAAU,UAAU,CAAA;AAAA,UAC7D;AAAA,QACF;AAAA,MACF,CAAA,CAAA,MAAQ;AAAA,MAER;AAAA,IACF;AAGA,IAAA,IAAI;AACF,MAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,SAAA,EAAW;AAAA,QACjC,OAAA,EAAS,EAAE,YAAA,EAAc,iDAAA,EAAkD;AAAA,QAC3E,MAAA,EAAQ,WAAA,CAAY,OAAA,CAAQ,GAAK;AAAA,OAClC,CAAA;AACD,MAAA,IAAI,IAAI,EAAA,EAAI;AACV,QAAA,MAAM,OAAA,GAAU,MAAM,GAAA,CAAI,IAAA,EAAK;AAC/B,QAAA,MAAM,WAAW,OAAA,CAAQ,KAAA,CAAM,kCAAkC,CAAA,GAAI,CAAC,GAAG,IAAA,EAAK;AAC9E,QAAA,MAAM,KAAA,GAAQ,QACX,OAAA,CAAQ,qDAAA,EAAuD,EAAE,CAAA,CACjE,OAAA,CAAQ,kDAAA,EAAoD,EAAE,CAAA,CAC9D,OAAA,CAAQ,8CAA8C,EAAE,CAAA,CACxD,OAAA,CAAQ,qDAAA,EAAuD,EAAE,CAAA,CACjE,QAAQ,oCAAA,EAAsC,CAAC,CAAA,EAAG,GAAA,EAAK,GAAA,KAAQ;AAAA,EAAK,GAAA,CAAI,MAAA,CAAO,MAAA,CAAO,GAAG,CAAC,CAAC,CAAA,CAAA,EAAI,GAAA,CAAI,OAAA,CAAQ,UAAA,EAAY,EAAE,CAAA,CAAE,MAAM;AAAA,CAAI,CAAA,CACrI,OAAA,CAAQ,kDAAA,EAAoD,CAAC,GAAG,IAAA,KAAS;AAAA;AAAA,EAAa,KAAK,OAAA,CAAQ,UAAA,EAAY,EAAE,CAAA,CAAE,MAAM;AAAA;AAAA,CAAY,CAAA,CACrI,OAAA,CAAQ,2BAAA,EAA6B,CAAC,GAAG,CAAA,KAAM;AAAA,EAAK,EAAE,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA,CAAE,MAAM;AAAA,CAAI,CAAA,CACzF,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA,CACvB,OAAA,CAAQ,SAAA,EAAW,GAAG,CAAA,CACtB,OAAA,CAAQ,SAAA,EAAW,MAAM,EACzB,IAAA,EAAK;AAER,QAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,SAAA,EAAW,KAAA,EAAO,UAAU,QAAQ,CAAA;AAAA,MAClE;AAAA,IACF,SAAS,CAAA,EAAQ;AACf,MAAA,OAAA,CAAQ,KAAK,CAAA,qCAAA,EAAwC,SAAS,CAAA,CAAA,CAAA,EAAK,CAAA,EAAG,WAAW,CAAC,CAAA;AAAA,IACpF;AAEA,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,kCAAA,EAAqC,SAAS,CAAA,CAAE,CAAA;AAAA,EAClE;AAAA,EAEQ,eAAA,CACN,GAAA,EACA,QAAA,EACA,QAAA,EACA,aAAA,EACiB;AACjB,IAAA,MAAM,aAAa,QAAA,CAAS,KAAA,CAAM,aAAa,CAAA,IAAK,QAAA,CAAS,MAAM,mBAAmB,CAAA;AACtF,IAAA,MAAM,KAAA,GAAA,CAAS,kBAAkB,UAAA,GAAa,UAAA,CAAW,CAAC,CAAA,CAAG,IAAA,KAAS,GAAA,CAAI,KAAA,CAAM,GAAG,CAAA,CAAE,MAAA,CAAO,OAAO,CAAA,CAAE,GAAA,MAAS,UAAA,CAAA,EAAa,KAAA,CAAM,GAAG,GAAG,CAAA;AAGvI,IAAA,MAAM,mBAAmB,CAAC,GAAG,QAAA,CAAS,QAAA,CAAS,sCAAsC,CAAC,CAAA;AACtF,IAAA,MAAM,mBAAA,GAA4C,gBAAA,CAC/C,GAAA,CAAI,CAAC,CAAA,MAAO;AAAA,MACX,WAAW,CAAA,CAAE,CAAC,KAAK,EAAA,EAAI,IAAA,GAAO,WAAA,EAAY;AAAA,MAC1C,IAAA,EAAA,CAAO,CAAA,CAAE,CAAC,CAAA,IAAK,IAAI,IAAA;AAAK,KAC1B,CAAE,EACD,MAAA,CAAO,CAAC,MAAM,CAAA,CAAE,IAAA,CAAK,SAAS,CAAC,CAAA;AAClC,IAAA,MAAM,wBAAwB,mBAAA,CAAoB,GAAA,CAAI,CAAC,CAAA,KAAM,EAAE,IAAI,CAAA;AAGnE,IAAA,MAAM,WAAA,GACJ,SAAS,KAAA,CAAM,wDAAwD,KACvE,QAAA,CAAS,KAAA,CAAM,yBAAyB,CAAA,IACxC,EAAC;AACH,IAAA,MAAM,eAAA,GAAkB,WAAA,CAAY,GAAA,CAAI,CAAC,MAAM,CAAA,CAAE,IAAA,EAAM,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,SAAS,EAAE,CAAA;AAGpF,IAAA,MAAM,iBAAA,GAAoB,CAAC,GAAG,QAAA,CAAS,QAAA,CAAS,mBAAmB,CAAC,CAAA,CACjE,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,CAAC,CAAA,CAAG,IAAA,EAAM,CAAA,CACvB,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,MAAA,GAAS,CAAA,IAAK,CAAA,CAAE,MAAA,GAAS,GAAG,CAAA;AAE/C,IAAA,MAAM,WAAA,GAAc,0BAAA,CAA2B,QAAA,EAAU,CAAC,CAAA;AAE1D,IAAA,OAAO;AAAA,MACL,GAAA;AAAA,MACA,KAAA;AAAA,MACA,eAAA,EAAiB,QAAA;AAAA,MACjB,QAAA,EAAU,MAAA,CAAO,UAAA,CAAW,QAAA,EAAU,OAAO,CAAA;AAAA,MAC7C,qBAAA;AAAA,MACA,mBAAA;AAAA,MACA,eAAA;AAAA,MACA,iBAAA;AAAA,MACA,WAAA;AAAA,MACA,aAAa,eAAA,CAAgB,MAAA;AAAA,MAC7B,aAAA,EAAe;AAAA,KACjB;AAAA,EACF;AACF","file":"chunk-ON6VFCJO.js","sourcesContent":["export interface TranscriptSegment {\n startSec: number;\n durationSec: number;\n timestamp: string;\n text: string;\n}\n\nexport interface YouTubeTranscriptResult {\n videoId: string;\n url: string;\n title: string;\n author?: string;\n language: string;\n isAutoGenerated: boolean;\n segments: TranscriptSegment[];\n stitchedMarkdown: string;\n}\n\nconst SPONSOR_NOISE_PATTERNS = [\n /\\b(sponsored by|this video is brought to you by|use (?:promo )?code\\b|don't forget to like and subscribe|hit the bell icon|link in the description)\\b/i,\n /(nhớ nhấn like và đăng ký kênh|bấm chuông thông báo|video(?:\\s+này)?\\s+được tài trợ bởi|được tài trợ bởi|link mua hàng dưới phần mô tả)/i,\n];\n\nexport function extractYouTubeVideoId(urlOrId: string): string | null {\n const trimmed = urlOrId.trim();\n if (/^[a-zA-Z0-9_-]{11}$/.test(trimmed)) return trimmed;\n\n try {\n const u = new URL(trimmed);\n const host = u.hostname.replace(/^www\\./, '').toLowerCase();\n if (host === 'youtu.be') {\n const id = u.pathname.split('/').filter(Boolean)[0];\n return id && /^[a-zA-Z0-9_-]{11}$/.test(id) ? id : null;\n }\n if (host === 'youtube.com' || host === 'm.youtube.com' || host === 'youtube-nocookie.com') {\n const v = u.searchParams.get('v');\n if (v && /^[a-zA-Z0-9_-]{11}$/.test(v)) return v;\n const parts = u.pathname.split('/').filter(Boolean);\n if ((parts[0] === 'embed' || parts[0] === 'shorts' || parts[0] === 'v') && parts[1]) {\n return /^[a-zA-Z0-9_-]{11}$/.test(parts[1]) ? parts[1] : null;\n }\n }\n } catch {\n // not a valid URL\n }\n return null;\n}\n\nexport function isYouTubeUrl(url: string): boolean {\n return extractYouTubeVideoId(url) !== null;\n}\n\nexport function formatTimestamp(seconds: number): string {\n const total = Math.max(0, Math.floor(seconds));\n const hrs = Math.floor(total / 3600);\n const mins = Math.floor((total % 3600) / 60);\n const secs = total % 60;\n if (hrs > 0) {\n return `${String(hrs).padStart(2, '0')}:${String(mins).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;\n }\n return `${String(mins).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;\n}\n\nfunction decodeHtmlEntities(str: string): string {\n return str\n .replace(/&/g, '&')\n .replace(/</g, '<')\n .replace(/>/g, '>')\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/'/g, \"'\")\n .replace(/ /g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\n/**\n * Stitches short caption fragments into coherent timestamped paragraphs (every ~30-45 seconds or sentence boundary)\n * and strips sponsor/subscribe filler noise.\n */\nexport function stitchCaptionSegments(segments: Array<{ startSec: number; durationSec: number; text: string }>): TranscriptSegment[] {\n const cleaned = segments\n .map((s) => ({\n startSec: s.startSec,\n durationSec: s.durationSec,\n text: decodeHtmlEntities(s.text),\n }))\n .filter((s) => s.text.length > 0 && !SPONSOR_NOISE_PATTERNS.some((re) => re.test(s.text)));\n\n if (cleaned.length === 0) return [];\n\n const stitched: TranscriptSegment[] = [];\n let bucketStart = cleaned[0]!.startSec;\n let bucketEnd = cleaned[0]!.startSec + cleaned[0]!.durationSec;\n let bucketTexts: string[] = [];\n\n for (const seg of cleaned) {\n bucketTexts.push(seg.text);\n bucketEnd = Math.max(bucketEnd, seg.startSec + seg.durationSec);\n const joined = bucketTexts.join(' ');\n const elapsed = seg.startSec - bucketStart;\n const endsSentence = /[.!?…]$/.test(seg.text);\n\n if ((elapsed >= 30 && endsSentence) || elapsed >= 50 || joined.length >= 360) {\n stitched.push({\n startSec: bucketStart,\n durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),\n timestamp: formatTimestamp(bucketStart),\n text: joined,\n });\n bucketTexts = [];\n bucketStart = seg.startSec + seg.durationSec;\n }\n }\n\n if (bucketTexts.length > 0) {\n stitched.push({\n startSec: bucketStart,\n durationSec: Math.max(1, Math.round(bucketEnd - bucketStart)),\n timestamp: formatTimestamp(bucketStart),\n text: bucketTexts.join(' '),\n });\n }\n\n return stitched;\n}\n\n/**\n * YouTube Transcript Ingestion Service (Idea §3.1)\n */\nexport class YouTubeIngestionService {\n async fetchTranscript(urlOrId: string, preferredLanguages: string[] = ['vi', 'en']): Promise<YouTubeTranscriptResult> {\n const videoId = extractYouTubeVideoId(urlOrId);\n if (!videoId) {\n throw new Error(`Invalid YouTube URL or Video ID: ${urlOrId}`);\n }\n\n const canonicalUrl = `https://www.youtube.com/watch?v=${videoId}`;\n const res = await fetch(canonicalUrl, {\n headers: {\n 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',\n 'Accept-Language': 'vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7',\n },\n signal: AbortSignal.timeout(12000),\n });\n\n if (!res.ok) {\n throw new Error(`Failed to fetch YouTube watch page (${res.status})`);\n }\n\n const html = await res.text();\n const playerMatch = html.match(/ytInitialPlayerResponse\\s*=\\s*(\\{[\\s\\S]*?\\});\\s*(?:var\\s|window\\.|<\\/script>)/);\n let title = `YouTube Video (${videoId})`;\n let author: string | undefined;\n let shortDescription = '';\n let rawSegments: Array<{ startSec: number; durationSec: number; text: string }> = [];\n let selectedLang = 'en';\n let isAutoGenerated = false;\n\n if (playerMatch?.[1]) {\n try {\n const playerData = JSON.parse(playerMatch[1]);\n title = playerData?.videoDetails?.title || title;\n author = playerData?.videoDetails?.author;\n shortDescription = playerData?.videoDetails?.shortDescription || '';\n\n const tracks: any[] =\n playerData?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];\n\n if (tracks.length > 0) {\n // Prioritize manual tracks in preferredLanguages, then auto-generated in preferredLanguages, then first track\n let chosenTrack =\n tracks.find(\n (t) => preferredLanguages.includes(t.languageCode) && t.kind !== 'asr'\n ) ||\n tracks.find((t) => preferredLanguages.includes(t.languageCode)) ||\n tracks[0];\n\n if (chosenTrack?.baseUrl) {\n selectedLang = chosenTrack.languageCode || 'en';\n isAutoGenerated = chosenTrack.kind === 'asr';\n const trackRes = await fetch(chosenTrack.baseUrl, {\n signal: AbortSignal.timeout(10000),\n });\n if (trackRes.ok) {\n const xml = await trackRes.text();\n const textMatches = [\n ...xml.matchAll(/<text\\s+start=\"([\\d.]+)\"(?:\\s+dur=\"([\\d.]+)\")?[^>]*>([\\s\\S]*?)<\\/text>/g),\n ];\n for (const m of textMatches) {\n rawSegments.push({\n startSec: parseFloat(m[1] || '0'),\n durationSec: parseFloat(m[2] || '2'),\n text: (m[3] || '').replace(/<[^>]+>/g, ' '),\n });\n }\n }\n }\n }\n } catch {\n // Fallback to meta tags below\n }\n }\n\n if (title === `YouTube Video (${videoId})`) {\n const titleMeta = html.match(/<title>([\\s\\S]*?)<\\/title>/i);\n if (titleMeta?.[1]) {\n title = decodeHtmlEntities(titleMeta[1].replace(/\\s*-\\s*YouTube$/i, ''));\n }\n }\n\n const stitched = stitchCaptionSegments(rawSegments);\n const mdLines: string[] = [\n `# ${title}`,\n `> **Source:** ${canonicalUrl}${author ? ` | **Channel:** ${author}` : ''} | **Language:** \\`${selectedLang}\\`${isAutoGenerated ? ' (Auto-caption)' : ''}`,\n ];\n\n if (shortDescription.trim()) {\n mdLines.push(`\\n## Video Overview & Description\\n${shortDescription.trim().slice(0, 1500)}`);\n }\n\n if (stitched.length > 0) {\n mdLines.push(`\\n## Timestamped Transcript`);\n for (const seg of stitched) {\n mdLines.push(`- **[${seg.timestamp}]** ${seg.text}`);\n }\n }\n\n return {\n videoId,\n url: canonicalUrl,\n title,\n author,\n language: selectedLang,\n isAutoGenerated,\n segments: stitched,\n stitchedMarkdown: mdLines.join('\\n'),\n };\n }\n}\n","import { CustomCrawlerAdapter } from '../config/researchConfig';\nimport { isYouTubeUrl, YouTubeIngestionService } from './youtubeTranscript';\n\nexport interface ExtractedCodeBlock {\n language: string;\n code: string;\n}\n\nexport interface ScrapedDocument {\n url: string;\n title: string;\n markdownContent: string;\n byteSize: number;\n extractedCodeSnippets: string[];\n extractedCodeBlocks?: ExtractedCodeBlock[];\n extractedTables?: string[];\n extractedHeadings?: string[];\n keyExcerpts?: string[];\n tablesCount: number;\n fetchProvider: 'jina' | 'crawl4ai' | 'youtube' | 'direct' | 'custom';\n}\n\nexport interface WebScraperOptions {\n crawl4aiHost?: string;\n jinaKey?: string;\n allowEnvFallback?: boolean;\n customAdapter?: CustomCrawlerAdapter;\n}\n\n/**\n * Extracts top substantive paragraphs from Markdown (stripping navigation, links-only lines, and cookie banners).\n */\nexport function extractSubstantiveExcerpts(markdown: string, maxExcerpts = 6): string[] {\n const withoutCode = markdown.replace(/```[\\s\\S]*?```/g, '\\n');\n const blocks = withoutCode\n .split(/\\n{2,}/)\n .map((b) => b.replace(/\\s+/g, ' ').trim())\n .filter((b) => {\n if (b.length < 60 || b.length > 900) return false;\n if (b.startsWith('#') || b.startsWith('|')) return false;\n const linkRatio = (b.match(/\\[[^\\]]+\\]\\([^)]+\\)/g) || []).join('').length / b.length;\n if (linkRatio > 0.45) return false;\n if (/\\b(cookie policy|privacy policy|all rights reserved|terms of service|sign in|log in|subscribe to our newsletter)\\b/i.test(b)) {\n return false;\n }\n return true;\n });\n\n return blocks.slice(0, maxExcerpts);\n}\n\n/**\n * High-fidelity web content & YouTube transcript extractor with Custom Crawler & Jina Auth support\n */\nexport class WebScraper {\n private crawl4aiHost?: string;\n private jinaKey?: string;\n private allowEnvFallback: boolean;\n private customAdapter?: CustomCrawlerAdapter;\n private youtubeService: YouTubeIngestionService;\n\n constructor(options: WebScraperOptions = {}) {\n this.allowEnvFallback = options.allowEnvFallback ?? true;\n this.crawl4aiHost =\n options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : undefined) || 'http://localhost:11235';\n this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : undefined);\n this.customAdapter = options.customAdapter;\n this.youtubeService = new YouTubeIngestionService();\n }\n\n setCustomAdapter(adapter: CustomCrawlerAdapter) {\n this.customAdapter = adapter;\n }\n\n setJinaKey(key?: string) {\n this.jinaKey = key;\n }\n\n async scrapeUrl(\n targetUrl: string,\n options: { customAdapter?: CustomCrawlerAdapter; jinaKey?: string; preferredLanguages?: string[] } = {}\n ): Promise<ScrapedDocument> {\n const adapter = options.customAdapter || this.customAdapter;\n\n // 0. Custom Crawler Adapter (e.g. MCP bridge, Playwright, Headless Chrome)\n if (adapter) {\n try {\n const result = await adapter.crawl(targetUrl);\n if (result && result.markdownContent) {\n return this.packageDocument(targetUrl, result.markdownContent, 'custom', result.title);\n }\n } catch (e: any) {\n console.warn(`[WebScraper] Custom crawler adapter failed for ${targetUrl}:`, e?.message || e);\n }\n }\n\n // 0.5. YouTube Video URL -> Route to YouTubeIngestionService (Idea §3.1)\n if (isYouTubeUrl(targetUrl)) {\n try {\n const yt = await this.youtubeService.fetchTranscript(targetUrl, options.preferredLanguages);\n if (yt.stitchedMarkdown && yt.stitchedMarkdown.length > 80) {\n return this.packageDocument(yt.url, yt.stitchedMarkdown, 'youtube', yt.title);\n }\n } catch (e: any) {\n console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);\n }\n }\n\n // 1. Try Jina Reader API (with optional Bearer auth from host app or env)\n try {\n const jinaUrl = `https://r.jina.ai/${targetUrl}`;\n const jinaKey = options.jinaKey || this.jinaKey;\n const headers: Record<string, string> = {\n 'User-Agent': 'OrchableComposer/1.0',\n 'X-Return-Format': 'markdown',\n };\n if (jinaKey) {\n headers['Authorization'] = `Bearer ${jinaKey}`;\n }\n\n const res = await fetch(jinaUrl, {\n headers,\n signal: AbortSignal.timeout(8000),\n });\n\n if (res.ok) {\n const markdown = await res.text();\n if (markdown && markdown.length > 100) {\n return this.packageDocument(targetUrl, markdown, 'jina');\n }\n }\n } catch (e: any) {\n console.warn(`[WebScraper] Jina Reader failed for ${targetUrl}:`, e?.message || e);\n }\n\n // 2. Try Crawl4AI Local Server if reachable\n if (this.crawl4aiHost) {\n try {\n const res = await fetch(`${this.crawl4aiHost}/crawl`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({\n urls: targetUrl,\n priority: 10,\n }),\n signal: AbortSignal.timeout(15000),\n });\n\n if (res.ok) {\n const data = await res.json();\n const markdown = data?.results?.[0]?.markdown || data?.markdown;\n if (markdown) {\n return this.packageDocument(targetUrl, markdown, 'crawl4ai');\n }\n }\n } catch {\n // ignore if local crawl4ai is not running\n }\n }\n\n // 3. Fallback: Direct Fetch & Structured HTML-to-Markdown Conversion\n try {\n const res = await fetch(targetUrl, {\n headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' },\n signal: AbortSignal.timeout(10000),\n });\n if (res.ok) {\n const rawHtml = await res.text();\n const titleTag = rawHtml.match(/<title[^>]*>([\\s\\S]*?)<\\/title>/i)?.[1]?.trim();\n const clean = rawHtml\n .replace(/<script\\b[^<]*(?:(?!<\\/script>)<[^<]*)*<\\/script>/gi, '')\n .replace(/<style\\b[^<]*(?:(?!<\\/style>)<[^<]*)*<\\/style>/gi, '')\n .replace(/<nav\\b[^<]*(?:(?!<\\/nav>)<[^<]*)*<\\/nav>/gi, '')\n .replace(/<footer\\b[^<]*(?:(?!<\\/footer>)<[^<]*)*<\\/footer>/gi, '')\n .replace(/<h([1-3])[^>]*>([\\s\\S]*?)<\\/h\\1>/gi, (_, lvl, txt) => `\\n${'#'.repeat(Number(lvl))} ${txt.replace(/<[^>]+>/g, '').trim()}\\n`)\n .replace(/<pre[^>]*><code[^>]*>([\\s\\S]*?)<\\/code><\\/pre>/gi, (_, code) => `\\n\\`\\`\\`\\n${code.replace(/<[^>]+>/g, '').trim()}\\n\\`\\`\\`\\n`)\n .replace(/<p[^>]*>([\\s\\S]*?)<\\/p>/gi, (_, p) => `\\n${p.replace(/<[^>]+>/g, ' ').trim()}\\n`)\n .replace(/<[^>]+>/g, ' ')\n .replace(/[ \\t]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim();\n\n return this.packageDocument(targetUrl, clean, 'direct', titleTag);\n }\n } catch (e: any) {\n console.warn(`[WebScraper] Direct fetch failed for ${targetUrl}:`, e?.message || e);\n }\n\n throw new Error(`Unable to fetch content from URL: ${targetUrl}`);\n }\n\n private packageDocument(\n url: string,\n markdown: string,\n provider: 'jina' | 'crawl4ai' | 'youtube' | 'direct' | 'custom',\n explicitTitle?: string\n ): ScrapedDocument {\n const titleMatch = markdown.match(/^#\\s+(.+)$/m) || markdown.match(/^Title:\\s*(.+)$/im);\n const title = (explicitTitle || (titleMatch ? titleMatch[1]!.trim() : url.split('/').filter(Boolean).pop() || 'Document')).slice(0, 180);\n\n // Extract code blocks with language tags\n const codeBlockMatches = [...markdown.matchAll(/```([a-zA-Z0-9_+-]*)\\n([\\s\\S]*?)```/g)];\n const extractedCodeBlocks: ExtractedCodeBlock[] = codeBlockMatches\n .map((m) => ({\n language: (m[1] || '').trim().toLowerCase(),\n code: (m[2] || '').trim(),\n }))\n .filter((b) => b.code.length > 0);\n const extractedCodeSnippets = extractedCodeBlocks.map((b) => b.code);\n\n // Extract markdown tables (contiguous pipe lines with separator row)\n const tableBlocks =\n markdown.match(/(?:^\\|.+\\|\\s*\\n)+^\\|[-:| ]+\\|\\s*\\n(?:^\\|.+\\|\\s*\\n?)+/gm) ||\n markdown.match(/\\|[\\s\\S]*?\\|[\\s\\S]*?\\|/g) ||\n [];\n const extractedTables = tableBlocks.map((t) => t.trim()).filter((t) => t.length > 15);\n\n // Extract section headings\n const extractedHeadings = [...markdown.matchAll(/^#{1,3}\\s+(.+)$/gm)]\n .map((m) => m[1]!.trim())\n .filter((h) => h.length > 2 && h.length < 120);\n\n const keyExcerpts = extractSubstantiveExcerpts(markdown, 6);\n\n return {\n url,\n title,\n markdownContent: markdown,\n byteSize: Buffer.byteLength(markdown, 'utf-8'),\n extractedCodeSnippets,\n extractedCodeBlocks,\n extractedTables,\n extractedHeadings,\n keyExcerpts,\n tablesCount: extractedTables.length,\n fetchProvider: provider,\n };\n }\n}\n"]}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { extractSubstantiveExcerpts } from './chunk-
|
|
1
|
+
import { extractSubstantiveExcerpts } from './chunk-ON6VFCJO.js';
|
|
2
2
|
|
|
3
3
|
// src/evidence/evidenceScorer.ts
|
|
4
4
|
function tokenizeBilingualKeywords(text) {
|
|
@@ -158,5 +158,5 @@ var ProvenanceTracker = class {
|
|
|
158
158
|
};
|
|
159
159
|
|
|
160
160
|
export { EvidenceScorer, ProvenanceTracker, extractTopSentences };
|
|
161
|
-
//# sourceMappingURL=chunk-
|
|
162
|
-
//# sourceMappingURL=chunk-
|
|
161
|
+
//# sourceMappingURL=chunk-QFYM2NV4.js.map
|
|
162
|
+
//# sourceMappingURL=chunk-QFYM2NV4.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/evidence/evidenceScorer.ts","../src/evidence/provenanceTracker.ts"],"names":[],"mappings":";;;AAoBA,SAAS,0BAA0B,IAAA,EAAwB;AACzD,EAAA,OAAO,IAAA,CACJ,UAAU,KAAK,CAAA,CACf,aAAY,CACZ,OAAA,CAAQ,kCAAA,EAAoC,GAAG,CAAA,CAC/C,KAAA,CAAM,KAAK,CAAA,CACX,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,MAAA,IAAU,KAAK,CAAC,0FAAA,CAA2F,IAAA,CAAK,CAAC,CAAC,CAAA;AACvI;AAKO,SAAS,mBAAA,CAAoB,QAAA,EAAkB,WAAA,EAAqB,YAAA,GAAe,CAAA,EAAa;AACrG,EAAA,MAAM,QAAA,GAAW,0BAA0B,WAAW,CAAA;AACtD,EAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,iBAAA,EAAmB,GAAG,CAAA;AAC3D,EAAA,MAAM,YAAA,GAAe,YAClB,KAAA,CAAM,kBAAkB,EACxB,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,aAAA,EAAe,EAAE,CAAA,CAAE,OAAA,CAAQ,QAAQ,GAAG,CAAA,CAAE,MAAM,CAAA,CACnE,MAAA,CAAO,CAAC,CAAA,KAAM;AACb,IAAA,IAAI,EAAE,MAAA,GAAS,EAAA,IAAM,CAAA,CAAE,MAAA,GAAS,KAAK,OAAO,KAAA;AAC5C,IAAA,IAAI,CAAA,CAAE,WAAW,GAAG,CAAA,IAAK,EAAE,UAAA,CAAW,MAAM,GAAG,OAAO,KAAA;AACtD,IAAA,IAAI,oEAAA,CAAqE,IAAA,CAAK,CAAC,CAAA,EAAG,OAAO,KAAA;AACzF,IAAA,OAAO,IAAA;AAAA,EACT,CAAC,CAAA;AAEH,EAAA,IAAI,YAAA,CAAa,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AAEvC,EAAA,MAAM,MAAA,GAAS,YAAA,CAAa,GAAA,CAAI,CAAC,UAAU,GAAA,KAAQ;AACjD,IAAA,MAAM,KAAA,GAAQ,QAAA,CAAS,SAAA,CAAU,KAAK,EAAE,WAAA,EAAY;AACpD,IAAA,MAAM,YAAA,GAAe,SAAS,MAAA,CAAO,CAAC,MAAM,KAAA,CAAM,QAAA,CAAS,CAAC,CAAC,CAAA,CAAE,MAAA;AAC/D,IAAA,MAAM,gBAAgB,GAAA,GAAM,CAAA,GAAI,GAAA,GAAM,GAAA,GAAM,KAAK,GAAA,GAAM,CAAA;AACvD,IAAA,MAAM,iBAAiB,yGAAA,CAA0G,IAAA;AAAA,MAC/H;AAAA,QAEE,GAAA,GACA,CAAA;AACJ,IAAA,OAAO;AAAA,MACL,QAAA;AAAA,MACA,GAAA;AAAA,MACA,KAAA,EAAO,YAAA,GAAe,CAAA,GAAI,aAAA,GAAgB;AAAA,KAC5C;AAAA,EACF,CAAC,CAAA;AAED,EAAA,OAAO,MAAA,CACJ,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,CAAA,CAAE,KAAA,IAAS,CAAA,CAAE,GAAA,GAAM,CAAA,CAAE,GAAG,CAAA,CACjD,KAAA,CAAM,CAAA,EAAG,YAAY,CAAA,CACrB,IAAA,CAAK,CAAC,CAAA,EAAG,MAAM,CAAA,CAAE,GAAA,GAAM,CAAA,CAAE,GAAG,CAAA,CAC5B,GAAA,CAAI,CAAC,CAAA,KAAM,EAAE,QAAQ,CAAA;AAC1B;AAKO,IAAM,iBAAN,MAAqB;AAAA,EAC1B,aAAA,CAAc,GAAA,EAAsB,MAAA,EAAyB,WAAA,EAAqC;AAChG,IAAA,IAAI,UAAA,GAAa,CAAA;AACjB,IAAA,MAAM,OAAA,GAAU,GAAA,CAAI,qBAAA,CAAsB,MAAA,GAAS,CAAA;AACnD,IAAA,IAAI,SAAA,GAAY,OAAA,GAAU,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,GAAA,CAAI,qBAAA,CAAsB,MAAM,CAAC,CAAA,GAAI,CAAA;AAC5F,IAAA,IAAI,UAAA,GAAa,CAAA;AAEjB,IAAA,MAAM,eAAe,GAAA,CAAI,eAAA,CAAgB,SAAA,CAAU,KAAK,EAAE,WAAA,EAAY;AACtE,IAAA,MAAM,cAAA,GAAiB,0BAA0B,WAAW,CAAA;AAC5D,IAAA,MAAM,cAAA,GAAiB,eAAe,MAAA,CAAO,CAAC,MAAM,YAAA,CAAa,QAAA,CAAS,CAAC,CAAC,CAAA,CAAE,MAAA;AAC9E,IAAA,MAAM,aAAa,cAAA,CAAe,MAAA,GAAS,CAAA,GAAI,cAAA,GAAiB,eAAe,MAAA,GAAS,CAAA;AAExF,IAAA,IAAI,UAAA,IAAc,MAAM,UAAA,IAAc,CAAA;AAAA,SAAA,IAC7B,UAAA,IAAc,KAAK,UAAA,IAAc,CAAA;AAAA,SAAA,IACjC,cAAA,IAAkB,GAAG,UAAA,IAAc,CAAA;AAE5C,IAAA,IAAI,GAAA,CAAI,WAAA,GAAc,CAAA,EAAG,UAAA,IAAc,CAAA;AACvC,IAAA,IAAI,GAAA,CAAI,QAAA,GAAW,IAAA,EAAM,UAAA,IAAc,CAAA;AACvC,IAAA,IAAA,CAAK,GAAA,CAAI,iBAAA,EAAmB,MAAA,IAAU,CAAA,KAAM,GAAG,UAAA,IAAc,CAAA;AAC7D,IAAA,UAAA,GAAa,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,UAAU,CAAA;AAGpC,IAAA,IAAI,CAAC,OAAA,IAAW,GAAA,CAAI,QAAA,GAAW,IAAA,IAAQ,cAAc,GAAA,EAAK;AACxD,MAAA,SAAA,GAAY,CAAA;AAAA,IACd;AAEA,IAAA,IAAI,aAAa,QAAA,CAAS,WAAW,KAAK,YAAA,CAAa,QAAA,CAAS,eAAe,CAAA,EAAG;AAChF,MAAA,UAAA,IAAc,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,WAAW,UAAA,GAAa,IAAA,GAAO,SAAA,GAAY,IAAA,GAAO,aAAa,GAAA,GAAM,GAAA;AAC3E,IAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,IAAA,CAAK,KAAA,CAAM,QAAQ,CAAC,CAAC,CAAA;AAC5D,IAAA,MAAM,aAAa,KAAA,IAAS,CAAA;AAE5B,IAAA,MAAM,YAAA,GAAe,mBAAA,CAAoB,GAAA,CAAI,eAAA,EAAiB,aAAa,CAAC,CAAA;AAC5E,IAAA,MAAM,QAAA,GAAW,IAAI,WAAA,EAAa,MAAA,GAAS,IAAI,WAAA,GAAc,0BAAA,CAA2B,GAAA,CAAI,eAAA,EAAiB,CAAC,CAAA;AAC9G,IAAA,MAAM,YAAA,GAAe,aAAa,MAAA,GAAS,CAAA,GAAI,eAAe,QAAA,CAAS,KAAA,CAAM,GAAG,CAAC,CAAA;AAEjF,IAAA,MAAM,OAAA,GACJ,YAAA,CAAa,MAAA,GAAS,CAAA,GAClB,CAAA,EAAG,YAAA,CAAa,CAAC,CAAC,CAAA,WAAA,EAAc,KAAK,CAAA,SAAA,EAAS,GAAA,CAAI,qBAAA,CAAsB,MAAM,CAAA,kBAAA,EAAkB,GAAA,CAAI,WAAW,CAAA,QAAA,CAAA,GAC/G,CAAA,gBAAA,EAAmB,KAAK,CAAA,KAAA,EAAQ,UAAA,GAAa,UAAA,GAAa,UAAU,CAAA,MAAA,EAAS,MAAM,CAAA,EAAA,EAAK,GAAA,CAAI,QAAQ,CAAA,QAAA,CAAA;AAE1G,IAAA,OAAO;AAAA,MACL,GAAA;AAAA,MACA,KAAA;AAAA,MACA,UAAA;AAAA,MACA,cAAA,EAAgB,KAAA;AAAA,MAChB,SAAA,EAAW;AAAA,QACT,cAAA,EAAgB,UAAA;AAAA,QAChB,kBAAA,EAAoB,UAAA;AAAA,QACpB,kBAAA,EAAoB,SAAA;AAAA,QACpB,WAAA,EAAa;AAAA,OACf;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAA,CACJ,GAAA,EACA,QACA,WAAA,EACA,OAAA,GAAgE,EAAC,EACxC;AACzB,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,aAAA,CAAc,GAAA,EAAK,QAAQ,WAAW,CAAA;AAC5D,IAAA,IAAI,CAAC,QAAQ,GAAA,EAAK;AAChB,MAAA,OAAO,QAAA;AAAA,IACT;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS;AAAA,QACb,CAAA,mGAAA,EAAsG,WAAW,CAAA,WAAA,EAAc,MAAM,CAAA,EAAA,CAAA;AAAA,QACrI,OAAA,CAAQ,OAAA,EAAS,OAAA,GAAU,CAAA,yBAAA,EAA4B,QAAQ,OAAA,CAAQ,OAAA,CAAQ,CAAC,CAAC,IAAI,OAAA,CAAQ,OAAA,CAAQ,OAAA,CAAQ,CAAC,CAAC,CAAA,CAAA,GAAK,EAAA;AAAA,QACpH,QAAQ,OAAA,EAAS,gBAAA,GAAmB,sBAAsB,OAAA,CAAQ,OAAA,CAAQ,gBAAgB,CAAA,CAAA,GAAK,EAAA;AAAA,QAC/F,QAAQ,OAAA,EAAS,WAAA,GAAc,qBAAqB,OAAA,CAAQ,OAAA,CAAQ,WAAW,CAAA,CAAA,GAAK,EAAA;AAAA,QACpF;AAAA,gBAAA,EAAqB,IAAI,KAAK,CAAA,CAAA;AAAA,QAC9B,CAAA,cAAA,EAAiB,IAAI,GAAG,CAAA,CAAA;AAAA,QACxB,CAAA;AAAA,EAAA,CAAsB,GAAA,CAAI,WAAA,IAAe,EAAC,EAAG,KAAA,CAAM,GAAG,CAAC,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,QACnE,CAAA,mBAAA,EAAsB,GAAA,CAAI,qBAAA,CAAsB,MAAM,CAAA,CAAA;AAAA,QACtD;AAAA,0BAAA,CAAA;AAAA,QACA,CAAA,sKAAA;AAAA,OACF,CACG,MAAA,CAAO,OAAO,CAAA,CACd,KAAK,IAAI,CAAA;AAEZ,MAAA,MAAM,GAAA,GAAM,MAAM,OAAA,CAAQ,GAAA,CAAI,MAAA,EAAQ;AAAA,QACpC,MAAA,EAAQ,wEAAA;AAAA,QACR,WAAA,EAAa,GAAA;AAAA,QACb,QAAA,EAAU;AAAA,OACX,CAAA;AAED,MAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,CAAM,aAAa,CAAA;AACrC,MAAA,IAAI,KAAA,EAAO;AACT,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,KAAA,CAAM,CAAC,CAAC,CAAA;AAClC,QAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,KAAK,CAAA,IAAK,QAAA,CAAS,KAAK,CAAC,CAAA;AAC9E,QAAA,OAAO;AAAA,UACL,GAAA;AAAA,UACA,KAAA;AAAA,UACA,YAAY,KAAA,IAAS,CAAA;AAAA,UACrB,cAAA,EAAgB,IAAA;AAAA,UAChB,SAAA,EAAW;AAAA,YACT,GAAG,QAAA,CAAS,SAAA;AAAA,YACZ,oBAAA,EAAsB,MAAA,CAAO,MAAA,CAAO,oBAAoB,CAAA,IAAK;AAAA,WAC/D;AAAA,UACA,OAAA,EAAS,MAAA,CAAO,MAAA,CAAO,OAAA,IAAW,SAAS,OAAO,CAAA;AAAA,UAClD,YAAA,EAAc,KAAA,CAAM,OAAA,CAAQ,MAAA,CAAO,YAAY,CAAA,IAAK,MAAA,CAAO,YAAA,CAAa,MAAA,GAAS,IAC7E,MAAA,CAAO,YAAA,CAAa,GAAA,CAAI,MAAM,IAC9B,QAAA,CAAS;AAAA,SACf;AAAA,MACF;AAAA,IACF,SAAS,CAAA,EAAQ;AACf,MAAA,OAAA,CAAQ,IAAA,CAAK,6EAAA,EAA+E,CAAA,EAAG,OAAA,IAAW,CAAC,CAAA;AAAA,IAC7G;AAEA,IAAA,OAAO,QAAA;AAAA,EACT;AACF;;;AC7KO,IAAM,oBAAN,MAAwB;AAAA,EAC7B,eAAA,CAAgB,OAAe,OAAA,EAAgD;AAC7E,IAAA,MAAM,UAAA,GAAa,CAAA,KAAA,EAAQ,IAAA,CAAK,MAAA,EAAO,CAAE,QAAA,CAAS,EAAE,CAAA,CAAE,SAAA,CAAU,CAAA,EAAG,CAAC,CAAA,CAAE,aAAa,CAAA,CAAA;AACnF,IAAA,MAAM,WAAA,GAAA,iBAAc,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAE3C,IAAA,MAAM,OAAA,GAAU;AAAA,MACd,yCAAkC,KAAK,CAAA,CAAA;AAAA,MACvC,wBAAwB,UAAU,CAAA,yBAAA,EAA4B,WAAW,CAAA,mCAAA,EAAsC,QAAQ,MAAM,CAAA,EAAA,CAAA;AAAA,MAC7H;AAAA,oBAAA,CAAA;AAAA,MACA,CAAA,4EAAA,CAAA;AAAA,MACA,CAAA,yBAAA,CAAA;AAAA,MACA,GAAG,OAAA,CAAQ,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,IAAA,EAAO,CAAA,CAAE,UAAU,CAAA,OAAA,EAAU,CAAA,CAAE,MAAM,CAAA,cAAA,EAAY,CAAA,CAAE,cAAc,aAAa,CAAA,CAAE,QAAQ,CAAA,qBAAA,EAAwB,CAAA,CAAE,SAAS,CAAA,MAAA,EAAS,CAAA,CAAE,UAAA,CAAW,KAAA,CAAM,CAAA,EAAG,EAAE,CAAC,CAAA,IAAA,CAAM,CAAA;AAAA,MACvL;AAAA,sBAAA,CAAA;AAAA,MACA,CAAA,mHAAA,CAAA;AAAA,MACA,CAAA,sGAAA;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,WAAA;AAAA,MACA,cAAc,OAAA,CAAQ,MAAA;AAAA,MACtB,OAAA;AAAA,MACA,YAAA,EAAc,OAAA,CAAQ,IAAA,CAAK,IAAI;AAAA,KACjC;AAAA,EACF;AACF","file":"chunk-GKF3NDEX.js","sourcesContent":["import { ScrapedDocument, extractSubstantiveExcerpts } from '../harvester/webScraper';\nimport { TechnicalDomain } from '../harvester/domainClassifier';\nimport { CourseResearchContext, LlmTextFn } from '../config/researchConfig';\n\nexport interface ScoredEvidence {\n doc: ScrapedDocument;\n score: number; // 1 to 10\n isApproved: boolean; // >= 6 is approved\n isLlmEvaluated?: boolean;\n breakdown: {\n technicalDepth?: number; // legacy alias for backwards compatibility\n structuralRichness: number; // 1-10: format hygiene, byte size, headings, tables\n codeSnippetQuality: number; // 1-10\n noiseFactor: number; // 1-10 (lower is better)\n pedagogicalRelevance?: number; // 1-10 (when evaluated by LLM)\n };\n summary: string;\n keyTakeaways: string[];\n}\n\nfunction tokenizeBilingualKeywords(text: string): string[] {\n return text\n .normalize('NFC')\n .toLowerCase()\n .replace(/[.,;:!?()[\\]{}\"“”‘’/\\\\|`*_~#-]+/g, ' ')\n .split(/\\s+/)\n .filter((w) => w.length >= 2 && !/^(and|the|for|with|from|into|của|trong|được|những|các|cho|với|khi|và|hoặc|để|từ|đến|về)$/.test(w));\n}\n\n/**\n * Extracts top key sentences from a document ranked by keyword overlap and structural salience (TextRank-inspired).\n */\nexport function extractTopSentences(markdown: string, targetTopic: string, maxSentences = 3): string[] {\n const keywords = tokenizeBilingualKeywords(targetTopic);\n const withoutCode = markdown.replace(/```[\\s\\S]*?```/g, ' ');\n const rawSentences = withoutCode\n .split(/(?<=[.!?。\\n])\\s+/)\n .map((s) => s.replace(/^[-*•#>\\s]+/, '').replace(/\\s+/g, ' ').trim())\n .filter((s) => {\n if (s.length < 45 || s.length > 320) return false;\n if (s.startsWith('|') || s.startsWith('http')) return false;\n if (/\\b(cookie|privacy policy|sign in|subscribe|all rights reserved)\\b/i.test(s)) return false;\n return true;\n });\n\n if (rawSentences.length === 0) return [];\n\n const scored = rawSentences.map((sentence, idx) => {\n const lower = sentence.normalize('NFC').toLowerCase();\n const matchedCount = keywords.filter((k) => lower.includes(k)).length;\n const positionBonus = idx < 8 ? 1.5 : idx < 20 ? 0.5 : 0;\n const technicalBonus = /(`[^`]+`|\\b(?:api|function|class|actor|async|sensor|pin|loop|state|biến|hàm|cảm biến|vòng lặp|mạch)\\b)/i.test(\n sentence\n )\n ? 1.5\n : 0;\n return {\n sentence,\n idx,\n score: matchedCount * 2 + positionBonus + technicalBonus,\n };\n });\n\n return scored\n .sort((a, b) => b.score - a.score || a.idx - b.idx)\n .slice(0, maxSentences)\n .sort((a, b) => a.idx - b.idx)\n .map((s) => s.sentence);\n}\n\n/**\n * Evaluates and scores evidence on a 1-10 scale and extracts real substantive takeaways\n */\nexport class EvidenceScorer {\n scoreDocument(doc: ScrapedDocument, domain: TechnicalDomain, targetTopic: string): ScoredEvidence {\n let depthScore = 5;\n const hasCode = doc.extractedCodeSnippets.length > 0;\n let codeScore = hasCode ? Math.min(10, 7 + Math.min(3, doc.extractedCodeSnippets.length)) : 5;\n let noiseScore = 2;\n\n const lowerContent = doc.markdownContent.normalize('NFC').toLowerCase();\n const targetKeywords = tokenizeBilingualKeywords(targetTopic);\n const keywordMatches = targetKeywords.filter((k) => lowerContent.includes(k)).length;\n const matchRatio = targetKeywords.length > 0 ? keywordMatches / targetKeywords.length : 1;\n\n if (matchRatio >= 0.75) depthScore += 3;\n else if (matchRatio >= 0.4) depthScore += 2;\n else if (keywordMatches >= 1) depthScore += 1;\n\n if (doc.tablesCount > 0) depthScore += 1;\n if (doc.byteSize > 1500) depthScore += 1;\n if ((doc.extractedHeadings?.length || 0) >= 2) depthScore += 1;\n depthScore = Math.min(10, depthScore);\n\n // Conceptual, academic, or official overview pages without code fences shouldn't be rejected if rich in structure\n if (!hasCode && doc.byteSize > 1200 && matchRatio >= 0.5) {\n codeScore = 6;\n }\n\n if (lowerContent.includes('subscribe') && lowerContent.includes('advertisement')) {\n noiseScore += 3;\n }\n\n const rawTotal = depthScore * 0.55 + codeScore * 0.35 - noiseScore * 0.1 + 0.5;\n const score = Math.max(1, Math.min(10, Math.round(rawTotal)));\n const isApproved = score >= 6;\n\n const topSentences = extractTopSentences(doc.markdownContent, targetTopic, 3);\n const excerpts = doc.keyExcerpts?.length ? doc.keyExcerpts : extractSubstantiveExcerpts(doc.markdownContent, 3);\n const keyTakeaways = topSentences.length > 0 ? topSentences : excerpts.slice(0, 2);\n\n const summary =\n keyTakeaways.length > 0\n ? `${keyTakeaways[0]} [Evidence ${score}/10 · ${doc.extractedCodeSnippets.length} code blocks · ${doc.tablesCount} tables]`\n : `Evidence scored ${score}/10 (${isApproved ? 'APPROVED' : 'REJECTED'}) for ${domain} (${doc.byteSize} bytes).`;\n\n return {\n doc,\n score,\n isApproved,\n isLlmEvaluated: false,\n breakdown: {\n technicalDepth: depthScore,\n structuralRichness: depthScore,\n codeSnippetQuality: codeScore,\n noiseFactor: noiseScore,\n },\n summary,\n keyTakeaways,\n };\n }\n\n /**\n * Evaluates evidence with host LLM for genuine pedagogical relevance, conceptual depth,\n * and accuracy for the target learner age band.\n */\n async scoreDocumentWithLlm(\n doc: ScrapedDocument,\n domain: TechnicalDomain,\n targetTopic: string,\n options: { llm?: LlmTextFn; context?: CourseResearchContext } = {}\n ): Promise<ScoredEvidence> {\n const baseline = this.scoreDocument(doc, domain, targetTopic);\n if (!options.llm) {\n return baseline;\n }\n\n try {\n const prompt = [\n `You are a Senior Curriculum Researcher. Evaluate this harvested web document for the course topic \"${targetTopic}\" (Domain: ${domain}).`,\n options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : '',\n options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : '',\n options.context?.softwareEnv ? `Software / Tools: ${options.context.softwareEnv}` : '',\n `\\nDocument Title: ${doc.title}`,\n `Document URL: ${doc.url}`,\n `Sample Excerpts:\\n${(doc.keyExcerpts || []).slice(0, 3).join('\\n')}`,\n `Code Blocks Found: ${doc.extractedCodeSnippets.length}`,\n `\\nReturn ONLY a JSON object:`,\n `{\"score\": 8, \"isApproved\": true, \"pedagogicalRelevance\": 9, \"summary\": \"1-sentence evaluative summary\", \"keyTakeaways\": [\"Takeaway 1 grounded in text\", \"Takeaway 2\"]}`,\n ]\n .filter(Boolean)\n .join('\\n');\n\n const raw = await options.llm(prompt, {\n system: 'You are an educational curriculum auditor. Output strictly valid JSON.',\n temperature: 0.2,\n jsonMode: true,\n });\n\n const match = raw.match(/\\{[\\s\\S]*\\}/);\n if (match) {\n const parsed = JSON.parse(match[0]);\n const score = Math.max(1, Math.min(10, Number(parsed.score) || baseline.score));\n return {\n doc,\n score,\n isApproved: score >= 6,\n isLlmEvaluated: true,\n breakdown: {\n ...baseline.breakdown,\n pedagogicalRelevance: Number(parsed.pedagogicalRelevance) || score,\n },\n summary: String(parsed.summary || baseline.summary),\n keyTakeaways: Array.isArray(parsed.keyTakeaways) && parsed.keyTakeaways.length > 0\n ? parsed.keyTakeaways.map(String)\n : baseline.keyTakeaways,\n };\n }\n } catch (e: any) {\n console.warn('[EvidenceScorer] LLM evidence evaluation failed, using structural baseline:', e?.message || e);\n }\n\n return baseline;\n }\n}\n","export interface ProvenanceRecord {\n resourceId: string;\n sourceUrl: string;\n accessedAt: string;\n domain: string;\n relevanceScore: number;\n byteSize: number;\n}\n\nexport interface ProvenanceSidecar {\n topic: string;\n campaignId: string;\n generatedAt: string;\n totalRecords: number;\n records: ProvenanceRecord[];\n provenanceMd: string;\n}\n\n/**\n * Generates the mandatory *.provenance.md sidecar document (English Canonical)\n */\nexport class ProvenanceTracker {\n generateSidecar(topic: string, records: ProvenanceRecord[]): ProvenanceSidecar {\n const campaignId = `CAMP-${Math.random().toString(36).substring(2, 9).toUpperCase()}`;\n const generatedAt = new Date().toISOString();\n\n const mdLines = [\n `# 📜 Provenance & Audit Trail: ${topic}`,\n `> **Campaign ID:** \\`${campaignId}\\` | **Generated At:** \\`${generatedAt}\\` | **Total Verified Sources:** \\`${records.length}\\``,\n `\\n## Audit Trail Table`,\n `| Resource ID | Domain | Relevancy | Size (Bytes) | Source URL | Timestamp |`,\n `|---|---|---|---|---|---|`,\n ...records.map(r => `| **${r.resourceId}** | \\`${r.domain}\\` | ⭐ \\`${r.relevanceScore}/10\\` | \\`${r.byteSize}\\` | [Access Source](${r.sourceUrl}) | \\`${r.accessedAt.slice(0, 19)}\\` |`),\n `\\n## Integrity Guarantee`,\n `- Every pedagogical claim and technical snippet in the curriculum maps 1-to-1 to the physical sources listed above.`,\n `- Unverified citations or hallucinated sources are strictly prohibited under Curriculum OS Rule §5.`\n ];\n\n return {\n topic,\n campaignId,\n generatedAt,\n totalRecords: records.length,\n records,\n provenanceMd: mdLines.join('\\n'),\n };\n }\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../src/evidence/evidenceScorer.ts","../src/evidence/provenanceTracker.ts"],"names":[],"mappings":";;;AAoBA,SAAS,0BAA0B,IAAA,EAAwB;AACzD,EAAA,OAAO,IAAA,CACJ,UAAU,KAAK,CAAA,CACf,aAAY,CACZ,OAAA,CAAQ,kCAAA,EAAoC,GAAG,CAAA,CAC/C,KAAA,CAAM,KAAK,CAAA,CACX,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,MAAA,IAAU,KAAK,CAAC,0FAAA,CAA2F,IAAA,CAAK,CAAC,CAAC,CAAA;AACvI;AAKO,SAAS,mBAAA,CAAoB,QAAA,EAAkB,WAAA,EAAqB,YAAA,GAAe,CAAA,EAAa;AACrG,EAAA,MAAM,QAAA,GAAW,0BAA0B,WAAW,CAAA;AACtD,EAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,iBAAA,EAAmB,GAAG,CAAA;AAC3D,EAAA,MAAM,YAAA,GAAe,YAClB,KAAA,CAAM,kBAAkB,EACxB,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,aAAA,EAAe,EAAE,CAAA,CAAE,OAAA,CAAQ,QAAQ,GAAG,CAAA,CAAE,MAAM,CAAA,CACnE,MAAA,CAAO,CAAC,CAAA,KAAM;AACb,IAAA,IAAI,EAAE,MAAA,GAAS,EAAA,IAAM,CAAA,CAAE,MAAA,GAAS,KAAK,OAAO,KAAA;AAC5C,IAAA,IAAI,CAAA,CAAE,WAAW,GAAG,CAAA,IAAK,EAAE,UAAA,CAAW,MAAM,GAAG,OAAO,KAAA;AACtD,IAAA,IAAI,oEAAA,CAAqE,IAAA,CAAK,CAAC,CAAA,EAAG,OAAO,KAAA;AACzF,IAAA,OAAO,IAAA;AAAA,EACT,CAAC,CAAA;AAEH,EAAA,IAAI,YAAA,CAAa,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AAEvC,EAAA,MAAM,MAAA,GAAS,YAAA,CAAa,GAAA,CAAI,CAAC,UAAU,GAAA,KAAQ;AACjD,IAAA,MAAM,KAAA,GAAQ,QAAA,CAAS,SAAA,CAAU,KAAK,EAAE,WAAA,EAAY;AACpD,IAAA,MAAM,YAAA,GAAe,SAAS,MAAA,CAAO,CAAC,MAAM,KAAA,CAAM,QAAA,CAAS,CAAC,CAAC,CAAA,CAAE,MAAA;AAC/D,IAAA,MAAM,gBAAgB,GAAA,GAAM,CAAA,GAAI,GAAA,GAAM,GAAA,GAAM,KAAK,GAAA,GAAM,CAAA;AACvD,IAAA,MAAM,iBAAiB,yGAAA,CAA0G,IAAA;AAAA,MAC/H;AAAA,QAEE,GAAA,GACA,CAAA;AACJ,IAAA,OAAO;AAAA,MACL,QAAA;AAAA,MACA,GAAA;AAAA,MACA,KAAA,EAAO,YAAA,GAAe,CAAA,GAAI,aAAA,GAAgB;AAAA,KAC5C;AAAA,EACF,CAAC,CAAA;AAED,EAAA,OAAO,MAAA,CACJ,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,CAAA,CAAE,KAAA,IAAS,CAAA,CAAE,GAAA,GAAM,CAAA,CAAE,GAAG,CAAA,CACjD,KAAA,CAAM,CAAA,EAAG,YAAY,CAAA,CACrB,IAAA,CAAK,CAAC,CAAA,EAAG,MAAM,CAAA,CAAE,GAAA,GAAM,CAAA,CAAE,GAAG,CAAA,CAC5B,GAAA,CAAI,CAAC,CAAA,KAAM,EAAE,QAAQ,CAAA;AAC1B;AAKO,IAAM,iBAAN,MAAqB;AAAA,EAC1B,aAAA,CAAc,GAAA,EAAsB,MAAA,EAAyB,WAAA,EAAqC;AAChG,IAAA,IAAI,UAAA,GAAa,CAAA;AACjB,IAAA,MAAM,OAAA,GAAU,GAAA,CAAI,qBAAA,CAAsB,MAAA,GAAS,CAAA;AACnD,IAAA,IAAI,SAAA,GAAY,OAAA,GAAU,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,GAAA,CAAI,qBAAA,CAAsB,MAAM,CAAC,CAAA,GAAI,CAAA;AAC5F,IAAA,IAAI,UAAA,GAAa,CAAA;AAEjB,IAAA,MAAM,eAAe,GAAA,CAAI,eAAA,CAAgB,SAAA,CAAU,KAAK,EAAE,WAAA,EAAY;AACtE,IAAA,MAAM,cAAA,GAAiB,0BAA0B,WAAW,CAAA;AAC5D,IAAA,MAAM,cAAA,GAAiB,eAAe,MAAA,CAAO,CAAC,MAAM,YAAA,CAAa,QAAA,CAAS,CAAC,CAAC,CAAA,CAAE,MAAA;AAC9E,IAAA,MAAM,aAAa,cAAA,CAAe,MAAA,GAAS,CAAA,GAAI,cAAA,GAAiB,eAAe,MAAA,GAAS,CAAA;AAExF,IAAA,IAAI,UAAA,IAAc,MAAM,UAAA,IAAc,CAAA;AAAA,SAAA,IAC7B,UAAA,IAAc,KAAK,UAAA,IAAc,CAAA;AAAA,SAAA,IACjC,cAAA,IAAkB,GAAG,UAAA,IAAc,CAAA;AAE5C,IAAA,IAAI,GAAA,CAAI,WAAA,GAAc,CAAA,EAAG,UAAA,IAAc,CAAA;AACvC,IAAA,IAAI,GAAA,CAAI,QAAA,GAAW,IAAA,EAAM,UAAA,IAAc,CAAA;AACvC,IAAA,IAAA,CAAK,GAAA,CAAI,iBAAA,EAAmB,MAAA,IAAU,CAAA,KAAM,GAAG,UAAA,IAAc,CAAA;AAC7D,IAAA,UAAA,GAAa,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,UAAU,CAAA;AAGpC,IAAA,IAAI,CAAC,OAAA,IAAW,GAAA,CAAI,QAAA,GAAW,IAAA,IAAQ,cAAc,GAAA,EAAK;AACxD,MAAA,SAAA,GAAY,CAAA;AAAA,IACd;AAEA,IAAA,IAAI,aAAa,QAAA,CAAS,WAAW,KAAK,YAAA,CAAa,QAAA,CAAS,eAAe,CAAA,EAAG;AAChF,MAAA,UAAA,IAAc,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,WAAW,UAAA,GAAa,IAAA,GAAO,SAAA,GAAY,IAAA,GAAO,aAAa,GAAA,GAAM,GAAA;AAC3E,IAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,IAAA,CAAK,KAAA,CAAM,QAAQ,CAAC,CAAC,CAAA;AAC5D,IAAA,MAAM,aAAa,KAAA,IAAS,CAAA;AAE5B,IAAA,MAAM,YAAA,GAAe,mBAAA,CAAoB,GAAA,CAAI,eAAA,EAAiB,aAAa,CAAC,CAAA;AAC5E,IAAA,MAAM,QAAA,GAAW,IAAI,WAAA,EAAa,MAAA,GAAS,IAAI,WAAA,GAAc,0BAAA,CAA2B,GAAA,CAAI,eAAA,EAAiB,CAAC,CAAA;AAC9G,IAAA,MAAM,YAAA,GAAe,aAAa,MAAA,GAAS,CAAA,GAAI,eAAe,QAAA,CAAS,KAAA,CAAM,GAAG,CAAC,CAAA;AAEjF,IAAA,MAAM,OAAA,GACJ,YAAA,CAAa,MAAA,GAAS,CAAA,GAClB,CAAA,EAAG,YAAA,CAAa,CAAC,CAAC,CAAA,WAAA,EAAc,KAAK,CAAA,SAAA,EAAS,GAAA,CAAI,qBAAA,CAAsB,MAAM,CAAA,kBAAA,EAAkB,GAAA,CAAI,WAAW,CAAA,QAAA,CAAA,GAC/G,CAAA,gBAAA,EAAmB,KAAK,CAAA,KAAA,EAAQ,UAAA,GAAa,UAAA,GAAa,UAAU,CAAA,MAAA,EAAS,MAAM,CAAA,EAAA,EAAK,GAAA,CAAI,QAAQ,CAAA,QAAA,CAAA;AAE1G,IAAA,OAAO;AAAA,MACL,GAAA;AAAA,MACA,KAAA;AAAA,MACA,UAAA;AAAA,MACA,cAAA,EAAgB,KAAA;AAAA,MAChB,SAAA,EAAW;AAAA,QACT,cAAA,EAAgB,UAAA;AAAA,QAChB,kBAAA,EAAoB,UAAA;AAAA,QACpB,kBAAA,EAAoB,SAAA;AAAA,QACpB,WAAA,EAAa;AAAA,OACf;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAA,CACJ,GAAA,EACA,QACA,WAAA,EACA,OAAA,GAAgE,EAAC,EACxC;AACzB,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,aAAA,CAAc,GAAA,EAAK,QAAQ,WAAW,CAAA;AAC5D,IAAA,IAAI,CAAC,QAAQ,GAAA,EAAK;AAChB,MAAA,OAAO,QAAA;AAAA,IACT;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS;AAAA,QACb,CAAA,mGAAA,EAAsG,WAAW,CAAA,WAAA,EAAc,MAAM,CAAA,EAAA,CAAA;AAAA,QACrI,OAAA,CAAQ,OAAA,EAAS,OAAA,GAAU,CAAA,yBAAA,EAA4B,QAAQ,OAAA,CAAQ,OAAA,CAAQ,CAAC,CAAC,IAAI,OAAA,CAAQ,OAAA,CAAQ,OAAA,CAAQ,CAAC,CAAC,CAAA,CAAA,GAAK,EAAA;AAAA,QACpH,QAAQ,OAAA,EAAS,gBAAA,GAAmB,sBAAsB,OAAA,CAAQ,OAAA,CAAQ,gBAAgB,CAAA,CAAA,GAAK,EAAA;AAAA,QAC/F,QAAQ,OAAA,EAAS,WAAA,GAAc,qBAAqB,OAAA,CAAQ,OAAA,CAAQ,WAAW,CAAA,CAAA,GAAK,EAAA;AAAA,QACpF;AAAA,gBAAA,EAAqB,IAAI,KAAK,CAAA,CAAA;AAAA,QAC9B,CAAA,cAAA,EAAiB,IAAI,GAAG,CAAA,CAAA;AAAA,QACxB,CAAA;AAAA,EAAA,CAAsB,GAAA,CAAI,WAAA,IAAe,EAAC,EAAG,KAAA,CAAM,GAAG,CAAC,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,QACnE,CAAA,mBAAA,EAAsB,GAAA,CAAI,qBAAA,CAAsB,MAAM,CAAA,CAAA;AAAA,QACtD;AAAA,0BAAA,CAAA;AAAA,QACA,CAAA,sKAAA;AAAA,OACF,CACG,MAAA,CAAO,OAAO,CAAA,CACd,KAAK,IAAI,CAAA;AAEZ,MAAA,MAAM,GAAA,GAAM,MAAM,OAAA,CAAQ,GAAA,CAAI,MAAA,EAAQ;AAAA,QACpC,MAAA,EAAQ,wEAAA;AAAA,QACR,WAAA,EAAa,GAAA;AAAA,QACb,QAAA,EAAU;AAAA,OACX,CAAA;AAED,MAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,CAAM,aAAa,CAAA;AACrC,MAAA,IAAI,KAAA,EAAO;AACT,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,KAAA,CAAM,CAAC,CAAC,CAAA;AAClC,QAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,KAAK,CAAA,IAAK,QAAA,CAAS,KAAK,CAAC,CAAA;AAC9E,QAAA,OAAO;AAAA,UACL,GAAA;AAAA,UACA,KAAA;AAAA,UACA,YAAY,KAAA,IAAS,CAAA;AAAA,UACrB,cAAA,EAAgB,IAAA;AAAA,UAChB,SAAA,EAAW;AAAA,YACT,GAAG,QAAA,CAAS,SAAA;AAAA,YACZ,oBAAA,EAAsB,MAAA,CAAO,MAAA,CAAO,oBAAoB,CAAA,IAAK;AAAA,WAC/D;AAAA,UACA,OAAA,EAAS,MAAA,CAAO,MAAA,CAAO,OAAA,IAAW,SAAS,OAAO,CAAA;AAAA,UAClD,YAAA,EAAc,KAAA,CAAM,OAAA,CAAQ,MAAA,CAAO,YAAY,CAAA,IAAK,MAAA,CAAO,YAAA,CAAa,MAAA,GAAS,IAC7E,MAAA,CAAO,YAAA,CAAa,GAAA,CAAI,MAAM,IAC9B,QAAA,CAAS;AAAA,SACf;AAAA,MACF;AAAA,IACF,SAAS,CAAA,EAAQ;AACf,MAAA,OAAA,CAAQ,IAAA,CAAK,6EAAA,EAA+E,CAAA,EAAG,OAAA,IAAW,CAAC,CAAA;AAAA,IAC7G;AAEA,IAAA,OAAO,QAAA;AAAA,EACT;AACF;;;AC7KO,IAAM,oBAAN,MAAwB;AAAA,EAC7B,eAAA,CAAgB,OAAe,OAAA,EAAgD;AAC7E,IAAA,MAAM,UAAA,GAAa,CAAA,KAAA,EAAQ,IAAA,CAAK,MAAA,EAAO,CAAE,QAAA,CAAS,EAAE,CAAA,CAAE,SAAA,CAAU,CAAA,EAAG,CAAC,CAAA,CAAE,aAAa,CAAA,CAAA;AACnF,IAAA,MAAM,WAAA,GAAA,iBAAc,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAE3C,IAAA,MAAM,OAAA,GAAU;AAAA,MACd,yCAAkC,KAAK,CAAA,CAAA;AAAA,MACvC,wBAAwB,UAAU,CAAA,yBAAA,EAA4B,WAAW,CAAA,mCAAA,EAAsC,QAAQ,MAAM,CAAA,EAAA,CAAA;AAAA,MAC7H;AAAA,oBAAA,CAAA;AAAA,MACA,CAAA,4EAAA,CAAA;AAAA,MACA,CAAA,yBAAA,CAAA;AAAA,MACA,GAAG,OAAA,CAAQ,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,IAAA,EAAO,CAAA,CAAE,UAAU,CAAA,OAAA,EAAU,CAAA,CAAE,MAAM,CAAA,cAAA,EAAY,CAAA,CAAE,cAAc,aAAa,CAAA,CAAE,QAAQ,CAAA,qBAAA,EAAwB,CAAA,CAAE,SAAS,CAAA,MAAA,EAAS,CAAA,CAAE,UAAA,CAAW,KAAA,CAAM,CAAA,EAAG,EAAE,CAAC,CAAA,IAAA,CAAM,CAAA;AAAA,MACvL;AAAA,sBAAA,CAAA;AAAA,MACA,CAAA,mHAAA,CAAA;AAAA,MACA,CAAA,sGAAA;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,WAAA;AAAA,MACA,cAAc,OAAA,CAAQ,MAAA;AAAA,MACtB,OAAA;AAAA,MACA,YAAA,EAAc,OAAA,CAAQ,IAAA,CAAK,IAAI;AAAA,KACjC;AAAA,EACF;AACF","file":"chunk-QFYM2NV4.js","sourcesContent":["import { ScrapedDocument, extractSubstantiveExcerpts } from '../harvester/webScraper';\nimport { TechnicalDomain } from '../harvester/domainClassifier';\nimport { CourseResearchContext, LlmTextFn } from '../config/researchConfig';\n\nexport interface ScoredEvidence {\n doc: ScrapedDocument;\n score: number; // 1 to 10\n isApproved: boolean; // >= 6 is approved\n isLlmEvaluated?: boolean;\n breakdown: {\n technicalDepth?: number; // legacy alias for backwards compatibility\n structuralRichness: number; // 1-10: format hygiene, byte size, headings, tables\n codeSnippetQuality: number; // 1-10\n noiseFactor: number; // 1-10 (lower is better)\n pedagogicalRelevance?: number; // 1-10 (when evaluated by LLM)\n };\n summary: string;\n keyTakeaways: string[];\n}\n\nfunction tokenizeBilingualKeywords(text: string): string[] {\n return text\n .normalize('NFC')\n .toLowerCase()\n .replace(/[.,;:!?()[\\]{}\"“”‘’/\\\\|`*_~#-]+/g, ' ')\n .split(/\\s+/)\n .filter((w) => w.length >= 2 && !/^(and|the|for|with|from|into|của|trong|được|những|các|cho|với|khi|và|hoặc|để|từ|đến|về)$/.test(w));\n}\n\n/**\n * Extracts top key sentences from a document ranked by keyword overlap and structural salience (TextRank-inspired).\n */\nexport function extractTopSentences(markdown: string, targetTopic: string, maxSentences = 3): string[] {\n const keywords = tokenizeBilingualKeywords(targetTopic);\n const withoutCode = markdown.replace(/```[\\s\\S]*?```/g, ' ');\n const rawSentences = withoutCode\n .split(/(?<=[.!?。\\n])\\s+/)\n .map((s) => s.replace(/^[-*•#>\\s]+/, '').replace(/\\s+/g, ' ').trim())\n .filter((s) => {\n if (s.length < 45 || s.length > 320) return false;\n if (s.startsWith('|') || s.startsWith('http')) return false;\n if (/\\b(cookie|privacy policy|sign in|subscribe|all rights reserved)\\b/i.test(s)) return false;\n return true;\n });\n\n if (rawSentences.length === 0) return [];\n\n const scored = rawSentences.map((sentence, idx) => {\n const lower = sentence.normalize('NFC').toLowerCase();\n const matchedCount = keywords.filter((k) => lower.includes(k)).length;\n const positionBonus = idx < 8 ? 1.5 : idx < 20 ? 0.5 : 0;\n const technicalBonus = /(`[^`]+`|\\b(?:api|function|class|actor|async|sensor|pin|loop|state|biến|hàm|cảm biến|vòng lặp|mạch)\\b)/i.test(\n sentence\n )\n ? 1.5\n : 0;\n return {\n sentence,\n idx,\n score: matchedCount * 2 + positionBonus + technicalBonus,\n };\n });\n\n return scored\n .sort((a, b) => b.score - a.score || a.idx - b.idx)\n .slice(0, maxSentences)\n .sort((a, b) => a.idx - b.idx)\n .map((s) => s.sentence);\n}\n\n/**\n * Evaluates and scores evidence on a 1-10 scale and extracts real substantive takeaways\n */\nexport class EvidenceScorer {\n scoreDocument(doc: ScrapedDocument, domain: TechnicalDomain, targetTopic: string): ScoredEvidence {\n let depthScore = 5;\n const hasCode = doc.extractedCodeSnippets.length > 0;\n let codeScore = hasCode ? Math.min(10, 7 + Math.min(3, doc.extractedCodeSnippets.length)) : 5;\n let noiseScore = 2;\n\n const lowerContent = doc.markdownContent.normalize('NFC').toLowerCase();\n const targetKeywords = tokenizeBilingualKeywords(targetTopic);\n const keywordMatches = targetKeywords.filter((k) => lowerContent.includes(k)).length;\n const matchRatio = targetKeywords.length > 0 ? keywordMatches / targetKeywords.length : 1;\n\n if (matchRatio >= 0.75) depthScore += 3;\n else if (matchRatio >= 0.4) depthScore += 2;\n else if (keywordMatches >= 1) depthScore += 1;\n\n if (doc.tablesCount > 0) depthScore += 1;\n if (doc.byteSize > 1500) depthScore += 1;\n if ((doc.extractedHeadings?.length || 0) >= 2) depthScore += 1;\n depthScore = Math.min(10, depthScore);\n\n // Conceptual, academic, or official overview pages without code fences shouldn't be rejected if rich in structure\n if (!hasCode && doc.byteSize > 1200 && matchRatio >= 0.5) {\n codeScore = 6;\n }\n\n if (lowerContent.includes('subscribe') && lowerContent.includes('advertisement')) {\n noiseScore += 3;\n }\n\n const rawTotal = depthScore * 0.55 + codeScore * 0.35 - noiseScore * 0.1 + 0.5;\n const score = Math.max(1, Math.min(10, Math.round(rawTotal)));\n const isApproved = score >= 6;\n\n const topSentences = extractTopSentences(doc.markdownContent, targetTopic, 3);\n const excerpts = doc.keyExcerpts?.length ? doc.keyExcerpts : extractSubstantiveExcerpts(doc.markdownContent, 3);\n const keyTakeaways = topSentences.length > 0 ? topSentences : excerpts.slice(0, 2);\n\n const summary =\n keyTakeaways.length > 0\n ? `${keyTakeaways[0]} [Evidence ${score}/10 · ${doc.extractedCodeSnippets.length} code blocks · ${doc.tablesCount} tables]`\n : `Evidence scored ${score}/10 (${isApproved ? 'APPROVED' : 'REJECTED'}) for ${domain} (${doc.byteSize} bytes).`;\n\n return {\n doc,\n score,\n isApproved,\n isLlmEvaluated: false,\n breakdown: {\n technicalDepth: depthScore,\n structuralRichness: depthScore,\n codeSnippetQuality: codeScore,\n noiseFactor: noiseScore,\n },\n summary,\n keyTakeaways,\n };\n }\n\n /**\n * Evaluates evidence with host LLM for genuine pedagogical relevance, conceptual depth,\n * and accuracy for the target learner age band.\n */\n async scoreDocumentWithLlm(\n doc: ScrapedDocument,\n domain: TechnicalDomain,\n targetTopic: string,\n options: { llm?: LlmTextFn; context?: CourseResearchContext } = {}\n ): Promise<ScoredEvidence> {\n const baseline = this.scoreDocument(doc, domain, targetTopic);\n if (!options.llm) {\n return baseline;\n }\n\n try {\n const prompt = [\n `You are a Senior Curriculum Researcher. Evaluate this harvested web document for the course topic \"${targetTopic}\" (Domain: ${domain}).`,\n options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]}` : '',\n options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}` : '',\n options.context?.softwareEnv ? `Software / Tools: ${options.context.softwareEnv}` : '',\n `\\nDocument Title: ${doc.title}`,\n `Document URL: ${doc.url}`,\n `Sample Excerpts:\\n${(doc.keyExcerpts || []).slice(0, 3).join('\\n')}`,\n `Code Blocks Found: ${doc.extractedCodeSnippets.length}`,\n `\\nReturn ONLY a JSON object:`,\n `{\"score\": 8, \"isApproved\": true, \"pedagogicalRelevance\": 9, \"summary\": \"1-sentence evaluative summary\", \"keyTakeaways\": [\"Takeaway 1 grounded in text\", \"Takeaway 2\"]}`,\n ]\n .filter(Boolean)\n .join('\\n');\n\n const raw = await options.llm(prompt, {\n system: 'You are an educational curriculum auditor. Output strictly valid JSON.',\n temperature: 0.2,\n jsonMode: true,\n });\n\n const match = raw.match(/\\{[\\s\\S]*\\}/);\n if (match) {\n const parsed = JSON.parse(match[0]);\n const score = Math.max(1, Math.min(10, Number(parsed.score) || baseline.score));\n return {\n doc,\n score,\n isApproved: score >= 6,\n isLlmEvaluated: true,\n breakdown: {\n ...baseline.breakdown,\n pedagogicalRelevance: Number(parsed.pedagogicalRelevance) || score,\n },\n summary: String(parsed.summary || baseline.summary),\n keyTakeaways: Array.isArray(parsed.keyTakeaways) && parsed.keyTakeaways.length > 0\n ? parsed.keyTakeaways.map(String)\n : baseline.keyTakeaways,\n };\n }\n } catch (e: any) {\n console.warn('[EvidenceScorer] LLM evidence evaluation failed, using structural baseline:', e?.message || e);\n }\n\n return baseline;\n }\n}\n","export interface ProvenanceRecord {\n resourceId: string;\n sourceUrl: string;\n accessedAt: string;\n domain: string;\n relevanceScore: number;\n byteSize: number;\n}\n\nexport interface ProvenanceSidecar {\n topic: string;\n campaignId: string;\n generatedAt: string;\n totalRecords: number;\n records: ProvenanceRecord[];\n provenanceMd: string;\n}\n\n/**\n * Generates the mandatory *.provenance.md sidecar document (English Canonical)\n */\nexport class ProvenanceTracker {\n generateSidecar(topic: string, records: ProvenanceRecord[]): ProvenanceSidecar {\n const campaignId = `CAMP-${Math.random().toString(36).substring(2, 9).toUpperCase()}`;\n const generatedAt = new Date().toISOString();\n\n const mdLines = [\n `# 📜 Provenance & Audit Trail: ${topic}`,\n `> **Campaign ID:** \\`${campaignId}\\` | **Generated At:** \\`${generatedAt}\\` | **Total Verified Sources:** \\`${records.length}\\``,\n `\\n## Audit Trail Table`,\n `| Resource ID | Domain | Relevancy | Size (Bytes) | Source URL | Timestamp |`,\n `|---|---|---|---|---|---|`,\n ...records.map(r => `| **${r.resourceId}** | \\`${r.domain}\\` | ⭐ \\`${r.relevanceScore}/10\\` | \\`${r.byteSize}\\` | [Access Source](${r.sourceUrl}) | \\`${r.accessedAt.slice(0, 19)}\\` |`),\n `\\n## Integrity Guarantee`,\n `- Every pedagogical claim and technical snippet in the curriculum maps 1-to-1 to the physical sources listed above.`,\n `- Unverified citations or hallucinated sources are strictly prohibited under Curriculum OS Rule §5.`\n ];\n\n return {\n topic,\n campaignId,\n generatedAt,\n totalRecords: records.length,\n records,\n provenanceMd: mdLines.join('\\n'),\n };\n }\n}\n"]}
|