antri_cli 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -0
- package/bin/antri.js +2 -0
- package/dist/cli/banner.d.ts +22 -0
- package/dist/cli/banner.d.ts.map +1 -0
- package/dist/cli/banner.js +133 -0
- package/dist/cli/banner.js.map +1 -0
- package/dist/cli/dialogs/filePicker.d.ts +23 -0
- package/dist/cli/dialogs/filePicker.d.ts.map +1 -0
- package/dist/cli/dialogs/filePicker.js +106 -0
- package/dist/cli/dialogs/filePicker.js.map +1 -0
- package/dist/cli/dialogs/modelPicker.d.ts +10 -0
- package/dist/cli/dialogs/modelPicker.d.ts.map +1 -0
- package/dist/cli/dialogs/modelPicker.js +205 -0
- package/dist/cli/dialogs/modelPicker.js.map +1 -0
- package/dist/cli/dialogs/profilePicker.d.ts +11 -0
- package/dist/cli/dialogs/profilePicker.d.ts.map +1 -0
- package/dist/cli/dialogs/profilePicker.js +185 -0
- package/dist/cli/dialogs/profilePicker.js.map +1 -0
- package/dist/cli/dialogs/providerPicker.d.ts +19 -0
- package/dist/cli/dialogs/providerPicker.d.ts.map +1 -0
- package/dist/cli/dialogs/providerPicker.js +205 -0
- package/dist/cli/dialogs/providerPicker.js.map +1 -0
- package/dist/cli/prompt.d.ts +3 -0
- package/dist/cli/prompt.d.ts.map +1 -0
- package/dist/cli/prompt.js +50 -0
- package/dist/cli/prompt.js.map +1 -0
- package/dist/cli/promptToolkit.d.ts +14 -0
- package/dist/cli/promptToolkit.d.ts.map +1 -0
- package/dist/cli/promptToolkit.js +389 -0
- package/dist/cli/promptToolkit.js.map +1 -0
- package/dist/cli/renderer.d.ts +21 -0
- package/dist/cli/renderer.d.ts.map +1 -0
- package/dist/cli/renderer.js +97 -0
- package/dist/cli/renderer.js.map +1 -0
- package/dist/cli/shortcuts.d.ts +13 -0
- package/dist/cli/shortcuts.d.ts.map +1 -0
- package/dist/cli/shortcuts.js +414 -0
- package/dist/cli/shortcuts.js.map +1 -0
- package/dist/core/agent.d.ts +15 -0
- package/dist/core/agent.d.ts.map +1 -0
- package/dist/core/agent.js +232 -0
- package/dist/core/agent.js.map +1 -0
- package/dist/core/citations.d.ts +19 -0
- package/dist/core/citations.d.ts.map +1 -0
- package/dist/core/citations.js +40 -0
- package/dist/core/citations.js.map +1 -0
- package/dist/core/config.d.ts +16 -0
- package/dist/core/config.d.ts.map +1 -0
- package/dist/core/config.js +172 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/crawler.d.ts +18 -0
- package/dist/core/crawler.d.ts.map +1 -0
- package/dist/core/crawler.js +83 -0
- package/dist/core/crawler.js.map +1 -0
- package/dist/core/debugger.d.ts +20 -0
- package/dist/core/debugger.d.ts.map +1 -0
- package/dist/core/debugger.js +105 -0
- package/dist/core/debugger.js.map +1 -0
- package/dist/core/dialectic.d.ts +23 -0
- package/dist/core/dialectic.d.ts.map +1 -0
- package/dist/core/dialectic.js +300 -0
- package/dist/core/dialectic.js.map +1 -0
- package/dist/core/goalLoop.d.ts +14 -0
- package/dist/core/goalLoop.d.ts.map +1 -0
- package/dist/core/goalLoop.js +104 -0
- package/dist/core/goalLoop.js.map +1 -0
- package/dist/core/history.d.ts +13 -0
- package/dist/core/history.d.ts.map +1 -0
- package/dist/core/history.js +53 -0
- package/dist/core/history.js.map +1 -0
- package/dist/core/metaOptimizer.d.ts +27 -0
- package/dist/core/metaOptimizer.d.ts.map +1 -0
- package/dist/core/metaOptimizer.js +98 -0
- package/dist/core/metaOptimizer.js.map +1 -0
- package/dist/core/sandbox.d.ts +29 -0
- package/dist/core/sandbox.d.ts.map +1 -0
- package/dist/core/sandbox.js +141 -0
- package/dist/core/sandbox.js.map +1 -0
- package/dist/core/scraper.d.ts +22 -0
- package/dist/core/scraper.d.ts.map +1 -0
- package/dist/core/scraper.js +180 -0
- package/dist/core/scraper.js.map +1 -0
- package/dist/core/search.d.ts +27 -0
- package/dist/core/search.d.ts.map +1 -0
- package/dist/core/search.js +222 -0
- package/dist/core/search.js.map +1 -0
- package/dist/core/skillSynthesizer.d.ts +43 -0
- package/dist/core/skillSynthesizer.d.ts.map +1 -0
- package/dist/core/skillSynthesizer.js +147 -0
- package/dist/core/skillSynthesizer.js.map +1 -0
- package/dist/core/tools.d.ts +12 -0
- package/dist/core/tools.d.ts.map +1 -0
- package/dist/core/tools.js +467 -0
- package/dist/core/tools.js.map +1 -0
- package/dist/core/updater.d.ts +9 -0
- package/dist/core/updater.d.ts.map +1 -0
- package/dist/core/updater.js +63 -0
- package/dist/core/updater.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +92 -0
- package/dist/index.js.map +1 -0
- package/dist/memory/consolidation.d.ts +17 -0
- package/dist/memory/consolidation.d.ts.map +1 -0
- package/dist/memory/consolidation.js +42 -0
- package/dist/memory/consolidation.js.map +1 -0
- package/dist/memory/episodic.d.ts +15 -0
- package/dist/memory/episodic.d.ts.map +1 -0
- package/dist/memory/episodic.js +97 -0
- package/dist/memory/episodic.js.map +1 -0
- package/dist/memory/manager.d.ts +24 -0
- package/dist/memory/manager.d.ts.map +1 -0
- package/dist/memory/manager.js +97 -0
- package/dist/memory/manager.js.map +1 -0
- package/dist/memory/profile.d.ts +21 -0
- package/dist/memory/profile.d.ts.map +1 -0
- package/dist/memory/profile.js +111 -0
- package/dist/memory/profile.js.map +1 -0
- package/dist/memory/semantic.d.ts +24 -0
- package/dist/memory/semantic.d.ts.map +1 -0
- package/dist/memory/semantic.js +102 -0
- package/dist/memory/semantic.js.map +1 -0
- package/dist/memory/types.d.ts +45 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +2 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/vectorStore.d.ts +18 -0
- package/dist/memory/vectorStore.d.ts.map +1 -0
- package/dist/memory/vectorStore.js +103 -0
- package/dist/memory/vectorStore.js.map +1 -0
- package/dist/profiles/profileManager.d.ts +25 -0
- package/dist/profiles/profileManager.d.ts.map +1 -0
- package/dist/profiles/profileManager.js +205 -0
- package/dist/profiles/profileManager.js.map +1 -0
- package/dist/providers/anthropic.d.ts +14 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +80 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/base.d.ts +7 -0
- package/dist/providers/base.d.ts.map +1 -0
- package/dist/providers/base.js +2 -0
- package/dist/providers/base.js.map +1 -0
- package/dist/providers/gemini.d.ts +14 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +77 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/index.d.ts +4 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +65 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/mock.d.ts +8 -0
- package/dist/providers/mock.d.ts.map +1 -0
- package/dist/providers/mock.js +62 -0
- package/dist/providers/mock.js.map +1 -0
- package/dist/providers/models.d.ts +8 -0
- package/dist/providers/models.d.ts.map +1 -0
- package/dist/providers/models.js +403 -0
- package/dist/providers/models.js.map +1 -0
- package/dist/providers/openai.d.ts +18 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +162 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/types.d.ts +116 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/logger.d.ts +23 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +26 -0
- package/dist/utils/logger.js.map +1 -0
- package/package.json +57 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
export class ConversationHistory {
|
|
3
|
+
messages = [];
|
|
4
|
+
maxHistoryTokens = 40000;
|
|
5
|
+
constructor() {
|
|
6
|
+
this.messages = [];
|
|
7
|
+
}
|
|
8
|
+
addMessage(message) {
|
|
9
|
+
this.messages.push({
|
|
10
|
+
...message,
|
|
11
|
+
timestamp: Date.now(),
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
getMessages() {
|
|
15
|
+
return [...this.messages];
|
|
16
|
+
}
|
|
17
|
+
getLastUserMessage() {
|
|
18
|
+
for (let i = this.messages.length - 1; i >= 0; i--) {
|
|
19
|
+
if (this.messages[i].role === 'user') {
|
|
20
|
+
return this.messages[i];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
clear() {
|
|
26
|
+
this.messages = [];
|
|
27
|
+
}
|
|
28
|
+
length() {
|
|
29
|
+
return this.messages.length;
|
|
30
|
+
}
|
|
31
|
+
exportMarkdown(filePath) {
|
|
32
|
+
const lines = [
|
|
33
|
+
`# ANTRI Code Chat Transcript`,
|
|
34
|
+
`*Generated on ${new Date().toLocaleString()}*`,
|
|
35
|
+
`---`,
|
|
36
|
+
'',
|
|
37
|
+
];
|
|
38
|
+
for (const msg of this.messages) {
|
|
39
|
+
if (msg.role === 'system')
|
|
40
|
+
continue;
|
|
41
|
+
const roleName = msg.role.toUpperCase();
|
|
42
|
+
lines.push(`### 👤 ${roleName}`);
|
|
43
|
+
lines.push(msg.content);
|
|
44
|
+
lines.push('');
|
|
45
|
+
}
|
|
46
|
+
const content = lines.join('\n');
|
|
47
|
+
if (filePath) {
|
|
48
|
+
fs.writeFileSync(filePath, content, 'utf-8');
|
|
49
|
+
}
|
|
50
|
+
return content;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=history.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/core/history.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,MAAM,OAAO,mBAAmB;IACtB,QAAQ,GAAkB,EAAE,CAAC;IAC7B,gBAAgB,GAAW,KAAK,CAAC;IAEzC;QACE,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,CAAC;IAEM,UAAU,CAAC,OAAoB;QACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACjB,GAAG,OAAO;YACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAEM,WAAW;QAChB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAEM,kBAAkB;QACvB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK;QACV,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,CAAC;IAEM,MAAM;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC9B,CAAC;IAEM,cAAc,CAAC,QAAiB;QACrC,MAAM,KAAK,GAAa;YACtB,8BAA8B;YAC9B,iBAAiB,IAAI,IAAI,EAAE,CAAC,cAAc,EAAE,GAAG;YAC/C,KAAK;YACL,EAAE;SACH,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ;gBAAE,SAAS;YACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,EAAE,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,QAAQ,EAAE,CAAC;YACb,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export interface MetaMetrics {
|
|
2
|
+
totalQueries: number;
|
|
3
|
+
successfulQueries: number;
|
|
4
|
+
toolCallStats: Record<string, {
|
|
5
|
+
calls: number;
|
|
6
|
+
errors: number;
|
|
7
|
+
totalDurationMs: number;
|
|
8
|
+
}>;
|
|
9
|
+
selfHealingRepairs: number;
|
|
10
|
+
synthesizedSkillsCount: number;
|
|
11
|
+
learnedPromptHeuristics: string[];
|
|
12
|
+
averageLatencyMs: number;
|
|
13
|
+
}
|
|
14
|
+
export declare class MetaOptimizer {
|
|
15
|
+
private metrics;
|
|
16
|
+
constructor();
|
|
17
|
+
private load;
|
|
18
|
+
private save;
|
|
19
|
+
recordToolExecution(toolName: string, durationMs: number, isError: boolean): void;
|
|
20
|
+
recordSelfHealing(): void;
|
|
21
|
+
recordQuerySuccess(durationMs: number): void;
|
|
22
|
+
addHeuristic(heuristic: string): void;
|
|
23
|
+
getMetrics(): MetaMetrics;
|
|
24
|
+
renderMetaStatus(): void;
|
|
25
|
+
}
|
|
26
|
+
export declare const metaOptimizer: MetaOptimizer;
|
|
27
|
+
//# sourceMappingURL=metaOptimizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metaOptimizer.d.ts","sourceRoot":"","sources":["../../src/core/metaOptimizer.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,WAAW;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1F,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAgBD,qBAAa,aAAa;IACxB,OAAO,CAAC,OAAO,CAAc;;IAM7B,OAAO,CAAC,IAAI;IAUZ,OAAO,CAAC,IAAI;IAQL,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI;IAWjF,iBAAiB,IAAI,IAAI;IAKzB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAO5C,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAOrC,UAAU,IAAI,WAAW;IAIzB,gBAAgB,IAAI,IAAI;CA2BhC;AAED,eAAO,MAAM,aAAa,eAAsB,CAAC"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
const META_FILE = path.join(os.homedir(), '.antri', 'memory', 'meta_optimization.json');
|
|
6
|
+
const DEFAULT_METRICS = {
|
|
7
|
+
totalQueries: 0,
|
|
8
|
+
successfulQueries: 0,
|
|
9
|
+
toolCallStats: {},
|
|
10
|
+
selfHealingRepairs: 0,
|
|
11
|
+
synthesizedSkillsCount: 0,
|
|
12
|
+
learnedPromptHeuristics: [
|
|
13
|
+
'For high-ambiguity questions, prefer dialectic debate synthesis over single-pass answers.',
|
|
14
|
+
'For local workspace operations, verify path existence before executing write_file.',
|
|
15
|
+
'When searching libraries, prioritize official documentation and version-specific release tags.',
|
|
16
|
+
],
|
|
17
|
+
averageLatencyMs: 1200,
|
|
18
|
+
};
|
|
19
|
+
export class MetaOptimizer {
|
|
20
|
+
metrics;
|
|
21
|
+
constructor() {
|
|
22
|
+
this.metrics = this.load();
|
|
23
|
+
}
|
|
24
|
+
load() {
|
|
25
|
+
try {
|
|
26
|
+
if (fs.existsSync(META_FILE)) {
|
|
27
|
+
const raw = fs.readFileSync(META_FILE, 'utf-8');
|
|
28
|
+
return { ...DEFAULT_METRICS, ...JSON.parse(raw) };
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
catch { }
|
|
32
|
+
return { ...DEFAULT_METRICS };
|
|
33
|
+
}
|
|
34
|
+
save() {
|
|
35
|
+
try {
|
|
36
|
+
const dir = path.dirname(META_FILE);
|
|
37
|
+
if (!fs.existsSync(dir))
|
|
38
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
39
|
+
fs.writeFileSync(META_FILE, JSON.stringify(this.metrics, null, 2), 'utf-8');
|
|
40
|
+
}
|
|
41
|
+
catch { }
|
|
42
|
+
}
|
|
43
|
+
recordToolExecution(toolName, durationMs, isError) {
|
|
44
|
+
if (!this.metrics.toolCallStats[toolName]) {
|
|
45
|
+
this.metrics.toolCallStats[toolName] = { calls: 0, errors: 0, totalDurationMs: 0 };
|
|
46
|
+
}
|
|
47
|
+
const stat = this.metrics.toolCallStats[toolName];
|
|
48
|
+
stat.calls += 1;
|
|
49
|
+
if (isError)
|
|
50
|
+
stat.errors += 1;
|
|
51
|
+
stat.totalDurationMs += durationMs;
|
|
52
|
+
this.save();
|
|
53
|
+
}
|
|
54
|
+
recordSelfHealing() {
|
|
55
|
+
this.metrics.selfHealingRepairs += 1;
|
|
56
|
+
this.save();
|
|
57
|
+
}
|
|
58
|
+
recordQuerySuccess(durationMs) {
|
|
59
|
+
this.metrics.totalQueries += 1;
|
|
60
|
+
this.metrics.successfulQueries += 1;
|
|
61
|
+
this.metrics.averageLatencyMs = Math.round((this.metrics.averageLatencyMs + durationMs) / 2);
|
|
62
|
+
this.save();
|
|
63
|
+
}
|
|
64
|
+
addHeuristic(heuristic) {
|
|
65
|
+
if (!this.metrics.learnedPromptHeuristics.includes(heuristic)) {
|
|
66
|
+
this.metrics.learnedPromptHeuristics.push(heuristic);
|
|
67
|
+
this.save();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
getMetrics() {
|
|
71
|
+
return this.metrics;
|
|
72
|
+
}
|
|
73
|
+
renderMetaStatus() {
|
|
74
|
+
const totalCalls = Object.values(this.metrics.toolCallStats).reduce((acc, curr) => acc + curr.calls, 0);
|
|
75
|
+
const totalErrors = Object.values(this.metrics.toolCallStats).reduce((acc, curr) => acc + curr.errors, 0);
|
|
76
|
+
const successRate = totalCalls > 0 ? Math.round(((totalCalls - totalErrors) / totalCalls) * 100) : 100;
|
|
77
|
+
console.log(chalk.bold.hex('#c084fc')('\n🧬 Meta-Optimization & Self-Evolution Metrics'));
|
|
78
|
+
console.log(chalk.hex('#334155')('─'.repeat(72)));
|
|
79
|
+
console.log(`• ${chalk.bold('Overall Tool Success Rate:')} ${chalk.green(`${successRate}%`)} (${totalCalls - totalErrors}/${totalCalls} calls)`);
|
|
80
|
+
console.log(`• ${chalk.bold('Autonomous Self-Healings:')} ${chalk.cyan(this.metrics.selfHealingRepairs)} automatic repairs`);
|
|
81
|
+
console.log(`• ${chalk.bold('Average Latency:')} ${chalk.cyan(`${this.metrics.averageLatencyMs}ms`)}`);
|
|
82
|
+
console.log(`• ${chalk.bold('Learned Strategy Heuristics:')} ${chalk.cyan(this.metrics.learnedPromptHeuristics.length)} heuristics`);
|
|
83
|
+
if (Object.keys(this.metrics.toolCallStats).length > 0) {
|
|
84
|
+
console.log(chalk.bold.hex('#a5b4fc')('\n📊 Tool Performance Breakdown:'));
|
|
85
|
+
for (const [tool, stat] of Object.entries(this.metrics.toolCallStats)) {
|
|
86
|
+
const rate = stat.calls > 0 ? Math.round(((stat.calls - stat.errors) / stat.calls) * 100) : 100;
|
|
87
|
+
const avg = Math.round(stat.totalDurationMs / (stat.calls || 1));
|
|
88
|
+
console.log(` - ${chalk.cyan(tool.padEnd(16))} Rate: ${rate >= 90 ? chalk.green(`${rate}%`) : chalk.yellow(`${rate}%`)} · Avg: ${avg}ms · Total: ${stat.calls}`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
console.log(chalk.bold.hex('#a5b4fc')('\n🧠 Meta-Prompt Heuristics:'));
|
|
92
|
+
this.metrics.learnedPromptHeuristics.forEach((h) => console.log(` - ${chalk.hex('#94a3b8')(h)}`));
|
|
93
|
+
console.log(chalk.hex('#334155')('─'.repeat(72)));
|
|
94
|
+
console.log();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
export const metaOptimizer = new MetaOptimizer();
|
|
98
|
+
//# sourceMappingURL=metaOptimizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metaOptimizer.js","sourceRoot":"","sources":["../../src/core/metaOptimizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,wBAAwB,CAAC,CAAC;AAYxF,MAAM,eAAe,GAAgB;IACnC,YAAY,EAAE,CAAC;IACf,iBAAiB,EAAE,CAAC;IACpB,aAAa,EAAE,EAAE;IACjB,kBAAkB,EAAE,CAAC;IACrB,sBAAsB,EAAE,CAAC;IACzB,uBAAuB,EAAE;QACvB,2FAA2F;QAC3F,oFAAoF;QACpF,gGAAgG;KACjG;IACD,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF,MAAM,OAAO,aAAa;IAChB,OAAO,CAAc;IAE7B;QACE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAChD,OAAO,EAAE,GAAG,eAAe,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACV,OAAO,EAAE,GAAG,eAAe,EAAE,CAAC;IAChC,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC9E,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC;IAEM,mBAAmB,CAAC,QAAgB,EAAE,UAAkB,EAAE,OAAgB;QAC/E,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;QACrF,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,OAAO;YAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,eAAe,IAAI,UAAU,CAAC;QACnC,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAEM,iBAAiB;QACtB,IAAI,CAAC,OAAO,CAAC,kBAAkB,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAEM,kBAAkB,CAAC,UAAkB;QAC1C,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAEM,YAAY,CAAC,SAAiB;QACnC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEM,gBAAgB;QACrB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACxG,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAEvG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,iDAAiD,CAAC,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,WAAW,GAAG,CAAC,KAAK,UAAU,GAAG,WAAW,IAAI,UAAU,SAAS,CAAC,CAAC;QACnJ,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;QAChI,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC,CAAC;QACnH,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,8BAA8B,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAErI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC;YAC3E,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;gBACtE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBAChG,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,GAAG,eAAe,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YACpK,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEnG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface ExecutionResult {
|
|
2
|
+
stdout: string;
|
|
3
|
+
stderr: string;
|
|
4
|
+
exitCode: number;
|
|
5
|
+
durationMs: number;
|
|
6
|
+
sandboxed: boolean;
|
|
7
|
+
isolationType: 'docker' | 'subprocess' | 'python';
|
|
8
|
+
}
|
|
9
|
+
export declare class SandboxEngine {
|
|
10
|
+
private static pythonCommand;
|
|
11
|
+
private static dockerAvailable;
|
|
12
|
+
/**
|
|
13
|
+
* Auto-detect available Python command (python, python3, py)
|
|
14
|
+
*/
|
|
15
|
+
static getPythonCommand(): Promise<string>;
|
|
16
|
+
/**
|
|
17
|
+
* Check if Docker is installed and running
|
|
18
|
+
*/
|
|
19
|
+
static isDockerAvailable(): Promise<boolean>;
|
|
20
|
+
/**
|
|
21
|
+
* Executes Python code in an isolated temporary runtime environment
|
|
22
|
+
*/
|
|
23
|
+
static executePython(code: string, workingDir?: string, timeoutMs?: number): Promise<ExecutionResult>;
|
|
24
|
+
/**
|
|
25
|
+
* Executes Shell command with subprocess/docker isolation
|
|
26
|
+
*/
|
|
27
|
+
static executeShell(command: string, workingDir?: string, useDockerIfAvailable?: boolean, timeoutMs?: number): Promise<ExecutionResult>;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=sandbox.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/core/sandbox.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,QAAQ,GAAG,YAAY,GAAG,QAAQ,CAAC;CACnD;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAuB;IACnD,OAAO,CAAC,MAAM,CAAC,eAAe,CAAwB;IAEtD;;OAEG;WACiB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IAiBvD;;OAEG;WACiB,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAYzD;;OAEG;WACiB,aAAa,CAC/B,IAAI,EAAE,MAAM,EACZ,UAAU,SAAgB,EAC1B,SAAS,SAAQ,GAChB,OAAO,CAAC,eAAe,CAAC;IA2C3B;;OAEG;WACiB,YAAY,CAC9B,OAAO,EAAE,MAAM,EACf,UAAU,SAAgB,EAC1B,oBAAoB,UAAQ,EAC5B,SAAS,SAAQ,GAChB,OAAO,CAAC,eAAe,CAAC;CAgD5B"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { exec } from 'child_process';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
import util from 'util';
|
|
6
|
+
const execPromise = util.promisify(exec);
|
|
7
|
+
export class SandboxEngine {
|
|
8
|
+
static pythonCommand = null;
|
|
9
|
+
static dockerAvailable = null;
|
|
10
|
+
/**
|
|
11
|
+
* Auto-detect available Python command (python, python3, py)
|
|
12
|
+
*/
|
|
13
|
+
static async getPythonCommand() {
|
|
14
|
+
if (this.pythonCommand)
|
|
15
|
+
return this.pythonCommand;
|
|
16
|
+
const candidates = ['python', 'python3', 'py'];
|
|
17
|
+
for (const cmd of candidates) {
|
|
18
|
+
try {
|
|
19
|
+
const { stdout } = await execPromise(`${cmd} --version`);
|
|
20
|
+
if (stdout || cmd) {
|
|
21
|
+
this.pythonCommand = cmd;
|
|
22
|
+
return cmd;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
catch { }
|
|
26
|
+
}
|
|
27
|
+
this.pythonCommand = 'python';
|
|
28
|
+
return 'python';
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Check if Docker is installed and running
|
|
32
|
+
*/
|
|
33
|
+
static async isDockerAvailable() {
|
|
34
|
+
if (this.dockerAvailable !== null)
|
|
35
|
+
return this.dockerAvailable;
|
|
36
|
+
try {
|
|
37
|
+
await execPromise('docker info', { timeout: 3000 });
|
|
38
|
+
this.dockerAvailable = true;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
this.dockerAvailable = false;
|
|
42
|
+
}
|
|
43
|
+
return this.dockerAvailable;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Executes Python code in an isolated temporary runtime environment
|
|
47
|
+
*/
|
|
48
|
+
static async executePython(code, workingDir = process.cwd(), timeoutMs = 30000) {
|
|
49
|
+
const startTime = Date.now();
|
|
50
|
+
const pyCmd = await this.getPythonCommand();
|
|
51
|
+
const tempDir = path.join(os.tmpdir(), 'antri_sandbox');
|
|
52
|
+
if (!fs.existsSync(tempDir)) {
|
|
53
|
+
fs.mkdirSync(tempDir, { recursive: true });
|
|
54
|
+
}
|
|
55
|
+
const scriptPath = path.join(tempDir, `script_${Date.now()}_${Math.random().toString(36).substring(2, 7)}.py`);
|
|
56
|
+
fs.writeFileSync(scriptPath, code, 'utf-8');
|
|
57
|
+
try {
|
|
58
|
+
const { stdout, stderr } = await execPromise(`"${pyCmd}" "${scriptPath}"`, {
|
|
59
|
+
cwd: workingDir,
|
|
60
|
+
timeout: timeoutMs,
|
|
61
|
+
maxBuffer: 1024 * 1024 * 5,
|
|
62
|
+
});
|
|
63
|
+
return {
|
|
64
|
+
stdout: stdout ? stdout.trim() : '',
|
|
65
|
+
stderr: stderr ? stderr.trim() : '',
|
|
66
|
+
exitCode: 0,
|
|
67
|
+
durationMs: Date.now() - startTime,
|
|
68
|
+
sandboxed: true,
|
|
69
|
+
isolationType: 'python',
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
return {
|
|
74
|
+
stdout: err.stdout ? err.stdout.trim() : '',
|
|
75
|
+
stderr: err.stderr ? err.stderr.trim() : (err.message || 'Execution error'),
|
|
76
|
+
exitCode: err.code || 1,
|
|
77
|
+
durationMs: Date.now() - startTime,
|
|
78
|
+
sandboxed: true,
|
|
79
|
+
isolationType: 'python',
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
try {
|
|
84
|
+
if (fs.existsSync(scriptPath))
|
|
85
|
+
fs.unlinkSync(scriptPath);
|
|
86
|
+
}
|
|
87
|
+
catch { }
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Executes Shell command with subprocess/docker isolation
|
|
92
|
+
*/
|
|
93
|
+
static async executeShell(command, workingDir = process.cwd(), useDockerIfAvailable = false, timeoutMs = 30000) {
|
|
94
|
+
const startTime = Date.now();
|
|
95
|
+
// Check Docker container isolation option
|
|
96
|
+
if (useDockerIfAvailable && (await this.isDockerAvailable())) {
|
|
97
|
+
try {
|
|
98
|
+
const dockerCmd = `docker run --rm -v "${workingDir}:/workspace" -w /workspace alpine/sh -c "${command.replace(/"/g, '\\"')}"`;
|
|
99
|
+
const { stdout, stderr } = await execPromise(dockerCmd, { timeout: timeoutMs });
|
|
100
|
+
return {
|
|
101
|
+
stdout: stdout.trim(),
|
|
102
|
+
stderr: stderr.trim(),
|
|
103
|
+
exitCode: 0,
|
|
104
|
+
durationMs: Date.now() - startTime,
|
|
105
|
+
sandboxed: true,
|
|
106
|
+
isolationType: 'docker',
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
// Fallback to native subprocess
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// Subprocess execution
|
|
114
|
+
try {
|
|
115
|
+
const { stdout, stderr } = await execPromise(command, {
|
|
116
|
+
cwd: workingDir,
|
|
117
|
+
timeout: timeoutMs,
|
|
118
|
+
maxBuffer: 1024 * 1024 * 5,
|
|
119
|
+
});
|
|
120
|
+
return {
|
|
121
|
+
stdout: (stdout || '').trim(),
|
|
122
|
+
stderr: (stderr || '').trim(),
|
|
123
|
+
exitCode: 0,
|
|
124
|
+
durationMs: Date.now() - startTime,
|
|
125
|
+
sandboxed: false,
|
|
126
|
+
isolationType: 'subprocess',
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
return {
|
|
131
|
+
stdout: (err.stdout || '').trim(),
|
|
132
|
+
stderr: (err.stderr || err.message || '').trim(),
|
|
133
|
+
exitCode: err.code || 1,
|
|
134
|
+
durationMs: Date.now() - startTime,
|
|
135
|
+
sandboxed: false,
|
|
136
|
+
isolationType: 'subprocess',
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
//# sourceMappingURL=sandbox.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/core/sandbox.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAS,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AAWzC,MAAM,OAAO,aAAa;IAChB,MAAM,CAAC,aAAa,GAAkB,IAAI,CAAC;IAC3C,MAAM,CAAC,eAAe,GAAmB,IAAI,CAAC;IAEtD;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,gBAAgB;QAClC,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,aAAa,CAAC;QAElD,MAAM,UAAU,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC;gBACzD,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBAClB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;oBACzB,OAAO,GAAG,CAAC;gBACb,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,iBAAiB;QACnC,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,eAAe,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/B,CAAC;QACD,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,aAAa,CAC/B,IAAY,EACZ,UAAU,GAAG,OAAO,CAAC,GAAG,EAAE,EAC1B,SAAS,GAAG,KAAK;QAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC;QAExD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/G,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,IAAI,KAAK,MAAM,UAAU,GAAG,EAAE;gBACzE,GAAG,EAAE,UAAU;gBACf,OAAO,EAAE,SAAS;gBAClB,SAAS,EAAE,IAAI,GAAG,IAAI,GAAG,CAAC;aAC3B,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;gBACnC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;gBACnC,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,QAAQ;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,OAAO;gBACL,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;gBAC3C,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,iBAAiB,CAAC;gBAC3E,QAAQ,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC;gBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,QAAQ;aACxB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;oBAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAC3D,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,YAAY,CAC9B,OAAe,EACf,UAAU,GAAG,OAAO,CAAC,GAAG,EAAE,EAC1B,oBAAoB,GAAG,KAAK,EAC5B,SAAS,GAAG,KAAK;QAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,0CAA0C;QAC1C,IAAI,oBAAoB,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,uBAAuB,UAAU,4CAA4C,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC;gBAC/H,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO;oBACL,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;oBACrB,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;oBACrB,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,SAAS,EAAE,IAAI;oBACf,aAAa,EAAE,QAAQ;iBACxB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE;gBACpD,GAAG,EAAE,UAAU;gBACf,OAAO,EAAE,SAAS;gBAClB,SAAS,EAAE,IAAI,GAAG,IAAI,GAAG,CAAC;aAC3B,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC7B,MAAM,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC7B,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,YAAY;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,OAAO;gBACL,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;gBACjC,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;gBAChD,QAAQ,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC;gBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,YAAY;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface ScrapedPage {
|
|
2
|
+
url: string;
|
|
3
|
+
title: string;
|
|
4
|
+
markdown: string;
|
|
5
|
+
links: string[];
|
|
6
|
+
charCount: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class WebScraper {
|
|
9
|
+
private static userAgent;
|
|
10
|
+
/**
|
|
11
|
+
* Scrapes a URL, extracts primary article/documentation content, and converts to clean Markdown
|
|
12
|
+
*/
|
|
13
|
+
static scrape(url: string, maxChars?: number): Promise<ScrapedPage>;
|
|
14
|
+
/**
|
|
15
|
+
* Trafilatura / Readability-style HTML cleaner and Markdown converter
|
|
16
|
+
*/
|
|
17
|
+
private static htmlToMarkdown;
|
|
18
|
+
private static extractLinks;
|
|
19
|
+
private static cleanText;
|
|
20
|
+
private static cleanEntities;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=scraper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/core/scraper.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAC,SAAS,CAC4F;IAEpH;;OAEG;WACiB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;IA6D/E;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,cAAc;IA+F7B,OAAO,CAAC,MAAM,CAAC,YAAY;IAwB3B,OAAO,CAAC,MAAM,CAAC,SAAS;IAIxB,OAAO,CAAC,MAAM,CAAC,aAAa;CAW7B"}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
export class WebScraper {
|
|
2
|
+
static userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
3
|
+
/**
|
|
4
|
+
* Scrapes a URL, extracts primary article/documentation content, and converts to clean Markdown
|
|
5
|
+
*/
|
|
6
|
+
static async scrape(url, maxChars = 10000) {
|
|
7
|
+
try {
|
|
8
|
+
const parsedUrl = new URL(url);
|
|
9
|
+
const res = await fetch(url, {
|
|
10
|
+
headers: {
|
|
11
|
+
'User-Agent': this.userAgent,
|
|
12
|
+
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
13
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
14
|
+
},
|
|
15
|
+
signal: AbortSignal.timeout(8000),
|
|
16
|
+
});
|
|
17
|
+
if (!res.ok) {
|
|
18
|
+
throw new Error(`HTTP ${res.status} ${res.statusText}`);
|
|
19
|
+
}
|
|
20
|
+
const contentType = res.headers.get('content-type') || '';
|
|
21
|
+
if (!contentType.includes('text') && !contentType.includes('html') && !contentType.includes('json')) {
|
|
22
|
+
return {
|
|
23
|
+
url,
|
|
24
|
+
title: 'Binary Content',
|
|
25
|
+
markdown: `[Content is binary/non-text: ${contentType}]`,
|
|
26
|
+
links: [],
|
|
27
|
+
charCount: 0,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const rawHtml = await res.text();
|
|
31
|
+
// Extract page title
|
|
32
|
+
const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(rawHtml);
|
|
33
|
+
const title = titleMatch ? this.cleanText(titleMatch[1]) : parsedUrl.hostname;
|
|
34
|
+
// Extract all internal and external links
|
|
35
|
+
const links = this.extractLinks(rawHtml, url);
|
|
36
|
+
// Extract and convert main content to markdown
|
|
37
|
+
let cleanMarkdown = this.htmlToMarkdown(rawHtml, parsedUrl);
|
|
38
|
+
if (cleanMarkdown.length > maxChars) {
|
|
39
|
+
cleanMarkdown = cleanMarkdown.slice(0, maxChars) + '\n\n... [Content Truncated]';
|
|
40
|
+
}
|
|
41
|
+
return {
|
|
42
|
+
url,
|
|
43
|
+
title,
|
|
44
|
+
markdown: cleanMarkdown,
|
|
45
|
+
links,
|
|
46
|
+
charCount: cleanMarkdown.length,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
catch (err) {
|
|
50
|
+
return {
|
|
51
|
+
url,
|
|
52
|
+
title: 'Error Scraping URL',
|
|
53
|
+
markdown: `Failed to scrape ${url}: ${err.message}`,
|
|
54
|
+
links: [],
|
|
55
|
+
charCount: 0,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Trafilatura / Readability-style HTML cleaner and Markdown converter
|
|
61
|
+
*/
|
|
62
|
+
static htmlToMarkdown(html, baseUrl) {
|
|
63
|
+
let content = html;
|
|
64
|
+
// 1. Remove non-content elements
|
|
65
|
+
content = content
|
|
66
|
+
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
|
|
67
|
+
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
|
|
68
|
+
.replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, '')
|
|
69
|
+
.replace(/<svg\b[^<]*(?:(?!<\/svg>)<[^<]*)*<\/svg>/gi, '')
|
|
70
|
+
.replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, '')
|
|
71
|
+
.replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, '')
|
|
72
|
+
.replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, '')
|
|
73
|
+
.replace(/<aside\b[^<]*(?:(?!<\/aside>)<[^<]*)*<\/aside>/gi, '')
|
|
74
|
+
.replace(/<form\b[^<]*(?:(?!<\/form>)<[^<]*)*<\/form>/gi, '')
|
|
75
|
+
.replace(/<!--[\s\S]*?-->/g, '');
|
|
76
|
+
// 2. Locate main content container if available
|
|
77
|
+
const mainMatch = /<main\b[^>]*>([\s\S]*?)<\/main>/i.exec(content) ||
|
|
78
|
+
/<article\b[^>]*>([\s\S]*?)<\/article>/i.exec(content) ||
|
|
79
|
+
/<div\b[^>]*(?:class|id)=["'][^"']*(?:content|docs|documentation|markdown|post|article|body)[^"']*["'][^>]*>([\s\S]*?)<\/div>/i.exec(content);
|
|
80
|
+
if (mainMatch && mainMatch[1].length > 400) {
|
|
81
|
+
content = mainMatch[1];
|
|
82
|
+
}
|
|
83
|
+
// 3. Preserve code blocks
|
|
84
|
+
const codeBlocks = [];
|
|
85
|
+
content = content.replace(/<pre[^>]*><code(?: class=["'](?:language-)?([a-zA-Z0-9_-]+)["'])?[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (match, lang, code) => {
|
|
86
|
+
const cleanCode = this.cleanEntities(code);
|
|
87
|
+
const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`;
|
|
88
|
+
codeBlocks.push(`\`\`\`${lang || ''}\n${cleanCode}\n\`\`\``);
|
|
89
|
+
return placeholder;
|
|
90
|
+
});
|
|
91
|
+
content = content.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (match, code) => {
|
|
92
|
+
const cleanCode = this.cleanEntities(code);
|
|
93
|
+
return ` \`${cleanCode.trim()}\` `;
|
|
94
|
+
});
|
|
95
|
+
// 4. Convert headings
|
|
96
|
+
content = content
|
|
97
|
+
.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n\n# $1\n\n')
|
|
98
|
+
.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n\n## $1\n\n')
|
|
99
|
+
.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n\n### $1\n\n')
|
|
100
|
+
.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '\n\n#### $1\n\n')
|
|
101
|
+
.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '\n\n##### $1\n\n')
|
|
102
|
+
.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '\n\n###### $1\n\n');
|
|
103
|
+
// 5. Convert lists
|
|
104
|
+
content = content
|
|
105
|
+
.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '\n• $1')
|
|
106
|
+
.replace(/<\/(ul|ol)>/gi, '\n\n');
|
|
107
|
+
// 6. Convert links
|
|
108
|
+
content = content.replace(/<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (match, href, text) => {
|
|
109
|
+
const cleanText = this.cleanText(text);
|
|
110
|
+
if (!cleanText || cleanText.length < 2 || href.startsWith('#') || href.startsWith('javascript:')) {
|
|
111
|
+
return cleanText;
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
const fullUrl = new URL(href, baseUrl).toString();
|
|
115
|
+
return `[${cleanText}](${fullUrl})`;
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return cleanText;
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
// 7. Convert text styles
|
|
122
|
+
content = content
|
|
123
|
+
.replace(/<(b|strong)[^>]*>([\s\S]*?)<\/\1>/gi, '**$2**')
|
|
124
|
+
.replace(/<(i|em)[^>]*>([\s\S]*?)<\/\1>/gi, '*$2*')
|
|
125
|
+
.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, '\n> $1\n')
|
|
126
|
+
.replace(/<hr[^>]*>/gi, '\n---\n')
|
|
127
|
+
.replace(/<br\s*[\/]?>/gi, '\n')
|
|
128
|
+
.replace(/<p[^>]*>/gi, '\n\n')
|
|
129
|
+
.replace(/<\/p>/gi, '\n\n');
|
|
130
|
+
// 8. Strip remaining HTML tags
|
|
131
|
+
content = content.replace(/<[^>]+>/g, ' ');
|
|
132
|
+
// 9. Restore code blocks
|
|
133
|
+
codeBlocks.forEach((code, idx) => {
|
|
134
|
+
content = content.replace(`__CODE_BLOCK_${idx}__`, `\n\n${code}\n\n`);
|
|
135
|
+
});
|
|
136
|
+
// 10. Normalize whitespace and clean entities
|
|
137
|
+
content = this.cleanEntities(content);
|
|
138
|
+
content = content
|
|
139
|
+
.replace(/[ \t]+/g, ' ')
|
|
140
|
+
.replace(/\n\s*\n\s*\n+/g, '\n\n')
|
|
141
|
+
.trim();
|
|
142
|
+
return content;
|
|
143
|
+
}
|
|
144
|
+
static extractLinks(html, currentUrl) {
|
|
145
|
+
const links = new Set();
|
|
146
|
+
const linkRegex = /<a\b[^>]*href=["']([^"']+)["'][^>]*>/gi;
|
|
147
|
+
let match;
|
|
148
|
+
const base = new URL(currentUrl);
|
|
149
|
+
while ((match = linkRegex.exec(html)) !== null) {
|
|
150
|
+
const href = match[1];
|
|
151
|
+
if (href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) {
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
const resolved = new URL(href, base);
|
|
156
|
+
// Only keep http/https links
|
|
157
|
+
if (resolved.protocol === 'http:' || resolved.protocol === 'https:') {
|
|
158
|
+
links.add(resolved.toString());
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch { }
|
|
162
|
+
}
|
|
163
|
+
return Array.from(links);
|
|
164
|
+
}
|
|
165
|
+
static cleanText(text) {
|
|
166
|
+
return this.cleanEntities(text.replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim());
|
|
167
|
+
}
|
|
168
|
+
static cleanEntities(text) {
|
|
169
|
+
return text
|
|
170
|
+
.replace(/ /g, ' ')
|
|
171
|
+
.replace(/&/g, '&')
|
|
172
|
+
.replace(/</g, '<')
|
|
173
|
+
.replace(/>/g, '>')
|
|
174
|
+
.replace(/"/g, '"')
|
|
175
|
+
.replace(/'/g, "'")
|
|
176
|
+
.replace(/—/g, '—')
|
|
177
|
+
.replace(/–/g, '–');
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=scraper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/core/scraper.ts"],"names":[],"mappings":"AAQA,MAAM,OAAO,UAAU;IACb,MAAM,CAAC,SAAS,GACtB,iHAAiH,CAAC;IAEpH;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,QAAQ,GAAG,KAAK;QACtD,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC3B,OAAO,EAAE;oBACP,YAAY,EAAE,IAAI,CAAC,SAAS;oBAC5B,MAAM,EAAE,iEAAiE;oBACzE,iBAAiB,EAAE,gBAAgB;iBACpC;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;aAClC,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpG,OAAO;oBACL,GAAG;oBACH,KAAK,EAAE,gBAAgB;oBACvB,QAAQ,EAAE,gCAAgC,WAAW,GAAG;oBACxD,KAAK,EAAE,EAAE;oBACT,SAAS,EAAE,CAAC;iBACb,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAEjC,qBAAqB;YACrB,MAAM,UAAU,GAAG,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpE,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC;YAE9E,0CAA0C;YAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAE9C,+CAA+C;YAC/C,IAAI,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAE5D,IAAI,aAAa,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBACpC,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,6BAA6B,CAAC;YACnF,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,KAAK;gBACL,QAAQ,EAAE,aAAa;gBACvB,KAAK;gBACL,SAAS,EAAE,aAAa,CAAC,MAAM;aAChC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,oBAAoB;gBAC3B,QAAQ,EAAE,oBAAoB,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE;gBACnD,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;aACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,cAAc,CAAC,IAAY,EAAE,OAAY;QACtD,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,iCAAiC;QACjC,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC;aAClE,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC;aAC/D,OAAO,CAAC,2DAA2D,EAAE,EAAE,CAAC;aACxE,OAAO,CAAC,4CAA4C,EAAE,EAAE,CAAC;aACzD,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC;aAClE,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC;aAClE,OAAO,CAAC,4CAA4C,EAAE,EAAE,CAAC;aACzD,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC;aAC/D,OAAO,CAAC,+CAA+C,EAAE,EAAE,CAAC;aAC5D,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAEnC,gDAAgD;QAChD,MAAM,SAAS,GAAG,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC;YAChD,wCAAwC,CAAC,IAAI,CAAC,OAAO,CAAC;YACtD,+HAA+H,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEhK,IAAI,SAAS,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC3C,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,0BAA0B;QAC1B,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oGAAoG,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;YACpJ,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,gBAAgB,UAAU,CAAC,MAAM,IAAI,CAAC;YAC1D,UAAU,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE,KAAK,SAAS,UAAU,CAAC,CAAC;YAC7D,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,iCAAiC,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,MAAM,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,sBAAsB;QACtB,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,6BAA6B,EAAE,cAAc,CAAC;aACtD,OAAO,CAAC,6BAA6B,EAAE,eAAe,CAAC;aACvD,OAAO,CAAC,6BAA6B,EAAE,gBAAgB,CAAC;aACxD,OAAO,CAAC,6BAA6B,EAAE,iBAAiB,CAAC;aACzD,OAAO,CAAC,6BAA6B,EAAE,kBAAkB,CAAC;aAC1D,OAAO,CAAC,6BAA6B,EAAE,mBAAmB,CAAC,CAAC;QAE/D,mBAAmB;QACnB,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,6BAA6B,EAAE,QAAQ,CAAC;aAChD,OAAO,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QAEpC,mBAAmB;QACnB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,uDAAuD,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;YACvG,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACjG,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAClD,OAAO,IAAI,SAAS,KAAK,OAAO,GAAG,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,yBAAyB;QACzB,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC;aACxD,OAAO,CAAC,iCAAiC,EAAE,MAAM,CAAC;aAClD,OAAO,CAAC,6CAA6C,EAAE,UAAU,CAAC;aAClE,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC;aACjC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;aAC/B,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC;aAC7B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE9B,+BAA+B;QAC/B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAE3C,yBAAyB;QACzB,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;YAC/B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,GAAG,IAAI,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,8CAA8C;QAC9C,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,GAAG,OAAO;aACd,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC;aACjC,IAAI,EAAE,CAAC;QAEV,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,IAAY,EAAE,UAAkB;QAC1D,MAAM,KAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,wCAAwC,CAAC;QAC3D,IAAI,KAA6B,CAAC;QAElC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;QAEjC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACpH,SAAS;YACX,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACrC,6BAA6B;gBAC7B,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBACpE,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAEO,MAAM,CAAC,SAAS,CAAC,IAAY;QACnC,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACtF,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,IAAY;QACvC,OAAO,IAAI;aACR,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAC9B,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export interface SearchResult {
|
|
2
|
+
title: string;
|
|
3
|
+
url: string;
|
|
4
|
+
snippet: string;
|
|
5
|
+
source: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class WebSearchEngine {
|
|
8
|
+
private static userAgent;
|
|
9
|
+
/**
|
|
10
|
+
* Autonomous Multi-Provider Web Search (Zero API Key required)
|
|
11
|
+
*/
|
|
12
|
+
static search(query: string, maxResults?: number): Promise<SearchResult[]>;
|
|
13
|
+
/**
|
|
14
|
+
* DuckDuckGo HTML Search Scraper (Free, No API Key)
|
|
15
|
+
*/
|
|
16
|
+
private static searchDuckDuckGo;
|
|
17
|
+
/**
|
|
18
|
+
* DuckDuckGo Lite Search (Fast fallback, No API Key)
|
|
19
|
+
*/
|
|
20
|
+
private static searchDuckDuckGoLite;
|
|
21
|
+
/**
|
|
22
|
+
* Public SearXNG Instances (JSON API, No Key)
|
|
23
|
+
*/
|
|
24
|
+
private static searchSearXNG;
|
|
25
|
+
private static stripHtml;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/core/search.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,SAAS,CAC4F;IAEpH;;OAEG;WACiB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAkClF;;OAEG;mBACkB,gBAAgB;IA4FrC;;OAEG;mBACkB,oBAAoB;IAoDzC;;OAEG;mBACkB,aAAa;IA+BlC,OAAO,CAAC,MAAM,CAAC,SAAS;CAYzB"}
|