atom-agent 0.3.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +83 -32
- package/dist/App.js +2178 -318
- package/dist/adapters.js +146 -15
- package/dist/agent/gates.js +153 -0
- package/dist/agent/loop-guard.js +184 -0
- package/dist/agent/loop.js +908 -0
- package/dist/agent/normalize.js +144 -0
- package/dist/agent/types.js +1 -0
- package/dist/auth.js +2 -1
- package/dist/cli.js +68 -6
- package/dist/compact.js +6 -48
- package/dist/config.js +171 -0
- package/dist/context-manager.js +564 -0
- package/dist/kilo.js +343 -0
- package/dist/local-discovery.js +308 -0
- package/dist/policy.js +286 -0
- package/dist/prompt-cache.js +99 -0
- package/dist/providers.js +183 -2
- package/dist/rollback.js +21 -0
- package/dist/scheduler.js +247 -0
- package/dist/session.js +35 -3
- package/dist/skills.js +214 -43
- package/dist/snapshots.js +57 -2
- package/dist/system.js +8 -1
- package/dist/telemetry-dashboard.js +589 -0
- package/dist/telemetry-server.js +301 -0
- package/dist/telemetry.js +1056 -0
- package/dist/tools/dir-cache.js +207 -0
- package/dist/tools/filesystem.js +149 -0
- package/dist/tools/fingerprints.js +33 -0
- package/dist/tools/overflow.js +76 -0
- package/dist/tools/read-cache.js +160 -0
- package/dist/tools/registry.js +802 -0
- package/dist/tools/search.js +242 -0
- package/dist/tools/shared.js +31 -0
- package/dist/tools/shell.js +273 -0
- package/dist/tools/todo.js +191 -0
- package/dist/tools/web.js +454 -0
- package/dist/tools.js +17 -1863
- package/dist/ui/activity.js +51 -0
- package/dist/ui/diff-panel.js +55 -0
- package/dist/ui/diff-view.js +112 -0
- package/dist/ui/diff.js +422 -0
- package/dist/ui/errors.js +129 -0
- package/dist/ui/highlight.js +120 -0
- package/dist/ui/input-model.js +115 -0
- package/dist/ui/input.js +40 -0
- package/dist/ui/live-tail.js +15 -0
- package/dist/ui/markdown.js +525 -0
- package/dist/ui/modals.js +47 -0
- package/dist/ui/palette.js +70 -0
- package/dist/ui/pickers.js +32 -0
- package/dist/ui/side-by-side.js +144 -0
- package/dist/ui/status-bar.js +75 -0
- package/dist/ui/theme.js +128 -0
- package/dist/ui/todo-panel.js +30 -0
- package/dist/ui/tool-inspector.js +59 -0
- package/dist/ui/transcript.js +128 -0
- package/dist/zen.js +145 -666
- package/package.json +1 -1
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// Session checklist state + todowrite/todo_get/todo_update. Module-global
|
|
2
|
+
// list (session-scoped, ephemeral); invariants refuse as invalid calls.
|
|
3
|
+
import { err, invalidCall } from "./shared.js";
|
|
4
|
+
export const TODO_STATUSES = new Set(["pending", "in_progress", "completed"]);
|
|
5
|
+
export const TODO_PRIORITIES = new Set(["high", "medium", "low"]);
|
|
6
|
+
// Session-scoped, ephemeral (resets with the process — same lifetime as
|
|
7
|
+
// read fingerprints and background tasks). todowrite replaces the whole
|
|
8
|
+
// list per call (Claude Code / opencode); todo_get reads it back;
|
|
9
|
+
// todo_update patches one item by index (check/uncheck without rewrite).
|
|
10
|
+
let todoItems = [];
|
|
11
|
+
// Copy for UI/tests (the executor's echoed rendering is the model path).
|
|
12
|
+
export function getTodos() {
|
|
13
|
+
return todoItems.map((t) => ({ ...t }));
|
|
14
|
+
}
|
|
15
|
+
// Reset for /new (a fresh conversation in the same process starts with a
|
|
16
|
+
// fresh checklist; /clear keeps it — the session continues).
|
|
17
|
+
export function clearTodos() {
|
|
18
|
+
todoItems = [];
|
|
19
|
+
}
|
|
20
|
+
function renderTodos(items) {
|
|
21
|
+
if (items.length === 0)
|
|
22
|
+
return "Todo list is empty.";
|
|
23
|
+
const mark = (s) => s === "completed" ? "✅" : s === "in_progress" ? "🔧" : "○";
|
|
24
|
+
return (`Todo list (${items.length}):\n` +
|
|
25
|
+
items
|
|
26
|
+
.map((t, i) => `${i + 1}. ${mark(t.status)} [${t.status}] ${t.content}${t.priority ? ` (${t.priority})` : ""}`)
|
|
27
|
+
.join("\n"));
|
|
28
|
+
}
|
|
29
|
+
// Replace the session checklist. Malformed items are model mistakes
|
|
30
|
+
// (`invalid call`, never runs); runtime failures keep plain `Error: ...`.
|
|
31
|
+
// Runtime invariants (harness-enforced, not just prompt discipline):
|
|
32
|
+
// - at most ONE item may be in_progress (flip the current one to completed
|
|
33
|
+
// or back to pending first — parallel "current work" is impossible state);
|
|
34
|
+
// - a completed item keeps its status across rewrites: only an explicit
|
|
35
|
+
// todo_update status patch reopens one (silent un-completion via a full
|
|
36
|
+
// rewrite is refused with a pointer to todo_update).
|
|
37
|
+
// An empty array clears; all-completed clears.
|
|
38
|
+
export async function todowriteTool(args) {
|
|
39
|
+
try {
|
|
40
|
+
const list = args?.todos;
|
|
41
|
+
if (!Array.isArray(list))
|
|
42
|
+
return err("todos must be an array");
|
|
43
|
+
const next = [];
|
|
44
|
+
for (let i = 0; i < list.length; i++) {
|
|
45
|
+
const item = list[i];
|
|
46
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) {
|
|
47
|
+
return invalidCall(`todo item ${i} must be an object. Expected {content: string, status: "pending" | "in_progress" | "completed", priority?: "high" | "medium" | "low", activeForm?: string}`);
|
|
48
|
+
}
|
|
49
|
+
if (typeof item["content"] !== "string" || item["content"].length === 0) {
|
|
50
|
+
return invalidCall(`todo item ${i} field "content" must be a non-empty string`);
|
|
51
|
+
}
|
|
52
|
+
if (typeof item["status"] !== "string" || !TODO_STATUSES.has(item["status"])) {
|
|
53
|
+
return invalidCall(`todo item ${i} field "status" must be one of "pending", "in_progress", "completed" (got ${JSON.stringify(item["status"])})`);
|
|
54
|
+
}
|
|
55
|
+
const clean = {
|
|
56
|
+
content: item["content"],
|
|
57
|
+
status: item["status"],
|
|
58
|
+
};
|
|
59
|
+
if (item["priority"] !== undefined) {
|
|
60
|
+
if (typeof item["priority"] !== "string" || !TODO_PRIORITIES.has(item["priority"])) {
|
|
61
|
+
return invalidCall(`todo item ${i} field "priority" must be one of "high", "medium", "low" (got ${JSON.stringify(item["priority"])})`);
|
|
62
|
+
}
|
|
63
|
+
clean.priority = item["priority"];
|
|
64
|
+
}
|
|
65
|
+
if (item["activeForm"] !== undefined) {
|
|
66
|
+
if (typeof item["activeForm"] !== "string") {
|
|
67
|
+
return invalidCall(`todo item ${i} field "activeForm" must be a string`);
|
|
68
|
+
}
|
|
69
|
+
if (item["activeForm"].length > 0)
|
|
70
|
+
clean.activeForm = item["activeForm"];
|
|
71
|
+
}
|
|
72
|
+
next.push(clean);
|
|
73
|
+
}
|
|
74
|
+
// Invariant: at most one in_progress (parallel current work is impossible
|
|
75
|
+
// state — complete or pause the other one first).
|
|
76
|
+
const active = next
|
|
77
|
+
.map((t, i) => ({ t, i }))
|
|
78
|
+
.filter(({ t }) => t.status === "in_progress");
|
|
79
|
+
if (active.length > 1) {
|
|
80
|
+
const names = active.map(({ t, i }) => `${i + 1}. ${t.content}`).join("; ");
|
|
81
|
+
return invalidCall(`only one task may be in_progress at a time (got ${active.length}: ${names}). ` +
|
|
82
|
+
`Mark the finished one completed (or paused back to pending) first`);
|
|
83
|
+
}
|
|
84
|
+
// Invariant: completed stays completed across rewrites — a full-list
|
|
85
|
+
// rewrite may not silently reopen a content-identical completed item.
|
|
86
|
+
// Reopening is an explicit act: todo_update that item's status.
|
|
87
|
+
for (const prev of todoItems) {
|
|
88
|
+
if (prev.status !== "completed")
|
|
89
|
+
continue;
|
|
90
|
+
const reopened = next.find((t) => t.content === prev.content && t.status !== "completed");
|
|
91
|
+
if (reopened) {
|
|
92
|
+
return invalidCall(`task "${prev.content}" is already completed — a rewrite cannot reopen it. ` +
|
|
93
|
+
`To reopen it explicitly, call todo_update on its index with a new status`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const prevCount = todoItems.length;
|
|
97
|
+
todoItems = next;
|
|
98
|
+
if (next.length === 0) {
|
|
99
|
+
return prevCount === 0 ? "Todo list is empty." : `Todo list cleared (${prevCount} item(s) removed).`;
|
|
100
|
+
}
|
|
101
|
+
if (next.every((t) => t.status === "completed")) {
|
|
102
|
+
todoItems = [];
|
|
103
|
+
return `All ${next.length} task(s) completed — todo list cleared.\n${renderTodos(next)}`;
|
|
104
|
+
}
|
|
105
|
+
return ("Todos have been modified successfully. Ensure that you continue to use the todo list to track your progress. Please proceed with the current tasks if applicable.\n" +
|
|
106
|
+
renderTodos(next));
|
|
107
|
+
}
|
|
108
|
+
catch (e) {
|
|
109
|
+
return err(e instanceof Error ? e.message : String(e));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
// Read the session checklist (pure read — the todowrite echo is the
|
|
113
|
+
// write path). Never throws; unknown fields are ignored by the schema.
|
|
114
|
+
export async function todoGetTool() {
|
|
115
|
+
return renderTodos(getTodos());
|
|
116
|
+
}
|
|
117
|
+
// Patch ONE item by 1-based index (the check/uncheck verb; Claude
|
|
118
|
+
// TaskUpdate equivalent for a single item). Out-of-range indexes are
|
|
119
|
+
// model mistakes (`invalid call` — the list changed, so re-read with
|
|
120
|
+
// todo_get). Completing the last open item clears the list, like
|
|
121
|
+
// todowrite. Never throws.
|
|
122
|
+
export async function todoUpdateTool(args) {
|
|
123
|
+
try {
|
|
124
|
+
const a = (args ?? {});
|
|
125
|
+
const rawIndex = a["index"];
|
|
126
|
+
if (typeof rawIndex !== "number" || !Number.isFinite(rawIndex) || Math.floor(rawIndex) !== rawIndex) {
|
|
127
|
+
return invalidCall(`field "index" for tool "todo_update" must be an integer (got ${JSON.stringify(rawIndex)})`);
|
|
128
|
+
}
|
|
129
|
+
if (rawIndex < 1 || rawIndex > todoItems.length) {
|
|
130
|
+
return invalidCall(`todo_update index ${rawIndex} out of range (list has ${todoItems.length} item(s); call todo_get to refresh)`);
|
|
131
|
+
}
|
|
132
|
+
const hasPatch = a["status"] !== undefined ||
|
|
133
|
+
a["content"] !== undefined ||
|
|
134
|
+
a["priority"] !== undefined ||
|
|
135
|
+
a["activeForm"] !== undefined;
|
|
136
|
+
if (!hasPatch) {
|
|
137
|
+
return invalidCall(`tool "todo_update" needs at least one of "status", "content", "priority", "activeForm" to change`);
|
|
138
|
+
}
|
|
139
|
+
const next = { ...todoItems[rawIndex - 1] };
|
|
140
|
+
if (a["status"] !== undefined) {
|
|
141
|
+
if (typeof a["status"] !== "string" || !TODO_STATUSES.has(a["status"])) {
|
|
142
|
+
return invalidCall(`field "status" for tool "todo_update" must be one of "pending", "in_progress", "completed" (got ${JSON.stringify(a["status"])})`);
|
|
143
|
+
}
|
|
144
|
+
// Invariant: at most one in_progress — starting this one while another
|
|
145
|
+
// runs is impossible state (complete or pause the other first).
|
|
146
|
+
// Reopening a completed item here is the explicit reset the
|
|
147
|
+
// todowrite rewrite guard points to, so it stays allowed.
|
|
148
|
+
if (a["status"] === "in_progress") {
|
|
149
|
+
const other = todoItems.findIndex((t, i) => i !== rawIndex - 1 && t.status === "in_progress");
|
|
150
|
+
if (other !== -1) {
|
|
151
|
+
return invalidCall(`only one task may be in_progress at a time (item ${other + 1} "${todoItems[other].content}" is already in_progress). Complete it or pause it back to pending first`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
next.status = a["status"];
|
|
155
|
+
}
|
|
156
|
+
if (a["content"] !== undefined) {
|
|
157
|
+
if (typeof a["content"] !== "string" || a["content"].length === 0) {
|
|
158
|
+
return invalidCall(`field "content" for tool "todo_update" must be a non-empty string`);
|
|
159
|
+
}
|
|
160
|
+
next.content = a["content"];
|
|
161
|
+
}
|
|
162
|
+
if (a["priority"] !== undefined) {
|
|
163
|
+
if (typeof a["priority"] !== "string" || !TODO_PRIORITIES.has(a["priority"])) {
|
|
164
|
+
return invalidCall(`field "priority" for tool "todo_update" must be one of "high", "medium", "low" (got ${JSON.stringify(a["priority"])})`);
|
|
165
|
+
}
|
|
166
|
+
next.priority = a["priority"];
|
|
167
|
+
}
|
|
168
|
+
if (a["activeForm"] !== undefined) {
|
|
169
|
+
if (typeof a["activeForm"] !== "string") {
|
|
170
|
+
return invalidCall(`field "activeForm" for tool "todo_update" must be a string`);
|
|
171
|
+
}
|
|
172
|
+
if (a["activeForm"].length > 0) {
|
|
173
|
+
next.activeForm = a["activeForm"];
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
delete next.activeForm;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
todoItems[rawIndex - 1] = next;
|
|
180
|
+
if (todoItems.length > 0 && todoItems.every((t) => t.status === "completed")) {
|
|
181
|
+
const snapshot = renderTodos(todoItems);
|
|
182
|
+
const done = todoItems.length;
|
|
183
|
+
todoItems = [];
|
|
184
|
+
return `All ${done} task(s) completed — todo list cleared.\n${snapshot}`;
|
|
185
|
+
}
|
|
186
|
+
return `Todo ${rawIndex} updated.\n${renderTodos(todoItems)}`;
|
|
187
|
+
}
|
|
188
|
+
catch (e) {
|
|
189
|
+
return err(e instanceof Error ? e.message : String(e));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
// Web executors: webfetch retrieval (SSRF-gated per URL and redirect hop)
|
|
2
|
+
// and websearch discovery. Node builtins + global fetch only.
|
|
3
|
+
import { promises as dns } from "node:dns";
|
|
4
|
+
import * as net from "node:net";
|
|
5
|
+
import { loadAtomConfig } from "../config.js";
|
|
6
|
+
import { classifyIp, defaultNetworkPolicy, isRedirectStatus, zoneAllows, zoneForAddresses, } from "../policy.js";
|
|
7
|
+
import { appendOverflow } from "./overflow.js";
|
|
8
|
+
import { err, READ_CHAR_CAP } from "./shared.js";
|
|
9
|
+
// ---- Web tools (webfetch retrieval / websearch discovery) ----
|
|
10
|
+
const WEB_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36";
|
|
11
|
+
const WEBFETCH_DOWNLOAD_CAP = 1024 * 1024; // ~1MB download cap
|
|
12
|
+
const WEBSEARCH_QUERY_CAP = 500;
|
|
13
|
+
const WEBSEARCH_TIMEOUT_MS = 30000;
|
|
14
|
+
// Decode common named entities plus decimal/hex numeric refs. Unknown
|
|
15
|
+
// entities are left as-is.
|
|
16
|
+
function decodeHtmlEntities(s) {
|
|
17
|
+
const numeric = s
|
|
18
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (m, hex) => {
|
|
19
|
+
const cp = parseInt(hex, 16);
|
|
20
|
+
return cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m;
|
|
21
|
+
})
|
|
22
|
+
.replace(/&#([0-9]+);/g, (m, dec) => {
|
|
23
|
+
const cp = parseInt(dec, 10);
|
|
24
|
+
return cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m;
|
|
25
|
+
});
|
|
26
|
+
return numeric
|
|
27
|
+
.replace(/&/g, "&")
|
|
28
|
+
.replace(/</g, "<")
|
|
29
|
+
.replace(/>/g, ">")
|
|
30
|
+
.replace(/"/g, '"')
|
|
31
|
+
.replace(/'|'/g, "'")
|
|
32
|
+
.replace(/ /g, " ");
|
|
33
|
+
}
|
|
34
|
+
// Minimal HTML -> text: drop comments and script/style/noscript/template
|
|
35
|
+
// blocks, map block tags to line breaks, strip remaining tags to spaces,
|
|
36
|
+
// decode entities, collapse whitespace. Paragraph breaks (~double newline)
|
|
37
|
+
// are preserved.
|
|
38
|
+
function htmlToText(html) {
|
|
39
|
+
let s = html.replace(/<!--[\s\S]*?-->/g, " ");
|
|
40
|
+
s = s.replace(/<(script|style|noscript|template)[\s>][\s\S]*?<\/\1\s*>/gi, " ");
|
|
41
|
+
s = s.replace(/<\/?(?:p|div|br|li|[ou]l|h[1-6]|tr|t[bdh]|table|section|article|header|footer|main|nav|aside|figure|figcaption|blockquote|pre|hr|dd|dt|dl)[^>]*>/gi, "\n");
|
|
42
|
+
s = s.replace(/<[^<>]*>/g, " ");
|
|
43
|
+
s = decodeHtmlEntities(s);
|
|
44
|
+
s = s.replace(/\r\n?/g, "\n");
|
|
45
|
+
s = s.replace(/[ \t\f\v ]+/g, " ");
|
|
46
|
+
s = s.replace(/ *\n */g, "\n");
|
|
47
|
+
s = s.replace(/\n{3,}/g, "\n\n");
|
|
48
|
+
return s.trim();
|
|
49
|
+
}
|
|
50
|
+
// Read a fetch Response body, aborting (cancelling the reader) once ~cap
|
|
51
|
+
// bytes are buffered. Falls back to res.text() when the body is not a
|
|
52
|
+
// stream (null-body responses, non-standard fetch mocks).
|
|
53
|
+
async function readBodyCapped(res, capBytes) {
|
|
54
|
+
const body = res.body;
|
|
55
|
+
if (body != null && typeof body.getReader === "function") {
|
|
56
|
+
const reader = body.getReader();
|
|
57
|
+
const chunks = [];
|
|
58
|
+
let total = 0;
|
|
59
|
+
let truncated = false;
|
|
60
|
+
try {
|
|
61
|
+
for (;;) {
|
|
62
|
+
const { done, value } = await reader.read();
|
|
63
|
+
if (done)
|
|
64
|
+
break;
|
|
65
|
+
if (!value || value.byteLength === 0)
|
|
66
|
+
continue;
|
|
67
|
+
if (total + value.byteLength > capBytes) {
|
|
68
|
+
const keep = capBytes - total;
|
|
69
|
+
if (keep > 0) {
|
|
70
|
+
chunks.push(value.slice(0, keep));
|
|
71
|
+
total += keep;
|
|
72
|
+
}
|
|
73
|
+
truncated = true;
|
|
74
|
+
try {
|
|
75
|
+
await reader.cancel?.();
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// ignore cancel errors
|
|
79
|
+
}
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
chunks.push(value);
|
|
83
|
+
total += value.byteLength;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
try {
|
|
88
|
+
reader.releaseLock?.();
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
// ignore
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const buf = Buffer.concat(chunks.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength)));
|
|
95
|
+
return { text: buf.toString("utf8"), truncated };
|
|
96
|
+
}
|
|
97
|
+
const text = await res.text();
|
|
98
|
+
if (text.length > capBytes)
|
|
99
|
+
return { text: text.slice(0, capBytes), truncated: true };
|
|
100
|
+
return { text, truncated: false };
|
|
101
|
+
}
|
|
102
|
+
const WEBFETCH_MAX_REDIRECTS = 5;
|
|
103
|
+
// Default hostname resolver: literal IPs classify directly (no DNS);
|
|
104
|
+
// "localhost" pins loopback without depending on resolver config; everything
|
|
105
|
+
// else resolves via the system resolver (all addresses — a hostname
|
|
106
|
+
// straddling zones is gated by its most sensitive one). Never throws: DNS
|
|
107
|
+
// failure yields [] (the caller blocks unresolvable hosts fail-closed).
|
|
108
|
+
async function defaultResolveHost(host) {
|
|
109
|
+
try {
|
|
110
|
+
if (host.toLowerCase() === "localhost")
|
|
111
|
+
return ["127.0.0.1"];
|
|
112
|
+
const records = await dns.lookup(host, { all: true });
|
|
113
|
+
return records.map((r) => r.address);
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return [];
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function loadNetworkPolicy() {
|
|
120
|
+
try {
|
|
121
|
+
const file = loadAtomConfig().config.network;
|
|
122
|
+
if (!file)
|
|
123
|
+
return defaultNetworkPolicy();
|
|
124
|
+
return { ...defaultNetworkPolicy(), ...file };
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return defaultNetworkPolicy();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// Gate one URL against the network policy (initial URL and EVERY redirect
|
|
131
|
+
// hop — redirects must never bypass it). Checks, in order: parseable,
|
|
132
|
+
// http(s) scheme, no embedded credentials, resolvable/classifiable host,
|
|
133
|
+
// zone allowed by policy. Known limitation (documented, not solved here):
|
|
134
|
+
// DNS rebinding between this check and fetch (TOCTOU) — mitigating that
|
|
135
|
+
// needs connection-level IP pinning, which global fetch does not offer.
|
|
136
|
+
export async function checkUrlAgainstPolicy(rawUrl, policy, resolve = defaultResolveHost) {
|
|
137
|
+
let parsed;
|
|
138
|
+
try {
|
|
139
|
+
parsed = new URL(rawUrl);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
return { ok: false, error: `Error: invalid URL: ${rawUrl}` };
|
|
143
|
+
}
|
|
144
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
145
|
+
return { ok: false, error: `Error: unsupported URL scheme (only http/https allowed): ${parsed.protocol}` };
|
|
146
|
+
}
|
|
147
|
+
// Credential-bearing URLs are an exfiltration shape (model-generated URLs
|
|
148
|
+
// should never carry userinfo); reject before any DNS or fetch.
|
|
149
|
+
if (parsed.username || parsed.password) {
|
|
150
|
+
return { ok: false, error: `Error: URLs with credentials are not allowed` };
|
|
151
|
+
}
|
|
152
|
+
let host = parsed.hostname.toLowerCase();
|
|
153
|
+
if (host.endsWith("."))
|
|
154
|
+
host = host.slice(0, -1);
|
|
155
|
+
if (!host)
|
|
156
|
+
return { ok: false, error: `Error: invalid URL: ${rawUrl}` };
|
|
157
|
+
// Bracketed IPv6 literals as carried by URL.hostname.
|
|
158
|
+
const bare = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
|
159
|
+
let zone;
|
|
160
|
+
if (net.isIP(bare) !== 0) {
|
|
161
|
+
const direct = classifyIp(bare);
|
|
162
|
+
zone = direct === "invalid" ? "blocked" : direct;
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
let addresses;
|
|
166
|
+
try {
|
|
167
|
+
addresses = await resolve(host);
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
addresses = [];
|
|
171
|
+
}
|
|
172
|
+
if (addresses.length === 0) {
|
|
173
|
+
return { ok: false, error: `Error: cannot resolve host: ${host}` };
|
|
174
|
+
}
|
|
175
|
+
zone = zoneForAddresses(addresses);
|
|
176
|
+
}
|
|
177
|
+
if (zone === "blocked" || !zoneAllows(zone, policy)) {
|
|
178
|
+
const key = zone === "public"
|
|
179
|
+
? "allowPublic"
|
|
180
|
+
: zone === "localhost"
|
|
181
|
+
? "allowLocalhost"
|
|
182
|
+
: zone === "private"
|
|
183
|
+
? "allowPrivate"
|
|
184
|
+
: "allowLinkLocal";
|
|
185
|
+
const reason = zone === "blocked"
|
|
186
|
+
? "unresolvable address"
|
|
187
|
+
: `network policy blocks ${zone} URLs (atom.json network.${key})`;
|
|
188
|
+
return { ok: false, error: `Error: ${reason}: ${rawUrl}` };
|
|
189
|
+
}
|
|
190
|
+
return { ok: true, zone, url: parsed.toString() };
|
|
191
|
+
}
|
|
192
|
+
// Fetch a page (retrieval). http:// is auto-upgraded to https:// (noted);
|
|
193
|
+
// only http/https schemes are allowed. Downloads are capped at ~1MB and
|
|
194
|
+
// output at ~64KB (both noted when truncated). markdown/text return page
|
|
195
|
+
// text (non-HTML content-types pass through as text); html returns the raw
|
|
196
|
+
// body. Network policy (SSRF): the initial URL and EVERY redirect hop are
|
|
197
|
+
// gated by zone (public/localhost/private/link-local/blocked) — redirects
|
|
198
|
+
// are followed manually (cap 5, loop-detected) so a public URL can never
|
|
199
|
+
// bounce to localhost/metadata unseen. Error strings, never throws.
|
|
200
|
+
export async function webfetchTool(args, opts) {
|
|
201
|
+
try {
|
|
202
|
+
const rawUrl = typeof args?.url === "string" ? args.url.trim() : "";
|
|
203
|
+
if (!rawUrl)
|
|
204
|
+
return err("url must be a non-empty string");
|
|
205
|
+
let parsed;
|
|
206
|
+
try {
|
|
207
|
+
parsed = new URL(rawUrl);
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
return err(`invalid URL: ${rawUrl}`);
|
|
211
|
+
}
|
|
212
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
213
|
+
return err(`unsupported URL scheme (only http/https allowed): ${parsed.protocol}`);
|
|
214
|
+
}
|
|
215
|
+
const format = args?.format ?? "markdown";
|
|
216
|
+
if (format !== "markdown" && format !== "text" && format !== "html") {
|
|
217
|
+
return err('format must be "markdown", "text", or "html"');
|
|
218
|
+
}
|
|
219
|
+
const t = args?.timeoutMs;
|
|
220
|
+
const timeoutMs = typeof t === "number" && Number.isFinite(t)
|
|
221
|
+
? Math.min(Math.max(Math.floor(t), 1), 120000)
|
|
222
|
+
: 30000;
|
|
223
|
+
let target = parsed.toString();
|
|
224
|
+
let upgraded = false;
|
|
225
|
+
if (parsed.protocol === "http:") {
|
|
226
|
+
parsed.protocol = "https:";
|
|
227
|
+
target = parsed.toString();
|
|
228
|
+
upgraded = true;
|
|
229
|
+
}
|
|
230
|
+
const policy = opts?.policy ?? loadNetworkPolicy();
|
|
231
|
+
const resolve = opts?.resolveHost ?? defaultResolveHost;
|
|
232
|
+
const first = await checkUrlAgainstPolicy(target, policy, resolve);
|
|
233
|
+
if (!first.ok)
|
|
234
|
+
return first.error;
|
|
235
|
+
target = first.url;
|
|
236
|
+
const visited = new Set([target]);
|
|
237
|
+
const ctrl = new AbortController();
|
|
238
|
+
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
239
|
+
let res;
|
|
240
|
+
let hops = 0;
|
|
241
|
+
try {
|
|
242
|
+
for (;;) {
|
|
243
|
+
res = await fetch(target, {
|
|
244
|
+
redirect: "manual",
|
|
245
|
+
signal: ctrl.signal,
|
|
246
|
+
headers: {
|
|
247
|
+
"User-Agent": WEB_UA,
|
|
248
|
+
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
249
|
+
},
|
|
250
|
+
});
|
|
251
|
+
const status = typeof res.status === "number" ? res.status : 0;
|
|
252
|
+
if (!isRedirectStatus(status))
|
|
253
|
+
break;
|
|
254
|
+
// Best-effort socket release before following (mocks may lack a body).
|
|
255
|
+
try {
|
|
256
|
+
await res.body?.cancel?.();
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// ignore — the next fetch proceeds regardless
|
|
260
|
+
}
|
|
261
|
+
const loc = res.headers?.get?.("location") ?? null;
|
|
262
|
+
if (!loc)
|
|
263
|
+
return err(`webfetch redirect without location: ${target}`);
|
|
264
|
+
let next;
|
|
265
|
+
try {
|
|
266
|
+
next = new URL(loc, target);
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
return err(`webfetch invalid redirect location from ${target}`);
|
|
270
|
+
}
|
|
271
|
+
if (next.protocol !== "http:" && next.protocol !== "https:") {
|
|
272
|
+
return err(`webfetch redirect to unsupported scheme (${next.protocol}): ${target}`);
|
|
273
|
+
}
|
|
274
|
+
if (visited.has(next.toString())) {
|
|
275
|
+
return err(`webfetch redirect loop detected: ${target}`);
|
|
276
|
+
}
|
|
277
|
+
visited.add(next.toString());
|
|
278
|
+
hops += 1;
|
|
279
|
+
if (hops > WEBFETCH_MAX_REDIRECTS) {
|
|
280
|
+
return err(`webfetch too many redirects (>${WEBFETCH_MAX_REDIRECTS}): ${target}`);
|
|
281
|
+
}
|
|
282
|
+
const hop = await checkUrlAgainstPolicy(next.toString(), policy, resolve);
|
|
283
|
+
if (!hop.ok)
|
|
284
|
+
return hop.error;
|
|
285
|
+
target = hop.url;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
catch (e) {
|
|
289
|
+
if (e instanceof Error && e.name === "AbortError") {
|
|
290
|
+
return err(`webfetch timed out after ${timeoutMs}ms: ${target}`);
|
|
291
|
+
}
|
|
292
|
+
return err(`webfetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
293
|
+
}
|
|
294
|
+
finally {
|
|
295
|
+
clearTimeout(timer);
|
|
296
|
+
}
|
|
297
|
+
if (!res.ok)
|
|
298
|
+
return err(`webfetch HTTP ${res.status} for ${target}`);
|
|
299
|
+
let body;
|
|
300
|
+
let downloadTruncated = false;
|
|
301
|
+
try {
|
|
302
|
+
const capped = await readBodyCapped(res, WEBFETCH_DOWNLOAD_CAP);
|
|
303
|
+
body = capped.text;
|
|
304
|
+
downloadTruncated = capped.truncated;
|
|
305
|
+
}
|
|
306
|
+
catch (e) {
|
|
307
|
+
return err(`webfetch failed reading response: ${e instanceof Error ? e.message : String(e)}`);
|
|
308
|
+
}
|
|
309
|
+
const contentType = res.headers?.get?.("content-type") ?? "";
|
|
310
|
+
const isHtml = contentType.trim() === "" || /html|xhtml/i.test(contentType);
|
|
311
|
+
const out = format === "html" || !isHtml ? body : htmlToText(body);
|
|
312
|
+
const prefix = upgraded ? "[note: upgraded http:// to https://]\n" : "";
|
|
313
|
+
const notes = [];
|
|
314
|
+
if (hops > 0)
|
|
315
|
+
notes.push(`[note: followed ${hops} redirect(s) to ${target}]`);
|
|
316
|
+
let text = out;
|
|
317
|
+
if (downloadTruncated)
|
|
318
|
+
notes.push("[truncated: download exceeded ~1MB]");
|
|
319
|
+
if (text.length > READ_CHAR_CAP) {
|
|
320
|
+
const full = text;
|
|
321
|
+
const head = full.slice(0, READ_CHAR_CAP);
|
|
322
|
+
text = head;
|
|
323
|
+
notes.push("[truncated: output exceeded 64KB]");
|
|
324
|
+
// Single spill: recover the pointer line from the composed tail.
|
|
325
|
+
const tailed = appendOverflow(head, "\n[truncated: output exceeded 64KB]", "converted page text", full);
|
|
326
|
+
const overflowLine = tailed.slice((head + "\n[truncated: output exceeded 64KB]\n").length);
|
|
327
|
+
if (overflowLine.startsWith("[overflow:"))
|
|
328
|
+
notes.push(overflowLine);
|
|
329
|
+
}
|
|
330
|
+
return prefix + text + (notes.length > 0 ? "\n" + notes.join("\n") : "");
|
|
331
|
+
}
|
|
332
|
+
catch (e) {
|
|
333
|
+
return err(e instanceof Error ? e.message : String(e));
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
// Unwrap a DuckDuckGo /l/ redirect (?uddg=<encoded target>) to the real
|
|
337
|
+
// target URL; pass direct http(s) hrefs through. Anything else is dropped.
|
|
338
|
+
function cleanDdgUrl(href) {
|
|
339
|
+
const h = decodeHtmlEntities(href.trim());
|
|
340
|
+
if (!h)
|
|
341
|
+
return "";
|
|
342
|
+
const abs = h.startsWith("//") ? `https:${h}` : h;
|
|
343
|
+
try {
|
|
344
|
+
const u = new URL(abs, "https://html.duckduckgo.com");
|
|
345
|
+
const uddg = u.searchParams.get("uddg");
|
|
346
|
+
if (uddg)
|
|
347
|
+
return uddg;
|
|
348
|
+
if (u.protocol === "http:" || u.protocol === "https:")
|
|
349
|
+
return u.toString();
|
|
350
|
+
return "";
|
|
351
|
+
}
|
|
352
|
+
catch {
|
|
353
|
+
return "";
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
function oneLine(s) {
|
|
357
|
+
return s.replace(/\s+/g, " ").trim();
|
|
358
|
+
}
|
|
359
|
+
// Parse DuckDuckGo HTML endpoint results with regex: split on result
|
|
360
|
+
// container divs, then take the first result__a anchor (title/url) and the
|
|
361
|
+
// result__snippet (a or div) per block. Blocks without a usable title/url
|
|
362
|
+
// are skipped.
|
|
363
|
+
export function parseDdgResults(html) {
|
|
364
|
+
const out = [];
|
|
365
|
+
try {
|
|
366
|
+
const chunks = html.split(/<div\b[^>]*\bclass="result[\s"']/i);
|
|
367
|
+
for (let i = 1; i < chunks.length; i++) {
|
|
368
|
+
const chunk = chunks[i];
|
|
369
|
+
let title = "";
|
|
370
|
+
let url = "";
|
|
371
|
+
const anchors = chunk.matchAll(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi);
|
|
372
|
+
for (const m of anchors) {
|
|
373
|
+
if (!/\bresult__a\b/.test(m[1]))
|
|
374
|
+
continue;
|
|
375
|
+
const href = /href\s*=\s*"([^"]*)"/i.exec(m[1])?.[1] ?? "";
|
|
376
|
+
url = cleanDdgUrl(href);
|
|
377
|
+
title = oneLine(htmlToText(m[2] ?? ""));
|
|
378
|
+
break;
|
|
379
|
+
}
|
|
380
|
+
if (!title || !url)
|
|
381
|
+
continue;
|
|
382
|
+
const snip = /result__snippet"[^>]*>([\s\S]*?)<\/(?:a|div)>/i.exec(chunk);
|
|
383
|
+
const snippet = snip ? oneLine(htmlToText(snip[1] ?? "")) : "";
|
|
384
|
+
out.push({ title, url, snippet });
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
catch {
|
|
388
|
+
return out;
|
|
389
|
+
}
|
|
390
|
+
return out;
|
|
391
|
+
}
|
|
392
|
+
// Search the web (discovery) via the keyless DuckDuckGo HTML endpoint —
|
|
393
|
+
// best-effort: DDG bot protection may answer 403, surfaced as an error
|
|
394
|
+
// string. Returns numbered "title — url" + snippet blocks, or "No results.".
|
|
395
|
+
// Error strings, never throws.
|
|
396
|
+
export async function websearchTool(args) {
|
|
397
|
+
try {
|
|
398
|
+
const raw = typeof args?.query === "string" ? args.query.trim() : "";
|
|
399
|
+
if (!raw)
|
|
400
|
+
return err("query must be a non-empty string");
|
|
401
|
+
// Client-side domain scoping (no server-side filters on this backend):
|
|
402
|
+
// `site: "example.com"` appends a `site:` operator to the query.
|
|
403
|
+
const site = typeof args?.site === "string" ? args.site.trim() : "";
|
|
404
|
+
const scoped = site ? `${raw} site:${site}` : raw;
|
|
405
|
+
const query = scoped.length > WEBSEARCH_QUERY_CAP ? scoped.slice(0, WEBSEARCH_QUERY_CAP) : scoped;
|
|
406
|
+
const n = args?.numResults;
|
|
407
|
+
const numResults = typeof n === "number" && Number.isFinite(n)
|
|
408
|
+
? Math.min(Math.max(Math.floor(n), 1), 20)
|
|
409
|
+
: 8;
|
|
410
|
+
const endpoint = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
411
|
+
const ctrl = new AbortController();
|
|
412
|
+
const timer = setTimeout(() => ctrl.abort(), WEBSEARCH_TIMEOUT_MS);
|
|
413
|
+
let res;
|
|
414
|
+
try {
|
|
415
|
+
res = await fetch(endpoint, {
|
|
416
|
+
signal: ctrl.signal,
|
|
417
|
+
headers: { "User-Agent": WEB_UA, Accept: "text/html,*/*;q=0.8" },
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
catch (e) {
|
|
421
|
+
if (e instanceof Error && e.name === "AbortError") {
|
|
422
|
+
return err(`websearch timed out after ${WEBSEARCH_TIMEOUT_MS}ms`);
|
|
423
|
+
}
|
|
424
|
+
return err(`websearch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
425
|
+
}
|
|
426
|
+
finally {
|
|
427
|
+
clearTimeout(timer);
|
|
428
|
+
}
|
|
429
|
+
if (res.status === 403) {
|
|
430
|
+
return err("websearch blocked by DuckDuckGo bot protection (HTTP 403; best-effort search — retry later)");
|
|
431
|
+
}
|
|
432
|
+
if (!res.ok)
|
|
433
|
+
return err(`websearch HTTP ${res.status}`);
|
|
434
|
+
let html;
|
|
435
|
+
try {
|
|
436
|
+
html = await res.text();
|
|
437
|
+
}
|
|
438
|
+
catch (e) {
|
|
439
|
+
return err(`websearch failed reading response: ${e instanceof Error ? e.message : String(e)}`);
|
|
440
|
+
}
|
|
441
|
+
const results = parseDdgResults(html).slice(0, numResults);
|
|
442
|
+
if (results.length === 0)
|
|
443
|
+
return "No results.";
|
|
444
|
+
return results
|
|
445
|
+
.map((r, i) => {
|
|
446
|
+
const head = `${i + 1}. ${r.title} — ${r.url}`;
|
|
447
|
+
return r.snippet ? `${head}\n ${r.snippet}` : head;
|
|
448
|
+
})
|
|
449
|
+
.join("\n");
|
|
450
|
+
}
|
|
451
|
+
catch (e) {
|
|
452
|
+
return err(e instanceof Error ? e.message : String(e));
|
|
453
|
+
}
|
|
454
|
+
}
|