@ikarmic/forge-core 5.2.0 → 5.4.0

package/dist/tools/background-process.js

@@ -0,0 +1,150 @@
+/**
+ * `bg_start`, `bg_read`, `bg_stop`: commands that keep running while the model works (a dev server, a watcher).
+ *
+ * The run owns a `BackgroundProcesses` (on `ToolContext.processes`) and stops everything in it when the run ends,
+ * so nothing outlives the chat that started it. A child agent is never given these tools (sub-agent.ts).
+ * Starting needs approval like `run_command` and goes through the same checks (`prepareCommand`); reading and
+ * stopping only touch processes this run started. Output is kept in a bounded buffer and redacted when read.
+ */
+import { spawn } from 'node:child_process';
+import { z } from 'zod';
+import { redact } from './redact.js';
+import { killTree, prepareCommand } from './run-command.js';
+import { shellEnv } from './shell-env.js';
+import { runnerShell, shellInvocation } from './shell.js';
+const MAX_LIVE = 3;
+const BUFFER_CHARS = 200_000;
+/** The most one read returns; the cursor moves past anything older. */
+const READ_CHARS = 20_000;
+export class BackgroundProcesses {
+    entries = new Map();
+    nextId = 1;
+    maxBufferChars;
+    constructor(options = {}) {
+        this.maxBufferChars = options.maxBufferChars ?? BUFFER_CHARS;
+    }
+    /** How many are running now. */
+    get liveCount() {
+        return [...this.entries.values()].filter((e) => e.running).length;
+    }
+    start(command, cwd, ctx, cwdOverride) {
+        if (this.liveCount >= MAX_LIVE) {
+            throw new Error(`${MAX_LIVE} background processes are already running. Stop one with bg_stop before starting another.`);
+        }
+        const prepared = prepareCommand(command, cwdOverride, { ...ctx, cwd });
+        const inv = shellInvocation(runnerShell(ctx.shell), prepared.commandLine);
+        const options = {
+            cwd: prepared.cwd,
+            // Windows: never run a same-named program the repository ships in its root (see shell-env.ts).
+            env: shellEnv(process.env),
+            stdio: ['ignore', 'pipe', 'pipe'],
+            windowsHide: true,
+            // POSIX: own process group, so a stop reaches everything the shell started (see killTree).
+            detached: process.platform !== 'win32',
+        };
+        const child = inv.file ? spawn(inv.file, inv.args, options) : spawn(prepared.commandLine, { ...options, shell: true });
+        const entry = { id: `bg${this.nextId++}`, command, child, running: true, exitCode: null, buffer: '', total: 0 };
+        this.entries.set(entry.id, entry);
+        const take = (chunk) => {
+            entry.total += chunk.length;
+            entry.buffer += chunk;
+            if (entry.buffer.length > this.maxBufferChars)
+                entry.buffer = entry.buffer.slice(entry.buffer.length - this.maxBufferChars);
+        };
+        child.stdout.setEncoding('utf8').on('data', take);
+        child.stderr.setEncoding('utf8').on('data', take);
+        child.on('error', (err) => {
+            take(`[could not start: ${err.message}]\n`);
+            entry.running = false;
+        });
+        // Same as run_command: a grandchild that escaped the tree kill must not keep this entry "running".
+        child.on('exit', (code) => {
+            entry.running = false;
+            entry.exitCode = code;
+            setTimeout(() => {
+                child.stdout.destroy();
+                child.stderr.destroy();
+            }, 500).unref();
+        });
+        child.on('close', (code) => {
+            entry.running = false;
+            entry.exitCode = code;
+        });
+        return { id: entry.id, pid: child.pid };
+    }
+    get(id) {
+        const entry = this.entries.get(id);
+        if (!entry)
+            throw new Error(`No background process with id "${id}". Ids come from bg_start.`);
+        return entry;
+    }
+    read(id, since = 0) {
+        const e = this.get(id);
+        const firstKept = e.total - e.buffer.length;
+        const from = Math.max(since, firstKept, e.total - READ_CHARS);
+        const raw = e.buffer.slice(from - firstKept);
+        return {
+            id: e.id,
+            command: e.command,
+            running: e.running,
+            exitCode: e.exitCode,
+            output: redact(raw).text,
+            nextSince: e.total,
+            droppedChars: Math.max(0, from - since),
+        };
+    }
+    stop(id) {
+        const e = this.get(id);
+        const wasRunning = e.running;
+        if (wasRunning)
+            killTree(e.child);
+        return { id: e.id, wasRunning };
+    }
+    /** Stop everything still running; returns what was running. */
+    stopAll() {
+        const live = [...this.entries.values()].filter((e) => e.running);
+        for (const e of live)
+            killTree(e.child);
+        return live.map((e) => ({ id: e.id, command: e.command }));
+    }
+}
+function manager(ctx) {
+    if (!ctx.processes)
+        throw new Error('Background processes are not available in this run. Use run_command instead.');
+    return ctx.processes;
+}
+const startParams = z.object({
+    command: z.string().describe('The shell command to keep running, for example "npm run dev".'),
+    cwd: z.string().optional().describe('Working directory. Defaults to the project root.'),
+});
+export const bgStartTool = {
+    name: 'bg_start',
+    description: 'Start a command that keeps running (a dev server, a file watcher) and return at once with an id. Read its output with bg_read and end it with bg_stop. ' +
+        `At most ${MAX_LIVE} at a time. Everything still running is stopped when this run ends, so it is for use within one task. Needs approval like run_command.`,
+    parameters: startParams,
+    destructive: true,
+    execute: async (args, ctx) => {
+        const { id, pid } = manager(ctx).start(args.command, ctx.cwd, ctx, args.cwd);
+        return { id, pid, note: `Started ${id}. Use bg_read with this id to see its output, and bg_stop when you are done.` };
+    },
+};
+const readParams = z.object({
+    id: z.string().describe('The id returned by bg_start.'),
+    since: z.number().int().nonnegative().optional().describe('The nextSince value of your last read, to get only new output.'),
+});
+export const bgReadTool = {
+    name: 'bg_read',
+    description: 'Read the output of a background process started with bg_start, and whether it is still running. Pass nextSince from the last read as since to get only new output.',
+    parameters: readParams,
+    execute: async (args, ctx) => {
+        const r = manager(ctx).read(args.id, args.since ?? 0);
+        return r.droppedChars > 0 ? { ...r, note: `${r.droppedChars} characters of older output are not shown.` } : r;
+    },
+};
+const stopParams = z.object({ id: z.string().describe('The id returned by bg_start.') });
+export const bgStopTool = {
+    name: 'bg_stop',
+    description: 'Stop a background process started with bg_start, together with everything it started.',
+    parameters: stopParams,
+    execute: async (args, ctx) => manager(ctx).stop(args.id),
+};

package/dist/tools/command-sandbox.d.ts

@@ -0,0 +1,24 @@
+import type { CommandSandbox } from '../types.js';
+export type SandboxedCommand = {
+    kind: 'wrapped';
+    command: string;
+} | {
+    kind: 'unavailable';
+    command: string;
+} | {
+    kind: 'refused';
+    reason: string;
+};
+/** Quote one argument for /bin/sh. */
+export declare function shellQuote(arg: string): string;
+interface Options {
+    cwd: string;
+    sandbox: CommandSandbox;
+    platform: NodeJS.Platform;
+    tmpDir: string;
+    findBinary?: (name: string) => string | undefined;
+    realpath?: (p: string) => string;
+}
+/** The command line that runs `command` inside the sandbox, or why it cannot be sandboxed. */
+export declare function sandboxCommand(command: string, opts: Options): SandboxedCommand;
+export {};

package/dist/tools/command-sandbox.js

@@ -0,0 +1,73 @@
+/**
+ * OS sandbox for run_command (after Kilo's kilo-sandbox): bubblewrap on Linux, sandbox-exec (Seatbelt) on macOS.
+ * Writes are confined to the workspace, the temp folder and any extra folders the user named; the workspace's .git
+ * stays read-only, since a hook written there would later run outside the sandbox; the network is cut unless
+ * allowed. Windows has no comparable tool, so there the command runs as before (approval still applies).
+ */
+import { existsSync, realpathSync } from 'node:fs';
+import path from 'node:path';
+import { resolveOnPath } from '../lsp/client.js';
+/** Quote one argument for /bin/sh. */
+export function shellQuote(arg) {
+    return `'${arg.replace(/'/g, `'\\''`)}'`;
+}
+function defaultFind(name) {
+    for (const dir of ['/usr/bin', '/bin', '/usr/local/bin']) {
+        const full = path.posix.join(dir, name);
+        if (existsSync(full))
+            return full;
+    }
+    return resolveOnPath(name, process.env, process.platform);
+}
+function safeRealpath(p) {
+    try {
+        return realpathSync(p);
+    }
+    catch {
+        return p;
+    }
+}
+/** The command line that runs `command` inside the sandbox, or why it cannot be sandboxed. */
+export function sandboxCommand(command, opts) {
+    const find = opts.findBinary ?? defaultFind;
+    const real = opts.realpath ?? safeRealpath;
+    if (opts.platform === 'linux') {
+        const bwrap = find('bwrap');
+        if (!bwrap)
+            return { kind: 'refused', reason: 'The command sandbox is on, but bubblewrap (bwrap) is not installed. Install it (apt install bubblewrap) or turn the sandbox off.' };
+        const writable = [opts.cwd, opts.tmpDir, ...(opts.sandbox.extraWritable ?? [])];
+        const gitDir = path.join(opts.cwd, '.git');
+        const args = [
+            bwrap,
+            '--die-with-parent',
+            '--new-session',
+            '--unshare-pid',
+            ...(opts.sandbox.network ? [] : ['--unshare-net']),
+            '--ro-bind', '/', '/',
+            '--dev', '/dev',
+            '--proc', '/proc',
+            ...writable.flatMap((p) => ['--bind', p, p]),
+            ...(existsSync(gitDir) ? ['--ro-bind', gitDir, gitDir] : []),
+            '--chdir', opts.cwd,
+            '--', '/bin/sh', '-c', command,
+        ];
+        return { kind: 'wrapped', command: args.map(shellQuote).join(' ') };
+    }
+    if (opts.platform === 'darwin') {
+        const exec = find('sandbox-exec');
+        if (!exec)
+            return { kind: 'refused', reason: 'The command sandbox is on, but sandbox-exec was not found. Turn the sandbox off to run commands.' };
+        const quote = (p) => JSON.stringify(real(p));
+        const writable = [opts.cwd, opts.tmpDir, ...(opts.sandbox.extraWritable ?? [])].map((p) => `(subpath ${quote(p)})`);
+        const profile = [
+            '(version 1)',
+            '(allow default)',
+            '(deny file-write*)',
+            `(allow file-write* ${writable.join(' ')} (subpath "/private/var/folders") (literal "/dev/null") (literal "/dev/stdout") (literal "/dev/stderr") (regex #"^/dev/tty"))`,
+            `(deny file-write* (subpath ${quote(path.posix.join(opts.cwd, '.git'))}))`,
+            ...(opts.sandbox.network ? [] : ['(deny network* (remote ip))']),
+        ].join('\n');
+        return { kind: 'wrapped', command: [exec, '-p', profile, '/bin/sh', '-c', command].map(shellQuote).join(' ') };
+    }
+    return { kind: 'unavailable', command };
+}

package/dist/tools/edit-file.d.ts

@@ -12,4 +12,12 @@ export interface EditFileResult {
     bytes: number;
 }
 export declare const editFileTool: ToolDefinition<z.infer<typeof params>, EditFileResult>;
+/**
+ * What edit_file writes for `original`; throws the message the model sees when old_text cannot be placed.
+ * Shared with the chat's diff preview so the two never disagree.
+ */
+export declare function applyTextEdit(original: string, oldTextIn: string, newTextIn: string, replaceAll: boolean, file: string): {
+    content: string;
+    replacements: number;
+};
 export {};

package/dist/tools/edit-file.js

@@ -1,5 +1,6 @@
 import { readFile, writeFile } from 'node:fs/promises';
 import { z } from 'zod';
+import { fallbackMatch } from './edit-match.js';
 import { resolveAndCheckPath } from './shared.js';
 import { decodeText, encodeText } from './text-encoding.js';
 const params = z.object({
@@ -24,29 +25,51 @@ export const editFileTool = {
                 throw new Error(`File not found: ${args.path}`);
             throw err;
         });
-        const occurrences = countOccurrences(original, args.old_text);
-        if (occurrences === 0) {
-            throw new Error(`old_text not found in ${args.path}. Re-read the file (it may have changed) and try again.`);
-        }
-        const replaceAll = args.replace_all === true;
-        if (!replaceAll && occurrences > 1) {
-            throw new Error(`old_text matches ${occurrences} times in ${args.path}. Pass more context to make it unique, or set replace_all=true to replace all occurrences.`);
-        }
-        const updated = replaceAll
-            ? original.split(args.old_text).join(args.new_text)
-            // Use a replacer FUNCTION so new_text is inserted literally: a plain
-            // string 2nd arg to String.prototype.replace still honors special
-            // patterns like $&, $1, $$ even though old_text isn't a regex.
-            : original.replace(args.old_text, () => args.new_text);
-        const bytes = encodeText(updated, encoding);
+        const { content, replacements } = applyTextEdit(original, args.old_text, args.new_text, args.replace_all === true, args.path);
+        const bytes = encodeText(content, encoding);
         await writeFile(resolved, bytes);
         return {
             path: resolved,
-            replacements: replaceAll ? occurrences : 1,
+            replacements,
             bytes: bytes.length,
         };
     },
 };
+/**
+ * What edit_file writes for `original`; throws the message the model sees when old_text cannot be placed.
+ * Shared with the chat's diff preview so the two never disagree.
+ */
+export function applyTextEdit(original, oldTextIn, newTextIn, replaceAll, file) {
+    // read_file shows a CRLF file without its \r, so the model sends LF text: match on LF, write back the file's own ending.
+    const crlf = original.includes('\r\n');
+    const content = crlf ? toLf(original) : original;
+    const oldText = toLf(oldTextIn);
+    const newText = toLf(newTextIn);
+    const occurrences = countOccurrences(content, oldText);
+    let updated;
+    if (occurrences === 0) {
+        const span = replaceAll ? undefined : fallbackMatch(content, oldText, file);
+        if (!span) {
+            throw new Error(`old_text not found in ${file}. Re-read the file (it may have changed) and try again.`);
+        }
+        updated = content.slice(0, span.start) + newText + content.slice(span.end);
+    }
+    else {
+        if (!replaceAll && occurrences > 1) {
+            throw new Error(`old_text matches ${occurrences} times in ${file}. Pass more context to make it unique, or set replace_all=true to replace all occurrences.`);
+        }
+        updated = replaceAll
+            ? content.split(oldText).join(newText)
+            // Use a replacer FUNCTION so new_text is inserted literally: a plain
+            // string 2nd arg to String.prototype.replace still honors special
+            // patterns like $&, $1, $$ even though old_text isn't a regex.
+            : content.replace(oldText, () => newText);
+    }
+    return { content: crlf ? updated.replace(/\n/g, '\r\n') : updated, replacements: replaceAll ? occurrences : 1 };
+}
+function toLf(text) {
+    return text.replace(/\r\n/g, '\n');
+}
 function countOccurrences(haystack, needle) {
     if (needle.length === 0)
         return 0;

package/dist/tools/edit-match.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Ways to place a model's `old_text` in a file when the exact text is absent. A model often gets indentation, inner
+ * spacing, escapes or a middle line slightly wrong. Kilo tries several of these in a chain; so do we (G15).
+ *
+ * Each finder returns every place that fits. Exactly one place is used; more than one is an error, so a loose match
+ * never chooses between two candidates, and a block whose middle is nothing like old_text is never replaced.
+ * All text here is LF: `applyTextEdit` normalises line endings before calling in.
+ */
+export type Span = {
+    start: number;
+    end: number;
+};
+/** 1 for equal strings, towards 0 as they differ (edit distance over the longer length). */
+export declare function similarity(a: string, b: string): number;
+/** The one place old_text fits, or undefined when none does. Throws when a finder finds two. */
+export declare function fallbackMatch(content: string, oldText: string, file: string): Span | undefined;

package/dist/tools/edit-match.js

@@ -0,0 +1,115 @@
+/**
+ * Ways to place a model's `old_text` in a file when the exact text is absent. A model often gets indentation, inner
+ * spacing, escapes or a middle line slightly wrong. Kilo tries several of these in a chain; so do we (G15).
+ *
+ * Each finder returns every place that fits. Exactly one place is used; more than one is an error, so a loose match
+ * never chooses between two candidates, and a block whose middle is nothing like old_text is never replaced.
+ * All text here is LF: `applyTextEdit` normalises line endings before calling in.
+ */
+const NEWLINE = '\n';
+/** Offset of the start of line `line` (0-based) in text split on newlines. */
+function lineOffset(lines, line) {
+    let n = 0;
+    for (let i = 0; i < line; i++)
+        n += lines[i].length + 1;
+    return n;
+}
+/** old_text's lines without a trailing empty one. */
+function wantedLines(oldText) {
+    const lines = oldText.split(NEWLINE);
+    if (lines.length > 1 && lines[lines.length - 1] === '')
+        lines.pop();
+    return lines;
+}
+/** The block of whole lines whose trimmed text equals old_text's trimmed lines (wrong indentation, trailing spaces). */
+const trimmedLines = (content, oldText) => {
+    const want = wantedLines(oldText).map((l) => l.trim());
+    if (want.every((l) => l === ''))
+        return [];
+    const lines = content.split(NEWLINE);
+    const spans = [];
+    for (let i = 0; i + want.length <= lines.length; i++) {
+        if (!want.every((w, j) => lines[i + j].trim() === w))
+            continue;
+        const last = i + want.length - 1;
+        spans.push({ start: lineOffset(lines, i), end: lineOffset(lines, last) + lines[last].length });
+    }
+    return spans;
+};
+const REGEX_SPECIAL = /[.*+?^${}()|[\]\\]/g;
+/** The same words in the same order, whatever the spaces and tabs between them. */
+const collapsedSpace = (content, oldText) => {
+    const words = oldText.split(/\s+/).filter(Boolean);
+    if (words.length < 2)
+        return [];
+    const re = new RegExp(words.map((w) => w.replace(REGEX_SPECIAL, (c) => `\\${c}`)).join('\\s+'), 'g');
+    return [...content.matchAll(re)].map((m) => ({ start: m.index, end: m.index + m[0].length }));
+};
+const ESCAPES = { n: '\n', t: '\t', r: '\r' };
+/** old_text written with escape sequences (a backslash and an n) where the file has the real character. */
+const unescaped = (content, oldText) => {
+    const plain = oldText.replace(/\\(n|t|r|"|'|`|\\)/g, (_m, c) => ESCAPES[c] ?? c);
+    if (plain === oldText)
+        return [];
+    const spans = [];
+    for (let i = content.indexOf(plain); i !== -1; i = content.indexOf(plain, i + plain.length))
+        spans.push({ start: i, end: i + plain.length });
+    return spans;
+};
+const ANCHOR_MIN_LINES = 3;
+const ANCHOR_MAX_CHARS = 4000;
+const ANCHOR_MIN_SIMILARITY = 0.6;
+/** A block of the same number of lines whose first and last lines match and whose middle is close to old_text's. */
+const blockAnchor = (content, oldText) => {
+    const want = wantedLines(oldText);
+    if (want.length < ANCHOR_MIN_LINES)
+        return [];
+    const first = want[0].trim();
+    const last = want[want.length - 1].trim();
+    if (!first || !last)
+        return [];
+    const middle = want.slice(1, -1).join('').replace(/\s+/g, '');
+    if (middle.length > ANCHOR_MAX_CHARS)
+        return [];
+    const lines = content.split(NEWLINE);
+    const spans = [];
+    for (let i = 0; i + want.length <= lines.length; i++) {
+        const end = i + want.length - 1;
+        if (lines[i].trim() !== first || lines[end].trim() !== last)
+            continue;
+        const have = lines.slice(i + 1, end).join('').replace(/\s+/g, '');
+        if (similarity(middle, have) >= ANCHOR_MIN_SIMILARITY)
+            spans.push({ start: lineOffset(lines, i), end: lineOffset(lines, end) + lines[end].length });
+    }
+    return spans;
+};
+/** 1 for equal strings, towards 0 as they differ (edit distance over the longer length). */
+export function similarity(a, b) {
+    if (a === b)
+        return 1;
+    const longest = Math.max(a.length, b.length);
+    if (longest === 0)
+        return 1;
+    let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
+    for (let i = 1; i <= a.length; i++) {
+        const row = [i];
+        for (let j = 1; j <= b.length; j++)
+            row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
+        prev = row;
+    }
+    return 1 - prev[b.length] / longest;
+}
+/** In order of how little each assumes. */
+const FINDERS = [trimmedLines, collapsedSpace, unescaped, blockAnchor];
+/** The one place old_text fits, or undefined when none does. Throws when a finder finds two. */
+export function fallbackMatch(content, oldText, file) {
+    for (const find of FINDERS) {
+        const spans = find(content, oldText);
+        if (spans.length === 1)
+            return spans[0];
+        if (spans.length > 1) {
+            throw new Error(`old_text matches more than one place in ${file} once spacing is ignored. Pass more context to make it unique.`);
+        }
+    }
+    return undefined;
+}

package/dist/tools/fetch-url.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import type { ToolDefinition } from '../types.js';
+import type { ToolContext, ToolDefinition } from '../types.js';
 declare const params: z.ZodObject<{
     url: z.ZodString;
     max_bytes: z.ZodOptional<z.ZodNumber>;
@@ -27,5 +27,19 @@ export interface FetchUrlResult {
  * loopback services or cloud metadata endpoints (e.g. 169.254.169.254).
  */
 export declare function isBlockedAddress(ip: string): boolean;
+/**
+ * Reject requests targeting loopback / link-local / private addresses:
+ * this closes an SSRF gap that otherwise exists with zero configuration
+ * (a bare `fetch_url({url: "http://169.254.169.254/..."})` could reach a
+ * cloud metadata endpoint). Applies by default, even with no ToolPolicy
+ * configured; a `policy.allowedUrls` match is treated as the caller's
+ * explicit opt-in and bypasses this check (e.g. a test or internal tool
+ * that deliberately targets a local service).
+ *
+ * Resolves the hostname via DNS (unless it's already an IP literal) so a
+ * public-looking hostname that resolves to a private address is caught
+ * too, not just literal IPs.
+ */
+export declare function assertUrlNotBlocked(rawUrl: string, policy: ToolContext['policy']): Promise<void>;
 export declare const fetchUrlTool: ToolDefinition<z.infer<typeof params>, FetchUrlResult>;
 export {};

package/dist/tools/fetch-url.js

@@ -2,6 +2,7 @@ import { lookup } from 'node:dns/promises';
 import net from 'node:net';
 import { z } from 'zod';
 import { installedFetch } from '../policy/proxy.js';
+import { decodeEntities, htmlToMarkdown } from './html-to-markdown.js';
 import { checkPolicyViolation } from './policy.js';
 const params = z.object({
     url: z
@@ -34,7 +35,8 @@ const params = z.object({
         .optional()
         .describe('Follow 3xx redirects. Default true.'),
 });
-const DEFAULT_MAX_BYTES = 200_000;
+// A page is read once and rarely needs more; the model can ask for a larger max_bytes. Matches read_file's 50 KB (G6).
+const DEFAULT_MAX_BYTES = 50_000;
 const DEFAULT_TIMEOUT_MS = 10_000;
 const EXCERPT_MAX_CHARS = 8_000;
 const USER_AGENT = 'ikarmic-forge/0.1 (fetch_url)';
@@ -104,7 +106,7 @@ export function isBlockedAddress(ip) {
  * public-looking hostname that resolves to a private address is caught
  * too, not just literal IPs.
  */
-async function assertUrlNotBlocked(rawUrl, policy) {
+export async function assertUrlNotBlocked(rawUrl, policy) {
     if (policy?.allowedUrls?.some((re) => re.test(rawUrl)))
         return;
     let hostname;
@@ -137,7 +139,7 @@ async function assertUrlNotBlocked(rawUrl, policy) {
 }
 export const fetchUrlTool = {
     name: 'fetch_url',
-    description: 'Fetch a URL over HTTP(S) and return its body as readable text. HTML is converted to plain text (scripts/styles stripped); JSON is pretty-printed; text is passed through. Follows up to 5 redirects and respects max_bytes/timeout. Does not follow links or execute JavaScript.',
+    description: 'Fetch a URL over HTTP(S) and return its body as readable text. HTML is converted to Markdown (headings, links, lists, tables and code kept; scripts, styles and hidden parts dropped); JSON is pretty-printed; text is passed through. Follows up to 5 redirects and respects max_bytes/timeout. Does not follow links or execute JavaScript.',
     parameters: params,
     destructive: false,
     execute: async (args, ctx) => {
@@ -214,7 +216,7 @@ export const fetchUrlTool = {
         let title;
         if (isHtml) {
             title = extractTitle(text);
-            body = htmlToText(text);
+            body = htmlToMarkdown(text, finalUrl);
         }
         else if (isJson) {
             body = prettyJson(text);
@@ -271,54 +273,12 @@ async function readWithCap(response, cap) {
     chunks.push(decoder.decode());
     return { text: chunks.join(''), bytes: total, truncated };
 }
-/**
- * Tiny HTML → text converter. Good enough for inspection; not a readability
- * replacement. Strips scripts, styles, comments, and tags; decodes the most
- * common entities; collapses whitespace. This is intentionally simple: we
- * want to avoid pulling in a 200KB HTML parser dependency for what's
- * essentially a "summarize this page" use case.
- */
-function htmlToText(html) {
-    let s = html;
-    // Drop <script>...</script> and <style>...</style> blocks first (their
-    // contents aren't part of the visible text).
-    s = s.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi, ' ');
-    s = s.replace(/<style\b[^>]*>[\s\S]*?<\/style\s*>/gi, ' ');
-    // Drop conditional comments and HTML comments.
-    s = s.replace(/<!--[\s\S]*?-->/g, ' ');
-    // Drop the tags themselves.
-    s = s.replace(/<\/?[a-zA-Z][^>]*>/g, ' ');
-    // Common named entities.
-    s = s
-        .replace(/&nbsp;/g, ' ')
-        .replace(/&amp;/g, '&')
-        .replace(/&lt;/g, '<')
-        .replace(/&gt;/g, '>')
-        .replace(/&quot;/g, '"')
-        .replace(/&apos;/g, "'")
-        .replace(/&#39;/g, "'")
-        .replace(/&hellip;/g, '…')
-        .replace(/&mdash;/g, '—')
-        .replace(/&ndash;/g, '–');
-    // Numeric entities (decimal and hex): limited to the common ones.
-    s = s.replace(/&#(\d+);/g, (_, code) => {
-        const n = Number(code);
-        return n > 0x1f && n < 0x7f ? String.fromCharCode(n) : ' ';
-    });
-    s = s.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => {
-        const n = parseInt(hex, 16);
-        return n > 0x1f && n < 0x7f ? String.fromCharCode(n) : ' ';
-    });
-    // Collapse runs of whitespace.
-    s = s.replace(/[ \t\f\v]+/g, ' ').replace(/\s*\n\s*/g, '\n').replace(/\n{2,}/g, '\n\n');
-    return s.trim();
-}
 /** Pull the `<title>` from the head, if present. */
 function extractTitle(html) {
     const m = html.match(/<title[^>]*>([\s\S]*?)<\/title\s*>/i);
     if (!m)
         return undefined;
-    return htmlToText(m[1]).trim() || undefined;
+    return decodeEntities(m[1].replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim() || undefined;
 }
 /** Pretty-print JSON if it parses; otherwise return the raw text. */
 function prettyJson(text) {

package/dist/tools/get-diagnostics.d.ts

@@ -0,0 +1,14 @@
+import { z } from 'zod';
+import type { ToolDefinition } from '../types.js';
+declare const params: z.ZodObject<{
+    paths: z.ZodArray<z.ZodString>;
+    include_warnings: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>;
+export interface DiagnosticsResult {
+    /** Problems found. */
+    problems: number;
+    /** One line per problem as file:line:column: message, or a sentence saying why there is nothing to report. */
+    text: string;
+}
+export declare const getDiagnosticsTool: ToolDefinition<z.infer<typeof params>, DiagnosticsResult>;
+export {};

package/dist/tools/get-diagnostics.js

@@ -0,0 +1,44 @@
+/**
+ * `get_diagnostics`: compiler and linter problems for some files, from the language server for their type.
+ *
+ * It asks first (RB-41). A language server reads the repository's own configuration when it starts (tsconfig.json,
+ * plugins, a rust-analyzer build script), and Forge keeps no server running between calls, so every call starts one.
+ */
+import path from 'node:path';
+import { z } from 'zod';
+import { collectErrors, formatErrors, hasServerFor } from '../lsp/index.js';
+import { resolveAndCheckPath } from './shared.js';
+const MAX_FILES = 20;
+const params = z.object({
+    paths: z
+        .array(z.string())
+        .min(1)
+        .max(MAX_FILES)
+        .describe(`Files to check, relative to the project root (at most ${MAX_FILES}).`),
+    include_warnings: z.boolean().optional().describe('Also list warnings. Default: errors only.'),
+});
+export const getDiagnosticsTool = {
+    name: 'get_diagnostics',
+    description: 'Get compiler and linter problems for some files from their language server (TypeScript, Python, Go, Rust), without running a build. ' +
+        "Starting a language server follows the project's own configuration, so it asks first. If no server is installed it says so; then use run_command with the project's own check instead.",
+    parameters: params,
+    destructive: true,
+    execute: async (args, ctx) => {
+        const files = args.paths.map((p) => path.relative(ctx.cwd, resolveAndCheckPath(p, ctx)) || p);
+        const config = { ...(ctx.lspConfig ?? {}), ...(args.include_warnings ? { includeWarnings: true } : {}) };
+        const covered = files.filter((f) => hasServerFor(f, config));
+        const notes = [];
+        const unsupported = files.filter((f) => !covered.includes(f));
+        if (unsupported.length > 0)
+            notes.push(`No language server for: ${unsupported.join(', ')}.`);
+        if (covered.length === 0)
+            return { problems: 0, text: notes.join('\n') };
+        const report = await collectErrors(covered, ctx.cwd, config);
+        if (report.skipped.length > 0) {
+            notes.push(`Could not start: ${[...new Set(report.skipped)].join(', ')} (not installed, or it failed to start). Use run_command with the project's own type-check or lint command instead.`);
+        }
+        const problems = report.errors.reduce((n, f) => n + f.diagnostics.length, 0);
+        const found = problems > 0 ? formatErrors(report.errors) : report.skipped.length > 0 ? '' : 'No problems found.';
+        return { problems, text: [found, ...notes].filter(Boolean).join('\n') };
+    },
+};

package/dist/tools/glob-filter.d.ts

@@ -0,0 +1,7 @@
+/**
+ * A file-name filter for tools that take a glob from the model (`search_files` include, `list_files` pattern).
+ * A slash-less glob such as "*.ts" names files at any depth, as in ripgrep -g; a glob with a slash is matched
+ * against the path relative to the searched directory. Do not use `globToRegex` directly for this: it anchors a
+ * slash-less pattern to the top level on purpose, for permission rules (RB-64).
+ */
+export declare function fileGlobMatcher(glob: string): (rel: string, name: string) => boolean;