@adecore/agents 0.31.11 → 0.32.0

package/src/skills.ts

@@ -1,5 +1,5 @@
 import { existsSync, type Dirent } from 'node:fs';
-import { readdir, readFile } from 'node:fs/promises';
+import { open, readdir, readFile, realpath, stat } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { dirname, join, parse } from 'node:path';
 import type { AgentKind, ChatSkill, ChatSkillSource } from '@adecore/agent-contracts';
@@ -10,8 +10,57 @@ export interface SkillRoot {
     source: ChatSkillSource;
     // A plugin names its skills `<plugin>:<skill>`, the way the CLI's init frame does.
     prefix?: string;
+    // How many folders below `dir` a skill folder may sit; absent is one, right under it.
+    depth?: number;
 }
 
+/* How far one scan reaches, so a home or a project of any size answers quickly. */
+export interface SkillScanLimits {
+    // Folders listed under one root, nearest first, so a deep tree never costs the skills near the top.
+    directories: number;
+    skills: number;
+    // Entries of one folder looked at, by name; absent is `ENTRIES_PER_FOLDER`.
+    entries?: number;
+}
+
+const SCAN_LIMITS: SkillScanLimits = { directories: 2000, skills: 1000 };
+
+const ENTRIES_PER_FOLDER = 1000;
+
+// Paths one folder resolves or reads at once, so a large folder never runs the process out of file handles.
+const SCAN_CONCURRENCY = 16;
+
+// A process short of file handles gets them back as other work closes its files, so a few short waits cover it.
+const BUSY_RETRIES = 3;
+const BUSY_RETRY_MS = 50;
+
+// Codex finds skills up to five folders below a root (`MAX_SCAN_DEPTH` of 6 in its skill loader counts the root itself).
+const CODEX_SKILL_DEPTH = 5;
+
+// Folders a package manager or a build fills. One may hold a SKILL.md of its own, but nothing under it is listed.
+const DEPENDENCY_FOLDERS = new Set([
+    'node_modules',
+    'bower_components',
+    'jspm_packages',
+    'vendor',
+    'Pods',
+    'Carthage',
+    'DerivedData',
+    '__pycache__',
+    'venv',
+    'site-packages',
+    'target',
+    'dist',
+    'build',
+    'out',
+    'coverage'
+]);
+
+// What a scan reads of a SKILL.md; the front matter sits at its top.
+const HEAD_BYTES = 64 * 1024;
+
+const MAX_DESCRIPTION = 1024;
+
 export interface SkillRootOptions {
     // The person's home directory; a test points it at a temporary tree.
     home?: string;
@@ -118,9 +167,9 @@ export async function skillRootsFor(kind: AgentKind, cwd: string, options: Skill
     const home = options.home ?? homedir();
     if (kind === 'codex') {
         return [
-            { dir: join(home, '.agents', 'skills'), source: 'user' },
-            { dir: join(home, '.codex', 'skills'), source: 'user' },
-            ...projectDirs(cwd, home).map((dir): SkillRoot => ({ dir: join(dir, '.agents', 'skills'), source: 'project' }))
+            { dir: join(home, '.agents', 'skills'), source: 'user', depth: CODEX_SKILL_DEPTH },
+            { dir: join(home, '.codex', 'skills'), source: 'user', depth: CODEX_SKILL_DEPTH },
+            ...projectDirs(cwd, home).map((dir): SkillRoot => ({ dir: join(dir, '.agents', 'skills'), source: 'project', depth: CODEX_SKILL_DEPTH }))
         ];
     }
     const configDir = options.claudeConfigDir ?? join(home, '.claude');
@@ -131,37 +180,167 @@ export async function skillRootsFor(kind: AgentKind, cwd: string, options: Skill
     ];
 }
 
-async function readSkill(root: SkillRoot, folder: string): Promise<ChatSkill | null> {
-    let text: string;
+/* Too many files open, in this process or on the system: it says nothing about the path that was asked for. */
+function isBusy(e: unknown): boolean {
+    return typeof e === 'object' && e !== null && 'code' in e && (e.code === 'EMFILE' || e.code === 'ENFILE');
+}
+
+function sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/* Runs `operation` again a few times while the process is short of file handles, and throws if it still is. */
+export async function whileBusy<T>(operation: () => Promise<T>, wait: (ms: number) => Promise<void> = sleep): Promise<T> {
+    for (let attempt = 1; ; attempt++) {
+        try {
+            return await operation();
+        } catch (e) {
+            if (!isBusy(e) || attempt > BUSY_RETRIES) {
+                throw e;
+            }
+            await wait(BUSY_RETRY_MS * attempt);
+        }
+    }
+}
+
+/* `work` over every item with at most `limit` running at once, answered in the order of `items`. */
+export async function mapLimited<T, R>(items: readonly T[], limit: number, work: (item: T) => Promise<R>): Promise<R[]> {
+    const results = new Array<R>(items.length);
+    let next = 0;
+    const worker = async (): Promise<void> => {
+        while (next < items.length) {
+            const index = next++;
+            results[index] = await work(items[index]!);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return results;
+}
+
+/* The top of a folder's SKILL.md, or null when it has none. A process short of file handles throws instead of finding none. */
+async function skillHead(folder: string): Promise<string | null> {
+    let handle;
+    try {
+        handle = await whileBusy(() => open(join(folder, 'SKILL.md'), 'r'));
+    } catch (e) {
+        if (isBusy(e)) {
+            throw e;
+        }
+        return null;
+    }
     try {
-        text = await readFile(join(root.dir, folder, 'SKILL.md'), 'utf8');
+        if (!(await handle.stat()).isFile()) {
+            return null;
+        }
+        const buffer = Buffer.alloc(HEAD_BYTES);
+        const { bytesRead } = await handle.read(buffer, 0, HEAD_BYTES, 0);
+        return buffer.subarray(0, bytesRead).toString('utf8');
     } catch {
         return null;
+    } finally {
+        await handle.close();
     }
+}
+
+function skillOf(root: SkillRoot, folder: string, text: string): ChatSkill | null {
     const fields = parseFrontmatter(text);
     const own = fields.name && NAME.test(fields.name) ? fields.name : folder;
     const name = root.prefix ? `${root.prefix}:${own}` : own;
     if (!NAME.test(name)) {
         return null;
     }
-    return { name, description: fields.description ?? '', source: root.source };
+    return { name, description: (fields.description ?? '').slice(0, MAX_DESCRIPTION), source: root.source };
+}
+
+/* The folder a path leads to after every link, or null when it leads to no folder. */
+async function realFolder(path: string): Promise<string | null> {
+    try {
+        const real = await whileBusy(() => realpath(path));
+        return (await whileBusy(() => stat(real))).isDirectory() ? real : null;
+    } catch (e) {
+        if (isBusy(e)) {
+            throw e;
+        }
+        return null;
+    }
+}
+
+/*
+ * The skills under one root, nearest first, down to the root's depth. A folder with a SKILL.md is a skill and is not
+ * searched further. Each root keeps its own record of the folders it read or went into, by real path, so a link back
+ * up ends there, while a skill two plugins link in shows under each of them.
+ */
+async function scanRoot(root: SkillRoot, limits: SkillScanLimits): Promise<ChatSkill[]> {
+    const start = await realFolder(root.dir);
+    if (start === null) {
+        return [];
+    }
+    const visited = new Set([start]);
+    const depth = root.depth ?? 1;
+    const entriesPerFolder = limits.entries ?? ENTRIES_PER_FOLDER;
+    const skills: ChatSkill[] = [];
+    let level = [root.dir];
+    let listed = 0;
+    for (let below = 1; below <= depth && level.length > 0; below++) {
+        const next: string[] = [];
+        for (const folder of level) {
+            if (listed >= limits.directories || skills.length >= limits.skills) {
+                return skills;
+            }
+            listed++;
+            let entries: Dirent[];
+            try {
+                entries = await whileBusy(() => readdir(folder, { withFileTypes: true }));
+            } catch (e) {
+                if (isBusy(e)) {
+                    throw e;
+                }
+                continue;
+            }
+            // A skill manager links each skill in from a shared folder, so a link counts as much as a folder.
+            const candidates = entries
+                .filter((entry) => (entry.isDirectory() || entry.isSymbolicLink()) && !entry.name.startsWith('.'))
+                .map((entry) => ({ name: entry.name, path: join(folder, entry.name) }))
+                .sort((a, b) => a.name.localeCompare(b.name))
+                .slice(0, entriesPerFolder);
+            const reals = await mapLimited(candidates, SCAN_CONCURRENCY, (candidate) => realFolder(candidate.path));
+            const fresh = candidates.flatMap((candidate, index) => {
+                const real = reals[index];
+                return real === null || real === undefined || visited.has(real) ? [] : [{ ...candidate, real }];
+            });
+            const heads = await mapLimited(fresh, SCAN_CONCURRENCY, (candidate) => skillHead(candidate.path));
+            for (const [index, candidate] of fresh.entries()) {
+                // Two links in one folder can lead to the same place; the first one read it.
+                if (visited.has(candidate.real)) {
+                    continue;
+                }
+                const head = heads[index];
+                if (head !== null && head !== undefined) {
+                    visited.add(candidate.real);
+                    const skill = skillOf(root, candidate.name, head);
+                    if (skill !== null && skills.length < limits.skills) {
+                        skills.push(skill);
+                    }
+                } else if (below < depth && !DEPENDENCY_FOLDERS.has(candidate.name)) {
+                    visited.add(candidate.real);
+                    next.push(candidate.path);
+                }
+            }
+        }
+        level = next;
+    }
+    return skills;
 }
 
 /* Every skill under these roots, sorted by name; the first root that carries a name wins. */
-export async function scanSkillRoots(roots: SkillRoot[]): Promise<ChatSkill[]> {
+export async function scanSkillRoots(roots: SkillRoot[], limits: SkillScanLimits = SCAN_LIMITS): Promise<ChatSkill[]> {
     const found = new Map<string, ChatSkill>();
     for (const root of roots) {
-        let entries: Dirent[];
-        try {
-            entries = await readdir(root.dir, { withFileTypes: true });
-        } catch {
-            continue;
+        if (found.size >= limits.skills) {
+            break;
         }
-        // A skill manager links each skill in from a shared folder; a link to anything but a skill folder reads as no skill.
-        const folders = entries.filter((entry) => entry.isDirectory() || entry.isSymbolicLink());
-        const skills = await Promise.all(folders.map((entry) => readSkill(root, entry.name)));
-        for (const skill of skills) {
-            if (skill && !found.has(skill.name)) {
+        for (const skill of await scanRoot(root, limits)) {
+            if (!found.has(skill.name) && found.size < limits.skills) {
                 found.set(skill.name, skill);
             }
         }

package/src/testing/fake-acp-capture.ts

@@ -2,7 +2,8 @@
  * An ACP agent that answers with the frames a real one sent, recorded in a capture: its `initialize`, its
  * `session/new`, and every update of one prompt before the response. Every prompt gets the same turn. A
  * `{ permission }` entry among the updates is a `session/request_permission` the turn waits on until the
- * client answers it, an `{ update, _meta }` entry is a notification with a `_meta` of its own, and with
+ * client answers it, a `{ request: { method, params } }` entry is any other request of the agent the turn waits on the
+ * same way, an `{ update, _meta }` entry is a notification with a `_meta` of its own, and with
  * `titleFromPrompt` a `session_info_update` carries the prompt's own text as its title, as Copilot CLI names a session.
  */
 import type { FakeCli } from './fake-cli.ts';
@@ -18,6 +19,8 @@ export interface AcpCapture {
     setConfigOption: unknown;
     promptUpdates: unknown[];
     promptResponse: unknown;
+    // The JSON-RPC error a prompt fails with instead of the response, when set.
+    promptError?: unknown;
     titleFromPrompt?: boolean;
 }
 
@@ -36,16 +39,17 @@ export function fakeAcpCapture(capture: AcpCapture, answers: unknown[] = []): Fa
         const play = (updates: unknown[], promptId: unknown, text: unknown): void => {
             for (const [index, fields] of updates.entries()) {
                 const permission = (fields as Frame).permission;
-                if (permission !== undefined) {
+                const request = (fields as Frame).request as { method: string; params: unknown } | undefined;
+                if (permission !== undefined || request !== undefined) {
                     const rpcId = nextRpcId++;
                     waiting = { rpcId, rest: updates.slice(index + 1), promptId, text };
-                    send({ id: rpcId, method: 'session/request_permission', params: permission });
+                    send({ id: rpcId, method: request?.method ?? 'session/request_permission', params: request?.params ?? permission });
                     return;
                 }
                 const named = capture.titleFromPrompt === true && (fields as Frame).sessionUpdate === 'session_info_update';
                 update(named ? { ...(fields as Frame), title: text } : fields);
             }
-            send({ id: promptId, result: capture.promptResponse });
+            send(capture.promptError === undefined ? { id: promptId, result: capture.promptResponse } : { id: promptId, error: capture.promptError });
         };
         return {
             onLine: (line) => {

package/src/usage/accounts.ts

@@ -1,5 +1,5 @@
 import { USAGE_PROVIDERS, type ChatInfo, type ProviderAccounts, type UsageAccount, type UsageProvider } from '@adecore/agent-contracts';
-import { definedEnv } from '../providers/accounts/launch.ts';
+import { definedEnv, storedAccount } from '../providers/accounts/launch.ts';
 import type { ProviderAccountsService } from '../providers/accounts/service.ts';
 import type { LimitAccount, LimitAccounts } from './limits/monitor.ts';
 import { accountRoots, type UsageRootPath } from './roots.ts';
@@ -69,7 +69,8 @@ export function limitAccountsOf(
             machineAccounts(service.snapshot(), nameOf)
                 .filter((account) => account.isDefault || (account.enabled && account.signedIn))
                 .map((account): LimitAccount => ({ ...usageAccount(account), isDefault: account.isDefault })),
-        envFor: (kind, id) => definedEnv(service.envFor(kind, id, baseEnv)),
+        // A plan read is no start: the default account goes unnamed, so turning it off does not fail its read.
+        envFor: (kind, id) => definedEnv(service.envFor(kind, storedAccount(kind, id), baseEnv)),
         lastUsedAt: (id) => service.lastLaunchAt(id)
     };
 }