agent-sh 0.15.0 → 0.15.2

package/src/agent/skills.ts

@@ -0,0 +1,228 @@
+/**
+ * Skill discovery and loading.
+ *
+ * Follows the Agent Skills standard (agentskills.io):
+ *   - Skills are directories containing a SKILL.md with YAML frontmatter
+ *   - Frontmatter must include `name` and `description`
+ *   - Full content is loaded on-demand (only names/descriptions in system prompt)
+ *
+ * Discovery locations:
+ *   Global:  ~/.agent-sh/skills/ (default), plus skillPaths from settings
+ *   Project: .agents/skills/ in cwd and ancestor dirs (up to git root)
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { CONFIG_DIR, getSettings } from "../core/settings.js";
+
+export interface Skill {
+  name: string;
+  description: string;
+  filePath: string;
+  baseDir: string;
+}
+
+/** Parse YAML frontmatter from a SKILL.md file. Supports inline scalars
+ *  and block scalars (`>`, `>-`, `|`, `|-`) for multi-line descriptions. */
+function parseFrontmatter(content: string): { meta: Record<string, string>; body: string } | null {
+  const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/);
+  if (!match) return null;
+
+  const meta: Record<string, string> = {};
+  const lines = match[1].split("\n");
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i]!;
+    const indent = line.length - line.trimStart().length;
+    const colon = line.indexOf(":");
+    if (colon <= 0 || indent > 0) { i++; continue; }
+    const key = line.slice(0, colon).trim();
+    const rawValue = line.slice(colon + 1).trim();
+    const blockStyle = rawValue.match(/^([>|])([+-]?)\s*$/);
+    if (!blockStyle) {
+      meta[key] = rawValue;
+      i++;
+      continue;
+    }
+    const folded = blockStyle[1] === ">";
+    const chomp = blockStyle[2];
+    const body: string[] = [];
+    let blockIndent = -1;
+    let j = i + 1;
+    while (j < lines.length) {
+      const next = lines[j]!;
+      if (next.trim() === "") { body.push(""); j++; continue; }
+      const ind = next.length - next.trimStart().length;
+      if (blockIndent === -1) blockIndent = ind;
+      if (ind < blockIndent) break;
+      body.push(next.slice(blockIndent));
+      j++;
+    }
+    let end = body.length;
+    if (chomp !== "+") {
+      while (end > 0 && body[end - 1] === "") end--;
+      if (chomp !== "-" && end < body.length) end++;
+    }
+    const kept = body.slice(0, end);
+    meta[key] = folded
+      ? kept.join(" ").replace(/\s+/g, " ").trim()
+      : kept.join("\n");
+    i = j;
+  }
+
+  return { meta, body: match[2] };
+}
+
+/** Load a single skill from a SKILL.md file. */
+function loadSkillFromFile(filePath: string): Skill | null {
+  try {
+    const content = fs.readFileSync(filePath, "utf-8");
+    const parsed = parseFrontmatter(content);
+    if (!parsed) return null;
+
+    const name = parsed.meta.name;
+    const description = parsed.meta.description;
+    if (!name || !description) return null;
+
+    if (parsed.meta["disable-model-invocation"] === "true") return null;
+
+    return {
+      name,
+      description,
+      filePath,
+      baseDir: path.dirname(filePath),
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Recursively scan a directory for SKILL.md files. */
+function scanDir(dir: string): Skill[] {
+  const skills: Skill[] = [];
+
+  let entries: fs.Dirent[];
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return skills;
+  }
+
+  // If this directory has a SKILL.md, it's a skill root — don't recurse further
+  const skillMd = path.join(dir, "SKILL.md");
+  try {
+    fs.accessSync(skillMd);
+    const skill = loadSkillFromFile(skillMd);
+    if (skill) skills.push(skill);
+    return skills;
+  } catch {
+    // No SKILL.md here — check subdirectories
+  }
+
+  for (const entry of entries) {
+    if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
+
+    const fullPath = path.join(dir, entry.name);
+    const isDir = entry.isDirectory() ||
+      (entry.isSymbolicLink() && (() => { try { return fs.statSync(fullPath).isDirectory(); } catch { return false; } })());
+
+    if (isDir) {
+      skills.push(...scanDir(fullPath));
+    }
+  }
+
+  return skills;
+}
+
+/** Expand ~ to home directory. */
+function expandHome(p: string): string {
+  if (p.startsWith("~/") || p === "~") {
+    return path.join(os.homedir(), p.slice(1));
+  }
+  return p;
+}
+
+function addUnique(target: Skill[], source: Skill[], seen: Set<string>): void {
+  for (const skill of source) {
+    if (!seen.has(skill.name)) {
+      seen.add(skill.name);
+      target.push(skill);
+    }
+  }
+}
+
+// Global skill sources are stable within a session, so cache the result
+// to skip filesystem scans on every system-prompt:build.
+let _cachedGlobalSkills: Skill[] | null = null;
+
+/** Discover global skills (stable across cwd changes). Cached per-process. */
+export function discoverGlobalSkills(): Skill[] {
+  if (_cachedGlobalSkills) return _cachedGlobalSkills;
+
+  const seen = new Set<string>();
+  const skills: Skill[] = [];
+
+  addUnique(skills, scanDir(path.join(CONFIG_DIR, "skills")), seen);
+
+  const settings = getSettings();
+  for (const p of settings.skillPaths ?? []) {
+    addUnique(skills, scanDir(path.resolve(expandHome(p))), seen);
+  }
+
+  _cachedGlobalSkills = skills;
+  return skills;
+}
+
+export function invalidateGlobalSkillsCache(): void {
+  _cachedGlobalSkills = null;
+}
+
+/**
+ * Discover project-level skills from .agents/skills/ in cwd hierarchy.
+ * Walks from cwd up to $HOME (or filesystem root if cwd is outside HOME).
+ * Git boundaries are ignored — nested repos under a skills-bearing parent
+ * would otherwise hide the parent's skills.
+ */
+export function discoverProjectSkills(cwd: string): Skill[] {
+  const seen = new Set<string>();
+  const skills: Skill[] = [];
+  const home = path.resolve(os.homedir());
+  let current = path.resolve(cwd);
+
+  while (true) {
+    addUnique(skills, scanDir(path.join(current, ".agents", "skills")), seen);
+    if (current === home) break;
+    const parent = path.dirname(current);
+    if (parent === current) break;
+    current = parent;
+  }
+
+  return skills;
+}
+
+/**
+ * Discover all skills (global + project).
+ */
+export function discoverSkills(cwd: string): Skill[] {
+  const seen = new Set<string>();
+  const skills: Skill[] = [];
+  addUnique(skills, discoverGlobalSkills(), seen);
+  addUnique(skills, discoverProjectSkills(cwd), seen);
+  return skills;
+}
+
+/**
+ * Load the full content of a skill (frontmatter stripped).
+ * Returns XML-wrapped content suitable for injection into conversation.
+ */
+export function loadSkillContent(skill: Skill): string | null {
+  try {
+    const content = fs.readFileSync(skill.filePath, "utf-8");
+    const parsed = parseFrontmatter(content);
+    if (!parsed) return content;
+
+    return `<skill name="${skill.name}" location="${skill.filePath}">\nReferences are relative to ${skill.baseDir}.\n\n${parsed.body.trim()}\n</skill>`;
+  } catch {
+    return null;
+  }
+}

package/src/agent/store.ts

@@ -0,0 +1,310 @@
+import * as fs from "node:fs";
+import * as fsp from "node:fs/promises";
+import * as path from "node:path";
+import * as crypto from "node:crypto";
+
+export interface Entry {
+  id: string;
+  parentId?: string;
+  ts: number;
+  kind: string;
+  payload: Record<string, unknown>;
+}
+
+export interface AppendOpts {
+  /** Memory-only; never persisted. */
+  ephemeral?: boolean;
+}
+
+export interface SearchHit {
+  entry: Entry;
+  line: string;
+}
+
+/** Append-only — no edit or delete. Implementations may apply bulk
+ *  retention (front-truncation, GC), but strategies cannot remove a
+ *  specific entry. */
+export interface Store {
+  append(entries: Entry[], opts?: AppendOpts): Promise<void>;
+  findById(id: string): Promise<Entry | null>;
+  readRecent(n?: number): Promise<Entry[]>;
+  search(query: string): Promise<SearchHit[]>;
+}
+
+export interface TreeStore extends Store {
+  getBranch(leafId?: string): Promise<Entry[]>;
+  setLeaf(id: string): void;
+  getLeaf(): string;
+}
+
+export function newEntryId(): string {
+  return crypto.randomBytes(4).toString("hex");
+}
+
+export function isTreeStore(s: Store): s is TreeStore {
+  return (
+    typeof (s as TreeStore).setLeaf === "function" &&
+    typeof (s as TreeStore).getLeaf === "function" &&
+    typeof (s as TreeStore).getBranch === "function"
+  );
+}
+
+function escapeRegex(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function compileSearchRegex(query: string): RegExp {
+  return new RegExp(escapeRegex(query), "i");
+}
+
+function matchEntry(entry: Entry, re: RegExp): SearchHit | null {
+  const line = JSON.stringify(entry);
+  return re.test(line) ? { entry, line } : null;
+}
+
+export class NoopStore implements Store {
+  async append(): Promise<void> {}
+  async findById(): Promise<Entry | null> { return null; }
+  async readRecent(): Promise<Entry[]> { return []; }
+  async search(): Promise<SearchHit[]> { return []; }
+}
+
+export class InMemoryStore implements TreeStore {
+  private entries = new Map<string, Entry>();
+  private order: string[] = [];
+  private leaf: string;
+
+  constructor(opts?: { root?: Entry }) {
+    if (opts?.root) {
+      this.entries.set(opts.root.id, opts.root);
+      this.order.push(opts.root.id);
+      this.leaf = opts.root.id;
+    } else {
+      this.leaf = "";
+    }
+  }
+
+  async append(entries: Entry[]): Promise<void> {
+    for (const e of entries) {
+      this.entries.set(e.id, e);
+      this.order.push(e.id);
+    }
+  }
+
+  async findById(id: string): Promise<Entry | null> {
+    return this.entries.get(id) ?? null;
+  }
+
+  async readRecent(n?: number): Promise<Entry[]> {
+    const slice = n == null ? this.order : this.order.slice(-n);
+    return slice.map((id) => this.entries.get(id)!);
+  }
+
+  async search(query: string): Promise<SearchHit[]> {
+    if (!query.trim()) return [];
+    const re = compileSearchRegex(query);
+    const out: SearchHit[] = [];
+    for (let i = this.order.length - 1; i >= 0; i--) {
+      const m = matchEntry(this.entries.get(this.order[i]!)!, re);
+      if (m) out.push(m);
+    }
+    return out;
+  }
+
+  async getBranch(leafId: string = this.leaf): Promise<Entry[]> {
+    const out: Entry[] = [];
+    const seen = new Set<string>();
+    let cur: string | undefined = leafId;
+    while (cur && !seen.has(cur)) {
+      seen.add(cur);
+      const e = this.entries.get(cur);
+      if (!e) break;
+      out.push(e);
+      cur = e.parentId;
+    }
+    return out.reverse();
+  }
+
+  setLeaf(id: string): void {
+    if (!this.entries.has(id)) throw new Error(`unknown entry: ${id}`);
+    this.leaf = id;
+  }
+
+  getLeaf(): string {
+    return this.leaf;
+  }
+}
+
+/** Multi-writer JSONL Store. O_APPEND with PIPE_BUF-bounded line
+ *  writes for atomic concurrent appends; lock-based front-truncation
+ *  for retention; reads stream the tail for cheap recent slices. */
+const LOCK_STALE_MS = 10_000;
+const DEFAULT_MAX_BYTES = 50 * 1024 * 1024;
+
+export interface SharedFileStoreOpts {
+  filePath: string;
+  /** Front-truncate above this size; truncation fires at 150% of the
+   *  cap to avoid frequent rewrites. */
+  maxBytes?: number;
+}
+
+export class SharedFileStore implements Store {
+  private filePath: string;
+  private lockPath: string;
+  private maxBytes: number;
+
+  constructor(opts: SharedFileStoreOpts) {
+    this.filePath = opts.filePath;
+    this.lockPath = opts.filePath + ".lock";
+    this.maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    try { fs.mkdirSync(path.dirname(this.filePath), { recursive: true }); } catch { /* ignore */ }
+  }
+
+  async append(entries: Entry[], opts?: AppendOpts): Promise<void> {
+    if (entries.length === 0) return;
+    if (opts?.ephemeral) return; // memory-only writes are a no-op on a file-only store
+    const lines = entries.map((e) => JSON.stringify(e) + "\n").join("");
+    await fsp.appendFile(this.filePath, lines, { flag: "a" });
+    await this.maybeTruncate();
+  }
+
+  async findById(id: string): Promise<Entry | null> {
+    for await (const line of this.streamReverseLines()) {
+      try {
+        const e = JSON.parse(line) as Entry;
+        if (e.id === id) return e;
+      } catch { /* skip malformed */ }
+    }
+    return null;
+  }
+
+  async readRecent(n?: number): Promise<Entry[]> {
+    const want = n ?? Infinity;
+    const recent: Entry[] = []; // newest-first
+    for await (const line of this.streamReverseLines()) {
+      try {
+        const e = JSON.parse(line) as Entry;
+        if (!e.id) continue;
+        recent.push(e);
+        if (recent.length >= want) break;
+      } catch { /* skip malformed */ }
+    }
+    return recent.reverse();
+  }
+
+  async search(query: string): Promise<SearchHit[]> {
+    if (!query.trim()) return [];
+    const re = compileSearchRegex(query);
+    const budgetBytes = 20 * 1024 * 1024;
+    let scanned = 0;
+    const out: SearchHit[] = [];
+    for await (const line of this.streamReverseLines()) {
+      scanned += line.length + 1;
+      if (scanned > budgetBytes) break;
+      try {
+        const e = JSON.parse(line) as Entry;
+        const m = matchEntry(e, re);
+        if (m) out.push(m);
+      } catch { /* skip malformed */ }
+    }
+    return out;
+  }
+
+  /** Yield lines newest-first by reading reverse-chunked blocks,
+   *  stitching across boundaries. */
+  private async *streamReverseLines(chunkBytes = 1 << 20): AsyncGenerator<string> {
+    let handle: fsp.FileHandle;
+    let fileSize: number;
+    try {
+      const stat = await fsp.stat(this.filePath);
+      fileSize = stat.size;
+      if (fileSize === 0) return;
+      handle = await fsp.open(this.filePath, "r");
+    } catch {
+      return;
+    }
+    try {
+      let position = fileSize;
+      let pending: Buffer = Buffer.alloc(0);
+      while (position > 0) {
+        const readSize = Math.min(chunkBytes, position);
+        position -= readSize;
+        const buf = Buffer.alloc(readSize);
+        await handle.read(buf, 0, readSize, position);
+        const combined = Buffer.concat([buf, pending]);
+        const newlineIdxs: number[] = [];
+        for (let i = 0; i < combined.length; i++) {
+          if (combined[i] === 0x0A) newlineIdxs.push(i);
+        }
+        if (newlineIdxs.length === 0) { pending = combined; continue; }
+        const firstNl = newlineIdxs[0]!;
+        const lastNl = newlineIdxs[newlineIdxs.length - 1]!;
+        const trailing = combined.subarray(lastNl + 1);
+        if (trailing.length > 0) yield trailing.toString("utf-8");
+        for (let i = newlineIdxs.length - 1; i >= 1; i--) {
+          const seg = combined.subarray(newlineIdxs[i - 1]! + 1, newlineIdxs[i]!);
+          if (seg.length > 0) yield seg.toString("utf-8");
+        }
+        const leading = combined.subarray(0, firstNl);
+        if (position === 0) {
+          if (leading.length > 0) yield leading.toString("utf-8");
+          pending = Buffer.alloc(0);
+        } else {
+          pending = leading;
+        }
+      }
+      if (pending.length > 0) yield pending.toString("utf-8");
+    } finally {
+      await handle.close();
+    }
+  }
+
+  private async maybeTruncate(): Promise<void> {
+    let size = 0;
+    try { size = (await fsp.stat(this.filePath)).size; } catch { return; }
+    if (size <= this.maxBytes * 1.5) return;
+
+    if (!(await this.acquireLock())) return;
+    try {
+      let content: string;
+      try { content = await fsp.readFile(this.filePath, "utf-8"); }
+      catch { return; }
+
+      const lines = content.split("\n").filter(Boolean);
+      let totalBytes = Buffer.byteLength(content, "utf-8");
+      let dropCount = 0;
+      while (totalBytes > this.maxBytes && dropCount < lines.length - 1) {
+        totalBytes -= Buffer.byteLength(lines[dropCount]! + "\n", "utf-8");
+        dropCount++;
+      }
+      if (dropCount === 0) return;
+
+      const remaining = lines.slice(dropCount).join("\n") + "\n";
+      const tmpPath = this.filePath + ".tmp." + process.pid;
+      await fsp.writeFile(tmpPath, remaining);
+      await fsp.rename(tmpPath, this.filePath);
+    } finally {
+      await this.releaseLock();
+    }
+  }
+
+  private async acquireLock(): Promise<boolean> {
+    try {
+      try {
+        const stat = await fsp.stat(this.lockPath);
+        if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
+          await fsp.unlink(this.lockPath).catch(() => {});
+        }
+      } catch { /* lock absent — good */ }
+      const fd = await fsp.open(this.lockPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY);
+      await fd.close();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  private async releaseLock(): Promise<void> {
+    await fsp.unlink(this.lockPath).catch(() => {});
+  }
+}