@cr1ms0n/pi-subagent 0.8.1

package/src/config.ts

@@ -0,0 +1,254 @@
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import * as path from "node:path";
+import type { TaskProfile } from "./types.js";
+import { MODEL_POLICY_CONFIG_FILE, parseModelPolicy, type ModelPolicySnapshot } from "./model-policy.js";
+
+const THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh"] as const;
+export type ThinkingLevel = (typeof THINKING_LEVELS)[number];
+
+const WIDGET_MODES = ["background", "off"] as const;
+export type WidgetMode = (typeof WIDGET_MODES)[number];
+const NOTIFICATION_MODES = ["batched", "off"] as const;
+export type NotificationMode = (typeof NOTIFICATION_MODES)[number];
+
+/**
+ * Per-profile defaults applied when a task omits the field. Explicit request
+ * values always win; profile defaults beat parent-session inheritance for
+ * model/thinking so users can e.g. route all explore tasks to a cheap model.
+ */
+export interface TaskDefaults {
+  model?: string;
+  thinking?: ThinkingLevel;
+  maxTurns?: number;
+  maxCost?: number;
+  timeoutMs?: number;
+  /** Ordered backup models tried on transient provider failures. */
+  fallbackModels?: string[];
+  /** Extra attempts on transient failures. */
+  maxRetries?: number;
+}
+
+export type TaskDefaultsByProfile = Partial<Record<TaskProfile, TaskDefaults>>;
+
+export interface SubagentConfig {
+  maxTasksPerRun: number;
+  maxActiveProcesses: number;
+  maxQueuedTasks: number;
+  defaultTimeoutMs: number;
+  maxResultBytes: number;
+  maxResultLines: number;
+  maxDetailsTextBytes: number;
+  maxCompletedInMemory: number;
+  maxDepth: number;
+  /** Grace period between SIGTERM and SIGKILL for child process trees. */
+  killGraceMs: number;
+  sessionDir: string;
+  /** Durable root for subagent git worktrees (never a purgeable tmpdir). */
+  worktreeDir: string;
+  /** Durable root for machine-wide locks, slots, and run process records. */
+  lockDir: string;
+  /**
+   * Machine-wide concurrent child process cap across every Pi parent process.
+   * 0 disables the global limiter (per-session maxActiveProcesses still applies).
+   */
+  maxGlobalActive: number;
+  /** Days after which unchanged, orphaned worktrees are swept. 0 disables. */
+  worktreeRetentionDays: number;
+  /** Days after which unreferenced child session files are swept. Unset/0 disables. */
+  sessionRetentionDays?: number;
+  /** Days after which terminal run process records / dead locks are swept. */
+  lockRetentionDays: number;
+  /** Legacy per-profile defaults. Model/fallback fields are retained for config compatibility but ignored by model policy. */
+  taskDefaults?: TaskDefaultsByProfile;
+  /** Strict, user-owned model routing policy. Missing means new spawns are rejected. */
+  modelPolicy?: ModelPolicySnapshot;
+  /** Human-readable parse/read error for the model policy. */
+  modelPolicyError?: string;
+  /**
+   * Wrap-up grace turns after a max_turns/max_cost breach: the child is steered
+   * to produce a final answer and given this many extra turns before SIGTERM.
+   * 0 restores the old immediate-stop behavior.
+   */
+  graceTurns: number;
+  /** Milliseconds of protocol silence before a running child is flagged as stalled. 0 disables. */
+  stallAfterMs: number;
+  /** Additional silence after the stall flag before the child is killed (feeds retry). 0 disables kill. */
+  stallKillAfterMs: number;
+  /** Default extra attempts on transient failures (queued timeout, stall, provider error). */
+  maxRetries: number;
+  /**
+   * Ambient background-run widget above the editor. `"off"` clears any existing
+   * widget and skips refreshes.
+   */
+  widget: WidgetMode;
+  /**
+   * Batched completion followUp messages for async runs. `"off"` disables the
+   * CompletionBatcher so the parent is not notified on finish.
+   */
+  notifications: NotificationMode;
+}
+
+export const defaultConfig: SubagentConfig = {
+  maxTasksPerRun: 8,
+  maxActiveProcesses: 4,
+  maxQueuedTasks: 32,
+  defaultTimeoutMs: 15 * 60_000,
+  maxResultBytes: 50 * 1024,
+  maxResultLines: 2_000,
+  maxDetailsTextBytes: 10 * 1024,
+  maxCompletedInMemory: 20,
+  maxDepth: 2,
+  killGraceMs: 3_000,
+  sessionDir: path.join(os.homedir(), ".pi", "subagent-sessions"),
+  worktreeDir: path.join(os.homedir(), ".pi", "subagent-worktrees"),
+  lockDir: path.join(os.homedir(), ".pi", "subagent-locks"),
+  maxGlobalActive: 16,
+  worktreeRetentionDays: 7,
+  lockRetentionDays: 7,
+  graceTurns: 2,
+  stallAfterMs: 90_000,
+  stallKillAfterMs: 90_000,
+  maxRetries: 1,
+  widget: "background",
+  notifications: "batched",
+};
+
+/** User-facing config file. Env vars override file values; both override defaults. */
+export const CONFIG_FILE = path.join(os.homedir(), ".pi", "subagent.json");
+
+function positiveNumber(value: unknown, min = 1): number | undefined {
+  const parsed = typeof value === "string" ? Number(value) : value;
+  return typeof parsed === "number" && Number.isFinite(parsed) && parsed >= min ? parsed : undefined;
+}
+
+function nonEmptyString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function oneOf<T extends readonly string[]>(allowed: T, value: unknown): T[number] | undefined {
+  return typeof value === "string" && (allowed as readonly string[]).includes(value)
+    ? (value as T[number])
+    : undefined;
+}
+
+function prune<T extends object>(value: Partial<T>): Partial<T> {
+  return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined)) as Partial<T>;
+}
+
+function sanitizeTaskDefaults(raw: unknown): TaskDefaults | undefined {
+  if (!raw || typeof raw !== "object") return undefined;
+  const value = raw as Record<string, unknown>;
+  const thinking = typeof value.thinking === "string" && (THINKING_LEVELS as readonly string[]).includes(value.thinking)
+    ? (value.thinking as ThinkingLevel)
+    : undefined;
+  const fallbackModels = Array.isArray(value.fallbackModels)
+    ? value.fallbackModels.map(nonEmptyString).filter((model): model is string => !!model)
+    : undefined;
+  const defaults = prune<TaskDefaults>({
+    model: nonEmptyString(value.model),
+    thinking,
+    maxTurns: positiveNumber(value.maxTurns),
+    maxCost: positiveNumber(value.maxCost, 0),
+    timeoutMs: positiveNumber(value.timeoutMs),
+    fallbackModels: fallbackModels?.length ? fallbackModels : undefined,
+    maxRetries: positiveNumber(value.maxRetries, 0),
+  });
+  return Object.keys(defaults).length ? defaults : undefined;
+}
+
+function sanitizeTaskDefaultsByProfile(raw: unknown): TaskDefaultsByProfile | undefined {
+  if (!raw || typeof raw !== "object") return undefined;
+  const value = raw as Record<string, unknown>;
+  const result: TaskDefaultsByProfile = {};
+  for (const profile of ["explore", "review", "general"] as const) {
+    const defaults = sanitizeTaskDefaults(value[profile]);
+    if (defaults) result[profile] = defaults;
+  }
+  return Object.keys(result).length ? result : undefined;
+}
+
+/** Validate untrusted JSON overrides field-by-field; unknown keys are dropped. */
+export function sanitizeConfigOverrides(raw: unknown, source = MODEL_POLICY_CONFIG_FILE): Partial<SubagentConfig> {
+  if (!raw || typeof raw !== "object") return {};
+  const value = raw as Record<string, unknown>;
+  let modelPolicy: ModelPolicySnapshot | undefined;
+  let modelPolicyError: string | undefined;
+  if (Object.prototype.hasOwnProperty.call(value, "modelPolicy")) {
+    try {
+      modelPolicy = parseModelPolicy(value.modelPolicy, source);
+    } catch (error: any) {
+      modelPolicyError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  return prune<SubagentConfig>({
+    modelPolicy,
+    modelPolicyError,
+    taskDefaults: sanitizeTaskDefaultsByProfile(value.taskDefaults),
+    maxTasksPerRun: positiveNumber(value.maxTasksPerRun),
+    maxActiveProcesses: positiveNumber(value.maxActiveProcesses),
+    maxQueuedTasks: positiveNumber(value.maxQueuedTasks, 0),
+    defaultTimeoutMs: positiveNumber(value.defaultTimeoutMs),
+    maxResultBytes: positiveNumber(value.maxResultBytes, 1024),
+    maxResultLines: positiveNumber(value.maxResultLines, 10),
+    maxDetailsTextBytes: positiveNumber(value.maxDetailsTextBytes, 256),
+    maxCompletedInMemory: positiveNumber(value.maxCompletedInMemory),
+    maxDepth: positiveNumber(value.maxDepth, 0),
+    killGraceMs: positiveNumber(value.killGraceMs, 100),
+    sessionDir: nonEmptyString(value.sessionDir),
+    worktreeDir: nonEmptyString(value.worktreeDir),
+    lockDir: nonEmptyString(value.lockDir),
+    maxGlobalActive: positiveNumber(value.maxGlobalActive, 0),
+    worktreeRetentionDays: positiveNumber(value.worktreeRetentionDays, 0),
+    sessionRetentionDays: positiveNumber(value.sessionRetentionDays, 0),
+    lockRetentionDays: positiveNumber(value.lockRetentionDays, 0),
+    graceTurns: positiveNumber(value.graceTurns, 0),
+    stallAfterMs: positiveNumber(value.stallAfterMs, 0),
+    stallKillAfterMs: positiveNumber(value.stallKillAfterMs, 0),
+    maxRetries: positiveNumber(value.maxRetries, 0),
+    widget: oneOf(WIDGET_MODES, value.widget),
+    notifications: oneOf(NOTIFICATION_MODES, value.notifications),
+  });
+}
+
+export function configFromEnv(env: NodeJS.ProcessEnv = process.env): Partial<SubagentConfig> {
+  return prune<SubagentConfig>({
+    maxTasksPerRun: positiveNumber(env.PI_SUBAGENT_MAX_TASKS),
+    maxActiveProcesses: positiveNumber(env.PI_SUBAGENT_MAX_ACTIVE),
+    maxQueuedTasks: positiveNumber(env.PI_SUBAGENT_MAX_QUEUED, 0),
+    defaultTimeoutMs: positiveNumber(env.PI_SUBAGENT_TIMEOUT_MS),
+    maxDepth: positiveNumber(env.PI_SUBAGENT_MAX_DEPTH, 0),
+    killGraceMs: positiveNumber(env.PI_SUBAGENT_KILL_GRACE_MS, 100),
+    sessionDir: nonEmptyString(env.PI_SUBAGENT_SESSION_DIR),
+    worktreeDir: nonEmptyString(env.PI_SUBAGENT_WORKTREE_DIR),
+    lockDir: nonEmptyString(env.PI_SUBAGENT_LOCK_DIR),
+    maxGlobalActive: positiveNumber(env.PI_SUBAGENT_MAX_GLOBAL_ACTIVE, 0),
+    worktreeRetentionDays: positiveNumber(env.PI_SUBAGENT_WORKTREE_RETENTION_DAYS, 0),
+    sessionRetentionDays: positiveNumber(env.PI_SUBAGENT_SESSION_RETENTION_DAYS, 0),
+    lockRetentionDays: positiveNumber(env.PI_SUBAGENT_LOCK_RETENTION_DAYS, 0),
+    graceTurns: positiveNumber(env.PI_SUBAGENT_GRACE_TURNS, 0),
+    stallAfterMs: positiveNumber(env.PI_SUBAGENT_STALL_AFTER_MS, 0),
+    stallKillAfterMs: positiveNumber(env.PI_SUBAGENT_STALL_KILL_AFTER_MS, 0),
+    maxRetries: positiveNumber(env.PI_SUBAGENT_MAX_RETRIES, 0),
+    widget: oneOf(WIDGET_MODES, env.PI_SUBAGENT_WIDGET),
+    notifications: oneOf(NOTIFICATION_MODES, env.PI_SUBAGENT_NOTIFICATIONS),
+  });
+}
+
+/** defaults ← file overrides ← env overrides. Pure; suitable for tests. */
+export function loadConfig(
+  fileOverrides: Partial<SubagentConfig> = {},
+  env: NodeJS.ProcessEnv = process.env,
+): SubagentConfig {
+  return { ...defaultConfig, ...prune(fileOverrides), ...configFromEnv(env) };
+}
+
+/** Read + sanitize the optional user config file. Missing or invalid files yield {}. */
+export async function readConfigFile(file = CONFIG_FILE): Promise<Partial<SubagentConfig>> {
+  try {
+    return sanitizeConfigOverrides(JSON.parse(await fs.readFile(file, "utf8")), file);
+  } catch (error: any) {
+    if (error?.code === "ENOENT") return {};
+    return { modelPolicyError: `Could not read ${file} as JSON; model policy was not loaded.` };
+  }
+}

package/src/distill.ts

@@ -0,0 +1,222 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+
+/**
+ * Lifecycle-driven child-session distillation.
+ *
+ * A child session is "over" when its run reached a terminal state and nothing
+ * on the current parent branch references it for resume. At that point the
+ * meaningful impact of the session — what it was asked, what it answered,
+ * what it cost — is preserved in a small `.digest.json`, and the full
+ * transcript file is deleted. No wall-clock retention windows: the trigger is
+ * run lifecycle, with a short min-age guard purely against races with
+ * concurrent parents.
+ */
+
+export interface SessionDigest {
+  schemaVersion: 1;
+  sessionId: string;
+  file: string;
+  distilledAt: number;
+  startedAt?: number;
+  endedAt?: number;
+  durationMs?: number;
+  model?: string;
+  thinking?: string;
+  /** First user message (the task), capped. */
+  task?: string;
+  /** Last non-empty assistant text (the outcome), capped. */
+  finalOutput?: string;
+  assistantTurns: number;
+  toolCalls: number;
+  errors: number;
+  usage?: { input: number; output: number; cost: number };
+  /** Original transcript size in bytes. */
+  originalBytes: number;
+  /** Set when the transcript could not be parsed; digest is metadata-only. */
+  parseFailed?: boolean;
+}
+
+export interface LifecycleSweepReport {
+  distilled: string[];
+  kept: number;
+  failed: string[];
+}
+
+const TASK_CAP = 600;
+const OUTPUT_CAP = 8_192;
+/** Race guard for concurrent parents whose runs we cannot see. Not retention. */
+export const SESSION_MIN_AGE_MS = 60 * 60_000;
+
+function parseTimestamp(value: unknown): number | undefined {
+  if (typeof value === "number" && Number.isFinite(value)) return value > 1e12 ? value : value * 1000;
+  if (typeof value === "string") {
+    const ms = Date.parse(value);
+    if (Number.isFinite(ms)) return ms;
+  }
+  return undefined;
+}
+
+function textOf(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .filter((block: any) => block && block.type === "text" && typeof block.text === "string")
+    .map((block: any) => block.text)
+    .join("");
+}
+
+function countToolCalls(content: unknown): number {
+  if (!Array.isArray(content)) return 0;
+  return content.filter((block: any) => block && (block.type === "toolCall" || block.type === "toolUse" || block.type === "tool_use")).length;
+}
+
+function sessionIdFromFile(file: string): string {
+  const base = path.basename(file, ".jsonl");
+  const underscore = base.lastIndexOf("_");
+  return underscore >= 0 ? base.slice(underscore + 1) : base;
+}
+
+export function digestPathFor(file: string): string {
+  return file.replace(/\.jsonl$/, ".digest.json");
+}
+
+/** Extract the meaningful impact of one child session transcript. */
+export async function digestSessionFile(file: string): Promise<SessionDigest> {
+  const stat = await fs.stat(file);
+  const digest: SessionDigest = {
+    schemaVersion: 1,
+    sessionId: sessionIdFromFile(file),
+    file: path.basename(file),
+    distilledAt: Date.now(),
+    assistantTurns: 0,
+    toolCalls: 0,
+    errors: 0,
+    originalBytes: stat.size,
+  };
+  let raw: string;
+  try {
+    raw = await fs.readFile(file, "utf8");
+  } catch {
+    digest.parseFailed = true;
+    return digest;
+  }
+
+  let usageInput = 0;
+  let usageOutput = 0;
+  let usageCost = 0;
+  let sawUsage = false;
+  let parsedAny = false;
+
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    let entry: any;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    parsedAny = true;
+    const ts = parseTimestamp(entry.timestamp ?? entry.ts);
+    if (ts !== undefined) {
+      if (digest.startedAt === undefined) digest.startedAt = ts;
+      digest.endedAt = ts;
+    }
+    if (entry.type === "model_change") {
+      const model = entry.modelId ?? entry.model;
+      if (typeof model === "string" && model) digest.model = model;
+    }
+    if (entry.type === "thinking_level_change" && typeof entry.thinkingLevel === "string") {
+      digest.thinking = entry.thinkingLevel;
+    }
+    const message = entry.type === "message" ? entry.message : undefined;
+    if (!message || typeof message !== "object") continue;
+    if (message.role === "user" && digest.task === undefined) {
+      const text = textOf(message.content).trim();
+      if (text) digest.task = text.slice(0, TASK_CAP);
+    }
+    if (message.role === "assistant") {
+      digest.assistantTurns++;
+      digest.toolCalls += countToolCalls(message.content);
+      if (message.stopReason === "error") digest.errors++;
+      const text = textOf(message.content).trim();
+      if (text) digest.finalOutput = text.slice(0, OUTPUT_CAP);
+      const usage = message.usage;
+      if (usage && typeof usage === "object") {
+        const input = Number(usage.input);
+        const output = Number(usage.output);
+        const cost = typeof usage.cost === "number" ? usage.cost : Number(usage.cost?.total);
+        if (Number.isFinite(input)) { usageInput += input; sawUsage = true; }
+        if (Number.isFinite(output)) { usageOutput += output; sawUsage = true; }
+        if (Number.isFinite(cost)) { usageCost += cost; sawUsage = true; }
+      }
+    }
+  }
+
+  if (!parsedAny) digest.parseFailed = true;
+  if (sawUsage) digest.usage = { input: usageInput, output: usageOutput, cost: usageCost };
+  if (digest.startedAt !== undefined && digest.endedAt !== undefined) {
+    digest.durationMs = Math.max(0, digest.endedAt - digest.startedAt);
+  }
+  return digest;
+}
+
+/**
+ * Distill one session file to a digest and remove the transcript.
+ * The transcript is only removed after the digest is durably written.
+ */
+export async function distillSessionFile(file: string): Promise<SessionDigest> {
+  const digest = await digestSessionFile(file);
+  const target = digestPathFor(file);
+  const tmp = `${target}.${process.pid}.tmp`;
+  await fs.writeFile(tmp, JSON.stringify(digest, null, 2), "utf8");
+  await fs.rename(tmp, target);
+  await fs.rm(file, { force: true });
+  return digest;
+}
+
+export interface LifecycleSweepOptions {
+  /** Session ids that must be kept (referenced by the live parent branch). */
+  keep: ReadonlySet<string>;
+  /** Session ids that are busy machine-wide (running run records, live locks). */
+  busy: ReadonlySet<string>;
+  minAgeMs?: number;
+  now?: number;
+}
+
+/**
+ * Distill every child session whose run is over. "Over" is a lifecycle fact:
+ * not referenced by the current branch, not owned by any running run record,
+ * not locked, and past a short race guard. Age plays no retention role.
+ */
+export async function sweepSessionsLifecycle(
+  sessionDir: string,
+  options: LifecycleSweepOptions,
+): Promise<LifecycleSweepReport> {
+  const report: LifecycleSweepReport = { distilled: [], kept: 0, failed: [] };
+  const minAge = options.minAgeMs ?? SESSION_MIN_AGE_MS;
+  const now = options.now ?? Date.now();
+  const entries = await fs.readdir(sessionDir, { withFileTypes: true }).catch(() => []);
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.endsWith(".jsonl")) continue;
+    const base = entry.name.slice(0, -".jsonl".length);
+    const isMatch = (id: string) => !!id && (base === id || base.includes(id));
+    if ([...options.keep].some(isMatch) || [...options.busy].some(isMatch)) {
+      report.kept++;
+      continue;
+    }
+    const file = path.join(sessionDir, entry.name);
+    const stat = await fs.stat(file).catch(() => undefined);
+    if (!stat || now - stat.mtimeMs < minAge) {
+      report.kept++;
+      continue;
+    }
+    try {
+      await distillSessionFile(file);
+      report.distilled.push(file);
+    } catch {
+      report.failed.push(file);
+    }
+  }
+  return report;
+}