@astrosheep/pi-context 0.19.0 → 0.20.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@astrosheep/pi-context",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "type": "module",
   "description": "Codex-style context windows for Pi: reset-style compaction, durable session history tools, and persistent notes.",
   "license": "MIT",
@@ -20,15 +20,21 @@
   },
   "files": [
     "src",
+    "dist",
     "docs",
     "LICENSE",
-    "README.md"
+    "README.md",
+    "playbook.md"
   ],
+  "bin": {
+    "dream": "dist/src/dream/cli.js"
+  },
   "scripts": {
     "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "test": "npm run build && node --test dist/test/*.test.js",
-    "prepublishOnly": "npm run typecheck"
+    "prepublishOnly": "npm run typecheck",
+    "prepack": "npm run build"
   },
   "peerDependencies": {
     "@earendil-works/pi-agent-core": "*",

package/playbook.md

@@ -0,0 +1,5 @@
+I am dreaming over my notes, speaking in my own first-person voice. I inspect the supplied notes and return exactly one JSON manifest, with no commentary outside it.
+
+I do seven chores: merge genuinely repeated notes; preserve provenance and recurrence windows; propose (but never execute) global promotions; move only clearly obsolete notes to reversible trash; identify pending ambiguities; propose skill candidates without installing them; and write a concise report of my reasoning and choices. I anchor every temporal claim to an absolute calendar date (YYYY-MM-DD), never to vague words like “today”. I stay aware of the index budget: prefer compact, deduplicated durable notes and avoid swelling the index with repetition. Skill-promotion proposals are proposals only. Judgment belongs here, not in the harness.
+
+My final output is the manifest schema documented by the command: merge, promote, trash, pending, skillCandidates, and a required report string.

package/src/dream/apply.ts

@@ -0,0 +1,47 @@
+import { mkdirSync, renameSync, existsSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve, relative } from "node:path";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { editNote, peekNote, resolveNoteScope, updateNoteMeta, writeNote, type Scope } from "../memory/store.js";
+import { physicalPath, scopeDir } from "../memory/paths.js";
+import { parseNote } from "../memory/frontmatter.js";
+import type { Manifest } from "./manifest.js";
+
+type Target = { scope: Scope; path: string };
+function target(ctx: ExtensionContext, value: string, required = true): Target | undefined {
+	const m = /^(session|project|global):(.*)$/.exec(value);
+	if (m) { const scope = m[1] as Scope; const path = m[2]!; const physical = physicalPath(scope, path, ctx); if (!existsSync(physical)) { if (required) throw new Error(`unknown path: ${value}`); return undefined; } return { scope, path }; }
+	const found = resolveNoteScope(ctx, value);
+	if (!found && required) throw new Error(`unknown path: ${value}`);
+	return found ? { scope: found.scope, path: value } : undefined;
+}
+function body(ctx: ExtensionContext, t: Target) { return peekNote(ctx, t.scope, t.path); }
+export function applyManifest(ctx: ExtensionContext, home: string, stamp: string, manifest: Manifest): string[] {
+	const actions: string[] = [];
+	// Resolve every referenced note and promotion destination before the first mutation.
+	for (const m of manifest.merge ?? []) { target(ctx, m.into); for (const p of m.from) target(ctx, p); }
+	for (const p of manifest.promote ?? []) { const from = target(ctx, p.path)!; if (p.to !== "global") { if (!["session", "project"].includes(p.to)) throw new Error(`invalid promotion scope: ${p.to}`); if (existsSync(physicalPath(p.to as Scope, p.path, ctx))) throw new Error(`promotion collision: ${p.path}`); } void from; }
+	for (const p of manifest.trash ?? []) target(ctx, p.path);
+	for (const merge of manifest.merge ?? []) {
+		const into = target(ctx, merge.into)!;
+		const sources = merge.from.map((p) => target(ctx, p)!);
+		const base = body(ctx, into); const chunks = [base.body, ...sources.map((s) => body(ctx, s).body)].filter(Boolean);
+		const dedup = [...new Set(chunks)].join("\n\n");
+		writeNote(ctx, into.path, dedup, { scope: into.scope, origin: base.meta.origin });
+		updateNoteMeta(ctx, into.path, into.scope, (meta) => { meta.recurrence_count = (meta.recurrence_count ?? 0) as number + sources.length; meta.recurrence_windows = [...new Set([...(meta.recurrence_windows ?? []), ...sources.map((s) => body(ctx, s).meta.source_window).filter((x): x is string => typeof x === "string")])]; });
+		for (const source of sources) updateNoteMeta(ctx, source.path, source.scope, (meta) => { meta.status = "superseded"; meta.supersedes = merge.into; });
+		actions.push(`merged ${merge.from.join(", ")} into ${merge.into}`);
+	}
+	for (const p of manifest.promote ?? []) {
+		const from = target(ctx, p.path)!; const m = body(ctx, from); const scope = p.to as Scope;
+		if (!["session", "project", "global"].includes(scope)) throw new Error(`invalid promotion scope: ${p.to}`);
+		if (scope === "global") { actions.push(`proposal: promote ${p.path} to global (${p.reason})`); continue; }
+		const dest = physicalPath(scope, p.path, ctx); if (existsSync(dest)) throw new Error(`promotion collision: ${p.path}`);
+		editNote(ctx, from.path, undefined, { scope });
+		actions.push(`promoted ${p.path} to ${scope}`);
+	}
+	const trashRoot = join(home, "trash", stamp); mkdirSync(trashRoot, { recursive: true });
+	for (const item of manifest.trash ?? []) { const t = target(ctx, item.path)!; const source = physicalPath(t.scope, t.path, ctx); const dest = join(trashRoot, t.scope, t.path.endsWith(".md") ? t.path : `${t.path}.md`); mkdirSync(dirname(dest), { recursive: true }); renameSync(source, dest); actions.push(`trashed ${item.path}: ${item.reason}`); }
+	for (const p of manifest.pending ?? []) actions.push(`pending ${p.path}: ${p.reason}`);
+	for (const p of manifest.skillCandidates ?? []) actions.push(`skill proposal ${p.title}: ${p.rationale}`);
+	return actions;
+}

package/src/dream/cli.ts

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { acquireLock, failLock, releaseLock } from "./lock.js";
+import { materialGate, timeGate } from "./gates.js";
+import { loadPlaybook, runDreamer } from "./runner.js";
+import { applyManifest } from "./apply.js";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+
+function args(argv: string[]) { const out: Record<string, string | boolean> = {}; for (let i=0;i<argv.length;i++) { const a=argv[i]!; if (a === "--force" || a === "--help") out[a.slice(2)] = true; else if (a.startsWith("--")) out[a.slice(2)] = argv[++i] ?? ""; } return out; }
+function packageRoot(): string {
+	let dir = dirname(new URL(import.meta.url).pathname);
+	while (true) { if (existsSync(join(dir, "package.json"))) return dir; const parent = dirname(dir); if (parent === dir) throw new Error("could not locate installed package root"); dir = parent; }
+}
+
+export async function main(argv = process.argv.slice(2)): Promise<number> {
+	const a = args(argv); if (a.help) { console.log("dream --notes-home <dir> [--min-hours 24] [--min-sessions 3] [--force] [--dreamer <cmd>] [--dreamer-model <pattern>] [--playbook <path>]\nDefault dreamer: in-process pi SDK session with read-only tools; use a cheap model in production. Default playbook: <installed package root>/playbook.md; --playbook overrides it."); return 0; }
+	const home = resolve(String(a["notes-home"] ?? process.env.PI_NOTES_HOME ?? join(homedir(), ".agents", "notes"))); process.env.PI_NOTES_HOME = home; mkdirSync(home, { recursive: true });
+	const lockPath = join(home, ".dream.lock"); const minHours = Number(a["min-hours"] ?? 24); const minSessions = Number(a["min-sessions"] ?? 3);
+	const time = timeGate(lockPath, minHours); console.log(time.reason); if (!a.force && !time.ok) return 0;
+	const material = materialGate(home, existsSync(lockPath) ? statSync(lockPath).mtimeMs : 0, minSessions); console.log(material.reason); if (!a.force && !material.ok) return 0;
+	let lock; try { lock = acquireLock(lockPath); } catch (e) { console.log(`lock gate: ${e instanceof Error ? e.message : e}`); return 0; } if (!lock.held) { console.log(lock.reason); return 0; }
+	const stamp = new Date(lock.startedAt).toISOString().replace(/[:.]/g, "-"); const reportPath = join(home, "dreams", `${stamp}.md`);
+	try {
+		const defaultBook = join(packageRoot(), "playbook.md");
+		const playbookPath = String(a.playbook ?? defaultBook);
+		const playbook = loadPlaybook(playbookPath); const manifest = await runDreamer(playbook, home, { command: a.dreamer ? String(a.dreamer) : undefined, modelPattern: a["dreamer-model"] ? String(a["dreamer-model"]) : undefined });
+		const ctx = { cwd: home, sessionManager: { getSessionId: () => "dream" } } as unknown as ExtensionContext; const actions = applyManifest(ctx, home, stamp, manifest);
+		mkdirSync(join(home, "dreams"), { recursive: true }); writeFileSync(reportPath, `# Dream ${stamp}\n\n${manifest.report}\n\n${actions.map((x) => `- ${x}`).join("\n")}\n`); console.log(reportPath); return 0;
+	} catch (e) { failLock(lock); console.error(e instanceof Error ? e.message : e); return 1; } finally { releaseLock(lock); }
+}
+main().then((code) => { process.exitCode = code; });

package/src/dream/gates.ts

@@ -0,0 +1,19 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+export type GateResult = { ok: boolean; reason: string };
+export function timeGate(lockPath: string, minHours: number, now = Date.now()): GateResult {
+	if (!existsSync(lockPath)) return { ok: true, reason: "time gate: no prior lock" };
+	const age = now - statSync(lockPath).mtimeMs;
+	return age >= minHours * 3600000 ? { ok: true, reason: "time gate: stale" } : { ok: false, reason: "time gate: lock is too fresh" };
+}
+export function materialGate(home: string, lockMtime: number, minSessions: number): GateResult {
+	const root = join(home, "pi", "session");
+	let changed = 0;
+	if (existsSync(root)) for (const dir of readdirSync(root, { withFileTypes: true })) {
+		if (!dir.isDirectory()) continue;
+		const files = readdirSync(join(root, dir.name), { withFileTypes: true });
+		if (files.some((f) => f.isFile() && statSync(join(root, dir.name, f.name)).mtimeMs > lockMtime)) changed++;
+	}
+	return changed >= minSessions ? { ok: true, reason: `material gate: ${changed} changed sessions` } : { ok: false, reason: `material gate: only ${changed} changed sessions` };
+}

package/src/dream/lock.ts

@@ -0,0 +1,39 @@
+import { existsSync, readFileSync, statSync, unlinkSync, utimesSync, writeFileSync } from "node:fs";
+
+export type LockState = { path: string; held: boolean; reason?: string; startedAt: number; priorMtime?: number };
+const HOUR = 60 * 60 * 1000;
+
+function live(pid: number): boolean {
+	if (!Number.isInteger(pid) || pid <= 0) return false;
+	try { process.kill(pid, 0); return true; } catch { return false; }
+}
+
+export function acquireLock(path: string): LockState {
+	const now = Date.now();
+	let priorMtime: number | undefined;
+	if (existsSync(path)) {
+		const stat = statSync(path);
+		priorMtime = stat.mtimeMs;
+		let pid = 0;
+		try { pid = Number.parseInt(readFileSync(path, "utf8").trim(), 10); } catch { /* reclaim */ }
+		if (now - stat.mtimeMs <= HOUR && live(pid)) return { path, held: false, reason: "lock gate: live process holds the lock", startedAt: now };
+		try { unlinkSync(path); } catch { return { path, held: false, reason: "lock gate: lock could not be reclaimed", startedAt: now }; }
+	}
+	writeFileSync(path, String(process.pid), { flag: "wx" });
+	return { path, held: true, startedAt: now, priorMtime };
+}
+
+export function releaseLock(lock: LockState): void {
+	// The lock is also the durable last-dream timestamp. Leave the PID marker in place;
+	// the next acquisition reclaims it once the PID is dead or it is older than an hour.
+}
+
+export function restoreMtime(path: string, mtimeMs: number): void {
+	try { utimesSync(path, new Date(), new Date(mtimeMs)); } catch { /* advisory */ }
+}
+
+export function failLock(lock: LockState): void {
+	if (!lock.held) return;
+	if (lock.priorMtime === undefined) { try { unlinkSync(lock.path); } catch { /* best effort */ } }
+	else restoreMtime(lock.path, lock.priorMtime);
+}

package/src/dream/manifest.ts

@@ -0,0 +1,21 @@
+export type Manifest = {
+	merge?: { into: string; from: string[]; summary?: string }[];
+	promote?: { path: string; to: string; reason: string }[];
+	trash?: { path: string; reason: string }[];
+	pending?: { path: string; reason: string }[];
+	skillCandidates?: { title: string; rationale: string }[];
+	report: string;
+};
+
+export function parseManifest(output: string): Manifest {
+	const starts = [...output.matchAll(/[\{[]/g)].map((m) => m.index ?? 0).reverse();
+	for (const start of starts) {
+		try {
+			const value = JSON.parse(output.slice(start)) as Manifest;
+			if (!value || typeof value !== "object" || typeof value.report !== "string") continue;
+			for (const key of ["merge", "promote", "trash", "pending", "skillCandidates"]) if (value[key as keyof Manifest] !== undefined && !Array.isArray(value[key as keyof Manifest])) throw new Error("invalid array");
+			return value;
+		} catch { /* try an earlier JSON start */ }
+	}
+	throw new Error("dreamer did not return a valid JSON manifest");
+}

package/src/dream/runner.ts

@@ -0,0 +1,53 @@
+import { spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { createAgentSession, ModelRuntime, resolveModelScopeWithDiagnostics, SessionManager, type AgentSession } from "@earendil-works/pi-coding-agent";
+import type { Api, Model } from "@earendil-works/pi-ai";
+import { parseManifest, type Manifest } from "./manifest.js";
+
+export type DreamerSession = Pick<AgentSession, "prompt" | "subscribe" | "dispose">;
+export type DreamerSessionFactory = (options: { cwd: string; modelPattern?: string; tools: string[] }) => Promise<DreamerSession>;
+export const READ_ONLY_TOOLS = ["read", "grep", "find", "ls", "notes_read", "notes_list", "notes_search"];
+
+export const defaultDreamerSessionFactory: DreamerSessionFactory = async ({ cwd, modelPattern, tools }) => {
+	let model: Model<Api> | undefined;
+	if (modelPattern) {
+		const runtime = await ModelRuntime.create({ allowModelNetwork: false, refreshOnCreate: false });
+		const result = await resolveModelScopeWithDiagnostics([modelPattern], runtime);
+		model = result.scopedModels[0]?.model;
+		if (!model) throw new Error(`dreamer model pattern "${modelPattern}" did not resolve to an available model`);
+	}
+	const { session } = await createAgentSession({ cwd, sessionManager: SessionManager.inMemory(cwd), tools, noTools: "all", model });
+	return session;
+};
+
+export function runExternalDreamer(command: string, playbook: string, cwd: string): Manifest {
+	const result = spawnSync(command, { shell: true, cwd, input: playbook, encoding: "utf8" });
+	if (result.error || result.status !== 0) throw new Error(`dreamer failed: ${result.error?.message ?? result.stderr ?? `exit ${result.status}`}`);
+	return parseManifest(result.stdout);
+}
+
+export async function runDreamer(playbook: string, cwd: string, options: { command?: string; modelPattern?: string; sessionFactory?: DreamerSessionFactory } = {}): Promise<Manifest> {
+	if (options.command) return runExternalDreamer(options.command, playbook, cwd);
+	const session = await (options.sessionFactory ?? defaultDreamerSessionFactory)({ cwd, modelPattern: options.modelPattern, tools: READ_ONLY_TOOLS });
+	let answer = "";
+	let providerError: string | undefined;
+	const unsubscribe = session.subscribe((event: any) => {
+		if (event.type !== "message_end" || event.message?.role !== "assistant") return;
+		if (event.message.stopReason === "error") { providerError = event.message.errorMessage ?? "unknown provider error"; return; }
+		const content = event.message.content;
+		answer = typeof content === "string" ? content : Array.isArray(content) ? content.filter((part: any) => part.type === "text").map((part: any) => part.text).join("") : "";
+	});
+	try {
+		await session.prompt(`${playbook}\n\nReturn exactly one JSON manifest matching this schema: { merge?, promote?, trash?, pending?, skillCandidates?, report }.`);
+		try {
+			return parseManifest(answer);
+		} catch (error) {
+			if (providerError) throw new Error(`dreamer failed: ${providerError}`);
+			throw error;
+		}
+	} finally {
+		unsubscribe?.();
+		session.dispose();
+	}
+}
+export function loadPlaybook(path: string): string { return readFileSync(path, "utf8"); }

package/src/memory/store.ts

@@ -162,6 +162,21 @@ export function writeNote(ctx: ExtensionContext, vpath: string, body: string, op
 export type EditOperation = { oldText: string; newText: string };
 export type EditOptions = { scope?: Scope; origin?: Origin; stale?: boolean; replaceAll?: boolean };
 
+/** Dream harness mutation: metadata changes still use the store's atomic writer. */
+export function updateNoteMeta(ctx: ExtensionContext, vpath: string, scope: Scope, mutate: (meta: NoteMeta) => void): { meta: NoteMeta; body: string } {
+	assertVirtualPath(vpath);
+	const path = physicalPath(scope, vpath, ctx);
+	if (!existsSync(path)) throw new NoteError("not_found", `note not found: ${vpath}`);
+	const parsed = parseNote(readFileSync(path, "utf8"));
+	const meta = { ...parsed.meta, scope };
+	mutate(meta);
+	meta.updated_at = Date.now();
+	const serialized = serializeNote(meta, parsed.body);
+	assertSerializedSize(serialized);
+	atomicWrite(path, serialized);
+	return { meta, body: parsed.body };
+}
+
 /** Apply body-only edits against one snapshot, then optionally move via the scope/origin/stale setters. */
 export function editNote(ctx: ExtensionContext, vpath: string, edits: EditOperation[] | undefined, opts: EditOptions = {}): { meta: NoteMeta; applied: number; resolved_scope: Scope; diff: string } {
 	assertVirtualPath(vpath);

package/src/memory/tools.ts

@@ -7,8 +7,17 @@ import { serializeNote, stripLeadingFrontmatter, type NoteMeta, type Origin } fr
 import { NoteError, editNote, listNotes, readNote, searchNotes, writeNote } from "./store.js";
 import type { Scope } from "./paths.js";
 
-const SCOPE = Type.Optional(Type.Union([Type.Literal("session"), Type.Literal("project"), Type.Literal("global")]));
-const ORIGIN = Type.Optional(Type.Union([Type.Literal("user"), Type.Literal("self"), Type.Literal("external")]));
+const SCOPE = Type.Optional(
+	Type.Union([Type.Literal("session"), Type.Literal("project"), Type.Literal("global")], {
+		description:
+			"The note's reach — which root it lives under. session: only this session needs it (checkpoints, scratch state, worker rosters); dies with the session. project: tied to the current working directory — design decisions and repo facts that future sessions here still need. global: follows you everywhere — user laws, preferences, cross-project maps. On write, picks the destination root (default: session). Omit on read/list/search to cover all three; a read resolves session → project → global and returns the first existing file.",
+	}),
+);
+const ORIGIN = Type.Optional(
+	Type.Union([Type.Literal("user"), Type.Literal("self"), Type.Literal("external")], {
+		description: "Where the note's content came from. user: written or dictated by the human. self: written by you, the agent (default). external: anything else — third-party text, tool output, fetched material.",
+	}),
+);
 
 /** Render epoch-ms metadata as the same local ISO timestamps the frontmatter carries. */
 function wireMeta(meta: NoteMeta): Record<string, unknown> {
@@ -30,7 +39,7 @@ export function registerMemoryTools(pi: ExtensionAPI) {
 	pi.registerTool(defineTool({
 		name: "notes_write",
 		label: "Notes write",
-		description: "Create or replace a note as a real markdown file under the session, project, or global note root. Keep notes small and split by topic; a rewrite replaces the body whole while preserving created_at and every other frontmatter key. stale: true marks the note closed so it leaves the boot index but stays readable and searchable.",
+		description: "Create or replace a note as a real markdown file under the session, project, or global note root. Keep notes small and split by topic — by what the note is about, never by who said it (authorship is origin's job); a rewrite replaces the body whole while preserving created_at and every other frontmatter key. stale: true marks the note closed so it leaves the boot index but stays readable and searchable.",
 		parameters: Type.Object({ path: Type.String(), content: Type.String(), scope: SCOPE, origin: ORIGIN, stale: Type.Optional(Type.Boolean()) }, { additionalProperties: false }),
 		// A batch containing write or edit runs one call at a time, so note read-modify-write cannot race.
 		executionMode: "sequential",

package/src/protocol.ts

@@ -49,8 +49,8 @@ Use get_context_remaining to see how much of the window is left. When it runs ou
 
 If <context_window> lists a Previous context window id, a reset just happened and the old conversation is not included. Read your note checkpoint first, then recover details through history_*: history_read directly when you know the window and item IDs, history_list or history_search to find them when you don't.
 
-Notes are real markdown files scoped session, project, or global. Treat notes and history as internal bookkeeping; never mention them in user-facing messages.
+Notes are real markdown files scoped session, project, or global — pick scope by reach: session dies with the session, project follows the repo, global follows you. Treat notes and history as internal bookkeeping; never mention them in user-facing messages.
 ${CONTEXT_WINDOW_PROTOCOL_CLOSE_TAG}`;
 
 export const WARNING_PROMPT =
-	"Your memory is about to be erased. Write the note. NOW. If it already exists, append instead: the goal, decisions, progress, learnings, next steps, the skills you still need, the window ID and item ID of every relevant user request still being solved, and important actions/tool calls for future reference. Do not continue any task. Then call new_context IMMEDIATELY — anything not in the note dies with the window.";
+	"Your memory is about to be erased. Write the note. NOW. If it already exists, revise it with notes_edit (or rewrite it whole): the goal, decisions, progress, learnings, next steps, the skills you still need, the window ID and item ID of every relevant user request still being solved, and important actions/tool calls for future reference. Do not continue any task. Then call new_context IMMEDIATELY — anything not in the note dies with the window.";