@pi-unipi/unipi 2.4.2 → 2.6.0

package/packages/memory/index.ts

@@ -35,6 +35,54 @@ import { isEmbeddingReady, hasModelChanged } from "./settings.js";
 /** Package version */
 const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
 
+interface MemoryReminderInput {
+  projectName: string;
+  memories: Array<{ title: string }>;
+  canSearch: boolean;
+  canStore: boolean;
+}
+
+/**
+ * Build the deterministic, model-visible first-turn memory reminder.
+ * Exported so provider-payload regressions can exercise the exact production
+ * content without initializing a storage backend.
+ */
+export function buildMemoryRecallReminder(input: MemoryReminderInput): string {
+  const lines = [
+    "## 🧠 Memory System Active",
+    "",
+    `You have ${input.memories.length} memories stored for project "${input.projectName}".`,
+  ];
+
+  if (input.canSearch && input.memories.length > 0) {
+    const titleList = input.memories.slice(0, 20).map((memory) => `- ${memory.title}`).join("\n");
+    const extra = input.memories.length > 20
+      ? `\n... and ${input.memories.length - 20} more`
+      : "";
+    lines.push(
+      "**BEFORE starting work**, call `memory_search` with relevant keywords to check for existing context.",
+      "",
+      "Available memories:",
+      titleList + extra,
+    );
+  }
+
+  if (input.canStore) {
+    lines.push(
+      "",
+      "**AFTER completing the task**, if you learned something non-obvious,",
+      "call `memory_store` to save it for future sessions.",
+    );
+  }
+
+  lines.push(
+    "",
+    "Guardrails: read max 10 memory results per search. Update existing memories instead of creating duplicates.",
+  );
+
+  return lines.join("\n");
+}
+
 /** Storage instance for current project */
 let projectStorage: MemoryStorage | null = null;
 
@@ -251,44 +299,18 @@ export default function (pi: ExtensionAPI) {
       return;
     }
 
-    const lines = [
-      "## 🧠 Memory System Active",
-      "",
-      `You have ${projectMemories.length} memories stored for project "${projectName}".`,
-    ];
-
-    if (canSearch && projectMemories.length > 0) {
-      const titleList = projectMemories.slice(0, 20).map(m => `- ${m.title}`).join("\n");
-      const extra = projectMemories.length > 20 ? `\n... and ${projectMemories.length - 20} more` : "";
-      lines.push(
-        "**BEFORE starting work**, call `memory_search` with relevant keywords to check for existing context.",
-        "",
-        "Available memories:",
-        titleList + extra,
-      );
-    } else {
-      recallDone = true;
-    }
-
-    if (canStore) {
-      lines.push(
-        "",
-        "**AFTER completing the task**, if you learned something non-obvious,",
-        "call `memory_store` to save it for future sessions.",
-      );
-    } else {
-      storeDone = true;
-    }
-
-    lines.push(
-      "",
-      "Guardrails: read max 10 memory results per search. Update existing memories instead of creating duplicates.",
-    );
+    if (!canSearch || projectMemories.length === 0) recallDone = true;
+    if (!canStore) storeDone = true;
 
     return {
       message: {
         customType: "unipi-memory-recall-reminder",
-        content: lines.join("\n"),
+        content: buildMemoryRecallReminder({
+          projectName,
+          memories: projectMemories,
+          canSearch,
+          canStore,
+        }),
         display: false,
       },
     };

package/packages/memory/mempalace.ts

@@ -12,6 +12,8 @@
  */
 
 import { spawnSync, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createRequire } from "node:module";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import * as os from "node:os";
@@ -27,11 +29,74 @@ const MIGRATED_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-mi
 const PING_VERIFIED_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-ping-verified");
 const PING_VERIFIED_TTL_MS = 24 * 60 * 60 * 1000; // 24h
 
-/** Path to the bundled bridge script. */
-const BRIDGE_PATH = path.join(dirname(fileURLToPath(import.meta.url)), "bridge", "mempalace_bridge.py");
+/** Migration marker schema. Increment when migration semantics change. */
+export const MIGRATION_STATE_VERSION = 2;
+
+export interface MigrationResult {
+  discovered: number;
+  imported: number;
+  updated: number;
+  skipped: number;
+  failed: number;
+  verified: number;
+  errors?: string[];
+}
+
+export interface MigrationState {
+  version: number;
+  completedAt: string;
+  sourceFingerprint: string;
+  result: MigrationResult;
+}
+
+let cachedBridgePath: string | null | undefined;
+
+function isReadableFile(candidate: string): boolean {
+  try {
+    return fs.statSync(candidate).isFile() && fs.accessSync(candidate, fs.constants.R_OK) === undefined;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Resolve the Python bridge in both supported layouts:
+ *
+ * - standalone @pi-unipi/memory: <package>/bridge/mempalace_bridge.py
+ * - bundled @pi-unipi/unipi:     <umbrella>/packages/memory/bridge/...
+ *
+ * The explicit environment override is useful for custom packagers. The
+ * package-resolution fallback handles npm layouts where dependencies are not
+ * hoisted beside the umbrella package.
+ */
+export function resolveMempalaceBridgePath(moduleUrl = import.meta.url): string | null {
+  const moduleDir = path.dirname(fileURLToPath(moduleUrl));
+  const candidates: string[] = [];
+  if (process.env.UNIPI_MEMPALACE_BRIDGE) {
+    candidates.push(path.resolve(process.env.UNIPI_MEMPALACE_BRIDGE));
+  }
+  candidates.push(
+    path.join(moduleDir, "bridge", "mempalace_bridge.py"),
+    path.join(moduleDir, "..", "memory", "bridge", "mempalace_bridge.py"),
+  );
+
+  try {
+    const require = createRequire(moduleUrl);
+    const memoryPackage = require.resolve("@pi-unipi/memory/package.json");
+    candidates.push(path.join(path.dirname(memoryPackage), "bridge", "mempalace_bridge.py"));
+  } catch { /* standalone/source layout may not expose package resolution */ }
+
+  return candidates.find(isReadableFile) ?? null;
+}
+
+function getBridgePath(): string | null {
+  if (cachedBridgePath === undefined) cachedBridgePath = resolveMempalaceBridgePath();
+  return cachedBridgePath;
+}
 
-function dirname(p: string): string {
-  return path.dirname(p);
+/** Clear bridge discovery cache (primarily for recovery/tests). */
+export function invalidateBridgePathCache(): void {
+  cachedBridgePath = undefined;
 }
 
 export interface BridgeResponse<T> {
@@ -196,22 +261,99 @@ export function invalidatePingVerified(): void {
   try { if (fs.existsSync(PING_VERIFIED_FLAG)) fs.unlinkSync(PING_VERIFIED_FLAG); } catch { /* ignore */ }
 }
 
-/** Has the one-way legacy migration been completed? */
-export function isMigrated(): boolean {
-  return fs.existsSync(MIGRATED_FLAG);
+/**
+ * Fingerprint all durable legacy sources. This makes migration catch-up
+ * automatic for existing installations instead of treating a years-old
+ * timestamp flag as permanently complete.
+ */
+export function getMemorySourceFingerprint(
+  sourceDir = path.join(os.homedir(), ".unipi", "memory"),
+): string {
+  const hash = createHash("sha256");
+  if (!fs.existsSync(sourceDir)) return hash.update("missing").digest("hex");
+
+  const visit = (dir: string): void => {
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true })
+        .sort((a, b) => a.name.localeCompare(b.name));
+    } catch {
+      hash.update(`unreadable:${dir}`);
+      return;
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        visit(full);
+        continue;
+      }
+      if (!entry.isFile() || (entry.name !== "memory.db" && !entry.name.endsWith(".md"))) continue;
+      try {
+        const stat = fs.statSync(full);
+        hash.update(`${path.relative(sourceDir, full)}\0${stat.size}\0${stat.mtimeMs}\n`);
+      } catch {
+        hash.update(`unreadable:${path.relative(sourceDir, full)}\n`);
+      }
+    }
+  };
+  visit(sourceDir);
+  return hash.digest("hex");
 }
 
-/** Mark the one-way legacy migration complete. */
-export function markMigrated(): void {
+/** Read a verified v2 migration state. Legacy timestamp markers return null. */
+export function readMigrationState(flagPath = MIGRATED_FLAG): MigrationState | null {
   try {
-    fs.mkdirSync(path.dirname(MIGRATED_FLAG), { recursive: true });
-    fs.writeFileSync(MIGRATED_FLAG, new Date().toISOString(), "utf-8");
-  } catch { /* ignore */ }
+    const parsed = JSON.parse(fs.readFileSync(flagPath, "utf-8")) as MigrationState;
+    if (
+      parsed?.version !== MIGRATION_STATE_VERSION ||
+      typeof parsed.completedAt !== "string" ||
+      typeof parsed.sourceFingerprint !== "string" ||
+      !parsed.result ||
+      parsed.result.failed !== 0 ||
+      parsed.result.verified !== parsed.result.discovered
+    ) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+/** Is the palace verified against the current durable source set? */
+export function isMigrated(
+  sourceFingerprint = getMemorySourceFingerprint(),
+  flagPath = MIGRATED_FLAG,
+): boolean {
+  return readMigrationState(flagPath)?.sourceFingerprint === sourceFingerprint;
+}
+
+/** Mark migration complete only after the caller has verified every record. */
+export function markMigrated(
+  sourceFingerprint: string,
+  result: MigrationResult,
+  flagPath = MIGRATED_FLAG,
+): boolean {
+  if (result.failed !== 0 || result.verified !== result.discovered) return false;
+  try {
+    fs.mkdirSync(path.dirname(flagPath), { recursive: true });
+    const state: MigrationState = {
+      version: MIGRATION_STATE_VERSION,
+      completedAt: new Date().toISOString(),
+      sourceFingerprint,
+      result,
+    };
+    const temp = `${flagPath}.${process.pid}.tmp`;
+    fs.writeFileSync(temp, JSON.stringify(state, null, 2), "utf-8");
+    fs.renameSync(temp, flagPath);
+    return true;
+  } catch {
+    return false;
+  }
 }
 
 /** Force re-migration by clearing the flag. */
-export function clearMigratedFlag(): void {
-  try { if (fs.existsSync(MIGRATED_FLAG)) fs.unlinkSync(MIGRATED_FLAG); } catch { /* ignore */ }
+export function clearMigratedFlag(flagPath = MIGRATED_FLAG): void {
+  try { if (fs.existsSync(flagPath)) fs.unlinkSync(flagPath); } catch { /* ignore */ }
 }
 
 /**
@@ -223,7 +365,10 @@ export function runBridge<T = unknown>(
   palace: string,
   cmd: string,
   args: Record<string, unknown> = {},
+  timeoutMs = 60_000,
 ): T | null {
+  const bridgePath = getBridgePath();
+  if (!bridgePath) return null;
   let argsJson: string;
   try {
     argsJson = JSON.stringify(args);
@@ -232,9 +377,9 @@ export function runBridge<T = unknown>(
   }
   let res;
   try {
-    res = spawnSync(install.python, [BRIDGE_PATH, palace, cmd, argsJson], {
+    res = spawnSync(install.python, [bridgePath, palace, cmd, argsJson], {
       encoding: "utf-8",
-      timeout: 60_000,
+      timeout: timeoutMs,
       maxBuffer: 64 * 1024 * 1024,
     });
   } catch {
@@ -266,8 +411,14 @@ export function runBridgeAsync<T = unknown>(
   palace: string,
   cmd: string,
   args: Record<string, unknown> = {},
+  timeoutMs = 60_000,
 ): Promise<T | null> {
   return new Promise((resolve) => {
+    const bridgePath = getBridgePath();
+    if (!bridgePath) {
+      resolve(null);
+      return;
+    }
     let argsJson: string;
     try {
       argsJson = JSON.stringify(args);
@@ -278,7 +429,7 @@ export function runBridgeAsync<T = unknown>(
 
     let child;
     try {
-      child = spawn(install.python, [BRIDGE_PATH, palace, cmd, argsJson], {
+      child = spawn(install.python, [bridgePath, palace, cmd, argsJson], {
         stdio: ["ignore", "pipe", "ignore"],
       });
     } catch {
@@ -298,7 +449,7 @@ export function runBridgeAsync<T = unknown>(
     const timer = setTimeout(() => {
       try { child.kill(); } catch { /* already gone */ }
       finish(null);
-    }, 60_000);
+    }, timeoutMs);
     // Do not hold the process open purely for a background bridge call.
     timer.unref?.();
 

package/packages/memory/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/memory",
-  "version": "2.4.1",
+  "version": "2.6.0",
   "description": "Persistent cross-session memory with MemPalace backend (auto-installed) and SQLite fallback for Pi coding agent",
   "type": "module",
   "main": "index.ts",
@@ -43,8 +43,8 @@
     "better-sqlite3": "^12.9.0",
     "sqlite-vec": "^0.1.9",
     "js-yaml": "^4.1.0",
-    "@pi-unipi/core": "2.4.1",
-    "@pi-unipi/info-screen": "2.4.1"
+    "@pi-unipi/core": "2.6.0",
+    "@pi-unipi/info-screen": "2.6.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",

package/packages/memory/storage.ts

@@ -20,6 +20,7 @@ import {
   runBridgeAsync,
   isMigrated,
   markMigrated,
+  getMemorySourceFingerprint,
   isPingVerified,
   markPingVerified,
   invalidatePingVerified,
@@ -29,6 +30,7 @@ import {
   type MempalaceSearchResult,
   type MempalaceListItem,
   type MempalaceListItemAll,
+  type MigrationResult,
 } from "./mempalace.js";
 
 export type MemoryBackend = "mempalace" | "sqlite";
@@ -102,6 +104,7 @@ export interface SearchResult {
 
 /** Memory file frontmatter */
 interface MemoryFrontmatter {
+  id?: string;
   title: string;
   tags: string[];
   project: string;
@@ -201,7 +204,7 @@ export function parseMemoryContent(content: string): MemoryRecord | null {
   const frontmatter = yaml.load(frontmatterStr) as MemoryFrontmatter;
 
   return {
-    id: "",
+    id: frontmatter.id || "",
     title: frontmatter.title,
     content: body.trim(),
     tags: frontmatter.tags || [],
@@ -217,6 +220,7 @@ export function parseMemoryContent(content: string): MemoryRecord | null {
  */
 export function writeMemoryFile(filePath: string, record: MemoryRecord): void {
   const frontmatter: MemoryFrontmatter = {
+    id: record.id,
     title: record.title,
     tags: record.tags,
     project: record.project,
@@ -383,16 +387,28 @@ export class MemoryStorage {
     this.mempalaceInstall = install;
     this.backend = "mempalace";
 
-    // One-way auto-migration of legacy memories (idempotent).
-    if (!isMigrated()) {
+    // Idempotent migration + automatic catch-up. The source fingerprint turns
+    // the old one-shot timestamp into a resumable state: new/changed markdown
+    // or SQLite sources trigger another verified upsert pass. Never mark a
+    // failed/partial run complete; it will retry on a later session.
+    const sourceFingerprint = getMemorySourceFingerprint(getMemoryBaseDir());
+    if (!isMigrated(sourceFingerprint)) {
       try {
-        runBridge(install, this.palacePath, "migrate", {
+        // First migrations can embed thousands of records. Give the bridge a
+        // practical bounded window rather than the normal per-operation 60s.
+        const result = runBridge<MigrationResult>(install, this.palacePath, "migrate", {
           source_dir: getMemoryBaseDir(),
-        });
-        markMigrated();
+        }, 15 * 60_000);
+        if (
+          result &&
+          result.failed === 0 &&
+          result.verified === result.discovered
+        ) {
+          markMigrated(sourceFingerprint, result);
+        }
       } catch {
-        // Migration failed — palace still usable for new stores; legacy
-        // memories can be re-migrated later. Do not block.
+        // Palace remains available for current writes; durable markdown/SQLite
+        // sources are untouched and migration retries because no state is set.
       }
     }
 
@@ -629,8 +645,9 @@ export class MemoryStorage {
       const record = parseMemoryFile(filePath);
       if (!record) continue;
 
-      // Generate ID from title (same logic as store())
-      const id = record.title.toLowerCase().replace(/[^a-z0-9]+/g, "_");
+      // New files preserve the authoritative store ID. Legacy files have no
+      // `id` frontmatter, so retain the historical title-derived fallback.
+      const id = record.id || record.title.toLowerCase().replace(/[^a-z0-9]+/g, "_");
 
       if (existingIds.has(id)) continue; // Already in DB
 

package/packages/milestone/README.md

@@ -15,9 +15,17 @@ Workflow operates at the task level — brainstorm, plan, work, review. Project
 
 ### Session Start
 
-On `before_agent_start`, milestone reads `.unipi/docs/MILESTONES.md` and appends a progress summary to the system prompt:
+On `before_agent_start`, milestone reads `.unipi/docs/MILESTONES.md` from `ctx.cwd` and appends a hidden `unipi-milestone-snapshot` custom message to the session:
 
 ```
+# UniPi Milestone Snapshot
+
+This snapshot supersedes all prior UniPi milestone snapshots; use only this snapshot for milestone status.
+
+Workspace: /path/to/project
+
+Status: active
+
 ## Project Milestones
 Overall progress: 5/10 items (50%)
   Phase 1: Foundation: 3/5 done
@@ -25,11 +33,11 @@ Overall progress: 5/10 items (50%)
 Current focus: Phase 1: Foundation
 ```
 
-If MILESTONES.md doesn't exist, no context is injected.
+Snapshots are append-only and hidden from the transcript. They keep the system-prompt prefix stable, persist milestone context in session history, and are deduplicated against the latest milestone custom message in the effective (compaction-aware) LLM context. If milestones disappear while an older active snapshot remains effective, an inactive snapshot is appended to supersede it. A clean workspace with no milestones and no effective snapshot receives no injected message.
 
 ### Session End
 
-On `session_shutdown`, milestone scans workflow docs modified during the session. Detects items that changed from `- [ ]` to `- [x]` and auto-updates MILESTONES.md using exact text matching.
+On `session_shutdown`, milestone scans workflow docs modified during the session. It uses the workspace captured from `session_start`'s `ctx.cwd`, detects items that changed from `- [ ]` to `- [x]`, and auto-updates MILESTONES.md using exact text matching.
 
 ### Coexist Triggers