omegon 0.6.0

package/extensions/project-memory/migration.ts

@@ -0,0 +1,293 @@
+/**
+ * Project Memory — Migration
+ *
+ * Migrates existing markdown-based memory (memory.md, archive/*.md, minds/*)
+ * into the SQLite fact store.
+ *
+ * Migration is idempotent — running it twice won't duplicate facts
+ * (dedup via content_hash handles this).
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { FactStore, type StoreFactOptions } from "./factstore.ts";
+import { SECTIONS, type SectionName } from "./template.ts";
+
+interface MigrationResult {
+  factsImported: number;
+  duplicatesSkipped: number;
+  archiveFactsImported: number;
+  mindsImported: number;
+  errors: string[];
+}
+
+/**
+ * Parse a markdown memory file into section → bullet[] map.
+ */
+function parseMarkdownMemory(content: string): Map<SectionName, string[]> {
+  const sectionBullets = new Map<SectionName, string[]>();
+  let currentSection: SectionName | null = null;
+
+  for (const line of content.split("\n")) {
+    const sectionMatch = line.match(/^## (.+)$/);
+    if (sectionMatch) {
+      const name = sectionMatch[1].trim();
+      if ((SECTIONS as readonly string[]).includes(name)) {
+        currentSection = name as SectionName;
+      } else {
+        currentSection = null;
+      }
+      continue;
+    }
+    if (currentSection && line.trim().startsWith("- ")) {
+      if (!sectionBullets.has(currentSection)) {
+        sectionBullets.set(currentSection, []);
+      }
+      // Strip the bullet prefix
+      const content = line.trim().replace(/^-\s*/, "").trim();
+      if (content) {
+        sectionBullets.get(currentSection)!.push(content);
+      }
+    }
+  }
+
+  return sectionBullets;
+}
+
+/**
+ * Parse an archive file for timestamped sections.
+ * Archive files have <!-- Archived YYYY-MM-DD --> markers and
+ * optionally [SectionName] prefixes on facts.
+ */
+function parseArchiveFile(content: string): { date: string; section: SectionName; content: string }[] {
+  const facts: { date: string; section: SectionName; content: string }[] = [];
+  let currentDate = "";
+
+  for (const line of content.split("\n")) {
+    const dateMatch = line.match(/<!--\s*Archived\s+(\d{4}-\d{2}-\d{2})\s*-->/);
+    if (dateMatch) {
+      currentDate = dateMatch[1];
+      continue;
+    }
+
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("<!--") || trimmed.startsWith("##")) continue;
+
+    // Try to extract [SectionName] prefix
+    const sectionMatch = trimmed.match(/^\[([^\]]+)\]\s*(.+)$/);
+    let section: SectionName = "Architecture"; // fallback
+    let factContent: string;
+
+    if (sectionMatch) {
+      const sectionName = sectionMatch[1].trim();
+      if ((SECTIONS as readonly string[]).includes(sectionName)) {
+        section = sectionName as SectionName;
+      }
+      factContent = sectionMatch[2].replace(/^-\s*/, "").trim();
+    } else {
+      factContent = trimmed.replace(/^-\s*/, "").trim();
+    }
+
+    if (factContent) {
+      facts.push({
+        date: currentDate || new Date().toISOString().split("T")[0],
+        section,
+        content: factContent,
+      });
+    }
+  }
+
+  return facts;
+}
+
+/**
+ * Migrate an entire .pi/memory directory into a FactStore.
+ */
+export function migrateToFactStore(memoryDir: string, store: FactStore): MigrationResult {
+  const result: MigrationResult = {
+    factsImported: 0,
+    duplicatesSkipped: 0,
+    archiveFactsImported: 0,
+    mindsImported: 0,
+    errors: [],
+  };
+
+  // 1. Migrate default memory.md
+  const defaultMemoryPath = path.join(memoryDir, "memory.md");
+  if (fs.existsSync(defaultMemoryPath)) {
+    try {
+      const content = fs.readFileSync(defaultMemoryPath, "utf8");
+      const sections = parseMarkdownMemory(content);
+
+      for (const [section, bullets] of sections) {
+        for (const bullet of bullets) {
+          const { duplicate } = store.storeFact({
+            mind: "default",
+            section,
+            content: bullet,
+            source: "migration",
+            // Give migrated facts a moderate reinforcement count
+            // They've survived in memory, so they're proven durable
+            reinforcement_count: 5,
+          });
+          if (duplicate) {
+            result.duplicatesSkipped++;
+          } else {
+            result.factsImported++;
+          }
+        }
+      }
+    } catch (err: any) {
+      result.errors.push(`Failed to migrate default memory: ${err.message}`);
+    }
+  }
+
+  // 2. Migrate archive files
+  const archiveDir = path.join(memoryDir, "archive");
+  if (fs.existsSync(archiveDir)) {
+    try {
+      const archiveFiles = fs.readdirSync(archiveDir)
+        .filter(f => f.endsWith(".md"))
+        .sort();
+
+      for (const file of archiveFiles) {
+        const content = fs.readFileSync(path.join(archiveDir, file), "utf8");
+        const archiveFacts = parseArchiveFile(content);
+
+        for (const af of archiveFacts) {
+          const { duplicate } = store.storeFact({
+            mind: "default",
+            section: af.section,
+            content: af.content,
+            source: "migration",
+            reinforcement_count: 2, // archived = less durable
+          });
+
+          if (!duplicate) {
+            // Mark as archived since these were already archived
+            // Find the fact we just inserted and archive it
+            // Actually, store it as active — let decay handle it naturally
+            // since these are older facts they'll have lower confidence
+            result.archiveFactsImported++;
+          } else {
+            result.duplicatesSkipped++;
+          }
+        }
+      }
+    } catch (err: any) {
+      result.errors.push(`Failed to migrate archive: ${err.message}`);
+    }
+  }
+
+  // 3. Migrate minds
+  const mindsDir = path.join(memoryDir, "minds");
+  if (fs.existsSync(mindsDir)) {
+    try {
+      // Read registry if it exists
+      const registryPath = path.join(mindsDir, "registry.json");
+      let registry: Record<string, any> = {};
+      try {
+        registry = JSON.parse(fs.readFileSync(registryPath, "utf8"));
+      } catch {
+        // No registry — scan directories
+      }
+
+      const entries = fs.readdirSync(mindsDir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isDirectory()) continue;
+        const mindName = entry.name;
+
+        // Skip state files
+        if (mindName === "." || mindName === "..") continue;
+
+        const metaPath = path.join(mindsDir, mindName, "meta.json");
+        const mindMemoryPath = path.join(mindsDir, mindName, "memory.md");
+
+        if (!fs.existsSync(mindMemoryPath)) continue;
+
+        try {
+          // Read metadata
+          let meta: any = {};
+          try {
+            meta = JSON.parse(fs.readFileSync(metaPath, "utf8"));
+          } catch {
+            meta = { name: mindName, description: "", status: "active" };
+          }
+
+          const regEntry = registry[mindName];
+
+          // Create mind in store if it doesn't exist
+          if (!store.mindExists(mindName)) {
+            store.createMind(mindName, meta.description ?? "", {
+              parent: meta.parent,
+              origin_type: regEntry?.origin?.type ?? meta.origin?.type ?? "local",
+              origin_path: regEntry?.origin?.path ?? meta.origin?.path,
+              readonly: meta.readonly ?? regEntry?.readonly ?? false,
+            });
+            result.mindsImported++;
+          }
+
+          // Migrate facts
+          const content = fs.readFileSync(mindMemoryPath, "utf8");
+          const sections = parseMarkdownMemory(content);
+
+          for (const [section, bullets] of sections) {
+            for (const bullet of bullets) {
+              const { duplicate } = store.storeFact({
+                mind: mindName,
+                section,
+                content: bullet,
+                source: "migration",
+                reinforcement_count: 5,
+              });
+              if (duplicate) {
+                result.duplicatesSkipped++;
+              } else {
+                result.factsImported++;
+              }
+            }
+          }
+        } catch (err: any) {
+          result.errors.push(`Failed to migrate mind "${mindName}": ${err.message}`);
+        }
+      }
+
+      // Migrate active mind state
+      const activeStatePath = path.join(mindsDir, "active.json");
+      if (fs.existsSync(activeStatePath)) {
+        try {
+          const state = JSON.parse(fs.readFileSync(activeStatePath, "utf8"));
+          if (state.activeMind) {
+            store.setActiveMind(state.activeMind);
+          }
+        } catch {
+          // Best effort
+        }
+      }
+    } catch (err: any) {
+      result.errors.push(`Failed to migrate minds: ${err.message}`);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Check if migration is needed — does markdown memory exist but no facts.db?
+ */
+export function needsMigration(memoryDir: string): boolean {
+  const hasMarkdown = fs.existsSync(path.join(memoryDir, "memory.md"));
+  const hasDb = fs.existsSync(path.join(memoryDir, "facts.db"));
+  return hasMarkdown && !hasDb;
+}
+
+/**
+ * Rename old markdown files after successful migration.
+ * Appends .migrated suffix so they're preserved but not re-migrated.
+ */
+export function markMigrated(memoryDir: string): void {
+  const memoryMd = path.join(memoryDir, "memory.md");
+  if (fs.existsSync(memoryMd)) {
+    fs.renameSync(memoryMd, memoryMd + ".migrated");
+  }
+}

package/extensions/project-memory/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "project-memory",
+  "version": "1.0.0",
+  "type": "commonjs",
+  "dependencies": {
+    "@cwilson613/pi-tui": "0.57.1-cwilson613.1",
+    "better-sqlite3": "^12.6.2"
+  }
+}

package/extensions/project-memory/sci-renderers.ts

@@ -0,0 +1,7 @@
+/**
+ * Sci-UI render helpers for project-memory tools.
+ *
+ * Re-exports the shared sci-ui primitives and adds memory-specific
+ * formatting helpers for structured card rendering.
+ */
+export { sciCall, sciOk, sciErr, sciExpanded, sciLoading } from "../sci-ui.ts";

package/extensions/project-memory/template.ts

@@ -0,0 +1,103 @@
+/**
+ * Project Memory — Default Template
+ */
+
+export const DEFAULT_TEMPLATE = `<!-- Project Memory — managed by project-memory extension -->
+<!-- Do not edit while a pi session is actively running -->
+
+## Architecture
+_System structure, component relationships, key abstractions_
+
+## Decisions
+_Choices made and their rationale_
+
+## Constraints
+_Requirements, limitations, environment details_
+
+## Known Issues
+_Bugs, flaky tests, workarounds_
+
+## Patterns & Conventions
+_Code style, project conventions, common approaches_
+
+## Specs
+_Active specifications, acceptance criteria, and design contracts driving current work_
+
+## Recent Work
+_What was done this week — ephemeral session receipts, decays within a business week_
+`;
+
+export const SECTIONS = [
+  "Architecture",
+  "Decisions",
+  "Constraints",
+  "Known Issues",
+  "Patterns & Conventions",
+  "Specs",
+  "Recent Work",
+] as const;
+
+export type SectionName = (typeof SECTIONS)[number];
+
+/**
+ * Check if a bullet already exists in the section (exact or near-duplicate).
+ * Normalizes whitespace and leading "- " for comparison.
+ */
+function isDuplicate(existingLines: string[], bullet: string): boolean {
+  const normalize = (s: string) =>
+    s.replace(/^-\s*/, "").trim().toLowerCase();
+  const normalized = normalize(bullet);
+  return existingLines.some((line) => {
+    if (!line.trim().startsWith("- ")) return false;
+    return normalize(line) === normalized;
+  });
+}
+
+/**
+ * Append a bullet to a specific section in the memory markdown.
+ * Returns the updated markdown string.
+ */
+export function appendToSection(markdown: string, section: SectionName, bullet: string): string {
+  const sectionHeader = `## ${section}`;
+  const lines = markdown.split("\n");
+  const headerIdx = lines.findIndex((l) => l.trim() === sectionHeader);
+
+  if (headerIdx === -1) {
+    // Section not found — append it
+    return markdown.trimEnd() + `\n\n${sectionHeader}\n\n${bullet}\n`;
+  }
+
+  // Find the next section header or end of file
+  let insertIdx = lines.length;
+  for (let i = headerIdx + 1; i < lines.length; i++) {
+    if (lines[i].match(/^## /)) {
+      insertIdx = i;
+      break;
+    }
+  }
+
+  // Walk back past blank lines to insert before the gap
+  while (insertIdx > headerIdx + 1 && lines[insertIdx - 1].trim() === "") {
+    insertIdx--;
+  }
+
+  // Check for duplicates in this section
+  const sectionLines = lines.slice(headerIdx + 1, insertIdx);
+  if (isDuplicate(sectionLines, bullet)) {
+    return markdown; // No change — already exists
+  }
+
+  lines.splice(insertIdx, 0, bullet);
+  return lines.join("\n");
+}
+
+/**
+ * Count non-empty, non-comment lines in markdown content.
+ * Filters out empty lines and HTML comment lines starting with <!--.
+ */
+export function countContentLines(content: string): number {
+  return content.split("\n").filter((l) => {
+    const trimmed = l.trim();
+    return trimmed !== "" && !trimmed.startsWith("<!--");
+  }).length;
+}

package/extensions/project-memory/triggers.ts

@@ -0,0 +1,52 @@
+/**
+ * Extraction trigger logic — determines when background extraction should run.
+ */
+
+import type { MemoryConfig } from "./types.ts";
+
+export interface ExtractionTriggerState {
+  lastExtractedTokens: number;
+  toolCallsSinceExtract: number;
+  manualStoresSinceExtract: number;
+  isInitialized: boolean;
+  isRunning: boolean;
+}
+
+export function createTriggerState(): ExtractionTriggerState {
+  return {
+    lastExtractedTokens: 0,
+    toolCallsSinceExtract: 0,
+    manualStoresSinceExtract: 0,
+    isInitialized: false,
+    isRunning: false,
+  };
+}
+
+export function shouldExtract(
+  state: ExtractionTriggerState,
+  currentTokens: number,
+  config: MemoryConfig,
+  consecutiveFailures: number = 0,
+): boolean {
+  if (state.isRunning) return false;
+
+  // Exponential backoff on consecutive failures: skip 2^n extraction opportunities
+  // (1 skip after 1 failure, 2 after 2, 4 after 3, cap at 16)
+  if (consecutiveFailures > 0) {
+    const backoffSlots = Math.min(1 << consecutiveFailures, 16);
+    if (state.toolCallsSinceExtract % backoffSlots !== 0) return false;
+  }
+
+  // Only suppress for manual stores after first extraction has established baseline.
+  if (state.isInitialized && state.manualStoresSinceExtract >= config.manualStoreThreshold) {
+    return false;
+  }
+
+  const tokenDelta = currentTokens - state.lastExtractedTokens;
+
+  if (!state.isInitialized) {
+    return currentTokens >= config.minimumTokensToInit;
+  }
+
+  return tokenDelta >= config.minimumTokensBetweenUpdate && state.toolCallsSinceExtract >= config.toolCallsBetweenUpdates;
+}

package/extensions/project-memory/types.ts

@@ -0,0 +1,102 @@
+/**
+ * Project Memory — Types
+ */
+
+export interface LifecycleMemoryMessage {
+  source: "design-tree" | "openspec" | "cleave";
+  context: string;
+  candidates: LifecycleMemoryCandidate[];
+}
+
+export interface LifecycleMemoryCandidate {
+  sourceKind: "design-decision" | "design-constraint" | "openspec-archive" | "openspec-assess" | "cleave-outcome" | "cleave-bug-fix";
+  authority: "explicit" | "inferred";
+  section: "Architecture" | "Decisions" | "Constraints" | "Known Issues" | "Patterns & Conventions" | "Specs";
+  content: string;
+  artifactRef?: {
+    type: "design-node" | "openspec-spec" | "openspec-baseline" | "cleave-review";
+    path: string;
+    subRef?: string;
+  };
+  supersedes?: string;
+  session?: string;
+}
+
+export interface MemoryConfig {
+  /** Max lines in active memory before extraction prunes */
+  maxLines: number;
+  /** Embedding backend used for semantic retrieval */
+  embeddingProvider: "voyage" | "openai" | "openai-compatible" | "ollama";
+  /** Embedding model to use for fact and episode vectors */
+  embeddingModel: string;
+  /** Minimum total message tokens before first extraction */
+  minimumTokensToInit: number;
+  /** Token delta required between extractions */
+  minimumTokensBetweenUpdate: number;
+  /** Minimum tool calls since last extraction */
+  toolCallsBetweenUpdates: number;
+  /** Skip auto-extraction if LLM stored >= this many facts since last extraction */
+  manualStoreThreshold: number;
+  /** Model to use for extraction subagent */
+  extractionModel: string;
+  /** Timeout for extraction subprocess in ms */
+  extractionTimeout: number;
+  /** Timeout for shutdown extraction in ms (shorter — blocks exit) */
+  shutdownExtractionTimeout: number;
+  /** Context % at which degeneracy pressure begins (gradient onset) */
+  pressureOnsetPercent: number;
+  /** Context % at which to warn the agent to consider compacting */
+  compactionWarningPercent: number;
+  /** Context % at which to auto-compact without asking */
+  compactionAutoPercent: number;
+  /** Use local model as fallback when cloud compaction fails */
+  compactionLocalFallback: boolean;
+  /** Try local model FIRST for compaction (cloud only if local unavailable) */
+  compactionLocalFirst: boolean;
+  /** Timeout for local model compaction inference in ms */
+  compactionLocalTimeout: number;
+  /** Enable intelligent fallback chain: local → gpt-5.3-codex-spark → haiku */
+  compactionFallbackChain: boolean;
+  /** Timeout for gpt-5.3-codex-spark compaction fallback in ms */
+  compactionCodexTimeout: number;
+  /** Timeout for retribution compaction fallback in ms */
+  compactionHaikuTimeout: number;
+  /** Max consecutive compaction retry attempts before giving up for the session */
+  compactionRetryLimit: number;
+  /** Enable Phase 2 global extraction (generalizes project facts to user-level store) */
+  globalExtractionEnabled: boolean;
+  /** Model to use for episode generation (first cloud attempt in fallback chain) */
+  episodeModel: string;
+  /** Per-attempt timeout for each step in the episode fallback chain (ms) */
+  episodeStepTimeout: number;
+  /** Enable fallback chain for episode generation: Ollama → cloud primary → cloud retribution → template */
+  episodeFallbackChain: boolean;
+}
+
+export const DEFAULT_CONFIG: MemoryConfig = {
+  maxLines: 50,
+  embeddingProvider: "voyage",
+  embeddingModel: "voyage-3-lite",
+  minimumTokensToInit: 10_000,
+  minimumTokensBetweenUpdate: 5_000,
+  toolCallsBetweenUpdates: 8,
+  manualStoreThreshold: 3,
+  extractionModel: "gpt-5.3-codex-spark",
+  extractionTimeout: 60_000,
+  shutdownExtractionTimeout: 15_000,
+  pressureOnsetPercent: 55,
+  compactionWarningPercent: 75,
+  compactionAutoPercent: 85,
+  compactionLocalFallback: true,
+  compactionLocalFirst: false,
+  compactionLocalTimeout: 45_000,
+  compactionFallbackChain: true,
+  compactionCodexTimeout: 60_000,
+  compactionHaikuTimeout: 30_000,
+  compactionRetryLimit: 3,
+  globalExtractionEnabled: false,
+  episodeModel: "gpt-5.3-codex-spark",
+  episodeStepTimeout: 8_000,
+  episodeFallbackChain: true,
+};
+