gsd-pi 2.33.1 → 2.34.0-dev.bbb5216

package/dist/resources/extensions/gsd/memory-extractor.js

@@ -0,0 +1,295 @@
+// GSD Memory Extractor — Background LLM extraction from activity logs
+//
+// After each unit completes, extracts durable knowledge from the session
+// transcript and stores it as memory entries. One extraction at a time
+// (mutex guard). Fire-and-forget — never blocks auto-mode.
+import { readFileSync, statSync } from 'node:fs';
+import { getActiveMemories, isUnitProcessed, markUnitProcessed, applyMemoryActions, decayStaleMemories, } from './memory-store.js';
+// ─── Concurrency Guard ──────────────────────────────────────────────────────
+let _extracting = false;
+let _lastExtractionTime = 0;
+const MIN_EXTRACTION_INTERVAL_MS = 30_000;
+// ─── Skip Conditions ────────────────────────────────────────────────────────
+const SKIP_TYPES = new Set([
+    'complete-slice',
+    'rewrite-docs',
+    'triage-captures',
+]);
+const MIN_ACTIVITY_SIZE = 1024; // 1KB
+// ─── Secret Redaction ───────────────────────────────────────────────────────
+const SECRET_PATTERNS = [
+    /(?:sk|pk|api[_-]?key|token|secret|password|credential|auth)[_-]?\w*[\s:=]+['"]?[\w\-./+=]{20,}['"]?/gi,
+    /AKIA[0-9A-Z]{16}/g,
+    /gh[pousr]_[A-Za-z0-9_]{36,}/g,
+    /[rsp]k_(?:live|test)_[A-Za-z0-9]{20,}/g,
+    /eyJ[A-Za-z0-9_-]{20,}\.eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]+/g,
+    /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g,
+    /(?:Bearer\s+)[A-Za-z0-9\-._~+/]+=*/gi,
+    /npm_[A-Za-z0-9]{36,}/g,
+    /sk-ant-[A-Za-z0-9\-_]{20,}/g,
+    /sk-[A-Za-z0-9]{40,}/g,
+];
+function redactSecrets(text) {
+    let result = text;
+    for (const pattern of SECRET_PATTERNS) {
+        // Reset lastIndex for global regexes
+        pattern.lastIndex = 0;
+        result = result.replace(pattern, '[REDACTED]');
+    }
+    return result;
+}
+// ─── Model Selection ────────────────────────────────────────────────────────
+/**
+ * Build an LLM call function using the cheapest available model (preferring Haiku).
+ * Returns null if no models available.
+ */
+export function buildMemoryLLMCall(ctx) {
+    try {
+        const available = ctx.modelRegistry.getAvailable();
+        if (!available || available.length === 0)
+            return null;
+        // Prefer Haiku by ID substring match
+        let model = available.find(m => m.id.toLowerCase().includes('haiku'));
+        // Fallback: cheapest by input cost
+        if (!model) {
+            model = [...available].sort((a, b) => a.cost.input - b.cost.input)[0];
+        }
+        if (!model)
+            return null;
+        const selectedModel = model;
+        return async (system, user) => {
+            const { completeSimple } = await import('@gsd/pi-ai');
+            const result = await completeSimple(selectedModel, {
+                systemPrompt: system,
+                messages: [{ role: 'user', content: [{ type: 'text', text: user }], timestamp: Date.now() }],
+            }, {
+                maxTokens: 2048,
+                temperature: 0,
+            });
+            // Extract text from response
+            const textParts = result.content
+                .filter((c) => c.type === 'text')
+                .map(c => c.text);
+            return textParts.join('');
+        };
+    }
+    catch {
+        return null;
+    }
+}
+// ─── Extraction Prompts ─────────────────────────────────────────────────────
+const EXTRACTION_SYSTEM = `You are a memory extraction agent for a software project. Analyze the session
+transcript and identify durable knowledge worth remembering for future sessions.
+
+Categories: architecture, convention, gotcha, preference, environment, pattern
+
+Actions (return JSON array):
+- CREATE: {"action": "CREATE", "category": "<cat>", "content": "<text>", "confidence": <0.6-0.95>}
+- UPDATE: {"action": "UPDATE", "id": "<MEM###>", "content": "<revised text>"}
+- REINFORCE: {"action": "REINFORCE", "id": "<MEM###>"}
+- SUPERSEDE: {"action": "SUPERSEDE", "id": "<MEM###>", "superseded_by": "<MEM###>"}
+
+Rules:
+- Don't create memories for one-off bug fixes or temporary state
+- Don't duplicate existing memories — use REINFORCE or UPDATE
+- Keep content to 1-3 sentences
+- Confidence: 0.6 tentative, 0.8 solid, 0.95 well-confirmed
+- Prefer fewer high-quality memories over many low-quality ones
+- Return empty array [] if nothing worth remembering
+- NEVER include secrets, API keys, or passwords
+
+Return ONLY a valid JSON array.`;
+function buildExtractionUserPrompt(unitType, unitId, existingMemories, transcript) {
+    let memoriesSection;
+    if (existingMemories.length === 0) {
+        memoriesSection = '(none yet)';
+    }
+    else {
+        memoriesSection = existingMemories
+            .map((m, i) => `${i + 1}. [${m.id}] (${m.category}) ${m.content}`)
+            .join('\n');
+    }
+    return `## Current Active Memories\n${memoriesSection}\n\n## Session Transcript (${unitType}: ${unitId})\n${transcript}`;
+}
+// ─── Activity JSONL Parsing ─────────────────────────────────────────────────
+/**
+ * Extract assistant message text from activity JSONL.
+ * Returns concatenated text content from assistant role entries.
+ */
+function extractTranscriptFromActivity(raw, maxChars = 30_000) {
+    const lines = raw.split('\n');
+    const parts = [];
+    let totalChars = 0;
+    for (const line of lines) {
+        if (!line.trim())
+            continue;
+        try {
+            const entry = JSON.parse(line);
+            if (entry.role !== 'assistant')
+                continue;
+            // Handle content array or direct text
+            if (Array.isArray(entry.content)) {
+                for (const block of entry.content) {
+                    if (block.type === 'text' && block.text) {
+                        const text = block.text;
+                        if (totalChars + text.length > maxChars) {
+                            parts.push(text.substring(0, maxChars - totalChars));
+                            return parts.join('\n\n');
+                        }
+                        parts.push(text);
+                        totalChars += text.length;
+                    }
+                }
+            }
+            else if (typeof entry.content === 'string') {
+                const text = entry.content;
+                if (totalChars + text.length > maxChars) {
+                    parts.push(text.substring(0, maxChars - totalChars));
+                    return parts.join('\n\n');
+                }
+                parts.push(text);
+                totalChars += text.length;
+            }
+        }
+        catch {
+            // Skip malformed lines
+        }
+    }
+    return parts.join('\n\n');
+}
+// ─── Response Parsing ───────────────────────────────────────────────────────
+/**
+ * Parse the LLM response into memory actions.
+ * Strips markdown fences, validates required fields.
+ * Returns [] on any parse failure.
+ */
+export function parseMemoryResponse(raw) {
+    try {
+        // Strip markdown code fences
+        let cleaned = raw.trim();
+        if (cleaned.startsWith('```')) {
+            cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?```\s*$/, '');
+        }
+        const parsed = JSON.parse(cleaned);
+        if (!Array.isArray(parsed))
+            return [];
+        const actions = [];
+        for (const item of parsed) {
+            if (!item || typeof item !== 'object' || !item.action)
+                continue;
+            switch (item.action) {
+                case 'CREATE':
+                    if (typeof item.category === 'string' && typeof item.content === 'string') {
+                        actions.push({
+                            action: 'CREATE',
+                            category: item.category,
+                            content: item.content,
+                            confidence: typeof item.confidence === 'number' ? item.confidence : undefined,
+                        });
+                    }
+                    break;
+                case 'UPDATE':
+                    if (typeof item.id === 'string' && typeof item.content === 'string') {
+                        actions.push({
+                            action: 'UPDATE',
+                            id: item.id,
+                            content: item.content,
+                            confidence: typeof item.confidence === 'number' ? item.confidence : undefined,
+                        });
+                    }
+                    break;
+                case 'REINFORCE':
+                    if (typeof item.id === 'string') {
+                        actions.push({ action: 'REINFORCE', id: item.id });
+                    }
+                    break;
+                case 'SUPERSEDE':
+                    if (typeof item.id === 'string' && typeof item.superseded_by === 'string') {
+                        actions.push({
+                            action: 'SUPERSEDE',
+                            id: item.id,
+                            superseded_by: item.superseded_by,
+                        });
+                    }
+                    break;
+            }
+        }
+        return actions;
+    }
+    catch {
+        return [];
+    }
+}
+// ─── Main Extraction Function ───────────────────────────────────────────────
+/**
+ * Extract memories from a completed unit's activity log.
+ * Fire-and-forget — never throws, mutex-guarded, respects rate limiting.
+ */
+export async function extractMemoriesFromUnit(activityFile, unitType, unitId, llmCallFn) {
+    // Mutex guard
+    if (_extracting)
+        return;
+    // Rate limit
+    const now = Date.now();
+    if (now - _lastExtractionTime < MIN_EXTRACTION_INTERVAL_MS)
+        return;
+    // Skip certain unit types
+    if (SKIP_TYPES.has(unitType))
+        return;
+    const unitKey = `${unitType}/${unitId}`;
+    // Already processed
+    if (isUnitProcessed(unitKey))
+        return;
+    // Check file size
+    try {
+        const stat = statSync(activityFile);
+        if (stat.size < MIN_ACTIVITY_SIZE)
+            return;
+    }
+    catch {
+        return;
+    }
+    _extracting = true;
+    _lastExtractionTime = now;
+    try {
+        // Read and parse activity file
+        const raw = readFileSync(activityFile, 'utf-8');
+        const transcript = extractTranscriptFromActivity(raw);
+        if (!transcript.trim())
+            return;
+        // Redact secrets
+        const safeTranscript = redactSecrets(transcript);
+        // Get current memories for context
+        const activeMemories = getActiveMemories().map(m => ({
+            id: m.id,
+            category: m.category,
+            content: m.content,
+        }));
+        // Build prompts
+        const userPrompt = buildExtractionUserPrompt(unitType, unitId, activeMemories, safeTranscript);
+        // Call LLM
+        const response = await llmCallFn(EXTRACTION_SYSTEM, userPrompt);
+        // Parse response
+        const actions = parseMemoryResponse(response);
+        // Apply actions
+        if (actions.length > 0) {
+            applyMemoryActions(actions, unitType, unitId);
+        }
+        // Decay stale memories periodically
+        decayStaleMemories(20);
+        // Mark unit as processed
+        markUnitProcessed(unitKey, activityFile);
+    }
+    catch {
+        // Non-fatal — memory extraction failure should never affect auto-mode
+    }
+    finally {
+        _extracting = false;
+    }
+}
+// ─── Testing Helpers ────────────────────────────────────────────────────────
+/** Reset extraction state (testing only). */
+export function _resetExtractionState() {
+    _extracting = false;
+    _lastExtractionTime = 0;
+}

package/dist/resources/extensions/gsd/memory-store.js

@@ -0,0 +1,351 @@
+// GSD Memory Store — CRUD, ranked queries, maintenance, and prompt formatting
+//
+// Storage layer for auto-learned project memories. Follows context-store.ts patterns.
+// All functions degrade gracefully: return empty results when DB unavailable, never throw.
+import { isDbAvailable, _getAdapter, transaction } from './gsd-db.js';
+// ─── Category Display Order ─────────────────────────────────────────────────
+const CATEGORY_PRIORITY = {
+    gotcha: 0,
+    convention: 1,
+    architecture: 2,
+    pattern: 3,
+    environment: 4,
+    preference: 5,
+};
+// ─── Row Mapping ────────────────────────────────────────────────────────────
+function rowToMemory(row) {
+    return {
+        seq: row['seq'],
+        id: row['id'],
+        category: row['category'],
+        content: row['content'],
+        confidence: row['confidence'],
+        source_unit_type: row['source_unit_type'] ?? null,
+        source_unit_id: row['source_unit_id'] ?? null,
+        created_at: row['created_at'],
+        updated_at: row['updated_at'],
+        superseded_by: row['superseded_by'] ?? null,
+        hit_count: row['hit_count'],
+    };
+}
+// ─── Query Functions ────────────────────────────────────────────────────────
+/**
+ * Get all memories where superseded_by IS NULL.
+ * Returns [] if DB is not available. Never throws.
+ */
+export function getActiveMemories() {
+    if (!isDbAvailable())
+        return [];
+    const adapter = _getAdapter();
+    if (!adapter)
+        return [];
+    try {
+        const rows = adapter.prepare('SELECT * FROM memories WHERE superseded_by IS NULL').all();
+        return rows.map(rowToMemory);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Get active memories ordered by ranking score: confidence * (1 + hit_count * 0.1).
+ * Higher-scored memories are more relevant and frequently confirmed.
+ */
+export function getActiveMemoriesRanked(limit = 30) {
+    if (!isDbAvailable())
+        return [];
+    const adapter = _getAdapter();
+    if (!adapter)
+        return [];
+    try {
+        const rows = adapter.prepare(`SELECT * FROM memories
+       WHERE superseded_by IS NULL
+       ORDER BY (confidence * (1.0 + hit_count * 0.1)) DESC
+       LIMIT :limit`).all({ ':limit': limit });
+        return rows.map(rowToMemory);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Generate the next memory ID: MEM + zero-padded 3-digit from MAX(seq).
+ * Returns MEM001 if no memories exist.
+ */
+export function nextMemoryId() {
+    if (!isDbAvailable())
+        return 'MEM001';
+    const adapter = _getAdapter();
+    if (!adapter)
+        return 'MEM001';
+    try {
+        const row = adapter
+            .prepare('SELECT MAX(seq) as max_seq FROM memories')
+            .get();
+        const maxSeq = row ? row['max_seq'] : null;
+        if (maxSeq == null || isNaN(maxSeq))
+            return 'MEM001';
+        const next = maxSeq + 1;
+        return `MEM${String(next).padStart(3, '0')}`;
+    }
+    catch {
+        return 'MEM001';
+    }
+}
+// ─── Mutation Functions ─────────────────────────────────────────────────────
+/**
+ * Insert a new memory with auto-assigned ID.
+ * Returns the assigned ID, or null on failure.
+ */
+export function createMemory(fields) {
+    if (!isDbAvailable())
+        return null;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return null;
+    try {
+        const id = nextMemoryId();
+        const now = new Date().toISOString();
+        adapter.prepare(`INSERT INTO memories (id, category, content, confidence, source_unit_type, source_unit_id, created_at, updated_at)
+       VALUES (:id, :category, :content, :confidence, :source_unit_type, :source_unit_id, :created_at, :updated_at)`).run({
+            ':id': id,
+            ':category': fields.category,
+            ':content': fields.content,
+            ':confidence': fields.confidence ?? 0.8,
+            ':source_unit_type': fields.source_unit_type ?? null,
+            ':source_unit_id': fields.source_unit_id ?? null,
+            ':created_at': now,
+            ':updated_at': now,
+        });
+        return id;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Update a memory's content and optionally its confidence.
+ */
+export function updateMemoryContent(id, content, confidence) {
+    if (!isDbAvailable())
+        return false;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return false;
+    try {
+        const now = new Date().toISOString();
+        if (confidence != null) {
+            adapter.prepare('UPDATE memories SET content = :content, confidence = :confidence, updated_at = :updated_at WHERE id = :id').run({ ':content': content, ':confidence': confidence, ':updated_at': now, ':id': id });
+        }
+        else {
+            adapter.prepare('UPDATE memories SET content = :content, updated_at = :updated_at WHERE id = :id').run({ ':content': content, ':updated_at': now, ':id': id });
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Reinforce a memory: increment hit_count, update timestamp.
+ */
+export function reinforceMemory(id) {
+    if (!isDbAvailable())
+        return false;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return false;
+    try {
+        adapter.prepare('UPDATE memories SET hit_count = hit_count + 1, updated_at = :updated_at WHERE id = :id').run({ ':updated_at': new Date().toISOString(), ':id': id });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Mark a memory as superseded by another.
+ */
+export function supersedeMemory(oldId, newId) {
+    if (!isDbAvailable())
+        return false;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return false;
+    try {
+        adapter.prepare('UPDATE memories SET superseded_by = :new_id, updated_at = :updated_at WHERE id = :old_id').run({ ':new_id': newId, ':updated_at': new Date().toISOString(), ':old_id': oldId });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Processed Unit Tracking ────────────────────────────────────────────────
+/**
+ * Check if a unit has already been processed for memory extraction.
+ */
+export function isUnitProcessed(unitKey) {
+    if (!isDbAvailable())
+        return false;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return false;
+    try {
+        const row = adapter.prepare('SELECT 1 FROM memory_processed_units WHERE unit_key = :key').get({ ':key': unitKey });
+        return row != null;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Record that a unit has been processed for memory extraction.
+ */
+export function markUnitProcessed(unitKey, activityFile) {
+    if (!isDbAvailable())
+        return false;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return false;
+    try {
+        adapter.prepare(`INSERT OR IGNORE INTO memory_processed_units (unit_key, activity_file, processed_at)
+       VALUES (:key, :file, :at)`).run({ ':key': unitKey, ':file': activityFile, ':at': new Date().toISOString() });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Maintenance ────────────────────────────────────────────────────────────
+/**
+ * Reduce confidence for memories not updated within the last N processed units.
+ * "Stale" = updated_at is older than the Nth most recent processed_at.
+ */
+export function decayStaleMemories(thresholdUnits = 20) {
+    if (!isDbAvailable())
+        return;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return;
+    try {
+        // Find the timestamp of the Nth most recent processed unit
+        const row = adapter.prepare(`SELECT processed_at FROM memory_processed_units
+       ORDER BY processed_at DESC
+       LIMIT 1 OFFSET :offset`).get({ ':offset': thresholdUnits - 1 });
+        if (!row)
+            return; // not enough processed units yet
+        const cutoff = row['processed_at'];
+        adapter.prepare(`UPDATE memories
+       SET confidence = MAX(0.1, confidence - 0.1), updated_at = :now
+       WHERE superseded_by IS NULL AND updated_at < :cutoff AND confidence > 0.1`).run({ ':now': new Date().toISOString(), ':cutoff': cutoff });
+    }
+    catch {
+        // non-fatal
+    }
+}
+/**
+ * Supersede lowest-ranked memories when count exceeds cap.
+ */
+export function enforceMemoryCap(max = 50) {
+    if (!isDbAvailable())
+        return;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return;
+    try {
+        const countRow = adapter.prepare('SELECT count(*) as cnt FROM memories WHERE superseded_by IS NULL').get();
+        const count = countRow?.['cnt'] ?? 0;
+        if (count <= max)
+            return;
+        const excess = count - max;
+        // Find the IDs of the lowest-ranked active memories
+        const rows = adapter.prepare(`SELECT id FROM memories
+       WHERE superseded_by IS NULL
+       ORDER BY (confidence * (1.0 + hit_count * 0.1)) ASC
+       LIMIT :limit`).all({ ':limit': excess });
+        const now = new Date().toISOString();
+        for (const row of rows) {
+            adapter.prepare('UPDATE memories SET superseded_by = :reason, updated_at = :now WHERE id = :id').run({ ':reason': 'CAP_EXCEEDED', ':now': now, ':id': row['id'] });
+        }
+    }
+    catch {
+        // non-fatal
+    }
+}
+// ─── Action Application ─────────────────────────────────────────────────────
+/**
+ * Process an array of memory actions in a transaction.
+ * Calls enforceMemoryCap at the end.
+ */
+export function applyMemoryActions(actions, unitType, unitId) {
+    if (!isDbAvailable() || actions.length === 0)
+        return;
+    try {
+        transaction(() => {
+            for (const action of actions) {
+                switch (action.action) {
+                    case 'CREATE':
+                        createMemory({
+                            category: action.category,
+                            content: action.content,
+                            confidence: action.confidence,
+                            source_unit_type: unitType,
+                            source_unit_id: unitId,
+                        });
+                        break;
+                    case 'UPDATE':
+                        updateMemoryContent(action.id, action.content, action.confidence);
+                        break;
+                    case 'REINFORCE':
+                        reinforceMemory(action.id);
+                        break;
+                    case 'SUPERSEDE':
+                        supersedeMemory(action.id, action.superseded_by);
+                        break;
+                }
+            }
+            enforceMemoryCap();
+        });
+    }
+    catch {
+        // non-fatal — transaction will have rolled back
+    }
+}
+// ─── Prompt Formatting ──────────────────────────────────────────────────────
+/**
+ * Format memories as categorized markdown for system prompt injection.
+ * Truncates to token budget (~4 chars per token).
+ */
+export function formatMemoriesForPrompt(memories, tokenBudget = 2000) {
+    if (memories.length === 0)
+        return '';
+    const charBudget = tokenBudget * 4;
+    const header = '## Project Memory (auto-learned)\n';
+    let output = header;
+    let remaining = charBudget - header.length;
+    // Group by category
+    const grouped = new Map();
+    for (const m of memories) {
+        const list = grouped.get(m.category) ?? [];
+        list.push(m);
+        grouped.set(m.category, list);
+    }
+    // Sort categories by priority
+    const sortedCategories = [...grouped.keys()].sort((a, b) => (CATEGORY_PRIORITY[a] ?? 99) - (CATEGORY_PRIORITY[b] ?? 99));
+    for (const category of sortedCategories) {
+        const items = grouped.get(category);
+        const catHeader = `\n### ${category.charAt(0).toUpperCase() + category.slice(1)}\n`;
+        if (remaining < catHeader.length + 10)
+            break;
+        output += catHeader;
+        remaining -= catHeader.length;
+        for (const item of items) {
+            const bullet = `- ${item.content}\n`;
+            if (remaining < bullet.length)
+                break;
+            output += bullet;
+            remaining -= bullet.length;
+        }
+    }
+    return output.trimEnd();
+}