@lumenflow/memory 2.2.1 → 2.3.1

package/dist/memory-store.js

@@ -1,5 +1,5 @@
 /**
- * Memory Store (WU-1463)
+ * Memory Store (WU-1463, WU-1238)
  *
  * JSONL-based memory store with load, query, and append operations.
  * Git-friendly format with one node per line for merge-safe diffs.
@@ -8,9 +8,11 @@
  * - Append-only writes (no full file rewrite)
  * - Indexed lookups by ID and WU
  * - Deterministic queryReady() ordering by priority then createdAt
+ * - WU-1238: Support for archived node filtering
+ * - WU-1238: Decay score-based sorting option
  *
- * @see {@link tools/lib/__tests__/memory-store.test.mjs} - Tests
- * @see {@link tools/lib/memory-schema.mjs} - Schema definitions
+ * @see {@link packages/@lumenflow/cli/src/lib/__tests__/memory-store.test.ts} - Tests
+ * @see {@link packages/@lumenflow/cli/src/lib/memory-schema.ts} - Schema definitions
  */
 import fs from 'node:fs/promises';
 import path from 'node:path';
@@ -68,6 +70,15 @@ function compareNodes(a, b) {
     // Tertiary: stable sort by ID for identical priority and timestamp
     return a.id.localeCompare(b.id);
 }
+/**
+ * Check if a node is archived (WU-1238).
+ *
+ * @param node - Memory node to check
+ * @returns True if node has metadata.status = 'archived'
+ */
+function isNodeArchived(node) {
+    return node.metadata?.status === 'archived';
+}
 /**
  * Loads memory from JSONL file and returns indexed nodes.
  *
@@ -86,71 +97,111 @@ function compareNodes(a, b) {
  * const memory = await loadMemory('/path/to/project');
  * const node = memory.byId.get('mem-abc1');
  * const wuNodes = memory.byWu.get('WU-1463') ?? [];
+ *
+ * @example
+ * // WU-1238: Include archived nodes
+ * const allMemory = await loadMemory('/path/to/project', { includeArchived: true });
  */
-export async function loadMemory(baseDir) {
+export async function loadMemory(baseDir, options = {}) {
+    const { includeArchived = false } = options;
     const filePath = path.join(baseDir, MEMORY_FILE_NAME);
-    const result = {
-        nodes: [],
-        byId: new Map(),
-        byWu: new Map(),
-    };
-    // Check if file exists
-    let content;
+    const content = await readMemoryFileOrEmpty(filePath);
+    if (content === null) {
+        return createEmptyIndexedMemory();
+    }
+    return parseAndIndexMemory(content, includeArchived);
+}
+/**
+ * Reads memory file content, returning null if file doesn't exist
+ */
+async function readMemoryFileOrEmpty(filePath) {
     try {
-        content = await fs.readFile(filePath, { encoding: 'utf-8' });
+        return await fs.readFile(filePath, { encoding: 'utf-8' });
     }
     catch (err) {
         const error = err;
         if (error.code === 'ENOENT') {
-            // File doesn't exist - return empty result
-            return result;
+            return null;
         }
         throw error;
     }
-    // Parse JSONL content
+}
+/**
+ * Creates an empty indexed memory result
+ */
+function createEmptyIndexedMemory() {
+    return {
+        nodes: [],
+        byId: new Map(),
+        byWu: new Map(),
+    };
+}
+/**
+ * Parses a single JSONL line and validates it
+ */
+function parseAndValidateLine(line, lineNumber) {
+    let parsed;
+    try {
+        parsed = JSON.parse(line);
+    }
+    catch (err) {
+        const errMsg = err instanceof Error ? err.message : String(err);
+        throw new Error(`Malformed JSON on line ${lineNumber}: ${errMsg}`);
+    }
+    const validation = validateMemoryNode(parsed);
+    if (!validation.success) {
+        const issues = validation.error.issues
+            .map((issue) => `${issue.path.join('.')}: ${issue.message}`)
+            .join(', ');
+        throw new Error(`Validation error on line ${lineNumber}: ${issues}`);
+    }
+    return validation.data;
+}
+/**
+ * Adds a node to the WU index
+ */
+function indexNodeByWu(result, node) {
+    if (!node.wu_id)
+        return;
+    if (!result.byWu.has(node.wu_id)) {
+        result.byWu.set(node.wu_id, []);
+    }
+    const wuNodes = result.byWu.get(node.wu_id);
+    if (wuNodes) {
+        wuNodes.push(node);
+    }
+}
+/**
+ * Parses JSONL content and builds indexed memory
+ */
+function parseAndIndexMemory(content, includeArchived) {
+    const result = createEmptyIndexedMemory();
     const lines = content.split('\n');
     for (let i = 0; i < lines.length; i++) {
-        const rawLine = lines[i];
-        const line = rawLine ? rawLine.trim() : '';
-        // Skip empty lines
-        if (!line) {
+        const line = lines[i]?.trim() ?? '';
+        if (!line)
+            continue;
+        const node = parseAndValidateLine(line, i + 1);
+        // WU-1238: Skip archived nodes unless includeArchived is true
+        if (!includeArchived && isNodeArchived(node)) {
             continue;
         }
-        // Parse JSON line
-        let parsed;
-        try {
-            parsed = JSON.parse(line);
-        }
-        catch (err) {
-            const errMsg = err instanceof Error ? err.message : String(err);
-            throw new Error(`Malformed JSON on line ${i + 1}: ${errMsg}`);
-        }
-        // Validate against schema
-        const validation = validateMemoryNode(parsed);
-        if (!validation.success) {
-            const issues = validation.error.issues
-                .map((issue) => `${issue.path.join('.')}: ${issue.message}`)
-                .join(', ');
-            throw new Error(`Validation error on line ${i + 1}: ${issues}`);
-        }
-        const node = validation.data;
-        // Add to nodes array
         result.nodes.push(node);
-        // Index by ID
         result.byId.set(node.id, node);
-        // Index by WU ID if present
-        if (node.wu_id) {
-            if (!result.byWu.has(node.wu_id)) {
-                result.byWu.set(node.wu_id, []);
-            }
-            const wuNodes = result.byWu.get(node.wu_id);
-            if (wuNodes) {
-                wuNodes.push(node);
-            }
-        }
+        indexNodeByWu(result, node);
     }
     return result;
 }
+/**
+ * Loads all memory including archived nodes.
+ * Convenience function for operations that need to see all nodes.
+ *
+ * @param baseDir - Directory containing memory.jsonl
+ * @returns Indexed memory nodes including archived
+ */
+export async function loadMemoryAll(baseDir) {
+    return loadMemory(baseDir, { includeArchived: true });
+}
 /**
  * Appends a single node to the memory file.
  *
@@ -206,9 +257,14 @@ export async function appendNode(baseDir, node) {
  * for (const node of ready) {
  *   await processNode(node);
  * }
+ *
+ * @example
+ * // WU-1238: Include archived nodes
+ * const all = await queryReady('/path/to/project', 'WU-1463', { includeArchived: true });
  */
-export async function queryReady(baseDir, wuId) {
-    const memory = await loadMemory(baseDir);
+export async function queryReady(baseDir, wuId, options = {}) {
+    const { includeArchived = false } = options;
+    const memory = await loadMemory(baseDir, { includeArchived });
     // Get nodes for this WU
     const wuNodes = memory.byWu.get(wuId) ?? [];
     // Return sorted copy (don't mutate original)
@@ -227,8 +283,13 @@ export async function queryReady(baseDir, wuId) {
  * @example
  * const nodes = await queryByWu('/path/to/project', 'WU-1463');
  * console.log(`Found ${nodes.length} nodes for WU-1463`);
+ *
+ * @example
+ * // WU-1238: Include archived nodes
+ * const all = await queryByWu('/path/to/project', 'WU-1463', { includeArchived: true });
  */
-export async function queryByWu(baseDir, wuId) {
-    const memory = await loadMemory(baseDir);
+export async function queryByWu(baseDir, wuId, options = {}) {
+    const { includeArchived = false } = options;
+    const memory = await loadMemory(baseDir, { includeArchived });
     return memory.byWu.get(wuId) ?? [];
 }

package/dist/signal-cleanup-core.js

@@ -0,0 +1,355 @@
+/**
+ * Signal Cleanup Core (WU-1204)
+ *
+ * TTL-based cleanup for signals to prevent unbounded growth.
+ * Implements configurable retention policies:
+ * - Read signals: 7 days default TTL
+ * - Unread signals: 30 days default TTL
+ * - Max entries: 500 default
+ * - Active WU protection: signals linked to in_progress/blocked WUs are never removed
+ *
+ * Reuses patterns from mem-cleanup-core.ts.
+ *
+ * @see {@link packages/@lumenflow/cli/src/signal-cleanup.ts} - CLI wrapper
+ * @see {@link packages/@lumenflow/memory/src/__tests__/signal-cleanup-core.test.ts} - Tests
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const ms = require('ms');
+import { LUMENFLOW_MEMORY_PATHS } from './paths.js';
+import { SIGNAL_FILE_NAME } from './mem-signal-core.js';
+/**
+ * Default TTL values in milliseconds
+ */
+const ONE_DAY_MS = 24 * 60 * 60 * 1000;
+/**
+ * Default signal cleanup configuration
+ */
+export const DEFAULT_SIGNAL_CLEANUP_CONFIG = {
+    ttl: 7 * ONE_DAY_MS,
+    unreadTtl: 30 * ONE_DAY_MS,
+    maxEntries: 500,
+};
+/**
+ * Parse a TTL duration string into milliseconds.
+ *
+ * Uses the `ms` package to parse human-readable duration strings.
+ *
+ * @param ttlString - TTL string (e.g., '7d', '30d', '24h', '60m')
+ * @returns TTL in milliseconds
+ * @throws If TTL format is invalid
+ *
+ * @example
+ * parseSignalTtl('7d');  // 604800000 (7 days in ms)
+ * parseSignalTtl('30d'); // 2592000000 (30 days in ms)
+ * parseSignalTtl('24h'); // 86400000 (24 hours in ms)
+ */
+export function parseSignalTtl(ttlString) {
+    if (!ttlString || typeof ttlString !== 'string') {
+        throw new Error('Invalid TTL format: TTL string is required');
+    }
+    const trimmed = ttlString.trim();
+    if (!trimmed) {
+        throw new Error('Invalid TTL format: TTL string is required');
+    }
+    // Use ms package to parse the duration
+    const result = ms(trimmed);
+    if (result === undefined || result <= 0) {
+        throw new Error(`Invalid TTL format: "${ttlString}" is not a valid duration`);
+    }
+    return result;
+}
+/**
+ * Check if a signal has expired based on TTL.
+ *
+ * @param signal - Signal to check
+ * @param ttlMs - TTL in milliseconds
+ * @param now - Current timestamp
+ * @returns True if signal is older than TTL
+ */
+function isSignalExpired(signal, ttlMs, now) {
+    if (!signal.created_at) {
+        return false; // No timestamp means we can't determine age - safer to retain
+    }
+    const createdAt = new Date(signal.created_at).getTime();
+    // Invalid date - safer to retain
+    if (Number.isNaN(createdAt)) {
+        return false;
+    }
+    const age = now - createdAt;
+    return age > ttlMs;
+}
+/**
+ * Check if a signal should be removed based on TTL and active WU protection.
+ *
+ * Policy rules (checked in order):
+ * 1. Active WU signals are always retained
+ * 2. Read signals older than TTL are removed
+ * 3. Unread signals older than unreadTtl are removed
+ * 4. Otherwise, signal is retained
+ *
+ * @param signal - Signal to check
+ * @param config - Cleanup configuration
+ * @param context - Removal context (now timestamp, active WU IDs)
+ * @returns Removal decision with reason
+ */
+export function shouldRemoveSignal(signal, config, context) {
+    const { now, activeWuIds } = context;
+    // Active WU protection: signals linked to in_progress/blocked WUs are always retained
+    if (signal.wu_id && activeWuIds.has(signal.wu_id)) {
+        return { remove: false, reason: 'active-wu-protected' };
+    }
+    // Check TTL based on read status
+    if (signal.read) {
+        // Read signals use the shorter TTL
+        if (isSignalExpired(signal, config.ttl, now)) {
+            return { remove: true, reason: 'ttl-expired' };
+        }
+    }
+    else {
+        // Unread signals use the longer unreadTtl
+        if (isSignalExpired(signal, config.unreadTtl, now)) {
+            return { remove: true, reason: 'unread-ttl-expired' };
+        }
+    }
+    // Default: retain
+    return { remove: false, reason: 'within-ttl' };
+}
+/**
+ * Calculate approximate byte size of a signal when serialized.
+ *
+ * @param signal - Signal to measure
+ * @returns Approximate byte size
+ */
+function estimateSignalBytes(signal) {
+    // JSON.stringify + newline character
+    return JSON.stringify(signal).length + 1;
+}
+/**
+ * Calculate compaction ratio (removed / total).
+ *
+ * @param removedCount - Number of removed signals
+ * @param totalCount - Total number of signals
+ * @returns Compaction ratio (0 to 1, or 0 if no signals)
+ */
+function getCompactionRatio(removedCount, totalCount) {
+    if (totalCount === 0) {
+        return 0;
+    }
+    return removedCount / totalCount;
+}
+/**
+ * Gets the signals file path for a project.
+ *
+ * @param baseDir - Project base directory
+ * @returns Full path to signals.jsonl
+ */
+function getSignalsPath(baseDir) {
+    return path.join(baseDir, LUMENFLOW_MEMORY_PATHS.MEMORY_DIR, SIGNAL_FILE_NAME);
+}
+/**
+ * Load signals directly from file (for cleanup, to avoid loadSignals filter limitations).
+ *
+ * @param baseDir - Project base directory
+ * @returns Array of all signals
+ */
+async function loadAllSignals(baseDir) {
+    const signalsPath = getSignalsPath(baseDir);
+    try {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- CLI tool reads known path
+        const content = await fs.readFile(signalsPath, { encoding: 'utf-8' });
+        const lines = content.split('\n').filter((line) => line.trim());
+        return lines.map((line) => JSON.parse(line));
+    }
+    catch (err) {
+        const error = err;
+        if (error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+}
+/**
+ * Write retained signals back to file.
+ *
+ * @param baseDir - Project base directory
+ * @param signals - Signals to write
+ */
+async function writeSignals(baseDir, signals) {
+    const signalsPath = getSignalsPath(baseDir);
+    const content = signals.map((s) => JSON.stringify(s)).join('\n') + (signals.length > 0 ? '\n' : '');
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- CLI tool writes known path
+    await fs.writeFile(signalsPath, content, { encoding: 'utf-8' });
+}
+/**
+ * Default function to get active WU IDs (returns empty set).
+ * Override in CLI to actually query WU status.
+ */
+async function defaultGetActiveWuIds() {
+    return new Set();
+}
+/**
+ * Build cleanup configuration from options.
+ *
+ * @param options - Cleanup options
+ * @returns Effective cleanup configuration
+ */
+function buildCleanupConfig(options) {
+    const { ttl, ttlMs: providedTtlMs, unreadTtl, unreadTtlMs: providedUnreadTtlMs, maxEntries, } = options;
+    let ttlMs = providedTtlMs ?? DEFAULT_SIGNAL_CLEANUP_CONFIG.ttl;
+    if (ttl && !providedTtlMs) {
+        ttlMs = parseSignalTtl(ttl);
+    }
+    let unreadTtlMs = providedUnreadTtlMs ?? DEFAULT_SIGNAL_CLEANUP_CONFIG.unreadTtl;
+    if (unreadTtl && !providedUnreadTtlMs) {
+        unreadTtlMs = parseSignalTtl(unreadTtl);
+    }
+    return {
+        ttl: ttlMs,
+        unreadTtl: unreadTtlMs,
+        maxEntries: maxEntries ?? DEFAULT_SIGNAL_CLEANUP_CONFIG.maxEntries,
+    };
+}
+/**
+ * Process signals for TTL-based removal decisions.
+ *
+ * @param signals - All signals to process
+ * @param config - Cleanup configuration
+ * @param context - Removal context
+ * @returns Cleanup state after TTL processing
+ */
+function processSignalsForTtl(signals, config, context) {
+    const state = {
+        removedIds: [],
+        retainedIds: [],
+        retainedSignals: [],
+        bytesFreed: 0,
+        breakdown: {
+            ttlExpired: 0,
+            unreadTtlExpired: 0,
+            countLimitExceeded: 0,
+            activeWuProtected: 0,
+        },
+    };
+    for (const signal of signals) {
+        const decision = shouldRemoveSignal(signal, config, context);
+        processSignalDecision(signal, decision, state);
+    }
+    return state;
+}
+/**
+ * Process a single signal's removal decision and update state.
+ *
+ * @param signal - Signal being processed
+ * @param decision - Removal decision for this signal
+ * @param state - Cleanup state to update
+ */
+function processSignalDecision(signal, decision, state) {
+    if (decision.remove) {
+        state.removedIds.push(signal.id);
+        state.bytesFreed += estimateSignalBytes(signal);
+        updateBreakdownForRemoval(decision.reason, state.breakdown);
+    }
+    else {
+        state.retainedIds.push(signal.id);
+        state.retainedSignals.push(signal);
+        if (decision.reason === 'active-wu-protected') {
+            state.breakdown.activeWuProtected++;
+        }
+    }
+}
+/**
+ * Update breakdown statistics for a removed signal.
+ *
+ * @param reason - Removal reason
+ * @param breakdown - Breakdown to update
+ */
+function updateBreakdownForRemoval(reason, breakdown) {
+    if (reason === 'ttl-expired') {
+        breakdown.ttlExpired++;
+    }
+    else if (reason === 'unread-ttl-expired') {
+        breakdown.unreadTtlExpired++;
+    }
+}
+/**
+ * Apply count-based pruning to keep only maxEntries signals.
+ *
+ * @param state - Current cleanup state (mutated in place)
+ * @param maxEntries - Maximum entries to retain
+ */
+function applyCountPruning(state, maxEntries) {
+    if (state.retainedSignals.length <= maxEntries) {
+        return;
+    }
+    // Sort by created_at (oldest first)
+    state.retainedSignals.sort((a, b) => {
+        const aTime = new Date(a.created_at).getTime();
+        const bTime = new Date(b.created_at).getTime();
+        return aTime - bTime;
+    });
+    const toRemove = state.retainedSignals.length - maxEntries;
+    for (let i = 0; i < toRemove; i++) {
+        // eslint-disable-next-line security/detect-object-injection -- Safe: i is a controlled loop index
+        const signal = state.retainedSignals[i];
+        const idIndex = state.retainedIds.indexOf(signal.id);
+        if (idIndex !== -1) {
+            state.retainedIds.splice(idIndex, 1);
+        }
+        state.removedIds.push(signal.id);
+        state.bytesFreed += estimateSignalBytes(signal);
+        state.breakdown.countLimitExceeded++;
+    }
+    state.retainedSignals.splice(0, toRemove);
+}
+/**
+ * Cleanup signals based on TTL and count limits.
+ *
+ * Removes signals according to policy:
+ * 1. Active WU signals (in_progress/blocked) are always retained
+ * 2. Read signals older than TTL (default 7d) are removed
+ * 3. Unread signals older than unreadTtl (default 30d) are removed
+ * 4. If over maxEntries, oldest signals are removed (keeping newest)
+ *
+ * In dry-run mode, no modifications are made but the result shows
+ * what would be removed.
+ *
+ * @param baseDir - Project base directory
+ * @param options - Cleanup options
+ * @returns Cleanup result with removed/retained IDs and metrics
+ *
+ * @example
+ * // Cleanup with dry-run to preview
+ * const preview = await cleanupSignals(baseDir, { dryRun: true });
+ * console.log(`Would remove ${preview.removedIds.length} signals`);
+ *
+ * @example
+ * // Cleanup with custom TTL
+ * const result = await cleanupSignals(baseDir, { ttl: '3d' });
+ */
+export async function cleanupSignals(baseDir, options = {}) {
+    const { dryRun = false, now = Date.now(), getActiveWuIds = defaultGetActiveWuIds } = options;
+    const config = buildCleanupConfig(options);
+    const signals = await loadAllSignals(baseDir);
+    const activeWuIds = await getActiveWuIds();
+    const state = processSignalsForTtl(signals, config, { now, activeWuIds });
+    applyCountPruning(state, config.maxEntries);
+    const compactionRatio = getCompactionRatio(state.removedIds.length, signals.length);
+    const baseResult = {
+        success: true,
+        removedIds: state.removedIds,
+        retainedIds: state.retainedIds,
+        bytesFreed: state.bytesFreed,
+        compactionRatio,
+        breakdown: state.breakdown,
+    };
+    if (dryRun) {
+        return { ...baseResult, dryRun: true };
+    }
+    if (state.removedIds.length > 0) {
+        await writeSignals(baseDir, state.retainedSignals);
+    }
+    return baseResult;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumenflow/memory",
-  "version": "2.2.1",
+  "version": "2.3.1",
   "description": "Memory layer for LumenFlow workflow framework - session tracking, context recovery, and agent coordination",
   "keywords": [
     "lumenflow",
@@ -25,6 +25,7 @@
   "exports": {
     ".": "./dist/index.js",
     "./checkpoint": "./dist/mem-checkpoint-core.js",
+    "./context": "./dist/mem-context-core.js",
     "./init": "./dist/mem-init-core.js",
     "./start": "./dist/mem-start-core.js",
     "./ready": "./dist/mem-ready-core.js",
@@ -33,6 +34,7 @@
     "./create": "./dist/mem-create-core.js",
     "./summarize": "./dist/mem-summarize-core.js",
     "./triage": "./dist/mem-triage-core.js",
+    "./index": "./dist/mem-index-core.js",
     "./schema": "./dist/memory-schema.js",
     "./store": "./dist/memory-store.js",
     "./dist/*": "./dist/*"
@@ -47,7 +49,7 @@
     "ms": "^2.1.3",
     "yaml": "^2.8.2",
     "zod": "^4.3.5",
-    "@lumenflow/core": "2.2.1"
+    "@lumenflow/core": "2.3.1"
   },
   "devDependencies": {
     "@vitest/coverage-v8": "^4.0.17",