@gregiteen/ssss-cli 0.11.2 → 0.12.0

package/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.12.0] - 2026-10-29
+### Added
+- `FileIdempotencyStore` now expires entries. It accepts `retentionMs` (default
+  24 hours, `0` disables) and `pruneEvery` (default every 500 puts), and exposes
+  `prune()`, which removes entries older than the window and returns the count.
+  Retention runs opportunistically inside `put` and never rejects a committed
+  operation. Previously entries were written and never removed, so a directory
+  only grew: one frequently rewritten document produced 14,438 records from a
+  single path, and a live vault reached 283k files / 1.2 GB, which slowed every
+  traversal of it. A retry arrives within minutes, so a bounded window loses
+  nothing that mattered. Lock and temp files are never treated as entries.
+
 ## [0.11.2] - 2026-10-19
 ### Added
 - `parseFrontmatter(fm, { strict: true })` throws a `SyntaxError` on lines that

package/VERSION

@@ -1 +1 @@
-0.11.2
+0.12.0

package/conformance/reference-bundle.ucw.json

@@ -106,7 +106,7 @@
     "file_count": 14,
     "provenance": {
       "content_hash": "sha256:7febd673938a3dc04c068e0bf08097a9783c09e612606d2df9a4db8dcb8907f5",
-      "exporter": "@gregiteen/ssss-cli@0.11.2"
+      "exporter": "@gregiteen/ssss-cli@0.12.0"
     }
   },
   "files": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gregiteen/ssss-cli",
-  "version": "0.11.2",
+  "version": "0.12.0",
   "description": "CLI tools and the reference engine for the Structured Semantic Syntax System",
   "main": "src/engine.mjs",
   "exports": {

package/scripts/conformance-09.mjs

@@ -1,4 +1,5 @@
 import fs from 'node:fs';
+import crypto from 'node:crypto';
 import { execFileSync, spawn, spawnSync } from 'node:child_process';
 import os from 'node:os';
 import path from 'node:path';
@@ -123,6 +124,32 @@ async function checkFileAdapterRaces(root, principal, check) {
   check('filesystem idempotency store accepts one of several processes putting the same key',
     winners(puts).length === 1 && stored?.request_hash === `racer-${winners(puts)[0]?.racer}`, describe(puts));
 
+  // Retention: an entry exists to reject a retry, which arrives within minutes,
+  // so entries expire rather than living forever. Without this a single
+  // frequently rewritten document produced 14,438 records and a live vault
+  // reached 283k files / 1.2 GB, slowing every traversal of the vault.
+  const retentionRoot = path.join(root, 'idempotency-retention');
+  const retention = new FileIdempotencyStore(retentionRoot, { retentionMs: 60_000, pruneEvery: 1 });
+  await retention.put('acme', 'fresh', { request_hash: 'sha256:fresh' });
+  const entryName = (workspaceId, idempotencyKey) =>
+    `${crypto.createHash('sha256').update(JSON.stringify([workspaceId, idempotencyKey])).digest('hex')}.json`;
+  const staleFile = path.join(retentionRoot, entryName('acme', 'stale'));
+  fs.writeFileSync(staleFile, '{"request_hash":"sha256:stale"}\n', { mode: 0o600 });
+  const aged = new Date(Date.now() - 10 * 60 * 1000);
+  fs.utimesSync(staleFile, aged, aged);
+  const removed = retention.prune();
+  check('filesystem idempotency store expires entries past the retention window',
+    removed === 1 && !fs.existsSync(staleFile), `removed ${removed}`);
+  check('filesystem idempotency store keeps entries inside the retention window',
+    (await retention.get('acme', 'fresh'))?.request_hash === 'sha256:fresh');
+  check('filesystem idempotency store skips retention when the window is disabled',
+    new FileIdempotencyStore(retentionRoot, { retentionMs: 0 }).prune() === 0);
+  const lockName = path.join(retentionRoot, `${entryName('acme', 'locked')}.lock`);
+  fs.writeFileSync(lockName, '');
+  const lockSurvived = retention.prune() === 0 && fs.existsSync(lockName);
+  fs.rmSync(lockName, { force: true });
+  check('filesystem idempotency store never treats a lock file as an entry', lockSurvived);
+
   const kernelRoot = path.join(root, 'kernel');
   const kernelPrincipal = { ...principal, capabilities: ['*:*'] };
   const stores = () => ({

package/scripts/conformance.mjs

@@ -346,6 +346,35 @@ async function runBundleConformance() {
     fs.rmSync(atomicVault, { recursive: true, force: true });
   }
 
+  const referenceVault = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-bundle-refs-'));
+  const missingVault = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-bundle-missing-'));
+  try {
+    const engine = createEngine({ registryDir: REGISTRY_DIR });
+    const envelope = (id, target, fields) => ({
+      type: 'operation', workspace_id: 'ws-refs', idempotency_key: id,
+      path: target, actor: { role: 'system' },
+      content: ['---', 'title: Reference test', 'description: Complete planned references.',
+        'timestamp: 2026-10-28T00:00:00Z', ...fields, '---', '', 'Exact payload.', ''].join('\n'),
+    });
+    const asset = envelope('ref-asset', 'assets/source.md', [
+      'type: asset', 'name: source', 'mime_type: text/plain', 'encoding: utf-8',
+      'resource_ref: resources/store.md',
+    ]);
+    const resource = envelope('ref-resource', 'resources/store.md', [
+      'type: resource', 'name: store', 'kind: storage', 'status: bound',
+    ]);
+    const imported = await importBundle([asset, resource], referenceVault, engine);
+    check('fresh import resolves a forward reference from the complete plan',
+      imported.ok && imported.committed === 2,
+      JSON.stringify(imported.results.map(item => item.validation?.errors || [])));
+    const missing = await importBundle([asset], missingVault, engine);
+    check('missing planned reference rejects import without partial writes',
+      !missing.ok && missing.committed === 0 && !fs.existsSync(path.join(missingVault, 'assets/source.md')));
+  } finally {
+    fs.rmSync(referenceVault, { recursive: true, force: true });
+    fs.rmSync(missingVault, { recursive: true, force: true });
+  }
+
   let pass = 0;
   for (const c of checks) {
     if (c.cond) { pass++; console.log(`  ✅ ${c.name}`); }

package/skills/ssss/SKILL.md

@@ -42,6 +42,11 @@ When an SSSS engine processes a write, it receives an **Envelope**:
 ### 4. Bundles (.ucw)
 An SSSS vault can be packaged into a `.ucw` (Universal Containerized Workspace) bundle. A bundle is a portable snapshot containing a manifest, deterministic file paths, and content hashes for cryptographic integrity.
 
+Bundle import validates the complete plan before committing. Hosts pass planned
+document lookup through the kernel context's `resolveReference` option so valid
+forward references resolve in an empty destination. Missing referenced documents
+must reject the whole import; never weaken constraints to bypass ordering.
+
 ## Validation 
 
 If you are writing tests or schemas in a host repository, ensure that:

package/src/bundle.mjs

@@ -508,7 +508,7 @@ export async function importBundle(plan, vaultRoot, engine) {
     });
   }
   const operationOptions = {
-    referenceResolver: (referencePath) => plannedDocuments.get(referencePath),
+    resolveReference: (referencePath) => plannedDocuments.get(referencePath),
   };
   // Two-phase import: validate the complete plan with in-plan references first.
   // A single invalid document prevents every commit, eliminating partial bundle

package/src/idempotency.mjs

@@ -6,6 +6,17 @@ import { withFileLock } from './file-lock.mjs';
 
 function key(workspaceId, idempotencyKey) { return JSON.stringify([workspaceId, idempotencyKey]); }
 
+/**
+ * How long a completed operation stays replayable. A retry of the same request
+ * arrives within minutes (a network timeout, an agent restart); nothing needs a
+ * day-old result, and retaining them forever is what filled a live vault with
+ * 283k files. Override per store with `new FileIdempotencyStore(root, { retentionMs })`.
+ */
+export const DEFAULT_RETENTION_MS = 24 * 60 * 60 * 1000;
+
+/** Prune once every N puts, so retention costs one directory scan per N writes. */
+export const DEFAULT_PRUNE_EVERY = 500;
+
 export class MemoryIdempotencyStore {
   #entries = new Map();
   async get(workspaceId, idempotencyKey) { return structuredClone(this.#entries.get(key(workspaceId, idempotencyKey)) || null); }
@@ -21,6 +32,18 @@ export class FileIdempotencyStore {
     fs.mkdirSync(root, { recursive: true });
     this.root = fs.realpathSync(root);
     this.lockOptions = options.lock || {};
+    // An idempotency entry exists to reject a retry of the same request. That
+    // window is minutes, not forever, so entries are retained for a bounded age
+    // and then removed. Without this the directory only grows: one frequently
+    // rewritten document created 14,438 records from a single path, and a live
+    // vault reached 283k files / 1.2 GB, which slowed every traversal of it.
+    this.retentionMs = Number.isFinite(options.retentionMs) && options.retentionMs >= 0
+      ? options.retentionMs
+      : DEFAULT_RETENTION_MS;
+    this.pruneEvery = Number.isInteger(options.pruneEvery) && options.pruneEvery > 0
+      ? options.pruneEvery
+      : DEFAULT_PRUNE_EVERY;
+    this.puts = 0;
   }
   #file(workspaceId, idempotencyKey) {
     const digest = crypto.createHash('sha256').update(key(workspaceId, idempotencyKey)).digest('hex');
@@ -44,8 +67,37 @@ export class FileIdempotencyStore {
         fs.writeFileSync(temp, `${JSON.stringify(value)}\n`, { flag: 'wx', mode: 0o600 });
         fs.renameSync(temp, file);
       } finally { fs.rmSync(temp, { force: true }); }
+      // Opportunistic retention: no scheduler, and the cost is amortised over
+      // putPushes. Never throws — a prune failure must not reject a committed
+      // operation, whose result the caller is about to rely on.
+      this.puts += 1;
+      if (this.retentionMs > 0 && this.puts % this.pruneEvery === 0) {
+        try { this.prune(); } catch { /* retention is best-effort */ }
+      }
     }, this.lockOptions);
   }
+  /**
+   * Remove entries older than the retention window. Returns the count removed.
+   * Only regular `*.json` files are considered; lock files and temp files are
+   * left alone so a concurrent writer is never disturbed.
+   */
+  prune(now = Date.now()) {
+    if (!(this.retentionMs > 0)) return 0;
+    const cutoff = now - this.retentionMs;
+    let removed = 0;
+    let entries;
+    try { entries = fs.readdirSync(this.root); } catch { return 0; }
+    for (const name of entries) {
+      if (!name.endsWith('.json')) continue;
+      const file = path.join(this.root, name);
+      try {
+        const stat = fs.lstatSync(file);
+        if (!stat.isFile() || stat.isSymbolicLink()) continue;
+        if (stat.mtimeMs < cutoff) { fs.rmSync(file, { force: true }); removed += 1; }
+      } catch { /* a vanished or unreadable entry is not a prune failure */ }
+    }
+    return removed;
+  }
 }
 
 export async function runIdempotencyContract(store) {