@gregiteen/ssss-cli 0.11.1 → 0.12.0

package/CHANGELOG.md

@@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.12.0] - 2026-10-29
+### Added
+- `FileIdempotencyStore` now expires entries. It accepts `retentionMs` (default
+  24 hours, `0` disables) and `pruneEvery` (default every 500 puts), and exposes
+  `prune()`, which removes entries older than the window and returns the count.
+  Retention runs opportunistically inside `put` and never rejects a committed
+  operation. Previously entries were written and never removed, so a directory
+  only grew: one frequently rewritten document produced 14,438 records from a
+  single path, and a live vault reached 283k files / 1.2 GB, which slowed every
+  traversal of it. A retry arrives within minutes, so a bounded window loses
+  nothing that mattered. Lock and temp files are never treated as entries.
+
+## [0.11.2] - 2026-10-19
+### Added
+- `parseFrontmatter(fm, { strict: true })` throws a `SyntaxError` on lines that
+  are not `key: value`, orphaned indentation, and malformed quoted or inline
+  values, so hosts such as Total Recall can report unreadable files. The default
+  stays lenient. Strict mode accepts all 30,518 files in 57 Total Recall vaults.
+
 ## [0.11.1] - 2026-10-19
 ### Fixed
 - The frontmatter parser (`@gregiteen/ssss-cli/frontmatter`) now reads the YAML

package/VERSION

@@ -1 +1 @@
-0.11.1
+0.12.0

package/conformance/reference-bundle.ucw.json

@@ -106,7 +106,7 @@
     "file_count": 14,
     "provenance": {
       "content_hash": "sha256:7febd673938a3dc04c068e0bf08097a9783c09e612606d2df9a4db8dcb8907f5",
-      "exporter": "@gregiteen/ssss-cli@0.11.1"
+      "exporter": "@gregiteen/ssss-cli@0.12.0"
     }
   },
   "files": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gregiteen/ssss-cli",
-  "version": "0.11.1",
+  "version": "0.12.0",
   "description": "CLI tools and the reference engine for the Structured Semantic Syntax System",
   "main": "src/engine.mjs",
   "exports": {

package/scripts/check-frontmatter.mjs

@@ -89,6 +89,16 @@ check('single-line files with literal \\n are repaired on read', () => {
   assert.equal(parseDocument('---\nk: a\\nb\n---\n').data.k, 'a\\nb');
 });
 
+check('strict mode throws on malformed YAML that lenient mode skips', () => {
+  const bad = ': : :\nnot: [valid\n';
+  assert.doesNotThrow(() => parseFrontmatter(bad));
+  assert.throws(() => parseFrontmatter(bad, { strict: true }), SyntaxError);
+  assert.throws(() => parseFrontmatter('k: "open\nj: 1\n', { strict: true }), SyntaxError);
+  assert.throws(() => parseFrontmatter('a: 1\n  b: 2\n', { strict: true }), SyntaxError);
+  assert.deepEqual(parseFrontmatter('a: [1, 2]\nb: {c: d}\n', { strict: true }), { a: [1, 2], b: { c: 'd' } });
+  assert.doesNotThrow(() => parseFrontmatter(bad)); // strict does not leak into later calls
+});
+
 check('serialize → parse round-trips tricky values', () => {
   const data = {
     'allowed-tools': ['Read', 'Bash'],

package/scripts/conformance-09.mjs

@@ -1,4 +1,5 @@
 import fs from 'node:fs';
+import crypto from 'node:crypto';
 import { execFileSync, spawn, spawnSync } from 'node:child_process';
 import os from 'node:os';
 import path from 'node:path';
@@ -123,6 +124,32 @@ async function checkFileAdapterRaces(root, principal, check) {
   check('filesystem idempotency store accepts one of several processes putting the same key',
     winners(puts).length === 1 && stored?.request_hash === `racer-${winners(puts)[0]?.racer}`, describe(puts));
 
+  // Retention: an entry exists to reject a retry, which arrives within minutes,
+  // so entries expire rather than living forever. Without this a single
+  // frequently rewritten document produced 14,438 records and a live vault
+  // reached 283k files / 1.2 GB, slowing every traversal of the vault.
+  const retentionRoot = path.join(root, 'idempotency-retention');
+  const retention = new FileIdempotencyStore(retentionRoot, { retentionMs: 60_000, pruneEvery: 1 });
+  await retention.put('acme', 'fresh', { request_hash: 'sha256:fresh' });
+  const entryName = (workspaceId, idempotencyKey) =>
+    `${crypto.createHash('sha256').update(JSON.stringify([workspaceId, idempotencyKey])).digest('hex')}.json`;
+  const staleFile = path.join(retentionRoot, entryName('acme', 'stale'));
+  fs.writeFileSync(staleFile, '{"request_hash":"sha256:stale"}\n', { mode: 0o600 });
+  const aged = new Date(Date.now() - 10 * 60 * 1000);
+  fs.utimesSync(staleFile, aged, aged);
+  const removed = retention.prune();
+  check('filesystem idempotency store expires entries past the retention window',
+    removed === 1 && !fs.existsSync(staleFile), `removed ${removed}`);
+  check('filesystem idempotency store keeps entries inside the retention window',
+    (await retention.get('acme', 'fresh'))?.request_hash === 'sha256:fresh');
+  check('filesystem idempotency store skips retention when the window is disabled',
+    new FileIdempotencyStore(retentionRoot, { retentionMs: 0 }).prune() === 0);
+  const lockName = path.join(retentionRoot, `${entryName('acme', 'locked')}.lock`);
+  fs.writeFileSync(lockName, '');
+  const lockSurvived = retention.prune() === 0 && fs.existsSync(lockName);
+  fs.rmSync(lockName, { force: true });
+  check('filesystem idempotency store never treats a lock file as an entry', lockSurvived);
+
   const kernelRoot = path.join(root, 'kernel');
   const kernelPrincipal = { ...principal, capabilities: ['*:*'] };
   const stores = () => ({

package/scripts/conformance.mjs

@@ -346,6 +346,35 @@ async function runBundleConformance() {
     fs.rmSync(atomicVault, { recursive: true, force: true });
   }
 
+  const referenceVault = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-bundle-refs-'));
+  const missingVault = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-bundle-missing-'));
+  try {
+    const engine = createEngine({ registryDir: REGISTRY_DIR });
+    const envelope = (id, target, fields) => ({
+      type: 'operation', workspace_id: 'ws-refs', idempotency_key: id,
+      path: target, actor: { role: 'system' },
+      content: ['---', 'title: Reference test', 'description: Complete planned references.',
+        'timestamp: 2026-10-28T00:00:00Z', ...fields, '---', '', 'Exact payload.', ''].join('\n'),
+    });
+    const asset = envelope('ref-asset', 'assets/source.md', [
+      'type: asset', 'name: source', 'mime_type: text/plain', 'encoding: utf-8',
+      'resource_ref: resources/store.md',
+    ]);
+    const resource = envelope('ref-resource', 'resources/store.md', [
+      'type: resource', 'name: store', 'kind: storage', 'status: bound',
+    ]);
+    const imported = await importBundle([asset, resource], referenceVault, engine);
+    check('fresh import resolves a forward reference from the complete plan',
+      imported.ok && imported.committed === 2,
+      JSON.stringify(imported.results.map(item => item.validation?.errors || [])));
+    const missing = await importBundle([asset], missingVault, engine);
+    check('missing planned reference rejects import without partial writes',
+      !missing.ok && missing.committed === 0 && !fs.existsSync(path.join(missingVault, 'assets/source.md')));
+  } finally {
+    fs.rmSync(referenceVault, { recursive: true, force: true });
+    fs.rmSync(missingVault, { recursive: true, force: true });
+  }
+
   let pass = 0;
   for (const c of checks) {
     if (c.cond) { pass++; console.log(`  ✅ ${c.name}`); }

package/skills/ssss/SKILL.md

@@ -42,6 +42,11 @@ When an SSSS engine processes a write, it receives an **Envelope**:
 ### 4. Bundles (.ucw)
 An SSSS vault can be packaged into a `.ucw` (Universal Containerized Workspace) bundle. A bundle is a portable snapshot containing a manifest, deterministic file paths, and content hashes for cryptographic integrity.
 
+Bundle import validates the complete plan before committing. Hosts pass planned
+document lookup through the kernel context's `resolveReference` option so valid
+forward references resolve in an empty destination. Missing referenced documents
+must reject the whole import; never weaken constraints to bypass ordering.
+
 ## Validation 
 
 If you are writing tests or schemas in a host repository, ensure that:

package/src/bundle.mjs

@@ -508,7 +508,7 @@ export async function importBundle(plan, vaultRoot, engine) {
     });
   }
   const operationOptions = {
-    referenceResolver: (referencePath) => plannedDocuments.get(referencePath),
+    resolveReference: (referencePath) => plannedDocuments.get(referencePath),
   };
   // Two-phase import: validate the complete plan with in-plan references first.
   // A single invalid document prevents every commit, eliminating partial bundle

package/src/frontmatter.mjs

@@ -170,16 +170,24 @@ function parseFlow(src) {
   return value;
 }
 
+// Set by parseFrontmatter(fm, { strict: true }): malformed input throws instead of
+// being skipped or kept as raw text.
+let strictMode = false;
+
 function coerce(raw) {
   const v = raw.trim();
   if (v === '') return '';
   if (v[0] === '"' || v[0] === "'") {
     const end = findClosingQuote(v);
     if (end > 0 && /^\s*(#.*)?$/.test(v.slice(end + 1))) return unquote(v.slice(0, end + 1));
+    if (strictMode) throw new SyntaxError(`Malformed quoted value: ${v.slice(0, 60)}`);
     return v;
   }
   if (v[0] === '[' || v[0] === '{') {
-    try { return parseFlow(v); } catch { return v; }
+    try { return parseFlow(v); } catch (err) {
+      if (strictMode) throw new SyntaxError(`Malformed inline value: ${v.slice(0, 60)}`);
+      return v;
+    }
   }
   return coercePlain(stripComment(v));
 }
@@ -214,6 +222,7 @@ function readScalar(lines, i, keyIndent, rest) {
     if (!line.trim()) { blanks++; j++; continue; }
     if (countIndent(line) <= keyIndent) break;
     if (!inline && line.trim().startsWith('#')) break;
+    if (strictMode && !inline && splitKeyValue(line.trim())) throw new SyntaxError(`Unexpected key in a wrapped value: ${line.trim().slice(0, 60)}`);
     const piece = inline ? line.trim() : stripComment(line.trim());
     text += blanks ? '\n'.repeat(blanks) + piece : ` ${piece}`;
     blanks = 0;
@@ -278,9 +287,17 @@ function parseMap(lines, start, indent) {
     if (!line.trim() || line.trim().startsWith('#')) { i++; continue; }
     const currentIndent = countIndent(line);
     if (currentIndent < indent) break;
-    if (currentIndent > indent) { i++; continue; }
+    if (currentIndent > indent) {
+      if (strictMode) throw new SyntaxError(`Unexpected indentation: ${line.trim().slice(0, 60)}`);
+      i++;
+      continue;
+    }
     const kv = splitKeyValue(line.slice(indent));
-    if (!kv) { i++; continue; }
+    if (!kv) {
+      if (strictMode) throw new SyntaxError(`Not a "key: value" line: ${line.trim().slice(0, 60)}`);
+      i++;
+      continue;
+    }
     const block = kv.rest.match(BLOCK_SCALAR_RE);
     if (block) {
       const parsed = parseBlockScalar(lines, i + 1, indent, block[1], block[2]);
@@ -369,12 +386,21 @@ function parseList(lines, start, indent) {
 }
 
 /**
- * Parse a frontmatter block (the inner YAML, no fences) into an object.
+ * Parse a frontmatter block (the inner YAML, no fences) into an object. The
+ * default is lenient: unreadable lines are skipped. `{ strict: true }` throws a
+ * SyntaxError on lines that are not `key: value`, orphaned indentation, and
+ * malformed quoted or inline values, so callers can report unreadable files.
  */
-export function parseFrontmatter(fm) {
+export function parseFrontmatter(fm, { strict = false } = {}) {
   if (!fm) return {};
-  // Drop the newline before the closing fence so it is not read as a blank line.
-  return parseMap(fm.replace(/\n$/, '').split('\n'), 0, 0).value;
+  const previous = strictMode;
+  strictMode = strict;
+  try {
+    // Drop the newline before the closing fence so it is not read as a blank line.
+    return parseMap(fm.replace(/\n$/, '').split('\n'), 0, 0).value;
+  } finally {
+    strictMode = previous;
+  }
 }
 
 /** Parse a full document (with fences) into { data, body }. */

package/src/idempotency.mjs

@@ -6,6 +6,17 @@ import { withFileLock } from './file-lock.mjs';
 
 function key(workspaceId, idempotencyKey) { return JSON.stringify([workspaceId, idempotencyKey]); }
 
+/**
+ * How long a completed operation stays replayable. A retry of the same request
+ * arrives within minutes (a network timeout, an agent restart); nothing needs a
+ * day-old result, and retaining them forever is what filled a live vault with
+ * 283k files. Override per store with `new FileIdempotencyStore(root, { retentionMs })`.
+ */
+export const DEFAULT_RETENTION_MS = 24 * 60 * 60 * 1000;
+
+/** Prune once every N puts, so retention costs one directory scan per N writes. */
+export const DEFAULT_PRUNE_EVERY = 500;
+
 export class MemoryIdempotencyStore {
   #entries = new Map();
   async get(workspaceId, idempotencyKey) { return structuredClone(this.#entries.get(key(workspaceId, idempotencyKey)) || null); }
@@ -21,6 +32,18 @@ export class FileIdempotencyStore {
     fs.mkdirSync(root, { recursive: true });
     this.root = fs.realpathSync(root);
     this.lockOptions = options.lock || {};
+    // An idempotency entry exists to reject a retry of the same request. That
+    // window is minutes, not forever, so entries are retained for a bounded age
+    // and then removed. Without this the directory only grows: one frequently
+    // rewritten document created 14,438 records from a single path, and a live
+    // vault reached 283k files / 1.2 GB, which slowed every traversal of it.
+    this.retentionMs = Number.isFinite(options.retentionMs) && options.retentionMs >= 0
+      ? options.retentionMs
+      : DEFAULT_RETENTION_MS;
+    this.pruneEvery = Number.isInteger(options.pruneEvery) && options.pruneEvery > 0
+      ? options.pruneEvery
+      : DEFAULT_PRUNE_EVERY;
+    this.puts = 0;
   }
   #file(workspaceId, idempotencyKey) {
     const digest = crypto.createHash('sha256').update(key(workspaceId, idempotencyKey)).digest('hex');
@@ -44,8 +67,37 @@ export class FileIdempotencyStore {
         fs.writeFileSync(temp, `${JSON.stringify(value)}\n`, { flag: 'wx', mode: 0o600 });
         fs.renameSync(temp, file);
       } finally { fs.rmSync(temp, { force: true }); }
+      // Opportunistic retention: no scheduler, and the cost is amortised over
+      // putPushes. Never throws — a prune failure must not reject a committed
+      // operation, whose result the caller is about to rely on.
+      this.puts += 1;
+      if (this.retentionMs > 0 && this.puts % this.pruneEvery === 0) {
+        try { this.prune(); } catch { /* retention is best-effort */ }
+      }
     }, this.lockOptions);
   }
+  /**
+   * Remove entries older than the retention window. Returns the count removed.
+   * Only regular `*.json` files are considered; lock files and temp files are
+   * left alone so a concurrent writer is never disturbed.
+   */
+  prune(now = Date.now()) {
+    if (!(this.retentionMs > 0)) return 0;
+    const cutoff = now - this.retentionMs;
+    let removed = 0;
+    let entries;
+    try { entries = fs.readdirSync(this.root); } catch { return 0; }
+    for (const name of entries) {
+      if (!name.endsWith('.json')) continue;
+      const file = path.join(this.root, name);
+      try {
+        const stat = fs.lstatSync(file);
+        if (!stat.isFile() || stat.isSymbolicLink()) continue;
+        if (stat.mtimeMs < cutoff) { fs.rmSync(file, { force: true }); removed += 1; }
+      } catch { /* a vanished or unreadable entry is not a prune failure */ }
+    }
+    return removed;
+  }
 }
 
 export async function runIdempotencyContract(store) {