@gregiteen/ssss-cli 0.10.1 → 0.10.3

package/CHANGELOG.md

@@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.10.3] - 2026-10-01
+### Fixed
+- Read event logs in 64 KiB chunks when building duplicate-ID indexes or
+  replaying events. Large histories no longer require one full-log string.
+  UTF-8 decoding, replay cursors, file locks and duplicate detection retain
+  their contracts. A regression check guards bounded reads and corrupt-log
+  rejection.
+
+## [0.10.2] - 2026-09-28
+### Fixed
+- The 0.10.0 notes promised pointer fields for large-file assets, but the
+  `asset` registry entry never declared them. It now declares `storage_uri`
+  and `resource_ref`. An asset with `encoding: none` is a pointer: it must set
+  `storage_uri` and be marked `x_portability: resource_bound` (or
+  `tenant_private`). `resource_ref` must name a `resource` document. Template
+  and sale exports replace `storage_uri` with `REQUIREMENT` and keep `hash` and
+  `size_bytes`, so the buyer can check the file they bind.
+
+### Behavior changes hosts should check
+- An `asset` with `encoding: none` and no `storage_uri` or no
+  `x_portability: resource_bound|tenant_private` is now rejected.
+- An `asset`'s `x_portability` may only be `resource_bound` or `tenant_private`.
+- `resource_ref` on an `asset` must point at an existing `resource` document.
+- Template/sale bundles containing pointer assets now declare a
+  `asset_storage_uri` parameter and a `bind-asset` provisioning step.
+- New conformance section: asset pointers (§5.4 asset, §5.5).
+
 ## [0.10.1] - 2026-09-28
 ### Fixed
 - A required list field set to an empty list (`deferred_steps: []`) was

package/VERSION

@@ -1 +1 @@
-0.10.1
+0.10.3

package/conformance/reference-bundle.ucw.json

@@ -106,7 +106,7 @@
     "file_count": 14,
     "provenance": {
       "content_hash": "sha256:7febd673938a3dc04c068e0bf08097a9783c09e612606d2df9a4db8dcb8907f5",
-      "exporter": "@gregiteen/ssss-cli@0.10.1"
+      "exporter": "@gregiteen/ssss-cli@0.10.3"
     }
   },
   "files": [

package/docs/ssss-spec.md

@@ -354,6 +354,7 @@ timestamp: 2026-05-16T14:00:00Z
 A text-encoded asset payload: source code, vector graphics, or base64-encoded binary content, or pointer metadata for large files.
 
 REQUIRED: `type`, `name`, `mime_type`, `encoding` (`base64|utf-8|none`).
+OPTIONAL: `slug`, `description`, `hash`, `size_bytes`, `storage_uri`, `resource_ref`.
 
 ```markdown
 ---
@@ -369,6 +370,33 @@ encoding: "base64"
 iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
 ```
 
+A file too large to carry in the vault is stored as a **pointer**: `encoding: none`
+and an empty body. A pointer REQUIRES `storage_uri`, the URI of the stored object,
+and `x_portability` set to `resource_bound` or `tenant_private` (§5.5), so a
+template or sale export never ships the seller's storage location: the exporter
+replaces `storage_uri` with `REQUIREMENT` and the buyer binds their own copy at
+provision. `resource_ref`, when present, is the vault path of the `resource`
+document (bucket, object store) that holds the object. `hash` (`sha256:<hex>`) and
+`size_bytes` pin the object's content and survive export, so the buyer can verify
+the file they supply.
+
+```markdown
+---
+type: asset
+name: "training-video"
+title: "Onboarding Video"
+description: "Staff onboarding video, stored in object storage."
+timestamp: 2026-09-28T00:00:00Z
+mime_type: "video/mp4"
+encoding: "none"
+storage_uri: "s3://media-bucket/training/onboarding.mp4"
+resource_ref: "resources/media-bucket.md"
+hash: "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+size_bytes: 734003200
+x_portability: resource_bound
+---
+```
+
 #### `resource`
 
 An external infrastructure, cloud asset, domain, database, or API dependency bound to the workspace. Carries `portability: resource_bound`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gregiteen/ssss-cli",
-  "version": "0.10.1",
+  "version": "0.10.3",
   "description": "CLI tools and the reference engine for the Structured Semantic Syntax System",
   "main": "src/engine.mjs",
   "exports": {
@@ -56,7 +56,7 @@
   ],
   "scripts": {
     "conformance": "node ./scripts/conformance.mjs conformance",
-    "test": "node ./scripts/conformance.mjs conformance --engine"
+    "test": "node ./scripts/conformance.mjs conformance --engine && node ./scripts/check-events-streaming.mjs"
   },
   "keywords": [
     "ssss",

package/registry/core.json

@@ -40,11 +40,19 @@
       "append_only": false,
       "portability": "structural",
       "required_fields": ["type", "name", "mime_type", "encoding"],
-      "optional_fields": ["slug", "hash", "size_bytes", "description"],
+      "optional_fields": ["slug", "hash", "size_bytes", "description", "storage_uri", "resource_ref", "x_portability"],
+      "required_when": {
+        "encoding==none": ["storage_uri", "x_portability"]
+      },
       "enums": {
-        "encoding": ["base64", "utf-8", "none"]
+        "encoding": ["base64", "utf-8", "none"],
+        "x_portability": ["resource_bound", "tenant_private"]
+      },
+      "references": {
+        "resource_ref": { "allowed_types": ["resource"] }
       },
-      "notes": "A text-encoded asset payload: source code, vector graphics, or base64-encoded binary content.",
+      "resource": { "kind": "storage", "binds": ["storage_uri"], "provision_relation": "stores_asset" },
+      "notes": "A text-encoded asset payload: source code, vector graphics, or base64-encoded binary content. With `encoding: none` the body is empty and the asset is a pointer to a large file: `storage_uri` locates the object, `resource_ref` optionally names the `resource` document (bucket, store) that holds it, and `hash`/`size_bytes` pin its content. A pointer MUST be marked `x_portability: resource_bound` (or `tenant_private`), so template/sale exports replace `storage_uri` with `REQUIREMENT` instead of shipping the seller's storage location.",
       "spec_ref": "§5.4 asset"
     },
     "resource": {

package/scripts/check-events-streaming.mjs

@@ -0,0 +1,41 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { JsonlEventStore, createCanonicalEvent } from '../src/events.mjs';
+
+const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-event-stream-'));
+const file = path.join(root, 'acme.jsonl');
+const make = index => createCanonicalEvent({
+  workspace_id: 'acme', action: 'event', subject: 'stream-test',
+  principal: { id: 'test', kind: 'service' }, operation_id: `operation-${index}`,
+  idempotency_key: `key-${index}`, payload: { text: 'a'.repeat(65500) + '🧠é'.repeat(25000) }
+});
+const events = [make(1), make(2)];
+const originals = { alloc: Buffer.alloc, readFileSync: fs.readFileSync, readSync: fs.readSync };
+try {
+  fs.writeFileSync(file, events.map(event => JSON.stringify(event)).join('\n') + '\n');
+  Buffer.alloc = (size, ...rest) => { assert.ok(size <= 65536, 'Event reader allocated the whole log'); return originals.alloc(size, ...rest); };
+  fs.readFileSync = (target, ...rest) => { assert.notEqual(target, file, 'Event reader loaded the whole log'); return originals.readFileSync(target, ...rest); };
+  let largestRead = 0;
+  fs.readSync = (fd, buffer, offset, length, position) => {
+    largestRead = Math.max(largestRead, length);
+    assert.ok(length <= 65536, 'Event reader requested the whole log');
+    return originals.readSync(fd, buffer, offset, Math.min(length, 4093), position);
+  };
+  const store = new JsonlEventStore(root);
+  await assert.rejects(store.append(events[0]), /Duplicate event_id/);
+  const third = make(3);
+  await store.append(third);
+  const replayed = [];
+  for await (const entry of store.replay({ workspaceId: 'acme', cursor: 1 })) replayed.push(entry);
+  assert.deepEqual(replayed.map(entry => entry.cursor), [2, 3]);
+  assert.deepEqual(replayed.map(entry => entry.event), [events[1], third]);
+  await assert.rejects(new JsonlEventStore(root).append(third), /Duplicate event_id/);
+  fs.appendFileSync(file, '{invalid JSON}\n');
+  await assert.rejects(new JsonlEventStore(root).append(make(4)), SyntaxError);
+  console.log(JSON.stringify({ ok: true, boundedReads: true, utf8: true, cursor: true, duplicate: true, corruptFailClosed: true, largestRead }));
+} finally {
+  Buffer.alloc = originals.alloc; fs.readFileSync = originals.readFileSync; fs.readSync = originals.readSync;
+  fs.rmSync(root, { recursive: true });
+}

package/scripts/conformance.mjs

@@ -480,6 +480,98 @@ async function runOperationContractRegressionConformance() {
   return pass === checks.length;
 }
 
+async function runAssetPointerConformance() {
+  const checks = [];
+  const check = (name, cond, detail = '') => { checks.push({ name, cond, detail }); };
+
+  const vault = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-asset-'));
+  const doc = (fields, body = '') => ['---', ...fields, '---', '', body, ''].join('\n');
+  const assetHead = [
+    'type: asset',
+    'name: training-video',
+    'title: Training Video',
+    'description: Onboarding video stored outside the vault.',
+    'timestamp: 2026-09-28T00:00:00Z',
+    'mime_type: video/mp4',
+  ];
+  let n = 0;
+  const write = (filePath, content) => engine.processOperation({
+    type: 'operation',
+    idempotency_key: `asset-${++n}`,
+    workspace_id: 'ws-asset',
+    path: filePath,
+    actor: { role: 'system' },
+    content,
+  }, vault);
+  const errorsOf = (res) => (res.validation?.errors || []).join('; ');
+  const engine = createEngine({ registryDir: REGISTRY_DIR });
+  try {
+    const bucket = await write('resources/media-bucket.md', doc([
+      'type: resource',
+      'name: Media Bucket',
+      'title: Media Bucket',
+      'description: Object storage for large media.',
+      'timestamp: 2026-09-28T00:00:00Z',
+      'kind: s3_bucket',
+      'status: bound',
+      'binds: s3://seller-media',
+    ]));
+    check('resource fixture commits', bucket.success === true, errorsOf(bucket));
+
+    const inline = await write('assets/logo.md', doc([
+      'type: asset', 'name: logo', 'title: Logo', 'description: Inline icon.',
+      'timestamp: 2026-09-28T00:00:00Z', 'mime_type: image/svg+xml', 'encoding: utf-8',
+    ], '<svg xmlns="http://www.w3.org/2000/svg"/>'));
+    check('inline asset needs no pointer fields', inline.success === true, errorsOf(inline));
+
+    const noUri = await write('assets/no-uri.md', doc([...assetHead, 'encoding: none', 'x_portability: resource_bound']));
+    check('pointer asset without storage_uri is rejected',
+      noUri.success === false && errorsOf(noUri).includes('storage_uri'), errorsOf(noUri));
+
+    const noClass = await write('assets/no-class.md', doc([...assetHead, 'encoding: none', 'storage_uri: s3://seller-media/v.mp4']));
+    check('pointer asset without x_portability is rejected',
+      noClass.success === false && errorsOf(noClass).includes('x_portability'), errorsOf(noClass));
+
+    const structural = await write('assets/structural.md', doc([...assetHead, 'encoding: none',
+      'storage_uri: s3://seller-media/v.mp4', 'x_portability: structural']));
+    check('pointer asset cannot be marked structural',
+      structural.success === false && errorsOf(structural).includes('x_portability'), errorsOf(structural));
+
+    const wrongRef = await write('assets/wrong-ref.md', doc([...assetHead, 'encoding: none',
+      'storage_uri: s3://seller-media/v.mp4', 'x_portability: resource_bound', 'resource_ref: assets/logo.md']));
+    check('resource_ref must name a resource document',
+      wrongRef.success === false && errorsOf(wrongRef).includes('resource_ref'), errorsOf(wrongRef));
+
+    const hash = `sha256:${'a'.repeat(64)}`;
+    const pointer = await write('assets/training-video.md', doc([...assetHead, 'encoding: none',
+      'storage_uri: s3://seller-media/training/v.mp4', 'resource_ref: resources/media-bucket.md',
+      `hash: "${hash}"`, 'size_bytes: 734003200', 'x_portability: resource_bound']));
+    check('pointer asset with storage_uri, resource_ref, hash and size_bytes commits',
+      pointer.success === true, errorsOf(pointer));
+
+    const sale = exportBundle(vault, { registryDir: REGISTRY_DIR, profile: 'sale' });
+    const exported = sale.files.find((f) => f.path === 'assets/training-video.md');
+    const exportedData = exported ? parseDocument(exported.content).data : {};
+    check('sale export replaces storage_uri with REQUIREMENT and keeps the content pin',
+      exportedData.storage_uri === 'REQUIREMENT' && exportedData.hash === hash &&
+      exportedData.size_bytes === 734003200 && !exported.content.includes('seller-media/training'),
+      exported?.content || 'pointer asset missing from sale bundle');
+    check('sale bundle with a pointer asset validates',
+      validateBundle(sale, { registryDir: REGISTRY_DIR }).valid === true,
+      validateBundle(sale, { registryDir: REGISTRY_DIR }).errors.join('; '));
+  } finally {
+    fs.rmSync(vault, { recursive: true, force: true });
+  }
+
+  let pass = 0;
+  for (const c of checks) {
+    if (c.cond) { pass++; console.log(`  ✅ ${c.name}`); }
+    else console.log(`  ❌ ${c.name}${c.detail ? ` — ${c.detail}` : ''}`);
+  }
+  console.log(`\n  ${pass}/${checks.length} asset pointer checks passed (§5.4 asset, §5.5)`);
+  return pass === checks.length;
+}
+
 async function runSemanticLocalizationConformance() {
   const checks = [];
   const check = (name, cond, detail = '') => { checks.push({ name, cond, detail }); };
@@ -996,6 +1088,8 @@ async function main() {
     const runtimeOk = await runRuntimeConformance();
     console.log('\nRunning operation regression conformance (src/engine.mjs, §6/§7) ...');
     const operationRegressionOk = await runOperationContractRegressionConformance();
+    console.log('\nRunning asset pointer conformance (registry/core.json asset, §5.4/§5.5) ...');
+    const assetPointerOk = await runAssetPointerConformance();
     console.log('\nRunning extension-registry conformance (src/registry.mjs) ...');
     const extensionRegistryOk = runRegistryExtensionConformance();
     console.log('\nRunning semantic/multilingual rendering conformance (src/semantic.mjs, §11.9) ...');
@@ -1007,7 +1101,7 @@ async function main() {
     console.log('\nRunning CLI smoke conformance (scripts/ssss.mjs) ...');
     const cliSmokeOk = await runCliSmokeConformance();
     process.exit(
-      engineOk && runtimeOk && operationRegressionOk && extensionRegistryOk &&
+      engineOk && runtimeOk && operationRegressionOk && assetPointerOk && extensionRegistryOk &&
       semanticOk && kernel09Ok && bundleOk && cliSmokeOk ? 0 : 1
     );
   } else {

package/src/events.mjs

@@ -2,6 +2,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
+import { StringDecoder } from 'node:string_decoder';
 import { withFileLock } from './file-lock.mjs';
 
 // UUIDv5 namespace for event ids derived from idempotency keys.
@@ -83,22 +84,33 @@ function sameStat(a, b) {
   return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs;
 }
 
-/** Read every event_id in a log, bounded to the bytes covered by the returned stat. */
+/** Decode one line at a time, never materializing the complete log as a string. */
+function* logLines(fd, size) {
+  const buffer = Buffer.alloc(64 * 1024);
+  const decoder = new StringDecoder('utf8');
+  let offset = 0, pending = '';
+  while (offset < size) {
+    const read = fs.readSync(fd, buffer, 0, Math.min(buffer.length, size - offset), offset);
+    if (read === 0) break;
+    offset += read;
+    const text = pending + decoder.write(buffer.subarray(0, read));
+    let start = 0, newline;
+    while ((newline = text.indexOf('\n', start)) !== -1) {
+      yield text.slice(start, newline);
+      start = newline + 1;
+    }
+    pending = text.slice(start);
+  }
+  pending += decoder.end();
+  if (pending) yield pending;
+}
+
+/** Read every event_id, bounded to the bytes covered by the returned stat. */
 function loadIndex(file) {
   const fd = openLog(file, fs.constants.O_RDONLY | NOFOLLOW);
   try {
-    const stat = fs.fstatSync(fd);
-    const bytes = Buffer.alloc(stat.size);
-    let offset = 0;
-    while (offset < bytes.length) {
-      const read = fs.readSync(fd, bytes, offset, bytes.length - offset, offset);
-      if (read === 0) break;
-      offset += read;
-    }
-    const ids = new Set();
-    for (const line of bytes.subarray(0, offset).toString('utf8').split('\n')) {
-      if (line.trim()) ids.add(JSON.parse(line).event_id);
-    }
+    const stat = fs.fstatSync(fd), ids = new Set();
+    for (const line of logLines(fd, stat.size)) if (line.trim()) ids.add(JSON.parse(line).event_id);
     return { ids, stat };
   } finally { fs.closeSync(fd); }
 }
@@ -168,12 +180,15 @@ export class JsonlEventStore {
     for (const file of files) {
       if (!fs.existsSync(file)) continue;
       if (fs.lstatSync(file).isSymbolicLink()) throw new Error('Symlinked event logs are forbidden.');
-      for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
-        if (!line.trim()) continue;
-        cursor++;
-        if (cursor <= (options.cursor || 0)) continue;
-        yield { cursor, event: JSON.parse(line) };
-      }
+      const fd = openLog(file, fs.constants.O_RDONLY | NOFOLLOW);
+      try {
+        for (const line of logLines(fd, fs.fstatSync(fd).size)) {
+          if (!line.trim()) continue;
+          cursor++;
+          if (cursor <= (options.cursor || 0)) continue;
+          yield { cursor, event: JSON.parse(line) };
+        }
+      } finally { fs.closeSync(fd); }
     }
   }
 }