@gregiteen/ssss-cli 0.10.2 → 0.10.3

package/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.10.3] - 2026-10-01
+### Fixed
+- Read event logs in 64 KiB chunks when building duplicate-ID indexes or
+  replaying events. Large histories no longer require one full-log string.
+  UTF-8 decoding, replay cursors, file locks and duplicate detection retain
+  their contracts. A regression check guards bounded reads and corrupt-log
+  rejection.
+
 ## [0.10.2] - 2026-09-28
 ### Fixed
 - The 0.10.0 notes promised pointer fields for large-file assets, but the

package/VERSION

@@ -1 +1 @@
-0.10.2
+0.10.3

package/conformance/reference-bundle.ucw.json

@@ -106,7 +106,7 @@
     "file_count": 14,
     "provenance": {
       "content_hash": "sha256:7febd673938a3dc04c068e0bf08097a9783c09e612606d2df9a4db8dcb8907f5",
-      "exporter": "@gregiteen/ssss-cli@0.10.2"
+      "exporter": "@gregiteen/ssss-cli@0.10.3"
     }
   },
   "files": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gregiteen/ssss-cli",
-  "version": "0.10.2",
+  "version": "0.10.3",
   "description": "CLI tools and the reference engine for the Structured Semantic Syntax System",
   "main": "src/engine.mjs",
   "exports": {
@@ -56,7 +56,7 @@
   ],
   "scripts": {
     "conformance": "node ./scripts/conformance.mjs conformance",
-    "test": "node ./scripts/conformance.mjs conformance --engine"
+    "test": "node ./scripts/conformance.mjs conformance --engine && node ./scripts/check-events-streaming.mjs"
   },
   "keywords": [
     "ssss",

package/scripts/check-events-streaming.mjs

@@ -0,0 +1,41 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { JsonlEventStore, createCanonicalEvent } from '../src/events.mjs';
+
+const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ssss-event-stream-'));
+const file = path.join(root, 'acme.jsonl');
+const make = index => createCanonicalEvent({
+  workspace_id: 'acme', action: 'event', subject: 'stream-test',
+  principal: { id: 'test', kind: 'service' }, operation_id: `operation-${index}`,
+  idempotency_key: `key-${index}`, payload: { text: 'a'.repeat(65500) + '🧠é'.repeat(25000) }
+});
+const events = [make(1), make(2)];
+const originals = { alloc: Buffer.alloc, readFileSync: fs.readFileSync, readSync: fs.readSync };
+try {
+  fs.writeFileSync(file, events.map(event => JSON.stringify(event)).join('\n') + '\n');
+  Buffer.alloc = (size, ...rest) => { assert.ok(size <= 65536, 'Event reader allocated the whole log'); return originals.alloc(size, ...rest); };
+  fs.readFileSync = (target, ...rest) => { assert.notEqual(target, file, 'Event reader loaded the whole log'); return originals.readFileSync(target, ...rest); };
+  let largestRead = 0;
+  fs.readSync = (fd, buffer, offset, length, position) => {
+    largestRead = Math.max(largestRead, length);
+    assert.ok(length <= 65536, 'Event reader requested the whole log');
+    return originals.readSync(fd, buffer, offset, Math.min(length, 4093), position);
+  };
+  const store = new JsonlEventStore(root);
+  await assert.rejects(store.append(events[0]), /Duplicate event_id/);
+  const third = make(3);
+  await store.append(third);
+  const replayed = [];
+  for await (const entry of store.replay({ workspaceId: 'acme', cursor: 1 })) replayed.push(entry);
+  assert.deepEqual(replayed.map(entry => entry.cursor), [2, 3]);
+  assert.deepEqual(replayed.map(entry => entry.event), [events[1], third]);
+  await assert.rejects(new JsonlEventStore(root).append(third), /Duplicate event_id/);
+  fs.appendFileSync(file, '{invalid JSON}\n');
+  await assert.rejects(new JsonlEventStore(root).append(make(4)), SyntaxError);
+  console.log(JSON.stringify({ ok: true, boundedReads: true, utf8: true, cursor: true, duplicate: true, corruptFailClosed: true, largestRead }));
+} finally {
+  Buffer.alloc = originals.alloc; fs.readFileSync = originals.readFileSync; fs.readSync = originals.readSync;
+  fs.rmSync(root, { recursive: true });
+}

package/src/events.mjs

@@ -2,6 +2,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
+import { StringDecoder } from 'node:string_decoder';
 import { withFileLock } from './file-lock.mjs';
 
 // UUIDv5 namespace for event ids derived from idempotency keys.
@@ -83,22 +84,33 @@ function sameStat(a, b) {
   return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs;
 }
 
-/** Read every event_id in a log, bounded to the bytes covered by the returned stat. */
+/** Decode one line at a time, never materializing the complete log as a string. */
+function* logLines(fd, size) {
+  const buffer = Buffer.alloc(64 * 1024);
+  const decoder = new StringDecoder('utf8');
+  let offset = 0, pending = '';
+  while (offset < size) {
+    const read = fs.readSync(fd, buffer, 0, Math.min(buffer.length, size - offset), offset);
+    if (read === 0) break;
+    offset += read;
+    const text = pending + decoder.write(buffer.subarray(0, read));
+    let start = 0, newline;
+    while ((newline = text.indexOf('\n', start)) !== -1) {
+      yield text.slice(start, newline);
+      start = newline + 1;
+    }
+    pending = text.slice(start);
+  }
+  pending += decoder.end();
+  if (pending) yield pending;
+}
+
+/** Read every event_id, bounded to the bytes covered by the returned stat. */
 function loadIndex(file) {
   const fd = openLog(file, fs.constants.O_RDONLY | NOFOLLOW);
   try {
-    const stat = fs.fstatSync(fd);
-    const bytes = Buffer.alloc(stat.size);
-    let offset = 0;
-    while (offset < bytes.length) {
-      const read = fs.readSync(fd, bytes, offset, bytes.length - offset, offset);
-      if (read === 0) break;
-      offset += read;
-    }
-    const ids = new Set();
-    for (const line of bytes.subarray(0, offset).toString('utf8').split('\n')) {
-      if (line.trim()) ids.add(JSON.parse(line).event_id);
-    }
+    const stat = fs.fstatSync(fd), ids = new Set();
+    for (const line of logLines(fd, stat.size)) if (line.trim()) ids.add(JSON.parse(line).event_id);
     return { ids, stat };
   } finally { fs.closeSync(fd); }
 }
@@ -168,12 +180,15 @@ export class JsonlEventStore {
     for (const file of files) {
       if (!fs.existsSync(file)) continue;
       if (fs.lstatSync(file).isSymbolicLink()) throw new Error('Symlinked event logs are forbidden.');
-      for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
-        if (!line.trim()) continue;
-        cursor++;
-        if (cursor <= (options.cursor || 0)) continue;
-        yield { cursor, event: JSON.parse(line) };
-      }
+      const fd = openLog(file, fs.constants.O_RDONLY | NOFOLLOW);
+      try {
+        for (const line of logLines(fd, fs.fstatSync(fd).size)) {
+          if (!line.trim()) continue;
+          cursor++;
+          if (cursor <= (options.cursor || 0)) continue;
+          yield { cursor, event: JSON.parse(line) };
+        }
+      } finally { fs.closeSync(fd); }
     }
   }
 }