@awebai/oats 0.22.6 → 0.22.7

package/docs/release-notes/v0.22.7.md

@@ -0,0 +1,33 @@
+# OATS v0.22.7
+
+Two resource fixes: index reconciliation streams journals instead of
+reading each one whole, so a full capture pass fits in memory on a store
+with very large session journals, and the packaged Desktop backend runs as
+Node rather than a second Electron runtime.
+
+## Record index: streamed reconciliation
+
+`RecordIndex.update()` used to materialize every journal as a buffer plus a
+parsed array before consulting its per-stream cursor; on a real store with
+an 887 MiB journal the indexing phase peaked above 2.3 GiB and had to be
+stopped. The store now offers `iterateStream()`, which reads 64 KiB chunks
+and yields one turn at a time with the same rules as `readStream()`:
+whitespace lines are skipped, an unparseable or `null` interior line is a
+corruption error carrying its absolute byte offset, and an unterminated
+final fragment is a torn tail left for the owner's append repair. The
+cursor, the single transaction and its rollback on corruption are
+unchanged. Measured on the same store: a full pass completes in 42 s with a
+peak of about 810 MiB under a 256 MB Node old-space budget.
+
+`readStream()` is unchanged for callers that need arrays.
+
+## Desktop backend runs as Node
+
+The packaged Desktop app spawns its backend server with its own
+executable, which is Electron. The child now runs with
+`ELECTRON_RUN_AS_NODE=1`, so the backend and the collector processes it
+starts run as plain Node instead of relaunching the app (verified headless
+against the signed 0.22.6 executable: the backend served `/api/version` at
+about 70 MiB). This does not explain the earlier GUI memory incident, whose
+snapshot held no Desktop process; it removes one way to multiply app
+runtimes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@awebai/oats",
-  "version": "0.22.6",
+  "version": "0.22.7",
   "description": "OATS (Open Agent Team Specification) — durable souls, disposable instances, targetable capability packages, and the runtime-neutral oats CLI/kernel.",
   "keywords": [
     "agents",

package/packages/record/lib/index-db.mjs

@@ -163,12 +163,13 @@ export class RecordIndex {
         "INSERT INTO index_state (stream, indexed_count) VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count = excluded.indexed_count",
       );
       for (const streamId of this.store.listStreams()) {
-        const turns = this.store.readStream(streamId);
         const done = Number(getState.get(streamId)?.indexed_count ?? 0);
-        for (let i = done; i < turns.length; i++) {
-          this.addTurn(turns[i], streamId, effectiveOwner);
+        let count = 0;
+        for (const turn of this.store.iterateStream(streamId)) {
+          if (count >= done) this.addTurn(turn, streamId, effectiveOwner);
+          count++;
         }
-        setState.run(streamId, turns.length);
+        setState.run(streamId, count);
       }
       this.db.exec("COMMIT");
     } catch (err) {

package/packages/record/lib/store.mjs

@@ -404,6 +404,43 @@ export class RecordStore {
     return parseJournal(readFileSync(path)).turns;
   }
 
+  // Bulk consumers need one turn at a time, not a buffer and parsed array
+  // for an entire historical journal (some exceed 800 MB). Keep at most
+  // one complete line plus read chunks, with the same corruption/torn-tail
+  // rules as readStream. Joining fragments only at a newline avoids
+  // repeatedly copying long lines as each chunk arrives.
+  *iterateStream(streamId) {
+    let fd;
+    try { fd = openSync(this.journalPath(streamId), "r"); }
+    catch (e) { if (e.code === "ENOENT") return; throw e; }
+    let pieces = [], size = 0, lineStart = 0;
+    try {
+      for (;;) {
+        const chunk = Buffer.allocUnsafe(65536);
+        const n = readSync(fd, chunk, 0, chunk.length, null);
+        if (n === 0) break;
+        let from = 0;
+        for (let nl = chunk.indexOf(10, from); nl >= 0 && nl < n; nl = chunk.indexOf(10, from)) {
+          pieces.push(chunk.subarray(from, nl + 1));
+          size += nl + 1 - from;
+          const line = pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
+          let parsed;
+          try { parsed = parseJournal(line); }
+          catch (e) {
+            if (e instanceof StoreError) throw new StoreError(`corrupt interior journal line at byte ${lineStart}`);
+            throw e;
+          }
+          lineStart += size;
+          pieces = []; size = 0; from = nl + 1;
+          yield* parsed.turns;
+        }
+        if (from < n) { pieces.push(chunk.subarray(from, n)); size += n - from; }
+      }
+      // A final fragment without a newline is a torn tail, even if its
+      // JSON is valid. Readers leave it for the owner's append repair.
+    } finally { closeSync(fd); }
+  }
+
   // Is this a session-content stream (`<owner>~<source>.<session-id>`)?
   // Their journals hold whole conversations and can be large, so bulk
   // reads exclude them unless asked; access them per-thread instead.