@volter/supercode-teams 0.3.155 → 0.3.157

package/README.md

@@ -52,6 +52,20 @@ Settled read bodies expire after 30 days, with explicit placeholders and retaine
 its originals. Expired read headers are capped at 1,000,000 and carrier/terminal event log tables at 10,000 rows each. Read the
 [dated qualification scope](../../docs/evidence/sqlite-mail-store-2026-10-08.md) before inferring platform support.
 
+Native mail-history reads process the opened file's initial extent in 64 KiB blocks,
+holding one unfinished record instead of the whole transcript and line array. Returned
+mail history and individual records can still be large. The
+[allocation evidence](../../docs/evidence/mail-transcript-stream-2026-10-08.md) names
+the source boundary and the remaining runtime qualification.
+
+The connector's redaction memo keeps serialized redacted JSON, with at most
+200,000 entries and 32 MiB of accounted string payload (two bytes per code unit,
+including keys and digests). An entry over 1 MiB is not retained. Oldest entries
+are evicted to fit; cache misses still go through the redaction worker. These are
+memo bounds, not total connector memory limits. The
+[source investigation](../../docs/evidence/connector-redaction-memo-2026-10-08.md)
+does not attribute the observed process peak or claim a measured runtime improvement.
+
 `supercode teams rpc mail.statistics` reads local storage counts and import/retention receipts. It is refused remotely
 and through the app door. Writer groups count stored envelope copies, not call attempts or quotas; database page
 counts exclude physical WAL size. Successful imports record their actual source reads, bytes, batch sizes and time.

package/connector/index.mjs

@@ -103,14 +103,30 @@ export function foldConnectorState(stateDir){
 }
 // What leaves for the server is redacted (@volter/teams/redact), and every digest this connector keeps or sends (what it
 // observed, a baseline, the carried prefix) is of that form: a digest of the local text named a prefix the server never
-// held, every carry was refused, and each session holding a secret went whole on every pass. A message is redacted once
-// per process, in the connector's redaction worker: by its digest, nothing kept for one without a secret (most), its
-// redacted form for one with. Each loaded message's digest (of the form sent) is taken once and read where needed.
-const REDACTED_MEMO=200000;
+// held, every carry was refused, and each session holding a secret went whole on every pass. A message is redacted
+// in the connector's redaction worker, with a bounded digest memo: no body for one without a secret (most), serialized
+// redacted JSON for one with. Evicted or oversized values are redacted again. Each loaded message's digest (of the form
+// sent) is taken once and read where needed. The byte budget counts two bytes per retained string code unit, including
+// keys and digests; the entry bound separately limits Map/record overhead. Neither is a total-process heap bound.
+const REDACTED_MEMO=200000,REDACTED_MEMO_BYTES=32*1024*1024,REDACTED_MEMO_ENTRY_BYTES=1024*1024;
 const redactions=new Map();
+let redactionBytes=0;
 const sentDigests=new WeakMap();
 const digestOf=message=>{let digest=sentDigests.get(message);if(digest===undefined){digest=hash(canonicalJson(message.value));sentDigests.set(message,digest);}return digest;};
-const remember=(digest,redacted)=>{if(redactions.size>=REDACTED_MEMO)redactions.clear();redactions.set(digest,redacted);};
+function remember(digest,redacted){
+  // Remove the earlier unchanged placeholder even if this redacted body is too large to cache: leaving it would
+  // make a later hit reuse the original, unredacted message.
+  const previous=redactions.get(digest);
+  if(previous){redactionBytes-=previous.bytes;redactions.delete(digest);}
+  const kept=redacted?{text:JSON.stringify(redacted.value),digest:redacted.digest}:null;
+  const bytes=2*(digest.length+(kept?.digest.length??0)+(kept?.text.length??0));
+  if(bytes>REDACTED_MEMO_ENTRY_BYTES)return;
+  while(redactions.size>=REDACTED_MEMO||redactionBytes+bytes>REDACTED_MEMO_BYTES){
+    const oldest=redactions.keys().next().value;
+    redactionBytes-=redactions.get(oldest).bytes;redactions.delete(oldest);
+  }
+  redactions.set(digest,{kept,bytes});redactionBytes+=bytes;
+}
 // A capture part is one request, and the server refuses a body over 8 MiB (volter-teams request-json.mjs): a part
 // holds at most PART_MESSAGES messages (the contract's bound) and PART_BYTES of them, so a run of large tool results
 // splits across parts instead of failing the whole session. A single message larger than that goes alone.
@@ -881,7 +897,7 @@ export class TeamsConnector {
     const out=new Array(raw.length),missing=[];
     raw.forEach((message,index)=>{
       const text=canonicalJson(message.value),digest=hash(text);
-      if(redactions.has(digest)){const kept=redactions.get(digest);out[index]=kept?{...message,value:kept.value}:message;sentDigests.set(out[index],kept?kept.digest:digest);return;}
+      if(redactions.has(digest)){const {kept}=redactions.get(digest);out[index]=kept?{...message,value:JSON.parse(kept.text)}:message;sentDigests.set(out[index],kept?kept.digest:digest);return;}
       if(held?.has(digest)){remember(digest,null);out[index]=message;sentDigests.set(message,digest);return;}
       missing.push([index,digest,text]);
     });

package/mail/transcript.mjs

@@ -1,17 +1,48 @@
 // What a session's transcript says about its mail (crates/harness/src/mail_transcript.rs, ported line for line): the
 // lines its own user typed, the lines it took from its harness's own channels, its answers, and when each filed message
-// first reached the model. The mailbox records a message as sent; only the transcript records it delivered. A transcript
-// is read only when a call names the message or the session it is about (one named read, never a sweep).
+// first reached the model. The mailbox records a message as sent; only the transcript records it delivered. Named mail
+// reads use this reader; agent upkeep also reads changed Claude transcripts to copy the person's typed lines.
 import { readdirSync, statSync } from 'node:fs';
-import { readFile } from 'node:fs/promises';
+import { open } from 'node:fs/promises';
 import { join } from 'node:path';
 import { blake3Hex } from './envelope.mjs';
 
-// A transcript is read off the daemon's event loop (its file read awaited) and parsed in turns of YIELD_LINES lines: one
-// large session's history never holds every other door while it is read.
+// Read one native record at a time. Reading the whole file and splitting it before the first yield retained every
+// tool result while parsing unrelated mail. The read buffer is bounded; one record and the returned mail remain.
+const READ_BYTES = 64 * 1024;
 const YIELD_LINES = 2000;
 const yieldTurn = () => new Promise((resolve) => setImmediate(resolve));
 
+async function* transcriptLines(path) {
+  const file = await open(path, 'r');
+  try {
+    // This pass reads the opened file's initial extent; an appended turn belongs to the next pass.
+    const { size } = await file.stat();
+    let position = 0, fragments = [];
+    while (position < size) {
+      const buffer = Buffer.allocUnsafe(Math.min(READ_BYTES, size - position));
+      const { bytesRead } = await file.read(buffer, 0, buffer.length, position);
+      if (!bytesRead) break;
+      position += bytesRead;
+      const chunk = buffer.subarray(0, bytesRead);
+      let start = 0;
+      for (let end = chunk.indexOf(10); end !== -1; end = chunk.indexOf(10, start)) {
+        const line = fragments.length
+          ? Buffer.concat([...fragments, chunk.subarray(start, end)]).toString('utf8')
+          : chunk.toString('utf8', start, end);
+        fragments = [];
+        start = end + 1;
+        yield line;
+      }
+      if (start < bytesRead) fragments.push(chunk.subarray(start));
+      // Yield even across a long record or many ignored lines, before another read/parse block.
+      await yieldTurn();
+    }
+    // Preserve the former split's final non-newline record, including its JSON-parse refusal if incomplete.
+    if (fragments.length) yield Buffer.concat(fragments).toString('utf8');
+  } finally { await file.close(); }
+}
+
 export const TYPED_ID_PREFIX = 'u-';
 const CHANNEL_ID_PREFIX = 'c-';
 const ANSWER_ID_PREFIX = 'a-';
@@ -88,10 +119,9 @@ export async function readClaude(path) {
   const takenIn = new Map(); // words → [times]
   const early = []; // [words, ids, index, at]
   const deliver = (id, at) => { if (!mail.delivered.has(id)) mail.delivered.set(id, at); };
-  const lines = (await readFile(path, 'utf8')).split('\n');
-  for (let index = 0; index < lines.length; index++) {
-    if (index && index % YIELD_LINES === 0) await yieldTurn();
-    const line = lines[index];
+  let lineIndex = 0;
+  for await (const line of transcriptLines(path)) {
+    if (++lineIndex % YIELD_LINES === 0) await yieldTurn();
     const prompt = line.includes('"promptSource"') || line.includes('"type":"user"');
     const queue = line.includes('"queue-operation"');
     const attachment = line.includes('"queued_command"');
@@ -214,10 +244,9 @@ function codexRenderedMailIds(line) {
 /** Final answers and delivered mail from a Codex session's own transcript (`read_codex_answers`). */
 export async function readCodex(path) {
   const mail = { typed: [], answers: [], channel: [], delivered: new Map() };
-  const lines = (await readFile(path, 'utf8')).split('\n');
-  for (let index = 0; index < lines.length; index++) {
-    if (index && index % YIELD_LINES === 0) await yieldTurn();
-    const line = lines[index];
+  let lineIndex = 0;
+  for await (const line of transcriptLines(path)) {
+    if (++lineIndex % YIELD_LINES === 0) await yieldTurn();
     const rendered = line.includes('cross-session-message id=');
     if (!rendered && !line.includes('"final_answer"')) continue;
     let record; try { record = JSON.parse(line); } catch { continue; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.3.155",
+  "version": "0.3.157",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",