@turns/dev 0.0.28 → 0.0.29

package/dist/agent-anchor.js

@@ -13,6 +13,7 @@ import { ACP_PROFILES, lettaHeadlessEnvironment, lettaSessionIdentity, PROCESS_H
 import { asyncQuestionInteractions, asyncQuestionReplyText, isAsyncQuestionInteraction, parseAsyncQuestionReplies, } from "./codex-async-questions.js";
 import { recordLettaSessionContext } from "./letta-session-context.js";
 import { recoverAsyncQuestion } from "./codex-async-question-recovery.js";
+import { existingTranscriptPath } from "./compressed-transcripts.js";
 import { BackgroundTaskState } from "./background-task-state.js";
 const socketIndex = process.argv.indexOf('--socket');
 const socketPath = socketIndex >= 0 ? path.resolve(process.argv[socketIndex + 1] || '') : '';
@@ -747,7 +748,9 @@ class CodexAppServerTransport extends LineTransport {
                     : 'workspace-write',
             approvalPolicy: this.anchor.permissionMode === 'bypassPermissions' ? 'never' : 'on-request',
         };
-        const resumePath = !createSession && fs.existsSync(this.anchor.transcriptPath) ? this.anchor.transcriptPath : '';
+        // Codex may have compressed or restored the rollout since discovery; it
+        // resumes either spelling and writes the plain `.jsonl` back.
+        const resumePath = createSession ? '' : existingTranscriptPath(this.anchor.transcriptPath);
         const opened = createSession
             ? await this.request('thread/start', {
                 ...configuration,

package/dist/codex-async-question-recovery.js

@@ -1,6 +1,8 @@
 import fs from 'node:fs';
 import readline from 'node:readline';
+import { Readable } from 'node:stream';
 import { ASYNC_QUESTION_TOOL, asyncQuestionInteractions, parseAsyncQuestionReplies } from "./codex-async-questions.js";
+import { existingTranscriptPath, isCompressedTranscript, readTranscriptSync } from "./compressed-transcripts.js";
 export function isAsyncQuestionId(value) {
     if (typeof value !== 'string')
         return false;
@@ -30,7 +32,22 @@ export async function recoverAsyncQuestion(transcriptPath, interactionId) {
     const [, callId] = JSON.parse(interactionId);
     let candidate = null;
     let delivered = false;
-    const stream = fs.createReadStream(transcriptPath, { encoding: 'utf8' });
+    // Codex compresses idle rollouts and restores them on resume, so read
+    // whichever spelling of this anchor's rollout exists now.
+    const file = existingTranscriptPath(transcriptPath);
+    if (!file)
+        return null;
+    let stream;
+    try {
+        stream = isCompressedTranscript(file)
+            ? Readable.from([readTranscriptSync(file).toString('utf8')])
+            : fs.createReadStream(file, { encoding: 'utf8' });
+    }
+    catch (error) {
+        if (error?.code === 'ENOENT')
+            return null;
+        throw error;
+    }
     const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
     try {
         for await (const line of lines) {

package/dist/compressed-transcripts.js

@@ -0,0 +1,99 @@
+import fs from 'node:fs';
+import { zstdDecompressSync } from 'node:zlib';
+/**
+ * Codex compresses idle rollouts in place (`rollout-*.jsonl` becomes
+ * `rollout-*.jsonl.zst`) and restores the plain file when the thread resumes.
+ * Both spellings are one logical transcript: descriptors keep the `.jsonl`
+ * source path so the turns.dev session ID is unchanged across that rename, and
+ * only the physical `file` names the compressed copy.
+ */
+const COMPRESSED_SUFFIX = '.zst';
+const PREFIX_CACHE_BYTES = 32 * 1024 * 1024;
+const MAX_PREFIX_BYTES = 1024 * 1024;
+export function isCompressedTranscript(file) {
+    return /\.jsonl\.zst$/iu.test(String(file || ''));
+}
+/** The plain `.jsonl` path a compressed rollout was made from. */
+export function logicalTranscriptPath(file) {
+    return isCompressedTranscript(file) ? file.slice(0, -COMPRESSED_SUFFIX.length) : file;
+}
+function isFile(file) {
+    try {
+        return fs.statSync(file).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+/** True when a compressed rollout should yield to its live plain sibling. */
+export function compressedTranscriptIsShadowed(file) {
+    return isCompressedTranscript(file) && isFile(logicalTranscriptPath(file));
+}
+/**
+ * The on-disk spelling of a logical transcript right now. Codex may compress
+ * or restore the file between discovery and use, so callers holding either
+ * path get whichever one exists, preferring the plain file.
+ */
+export function existingTranscriptPath(file) {
+    const plain = logicalTranscriptPath(file);
+    if (isFile(plain))
+        return plain;
+    if (/\.jsonl$/iu.test(plain) && isFile(`${plain}${COMPRESSED_SUFFIX}`))
+        return `${plain}${COMPRESSED_SUFFIX}`;
+    return isFile(file) ? file : '';
+}
+function statFingerprint(file) {
+    const stat = fs.statSync(file);
+    return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`;
+}
+// Sync zstd decoding cannot stop early, so every read decompresses the whole
+// frame. A compressed rollout is idle by construction, so metadata probes
+// (working directory, subagent markers, lineage) share a fingerprinted cache
+// of bounded prefixes instead of re-decoding on every inventory pass. Whole
+// bodies are never cached: a long rollout can decode to hundreds of MB.
+const prefixes = new Map();
+let prefixBytes = 0;
+/** Whole decoded contents of a transcript file, compressed or not. */
+export function readTranscriptSync(file) {
+    const contents = fs.readFileSync(file);
+    return isCompressedTranscript(file) ? zstdDecompressSync(contents) : contents;
+}
+function plainPrefix(file, maxBytes) {
+    const handle = fs.openSync(file, 'r');
+    try {
+        const buffer = Buffer.alloc(Math.max(0, Math.min(fs.fstatSync(handle).size, maxBytes)));
+        const bytesRead = fs.readSync(handle, buffer, 0, buffer.length, 0);
+        return buffer.subarray(0, bytesRead);
+    }
+    finally {
+        fs.closeSync(handle);
+    }
+}
+/** At most `maxBytes` decoded bytes from the start of a transcript file. */
+export function readTranscriptPrefixSync(file, maxBytes) {
+    if (!isCompressedTranscript(file))
+        return plainPrefix(file, maxBytes);
+    const fingerprint = statFingerprint(file);
+    const cached = prefixes.get(file);
+    if (cached?.fingerprint === fingerprint &&
+        (cached.prefix.length >= maxBytes || cached.prefix.length < MAX_PREFIX_BYTES)) {
+        prefixes.delete(file);
+        prefixes.set(file, cached);
+        return cached.prefix.subarray(0, maxBytes);
+    }
+    const contents = readTranscriptSync(file);
+    if (cached) {
+        prefixes.delete(file);
+        prefixBytes -= cached.prefix.length;
+    }
+    // Copy so the cache never pins a whole decoded rollout.
+    const prefix = Buffer.from(contents.subarray(0, MAX_PREFIX_BYTES));
+    prefixes.set(file, { fingerprint, prefix });
+    prefixBytes += prefix.length;
+    while (prefixBytes > PREFIX_CACHE_BYTES && prefixes.size > 1) {
+        const [oldest, entry] = prefixes.entries().next().value;
+        prefixes.delete(oldest);
+        prefixBytes -= entry.prefix.length;
+    }
+    return contents.subarray(0, maxBytes);
+}

package/dist/local-session-sources.js

@@ -4,6 +4,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { StringDecoder } from 'node:string_decoder';
+import { compressedTranscriptIsShadowed, existingTranscriptPath, isCompressedTranscript, logicalTranscriptPath, readTranscriptPrefixSync, } from "./compressed-transcripts.js";
 import { readDeepseekTrajectoryHeader } from "./deepseek-trajectories.js";
 import { lettaSessionIdentity } from "./harness-registry.js";
 import { readLettaSessionContext } from "./letta-session-context.js";
@@ -38,7 +39,8 @@ const EXTRA_FILE_STORES = {
     },
     codex: {
         root: path.join(process.env.CODEX_HOME || path.join(HOME, '.codex'), 'sessions'),
-        include: (file) => /[\\/]rollout-[^\\/]+\.jsonl$/i.test(file),
+        // Codex zstd-compresses idle rollouts in place; see compressed-transcripts.ts.
+        include: (file) => /[\\/]rollout-[^\\/]+\.jsonl(?:\.zst)?$/i.test(file),
     },
     grok: { root: path.join(HOME, '.grok', 'sessions'), include: (file) => path.basename(file) === 'chat_history.jsonl' },
     kimi: {
@@ -281,6 +283,10 @@ function descriptorNativeSessionId(descriptor) {
         nativeSessionIdFromPath(String(descriptor?.harness || ''), descriptor?.sourcePath || descriptor?.file));
 }
 function fileDescriptor(harness, file) {
+    // While Codex restores or recompresses a rollout both spellings can exist
+    // briefly; the plain file is the one a live writer appends to.
+    if (compressedTranscriptIsShadowed(file))
+        return null;
     const stat = safeStat(file);
     if (!stat?.isFile() || stat.size === 0)
         return null;
@@ -288,7 +294,7 @@ function fileDescriptor(harness, file) {
         harness,
         agent: harness,
         file,
-        sourcePath: displayPath(file),
+        sourcePath: displayPath(logicalTranscriptPath(file)),
         size: stat.size,
         mtimeMs: stat.mtimeMs,
         createdAt: (stat.birthtimeMs > 0 ? stat.birthtime : stat.mtime).toISOString(),
@@ -301,14 +307,16 @@ export function nativeDescriptorForSource(harness, sourcePath) {
     const store = Object.hasOwn(EXTRA_FILE_STORES, harness) ? EXTRA_FILE_STORES[harness] : null;
     if (!store || !sourcePath || sourcePath.includes('\0'))
         return null;
-    const file = sourcePath.startsWith('~/') ? path.join(HOME, sourcePath.slice(2)) : sourcePath;
-    if (!path.isAbsolute(file))
+    const logical = sourcePath.startsWith('~/') ? path.join(HOME, sourcePath.slice(2)) : sourcePath;
+    if (!path.isAbsolute(logical) || !store.include(logical))
         return null;
+    // A source path always names the plain transcript; follow it to the
+    // compressed copy when that is what exists on disk now.
+    const physical = existingTranscriptPath(logical);
+    const file = physical && store.include(physical) ? physical : logical;
     const relative = path.relative(store.root, file);
     if (!relative || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative))
         return null;
-    if (!store.include(file))
-        return null;
     // Discovery does not traverse symlinked entries. Keep this direct lookup
     // from admitting a transcript that an inventory walk would never discover.
     try {
@@ -383,6 +391,17 @@ function firstJsonDocument(file) {
     const stat = safeStat(file);
     if (!stat?.isFile())
         return null;
+    if (isCompressedTranscript(file)) {
+        try {
+            const source = readTranscriptPrefixSync(file, 1024 * 1024)
+                .toString('utf8')
+                .replace(/^\uFEFF/u, '');
+            return JSON.parse(source.split(/\r?\n/u, 1)[0]);
+        }
+        catch {
+            return null;
+        }
+    }
     let descriptor;
     try {
         descriptor = fs.openSync(file, 'r');
@@ -896,6 +915,17 @@ function filePrefix(file, maxBytes) {
     }
 }
 function recordPrefix(file, limit, maxBytes) {
+    if (isCompressedTranscript(file)) {
+        try {
+            // Callers split and cap the record count themselves.
+            return readTranscriptPrefixSync(file, maxBytes)
+                .toString('utf8')
+                .replace(/^\uFEFF/u, '');
+        }
+        catch {
+            return '';
+        }
+    }
     let handle;
     try {
         const stat = fs.statSync(file);

package/dist/upload-sessions.js

@@ -9,6 +9,7 @@ import WebSocket from 'ws';
 import { logError, logInfo } from "./logging.js";
 import { NativeInventoryWorker } from "./native-inventory-worker.js";
 import { QueueDispatchLifecycle } from "./queue-lifecycle.js";
+import { isCompressedTranscript, readTranscriptSync } from "./compressed-transcripts.js";
 import { streamDeepseekTrajectoryLines } from "./deepseek-trajectories.js";
 import { normalizeAllowedDirectories, workingDirectoryIsAllowed } from "./directory-policy.js";
 import { discoverGitAssociation } from "./git-association.js";
@@ -481,6 +482,22 @@ async function* completeNativeFileLines(file, startOffset, endOffset) {
         input.destroy();
     }
 }
+/** Complete lines of a compressed rollout. Offsets are decoded-byte offsets,
+ * which never compare with the compressed file size, so these files always
+ * rescan from the start (see canResumeNativeCursor). */
+function* completeDecodedLines(contents) {
+    let lineStart = 0;
+    for (;;) {
+        const newline = contents.indexOf(0x0a, lineStart);
+        if (newline < 0)
+            return;
+        let line = contents.subarray(lineStart, newline);
+        if (line.at(-1) === 0x0d)
+            line = line.subarray(0, -1);
+        yield { raw: line.toString('utf8'), endOffset: newline + 1 };
+        lineStart = newline + 1;
+    }
+}
 async function* nativeSourceRecords(descriptor, cursor) {
     if (descriptor.harness === 'deepseek') {
         for await (const raw of streamDeepseekTrajectoryLines(descriptor.file)) {
@@ -502,6 +519,10 @@ async function* nativeSourceRecords(descriptor, cursor) {
         }
         return;
     }
+    if (isCompressedTranscript(descriptor.file)) {
+        yield* completeDecodedLines(readTranscriptSync(descriptor.file));
+        return;
+    }
     yield* completeNativeFileLines(descriptor.file, Number(cursor.offset) || 0, Math.max(-1, (Number(descriptor.size) || 0) - 1));
 }
 /** Stream canonical raw events while retaining only one WebSocket batch. The
@@ -962,6 +983,7 @@ function nativeSourceIdentity(file) {
 function canResumeNativeCursor(descriptor, cursor, nextRow) {
     return Boolean(cursor &&
         !descriptor.file.endsWith('.json') &&
+        !isCompressedTranscript(descriptor.file) &&
         descriptor.harness !== 'deepseek' &&
         !descriptor.sourcePath.includes('.db#') &&
         cursor.file === descriptor.file &&

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@turns/dev",
-  "version": "0.0.28",
+  "version": "0.0.29",
   "description": "Upload local AI agent sessions to turns.dev",
   "license": "MIT",
   "type": "module",