@turns/dev 0.0.10 → 0.0.11

package/README.md

@@ -73,10 +73,17 @@ bounds a deliberately broad matcher: prose that merely looks like a path
 resolves to nothing and is never mentioned to the server, and no directory is
 ever scanned.
 
-An explicit file browse may upload up to 199 MB to the same private R2 prefix.
+An explicit file browse may upload up to 2 GiB to the same private R2 prefix.
+Files over 100 MiB use retryable 32 MiB multipart requests and report upload
+progress to the requesting browser over the control WebSocket.
 The daemon streams the file from disk and the Worker hands that request stream
 straight to R2, so neither process retains the complete file in memory.
 
+Range-based viewers can bypass that upload. A capable daemon advertises live
+file ranges, validates the path through the same allowlist, and streams a
+bounded window as ordered 32 KiB WebSocket chunks. The browser hex view pages
+64 KiB viewport windows, including arbitrary offset jumps.
+
 The systemd unit uses `KillMode=process`, and the macOS LaunchAgent uses
 `AbandonProcessGroup`: restarting the uploader leaves its detached agent-anchor
 processes running so in-flight harness turns continue.

package/dist/file-browsing.js

@@ -17,10 +17,19 @@ import { uploadFileStream } from "./path-resolution.js";
  */
 /** Kept in step with `MAX_DIRECTORY_ENTRIES` in the Worker's shared listing. */
 export const MAX_DIRECTORY_ENTRIES = 2_000;
-/** One MB below the Worker request-body ceiling, matching the R2 endpoint. */
-export const MAX_BROWSE_FILE_BYTES = 199 * 1024 * 1024;
+/** Kept in step with `MAX_FILE_BYTES` on the server. Multipart transport keeps
+ * each individual Worker request below the platform request-body ceiling. */
+export const MAX_BROWSE_FILE_BYTES = 2 * 1024 * 1024 * 1024;
 /** The derived label used in reader-facing errors. */
-export const BROWSE_FILE_LIMIT_LABEL = `${Math.round(MAX_BROWSE_FILE_BYTES / (1024 * 1024))} MB`;
+export const BROWSE_FILE_LIMIT_LABEL = `${MAX_BROWSE_FILE_BYTES / (1024 * 1024 * 1024)} GB`;
+/** Frequent filesystem chunks are useful for accurate accounting but noisy on
+ * a control socket. Four frames per second plus phase boundaries stays smooth. */
+export const BROWSE_PROGRESS_INTERVAL_MS = 250;
+/** One browser range request is deliberately small enough to remain cheap to
+ * retry or abandon when a reader scrolls quickly through a large file. */
+export const MAX_BROWSE_RANGE_BYTES = 256 * 1024;
+/** Raw bytes per WebSocket payload. Base64 expands this to roughly 43 KiB. */
+export const BROWSE_RANGE_CHUNK_BYTES = 32 * 1024;
 /** Enough data to distinguish source and prose from arbitrary binary bytes
  * without reading a large artifact into the daemon's memory. */
 export const FILE_TYPE_SAMPLE_BYTES = 64 * 1024;
@@ -173,7 +182,50 @@ export function readDirectory(directory) {
     // clock inside the body would make every read produce different bytes.
     return { path: directory, entries, truncated: names.length > MAX_DIRECTORY_ENTRIES };
 }
-export async function browseMachinePath({ requested, machine, baseUrl, options, workingDirectory = '', contentEtag = '', fetchImpl = fetch, }) {
+/** Stream one bounded file window without creating or updating its R2 object. */
+export async function streamMachineFileRange({ requested, offset, length, workingDirectory = '', allowedDirectories = [], onReady, onChunk, }) {
+    if (!Number.isSafeInteger(offset) || offset < 0)
+        throw new Error('Invalid file range offset.');
+    if (!Number.isSafeInteger(length) || length <= 0 || length > MAX_BROWSE_RANGE_BYTES) {
+        throw new Error('Invalid file range length.');
+    }
+    const resolved = resolveBrowsePath(requested, { workingDirectory, allowedDirectories });
+    if (!resolved.stat.isFile())
+        throw new Error(`Not a readable file: ${resolved.path}`);
+    if (resolved.stat.size > MAX_BROWSE_FILE_BYTES) {
+        throw new Error(`File is larger than the ${BROWSE_FILE_LIMIT_LABEL} browse limit.`);
+    }
+    if (offset > resolved.stat.size)
+        throw new Error('File range starts beyond the end of the file.');
+    const boundedLength = Math.min(length, resolved.stat.size - offset);
+    const sample = readFileTypeSample(resolved.path, resolved.stat.size);
+    const range = {
+        path: resolved.path,
+        offset,
+        length: boundedLength,
+        totalBytes: resolved.stat.size,
+        contentType: browseContentType(resolved.path, sample, sample.byteLength === resolved.stat.size),
+        modifiedAt: isoTime(resolved.stat.mtimeMs),
+    };
+    onReady(range);
+    if (!boundedLength)
+        return range;
+    const stream = fs.createReadStream(resolved.path, {
+        start: offset,
+        end: offset + boundedLength - 1,
+        highWaterMark: BROWSE_RANGE_CHUNK_BYTES,
+    });
+    let chunkOffset = offset;
+    for await (const value of stream) {
+        const chunk = value;
+        onChunk(chunkOffset, chunk);
+        chunkOffset += chunk.byteLength;
+    }
+    if (chunkOffset !== offset + boundedLength)
+        throw new Error('The file changed while its byte range was being read.');
+    return range;
+}
+export async function browseMachinePath({ requested, machine, baseUrl, options, workingDirectory = '', contentEtag = '', onUploadProgress, fetchImpl = fetch, }) {
     const resolved = resolveBrowsePath(requested, {
         workingDirectory,
         allowedDirectories: options.allowedDirectories || [],
@@ -201,6 +253,20 @@ export async function browseMachinePath({ requested, machine, baseUrl, options,
     const modifiedAt = isoTime(resolved.stat.mtimeMs);
     if (resolved.stat.size === 0)
         return { path: resolved.path, kind: 'file', bytes: 0, contentType, modifiedAt };
+    let lastProgressAt = 0;
+    let lastUploadedBytes = -1;
+    const reportProgress = (phase, uploadedBytes, force = false) => {
+        if (!onUploadProgress)
+            return;
+        const bounded = Math.max(lastUploadedBytes, Math.min(resolved.stat.size, Math.max(0, uploadedBytes)));
+        const now = Date.now();
+        if (!force && now - lastProgressAt < BROWSE_PROGRESS_INTERVAL_MS && bounded < resolved.stat.size)
+            return;
+        lastProgressAt = now;
+        lastUploadedBytes = bounded;
+        onUploadProgress({ path: resolved.path, phase, uploadedBytes: bounded, totalBytes: resolved.stat.size });
+    };
+    reportProgress('starting', 0, true);
     await uploadFileStream({
         baseUrl,
         options,
@@ -210,6 +276,8 @@ export async function browseMachinePath({ requested, machine, baseUrl, options,
         bytes: resolved.stat.size,
         modifiedAt,
         ...(contentEtag ? { contentEtag } : {}),
+        onProgress: (uploadedBytes) => reportProgress('uploading', uploadedBytes),
+        onFinalizing: () => reportProgress('finalizing', resolved.stat.size, true),
         fetchImpl,
     });
     return { path: resolved.path, kind: 'file', bytes: resolved.stat.size, contentType, modifiedAt };

package/dist/path-resolution.js

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { Transform } from 'node:stream';
 /**
  * Turning the paths an agent writes into things a reader can open.
  *
@@ -180,6 +181,178 @@ function fileUploadUrl(baseUrl, machine, absolutePath) {
             : encoded;
     return new URL(`/api/file/${encodeURIComponent(machine)}/${encodedPath}`, String(baseUrl));
 }
+/** Kept in step with the explicit-browse transport constants in shared/files.ts. */
+export const MAX_SINGLE_BROWSE_UPLOAD_BYTES = 100 * 1024 * 1024;
+export const BROWSE_MULTIPART_PART_BYTES = 32 * 1024 * 1024;
+export const MAX_BROWSE_MULTIPART_PARTS = 64;
+export function filePartRanges(bytes, partSize = BROWSE_MULTIPART_PART_BYTES) {
+    if (!Number.isSafeInteger(bytes) || bytes <= 0 || !Number.isSafeInteger(partSize) || partSize <= 0)
+        return [];
+    const ranges = [];
+    for (let start = 0, partNumber = 1; start < bytes; start += partSize, partNumber += 1) {
+        ranges.push({ partNumber, start, end: Math.min(bytes, start + partSize) });
+    }
+    return ranges;
+}
+function fileMultipartUrl(baseUrl, machine, absolutePath, suffix = '') {
+    const url = new URL(`/api/file-multipart/${encodeURIComponent(machine)}${suffix}`, String(baseUrl));
+    url.searchParams.set('path', absolutePath);
+    return url;
+}
+function progressStream(absolutePath, start, end, onProgress) {
+    const source = fs.createReadStream(absolutePath, { start, end: end - 1 });
+    let loaded = 0;
+    const body = new Transform({
+        transform(chunk, _encoding, callback) {
+            loaded += chunk.byteLength;
+            onProgress?.(loaded);
+            callback(null, chunk);
+        },
+    });
+    source.pipe(body);
+    return {
+        body,
+        destroy() {
+            source.destroy();
+            body.destroy();
+        },
+    };
+}
+async function uploadFailure(response, label) {
+    const detail = (await response.text().catch(() => '')).trim();
+    return new Error(`${label} (${response.status})${detail ? `: ${detail.slice(0, 500)}` : '.'}`);
+}
+function retryDelay(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function uploadMultipartFile({ baseUrl, options, machine, absolutePath, contentType, bytes, modifiedAt, contentEtag, onProgress, onFinalizing, fetchImpl, }) {
+    const createUrl = fileMultipartUrl(baseUrl, machine, absolutePath);
+    const createBody = JSON.stringify({
+        size: bytes,
+        contentType,
+        ...(modifiedAt ? { modifiedAt } : {}),
+        ...(contentEtag ? { contentEtag } : {}),
+    });
+    const created = await fetchImpl(createUrl, {
+        method: 'POST',
+        headers: {
+            ...uploadHeaders(options),
+            'content-type': 'application/json',
+            'content-length': String(Buffer.byteLength(createBody)),
+        },
+        body: createBody,
+    });
+    if (!created.ok)
+        throw await uploadFailure(created, 'Multipart file upload could not start');
+    const payload = (await created.json().catch(() => ({})));
+    const uploadId = String(payload.upload?.uploadId || '');
+    const partSize = Number(payload.upload?.partSize);
+    const receiptToken = String(payload.upload?.receiptToken || '');
+    if (!uploadId ||
+        !receiptToken ||
+        !Number.isSafeInteger(partSize) ||
+        partSize <= 0 ||
+        partSize > BROWSE_MULTIPART_PART_BYTES) {
+        throw new Error('Multipart file upload returned an invalid session.');
+    }
+    const ranges = filePartRanges(bytes, partSize);
+    if (!ranges.length || ranges.length > MAX_BROWSE_MULTIPART_PARTS) {
+        throw new Error('Multipart file upload returned an invalid part size.');
+    }
+    const parts = [];
+    let completed = 0;
+    try {
+        for (const range of ranges) {
+            const length = range.end - range.start;
+            let receipt = null;
+            let greatestLoaded = 0;
+            let lastError = null;
+            for (let attempt = 0; attempt < 3 && !receipt; attempt += 1) {
+                const stream = progressStream(absolutePath, range.start, range.end, (loaded) => {
+                    greatestLoaded = Math.max(greatestLoaded, loaded);
+                    onProgress?.(Math.min(bytes, completed + greatestLoaded));
+                });
+                try {
+                    const partUrl = fileMultipartUrl(baseUrl, machine, absolutePath, `/parts/${range.partNumber}`);
+                    partUrl.searchParams.set('upload_id', uploadId);
+                    const response = await fetchImpl(partUrl, {
+                        method: 'PUT',
+                        headers: {
+                            ...uploadHeaders(options),
+                            'content-type': 'application/octet-stream',
+                            'content-length': String(length),
+                        },
+                        body: stream.body,
+                        duplex: 'half',
+                    });
+                    if (!response.ok)
+                        throw await uploadFailure(response, `Multipart file part ${range.partNumber} failed`);
+                    const answer = (await response.json().catch(() => ({})));
+                    if (answer.part?.partNumber !== range.partNumber || !answer.part.etag) {
+                        throw new Error(`Multipart file part ${range.partNumber} returned no receipt.`);
+                    }
+                    receipt = { partNumber: range.partNumber, etag: String(answer.part.etag) };
+                }
+                catch (error) {
+                    lastError = error;
+                    if (attempt < 2)
+                        await retryDelay(250 * 2 ** attempt);
+                }
+                finally {
+                    stream.destroy();
+                }
+            }
+            if (!receipt)
+                throw lastError instanceof Error ? lastError : new Error(String(lastError || 'File part failed.'));
+            parts.push(receipt);
+            completed += length;
+            onProgress?.(completed);
+        }
+        const completeUrl = fileMultipartUrl(baseUrl, machine, absolutePath, '/complete');
+        completeUrl.searchParams.set('upload_id', uploadId);
+        const completeBody = JSON.stringify({ parts, size: bytes, receiptToken });
+        onFinalizing?.();
+        let completedUpload = false;
+        let completionError = null;
+        for (let attempt = 0; attempt < 3 && !completedUpload; attempt += 1) {
+            try {
+                const response = await fetchImpl(completeUrl, {
+                    method: 'POST',
+                    headers: {
+                        ...uploadHeaders(options),
+                        'content-type': 'application/json',
+                        'content-length': String(Buffer.byteLength(completeBody)),
+                    },
+                    body: completeBody,
+                });
+                if (!response.ok) {
+                    const failure = await uploadFailure(response, 'Multipart file upload could not finish');
+                    if (response.status < 500 && response.status !== 408 && response.status !== 429)
+                        throw failure;
+                    completionError = failure;
+                }
+                else {
+                    completedUpload = true;
+                }
+            }
+            catch (error) {
+                completionError = error;
+            }
+            if (!completedUpload && attempt < 2)
+                await retryDelay(250 * 2 ** attempt);
+        }
+        if (!completedUpload) {
+            throw completionError instanceof Error ? completionError : new Error('Multipart file upload could not finish.');
+        }
+        return true;
+    }
+    catch (error) {
+        const abortUrl = fileMultipartUrl(baseUrl, machine, absolutePath);
+        abortUrl.searchParams.set('upload_id', uploadId);
+        await fetchImpl(abortUrl, { method: 'DELETE', headers: uploadHeaders(options) }).catch(() => { });
+        throw error;
+    }
+}
 /**
  * Put bytes in the account's private prefix, under the path they came from.
  *
@@ -204,9 +377,31 @@ export async function uploadFileBytes({ baseUrl, options, machine, absolutePath,
     return true;
 }
 /** Stream one on-disk file to R2 without retaining its body in the daemon. */
-export async function uploadFileStream({ baseUrl, options, machine, absolutePath, contentType, bytes, modifiedAt, contentEtag, fetchImpl = fetch, }) {
+export async function uploadFileStream({ baseUrl, options, machine, absolutePath, contentType, bytes, modifiedAt, contentEtag, onProgress, onFinalizing, fetchImpl = fetch, }) {
+    if (bytes > MAX_SINGLE_BROWSE_UPLOAD_BYTES) {
+        return uploadMultipartFile({
+            baseUrl,
+            options,
+            machine,
+            absolutePath,
+            contentType,
+            bytes,
+            modifiedAt,
+            contentEtag,
+            onProgress,
+            onFinalizing,
+            fetchImpl,
+        });
+    }
     const url = fileUploadUrl(baseUrl, machine, absolutePath);
-    const body = fs.createReadStream(absolutePath);
+    let finalizing = false;
+    const stream = progressStream(absolutePath, 0, bytes, (uploadedBytes) => {
+        onProgress?.(uploadedBytes);
+        if (!finalizing && uploadedBytes >= bytes) {
+            finalizing = true;
+            onFinalizing?.();
+        }
+    });
     try {
         const response = await fetchImpl(url, {
             method: 'PUT',
@@ -217,7 +412,7 @@ export async function uploadFileStream({ baseUrl, options, machine, absolutePath
                 ...(modifiedAt ? { 'x-turns-file-modified-at': modifiedAt } : {}),
                 ...(contentEtag ? { 'x-turns-content-etag': contentEtag } : {}),
             },
-            body,
+            body: stream.body,
             // Node requires this opt-in for a streaming request body.
             duplex: 'half',
         });
@@ -226,7 +421,7 @@ export async function uploadFileStream({ baseUrl, options, machine, absolutePath
         return true;
     }
     finally {
-        body.destroy();
+        stream.destroy();
     }
 }
 /** Send one resolved file to the account's private prefix. */

package/dist/queue-worker.js

@@ -6,7 +6,7 @@ import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { anchorId, connectExistingAgentAnchors, launchAgentAnchor, waitForAgentTurn } from "./agent-anchor-client.js";
 import { assertAllowedWorkingDirectory, workingDirectoryIsAllowed } from "./directory-policy.js";
-import { browseMachinePath } from "./file-browsing.js";
+import { browseMachinePath, streamMachineFileRange } from "./file-browsing.js";
 import { applyFileEdit } from "./file-edits.js";
 import { searchMachinePaths } from "./file-search.js";
 import { readMachineGitBaseBlob, readMachineGitPatch, readMachineGitStatus } from "./git-status.js";
@@ -1052,6 +1052,14 @@ export async function watchQueuedPrompts({ rpc, options, machine, getDescriptors
                 baseUrl,
                 options,
                 workingDirectory: await browseWorkingDirectory(sessionId),
+                onUploadProgress: (progress) => {
+                    replyRpc.notify({
+                        type: 'session.file.browse.progress',
+                        requestId: message.requestId,
+                        sessionId,
+                        ...progress,
+                    });
+                },
             });
             replyRpc.notify({
                 type: 'session.file.browse.result',
@@ -1071,6 +1079,57 @@ export async function watchQueuedPrompts({ rpc, options, machine, getDescriptors
             });
         }
     };
+    /** Stream a small file window over the control socket. This never writes R2;
+     * range-based viewers can therefore paint before a multi-gigabyte upload. */
+    const handleFileRange = async (message, replyRpc) => {
+        const sessionId = String(message.sessionId || '');
+        let canonicalPath = '';
+        try {
+            await streamMachineFileRange({
+                requested: String(message.path || ''),
+                offset: Number(message.offset),
+                length: Number(message.length),
+                workingDirectory: await browseWorkingDirectory(sessionId),
+                allowedDirectories: options.allowedDirectories || [],
+                onReady: (range) => {
+                    canonicalPath = range.path;
+                    replyRpc.notify({
+                        type: 'session.file.range.ready',
+                        requestId: message.requestId,
+                        sessionId,
+                        ...range,
+                    });
+                },
+                onChunk: (offset, bytes) => {
+                    replyRpc.notify({
+                        type: 'session.file.range.chunk',
+                        requestId: message.requestId,
+                        sessionId,
+                        path: canonicalPath,
+                        offset,
+                        data: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64'),
+                    });
+                },
+            });
+            replyRpc.notify({
+                type: 'session.file.range.result',
+                requestId: message.requestId,
+                sessionId,
+                path: canonicalPath,
+                ok: true,
+            });
+        }
+        catch (error) {
+            replyRpc.notify({
+                type: 'session.file.range.result',
+                requestId: message.requestId,
+                sessionId,
+                ...(canonicalPath ? { path: canonicalPath } : {}),
+                ok: false,
+                error: redactor.redactText(error?.message || String(error)),
+            });
+        }
+    };
     /** Refresh recursive path-name matches for directory typeahead.
      *
      * Like browsing, this is a machine filesystem operation rather than a
@@ -1217,6 +1276,8 @@ export async function watchQueuedPrompts({ rpc, options, machine, getDescriptors
             void handleInteractionAnswer(message, rpc);
         if (message.type === 'session.file.browse.request')
             void handleFileBrowse(message, rpc);
+        if (message.type === 'session.file.range.request')
+            void handleFileRange(message, rpc);
         if (message.type === 'session.file.search.request')
             void handleFileSearch(message, rpc);
         if (message.type === 'session.git.read.request')
@@ -1235,7 +1296,7 @@ export async function watchQueuedPrompts({ rpc, options, machine, getDescriptors
                 type: 'machine.register',
                 machineId: machine,
                 machineHostname: os.hostname(),
-                capabilities: { fileSearch: true, git: true },
+                capabilities: { fileSearch: true, git: true, fileRanges: true },
                 workerId,
                 workerInstanceId,
                 anchorSessionIds: [...anchors.keys()],

package/dist/upload-sessions.js

@@ -946,8 +946,9 @@ function gitAssociationForSession(cwd, redactor) {
  * @param {number} [options.maxBatches]
  * @param {() => boolean} [options.shouldYield]
  * @param {boolean} [options.recoverRemoteAhead]
+ * @param {boolean} [options.forceReset]
  */
-export async function uploadNativeSession(rpc, machine, descriptor, cursor = {}, onProgress = () => { }, redactor = DEFAULT_REDACTOR, { maxBatches = Number.POSITIVE_INFINITY, shouldYield = () => false, recoverRemoteAhead = true, onPathCandidates = (_sessionId, _workingDirectory, _candidates) => { }, } = {}) {
+export async function uploadNativeSession(rpc, machine, descriptor, cursor = {}, onProgress = () => { }, redactor = DEFAULT_REDACTOR, { maxBatches = Number.POSITIVE_INFINITY, shouldYield = () => false, recoverRemoteAhead = true, forceReset = false, onPathCandidates = (_sessionId, _workingDirectory, _candidates) => { }, } = {}) {
     const workingDirectory = await nativeDescriptorWorkingDirectory(descriptor);
     const agentSessionId = await nativeDescriptorAgentSessionId(descriptor);
     const git = gitAssociationForSession(workingDirectory, redactor);
@@ -976,7 +977,14 @@ export async function uploadNativeSession(rpc, machine, descriptor, cursor = {},
     const uploadSessionId = String(opened.sessionId || sessionId);
     let nextRow = Number(opened.nextRow) || 0;
     let initialUpload = nextRow === 0 || cursor.initialUpload === true;
-    if (Number(opened.redactionVersion || 0) !== REDACTION_VERSION) {
+    if (forceReset && nextRow > 0) {
+        await rpc.request({ type: 'session.reset', sessionId: uploadSessionId, redactionVersion: REDACTION_VERSION });
+        nextRow = 0;
+        for (const key of Object.keys(cursor))
+            delete cursor[key];
+        initialUpload = true;
+    }
+    else if (Number(opened.redactionVersion || 0) !== REDACTION_VERSION) {
         await rpc.request({ type: 'session.reset', sessionId: uploadSessionId, redactionVersion: REDACTION_VERSION });
         nextRow = 0;
         for (const key of Object.keys(cursor))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@turns/dev",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "description": "Upload local AI agent sessions to turns.dev",
   "license": "MIT",
   "type": "module",