@1agh/maude 1.2.0 → 1.3.0

package/apps/studio/sync/file-plane.ts

@@ -46,13 +46,18 @@
 
 import { createHash } from 'node:crypto';
 import {
+  appendFileSync,
+  closeSync,
   type Dirent,
   existsSync,
   mkdirSync,
+  openSync,
   readdirSync,
   readFileSync,
+  readSync,
   realpathSync,
   renameSync,
+  rmSync,
   statSync,
   writeFileSync,
 } from 'node:fs';
@@ -77,8 +82,35 @@ const PUT_TIMEOUT_MS = 120_000;
 
 /** Refuse an implausible body rather than streaming it to disk. This is the
  *  PULL/receive cap and is deliberately generous — it bounds what we will
- *  accept, not what a hub will. */
-const MAX_FILE_BYTES = 512 * 1024 * 1024;
+ *  accept, not what a hub will. Plan T18: the project-file ceiling the hub
+ *  uses too (`apps/hub/src/file-limits.mjs`), so a real video moves. */
+export const MAX_FILE_BYTES = (() => {
+  const raw = Number(process.env.MAUDE_MAX_PROJECT_FILE_BYTES);
+  return Number.isFinite(raw) && raw >= 95 * 1024 * 1024
+    ? Math.min(raw, 8192 * 1024 * 1024)
+    : 2048 * 1024 * 1024;
+})();
+
+/** T18 — above this a download streams to disk (and resumes) instead of
+ *  being held in memory. */
+export const STREAM_DOWNLOAD_ABOVE_BYTES = 32 * 1024 * 1024;
+
+/** sha256 of a file, read in 1 MiB chunks (T18 — a video is never held whole). */
+export function sha256File(abs: string): string {
+  const hash = createHash('sha256');
+  const buf = Buffer.allocUnsafe(1024 * 1024);
+  const fd = openSync(abs, 'r');
+  try {
+    for (;;) {
+      const n = readSync(fd, buf, 0, buf.length, null);
+      if (n <= 0) break;
+      hash.update(n === buf.length ? buf : buf.subarray(0, n));
+    }
+  } finally {
+    closeSync(fd);
+  }
+  return hash.digest('hex');
+}
 
 /**
  * What the door accepts, until it tells us otherwise.
@@ -112,6 +144,8 @@ export const MIN_TRUSTED_MAX_FILE_BYTES = 1024 * 1024;
  *  relative to the hub's hourly quota window, so a reading can never strand a
  *  client for the rest of its life. */
 export const HUB_LIMITS_TTL_MS = 10 * 60_000;
+/** A failed limits read is asked again this soon, not after the full TTL. */
+export const HUB_LIMITS_RETRY_MS = 30_000;
 export const MAX_TRUSTED_QUOTA_PAUSE_MS = 2 * 60 * 60_000;
 
 /** How many files one pass will move. The remainder is the next pass's work. */
@@ -390,15 +424,35 @@ export interface FilePlaneOptions {
   onProgress?: () => void;
 }
 
-interface LocalFile {
+interface SyncableLocalFile {
   rel: string;
   abs: string;
   hash: string;
   size: number;
   mtimeMs: number;
   cls: FileClass;
+  oversized?: undefined;
 }
 
+/**
+ * Present here, eligible, and over the plane's ceiling (audit 2026-09-13 P1
+ * #4). It is IN the inventory — absent from it, it was indistinguishable from
+ * a local delete (a grown asset was deleted on the hub) and from a missing
+ * local copy (a hub copy overwrote it). Never hashed: reading half a gigabyte
+ * to label a row it cannot move is the cost the ceiling exists to avoid.
+ */
+interface OversizedLocalFile {
+  rel: string;
+  abs: string;
+  hash: null;
+  size: number;
+  mtimeMs: number;
+  cls: FileClass;
+  oversized: true;
+}
+
+type LocalFile = SyncableLocalFile | OversizedLocalFile;
+
 const sha256 = (bytes: Uint8Array | string): string =>
   createHash('sha256').update(bytes).digest('hex');
 
@@ -441,7 +495,6 @@ export function scanLocalFiles(
       } catch {
         continue;
       }
-      if (st.size > MAX_FILE_BYTES) continue;
       found.push({ rel: childRel, abs: childAbs, size: st.size, mtimeMs: st.mtimeMs });
     }
   };
@@ -456,10 +509,14 @@ export function scanLocalFiles(
   for (const f of found) {
     const cls = classifyProjectFile(f.rel, opts);
     if (!isFilePlaneClass(cls)) continue;
+    if (f.size > MAX_FILE_BYTES) {
+      out.set(f.rel, { ...f, cls, hash: null, oversized: true });
+      continue;
+    }
     let hash = ledger.cachedHash(f.rel, f.size, f.mtimeMs);
     if (hash === null) {
       try {
-        hash = sha256(readFileSync(f.abs));
+        hash = sha256File(f.abs);
       } catch {
         continue;
       }
@@ -578,6 +635,9 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     quotaResetsAt?: number;
     quotaUsed?: number;
     quotaBytesPerWindow?: number;
+    /** T18 — the hub takes resumable upload sessions up to this size. */
+    maxSessionBytes?: number;
+    partBytes?: number;
   } | null = null;
   /** When the limits were last learned. 0 = never. */
   let hubLimitsAt = 0;
@@ -607,12 +667,26 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
         headers: auth(),
         signal: AbortSignal.timeout(GET_TIMEOUT_MS),
       });
-      if (!res.ok) return;
+      if (res.status === 404) {
+        // A hub older than this route: its door takes one request per file
+        // and nothing more, which IS an answer.
+        hubLimits = { maxFileBytes: DEFAULT_HUB_MAX_FILE_BYTES };
+        return;
+      }
+      if (!res.ok) {
+        // Not an answer (a 429 behind a shared address, a 5xx): ask again
+        // soon instead of living on the fallback ceiling for the whole TTL.
+        hubLimitsAt = now() - HUB_LIMITS_TTL_MS + HUB_LIMITS_RETRY_MS;
+        return;
+      }
       const body = (await res.json()) as {
         maxFileBytes?: unknown;
         quotaResetsAt?: unknown;
         quotaUsed?: unknown;
         quotaBytesPerWindow?: unknown;
+        uploadSessions?: unknown;
+        maxSessionBytes?: unknown;
+        partBytes?: unknown;
       };
       if (
         Number.isFinite(body?.maxFileBytes) &&
@@ -630,18 +704,35 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
           ...(Number.isFinite(body?.quotaBytesPerWindow)
             ? { quotaBytesPerWindow: body.quotaBytesPerWindow as number }
             : {}),
+          // T18 — sessions, clamped like every other hub-supplied lever: never
+          // past our own receive cap, parts between 1 and 64 MiB.
+          ...(body?.uploadSessions === true && Number.isFinite(body?.maxSessionBytes)
+            ? {
+                maxSessionBytes: Math.min(body.maxSessionBytes as number, MAX_FILE_BYTES),
+                partBytes: Math.min(
+                  Math.max(Number(body.partBytes) || 8 * 1024 * 1024, 1024 * 1024),
+                  64 * 1024 * 1024
+                ),
+              }
+            : {}),
         };
       }
     } catch {
-      /* the fallback is the pre-existing behaviour */
+      /* the fallback is the pre-existing behaviour — and it is asked again soon */
+      hubLimitsAt = now() - HUB_LIMITS_TTL_MS + HUB_LIMITS_RETRY_MS;
     }
   }
 
-  /** The push ceiling in force right now. */
-  function pushCeiling(): number {
+  /** The single-PUT ceiling in force right now. */
+  function singlePutCeiling(): number {
     return hubLimits?.maxFileBytes ?? DEFAULT_HUB_MAX_FILE_BYTES;
   }
 
+  /** The push ceiling in force right now — sessions raise it (T18). */
+  function pushCeiling(): number {
+    return Math.max(singlePutCeiling(), hubLimits?.maxSessionBytes ?? 0);
+  }
+
   /**
    * Bytes still available inside the hub's current write window, or null when
    * the hub did not say.
@@ -720,20 +811,38 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     expectHash: string,
     ceiling: number
   ): Promise<
-    | { ok: true; bytes: Uint8Array }
+    | { ok: true; bytes: Uint8Array; file?: undefined; size: number }
+    | { ok: true; file: string; bytes?: undefined; size: number }
     | { ok: false; reason: string; overCap?: true; rateLimited?: true }
   > {
     let res: Response;
     requestsThisPass += 1;
+    // T18 — a partial download of these exact bytes resumes where it stopped.
+    const staged = path.join(designRoot, '_state', 'downloads', `${expectHash}.part`);
+    let resumeFrom = 0;
+    try {
+      resumeFrom = statSync(staged).size;
+    } catch {
+      /* nothing staged */
+    }
     try {
       res = await fetchImpl(
         `${base}/_project-file/${rel.split('/').map(encodeURIComponent).join('/')}`,
-        { headers: auth(), signal: AbortSignal.timeout(GET_TIMEOUT_MS) }
+        {
+          headers: { ...auth(), ...(resumeFrom > 0 ? { range: `bytes=${resumeFrom}-` } : {}) },
+          signal: AbortSignal.timeout(GET_TIMEOUT_MS),
+        }
       );
     } catch (err) {
       return { ok: false, reason: transportFailure(err, rel, log) };
     }
+    if (res.status === 416) {
+      // The staged part is not a prefix of what the hub holds now: start over.
+      rmSync(staged, { force: true });
+      return { ok: false, reason: 'the partial download no longer matches — restarting it' };
+    }
     if (!res.ok) return await refusal(res);
+    const resumed = res.status === 206 && resumeFrom > 0;
 
     // The cap is consulted BEFORE the body exists, and again as it arrives.
     // Buffering first and measuring after is not a cap at all: a hub answering
@@ -750,6 +859,49 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
       };
     }
 
+    // T18 — a large body streams to the staging file (appending after a
+    // resume) and is verified from disk; memory stays one chunk.
+    if (resumed || (Number.isFinite(declared) && declared > STREAM_DOWNLOAD_ABOVE_BYTES)) {
+      if (!resumed) rmSync(staged, { force: true });
+      mkdirSync(path.dirname(staged), { recursive: true });
+      const body = res.body;
+      if (!body?.getReader) return { ok: false, reason: 'no response stream' };
+      const reader = body.getReader();
+      let total = resumed ? resumeFrom : 0;
+      try {
+        for (;;) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          if (!value) continue;
+          total += value.byteLength;
+          if (total > cap) {
+            await reader.cancel().catch(() => {});
+            rmSync(staged, { force: true });
+            return {
+              ok: false,
+              reason: `over the cap mid-stream (${total} B > ${cap} B)`,
+              overCap: true,
+            };
+          }
+          appendFileSync(staged, value);
+        }
+      } catch (err) {
+        // What arrived stays staged: the next pass asks for the rest.
+        return { ok: false, reason: transportFailure(err, '<response body>', log) };
+      }
+      let got: string;
+      try {
+        got = sha256File(staged);
+      } catch {
+        return { ok: false, reason: 'could not read the downloaded file back' };
+      }
+      if (got !== expectHash) {
+        rmSync(staged, { force: true });
+        return { ok: false, reason: 'content hash mismatch (racing a write?)' };
+      }
+      return { ok: true, file: staged, size: total };
+    }
+
     const bytes = await readCapped(res, cap);
     if (!bytes.ok) return bytes;
     // The hub may REFUSE to serve; it must never be able to SUBSTITUTE.
@@ -757,7 +909,7 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     if (got !== expectHash) {
       return { ok: false, reason: 'content hash mismatch (racing a write?)' };
     }
-    return { ok: true, bytes: bytes.bytes };
+    return { ok: true, bytes: bytes.bytes, size: bytes.bytes.byteLength };
   }
 
   /**
@@ -931,13 +1083,18 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     return isFilePlaneClass(cls);
   }
 
-  /** Land bytes at `rel`, atomically. Throws on failure — `adoptAfter` catches. */
-  function materialize(rel: string, bytes: Uint8Array): void {
+  /** Land bytes at `rel`, atomically. Throws on failure — `adoptAfter` catches.
+   *  A streamed download (T18) is moved into place from its staging file. */
+  function materialize(rel: string, payload: { bytes?: Uint8Array; file?: string }): void {
     const target = safeTarget(rel);
     if (!target) throw new Error(`refusing to write ${rel} — it does not resolve inside the root`);
     mkdirSync(target.parent, { recursive: true });
+    if (payload.file) {
+      renameSync(payload.file, target.abs);
+      return;
+    }
     const tmp = `${target.abs}.part`;
-    writeFileSync(tmp, bytes);
+    writeFileSync(tmp, payload.bytes ?? new Uint8Array());
     renameSync(tmp, target.abs);
   }
 
@@ -964,7 +1121,7 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
    * Upload one file with a compare-and-swap against the state we decided from.
    */
   async function push(
-    local: LocalFile,
+    local: SyncableLocalFile,
     expect: string | null
   ): Promise<
     | { ok: true; seq: number | null }
@@ -997,6 +1154,12 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
         ...(hubLimits?.quotaResetsAt ? { quotaResetsAt: hubLimits.quotaResetsAt } : {}),
       };
     }
+    if (local.size > pushCeiling() && hubLimits === null) {
+      // The hub has not said what it takes (its answer was refused or lost):
+      // over the fallback ceiling is unknown, not too large. Retry; never
+      // park a file as terminally refused on a guess.
+      return { ok: false, reason: 'Waiting for the workspace to say how large a file it accepts' };
+    }
     if (local.size > pushCeiling()) {
       return {
         ok: false,
@@ -1004,6 +1167,8 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
         reason: `Too big for this workspace — ${mb(local.size)}, and the limit is ${mb(pushCeiling())}`,
       };
     }
+    // T18 — past one request: a resumable session, in verified parts.
+    if (local.size > singlePutCeiling()) return pushSession(local, expect);
     let bytes: Uint8Array;
     try {
       bytes = readFileSync(local.abs);
@@ -1053,6 +1218,104 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     }
   }
 
+  /**
+   * T18 — upload a large file as a resumable session. The hub makes creation
+   * idempotent for the same (person, path, bytes), so a restarted app resumes
+   * by asking again and sending only the parts the hub does not hold. Memory
+   * is one part; the whole object is verified by the hub before it lands.
+   */
+  async function pushSession(
+    local: SyncableLocalFile,
+    expect: string | null
+  ): Promise<Awaited<ReturnType<typeof push>>> {
+    const json = async (res: Response) =>
+      (await res.json().catch(() => ({}))) as Record<string, unknown>;
+    ledger.outboxAdd(local.hash);
+    requestsThisPass += 1;
+    try {
+      const created = await fetchImpl(`${base}/api/file-uploads`, {
+        method: 'POST',
+        headers: { ...auth(), 'content-type': 'application/json' },
+        body: JSON.stringify({
+          path: local.rel,
+          size: local.size,
+          sha256: local.hash,
+          expectHash: expect ?? 'none',
+        }),
+        signal: AbortSignal.timeout(GET_TIMEOUT_MS),
+      });
+      if (!created.ok) return await refusal(created);
+      const s = await json(created);
+      const id = typeof s.id === 'string' ? s.id : '';
+      const partBytes = Number(s.partBytes);
+      const parts = Number(s.parts);
+      if (!/^up_[0-9a-f]{32}$/.test(id) || !(partBytes > 0) || !(parts > 0)) {
+        return { ok: false, reason: 'The workspace answered the upload with nonsense' };
+      }
+      const have = new Set(Array.isArray(s.received) ? (s.received as number[]) : []);
+      const buf = Buffer.allocUnsafe(partBytes);
+      const fd = openSync(local.abs, 'r');
+      try {
+        for (let n = 0; n < parts; n += 1) {
+          if (have.has(n)) continue;
+          const len = Math.min(partBytes, local.size - n * partBytes);
+          const read = readSync(fd, buf, 0, len, n * partBytes);
+          if (read !== len) return { ok: false, reason: 'The file changed while it was uploading' };
+          const part = Buffer.from(buf.subarray(0, len));
+          let res: Response | null = null;
+          let lastErr: unknown = null;
+          for (let attempt = 0; attempt < 3 && !res?.ok; attempt += 1) {
+            try {
+              res = await fetchImpl(`${base}/api/file-uploads/${id}/${n}`, {
+                method: 'PUT',
+                headers: {
+                  ...auth(),
+                  'content-type': 'application/octet-stream',
+                  'x-maude-part-sha256': sha256(part),
+                },
+                body: part as unknown as BodyInit,
+                signal: AbortSignal.timeout(PUT_TIMEOUT_MS),
+              });
+              if (res.status === 429 || res.status === 507) return await refusal(res);
+            } catch (err) {
+              lastErr = err;
+              res = null;
+            }
+          }
+          if (!res) return { ok: false, reason: transportFailure(lastErr, local.rel, log) };
+          if (!res.ok) return await refusal(res);
+        }
+      } finally {
+        closeSync(fd);
+      }
+      const done = await fetchImpl(`${base}/api/file-uploads/${id}/complete`, {
+        method: 'POST',
+        headers: auth(),
+        signal: AbortSignal.timeout(PUT_TIMEOUT_MS),
+      });
+      if (done.status === 409) {
+        const body = await json(done);
+        if (Array.isArray(body.received)) {
+          // Parts the hub lost between our PUTs and the completion: the next
+          // pass resumes and sends exactly those.
+          return { ok: false, reason: 'Upload interrupted — it resumes on the next pass' };
+        }
+        return {
+          ok: false,
+          conflict: true,
+          current: typeof body.current === 'string' ? body.current : null,
+        };
+      }
+      if (!done.ok) return await refusal(done);
+      const body = await json(done);
+      return { ok: true, seq: typeof body.seq === 'number' ? body.seq : null };
+    } catch (err) {
+      return { ok: false, reason: transportFailure(err, local.rel, log) };
+    } finally {
+      ledger.outboxDone(local.hash);
+    }
+  }
+
   /**
    * Tell the hub a file is gone here — Increment 6.
    *
@@ -1316,7 +1579,7 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     let work: {
       rel: string;
       decision: ReturnType<typeof decideFile>;
-      local: LocalFile | undefined;
+      local: SyncableLocalFile | undefined;
       row: JournalEntry | undefined;
       remoteHash: string | null;
     }[] = [];
@@ -1331,6 +1594,12 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     for (const rel of paths) {
       const row = delta.get(rel);
       const here = local.get(rel);
+      // HELD, NOT ABSENT. A file this peer cannot move is still a file: no
+      // decision may read it as deleted here or missing here.
+      if (here?.oversized) {
+        holdOversized(out, ledger, here);
+        continue;
+      }
       // What the hub holds: this page when it spoke about the path, otherwise
       // what we last learned. `undefined` (never learned) reads as null only
       // after a full read has had the chance to say so.
@@ -1500,7 +1769,10 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     // Referenced assets first (DDR-223's strokes→bytes coupling): a picture a
     // just-arrived annotation points at is the one a person is staring at.
     const referenced = referencedAssetNames(designRoot);
-    work.sort((a, b) => rank(a.rel, referenced) - rank(b.rel, referenced));
+    // …then smallest first (T19): one 1 GB master never makes the forty
+    // images a canvas needs wait behind it; big media fills in last.
+    const sizeOf = (w: (typeof work)[number]) => Math.max(claimedSize(w.row), w.local?.size ?? 0);
+    work.sort((a, b) => rank(a.rel, referenced) - rank(b.rel, referenced) || sizeOf(a) - sizeOf(b));
 
     const budget = createPullBudget({
       label: 'sync/files',
@@ -1629,7 +1901,7 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     item: {
       rel: string;
       decision: ReturnType<typeof decideFile>;
-      local?: LocalFile;
+      local?: SyncableLocalFile;
       row?: JournalEntry;
       remoteHash: string | null;
     },
@@ -1676,10 +1948,10 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
             }
             return false;
           }
-          if (!chargeOverrun(budget, rel, got.bytes.byteLength, claim, out)) return false;
+          if (!chargeOverrun(budget, rel, got.size, claim, out)) return false;
           const copyRel = conflictCopyName(rel, now(), 'hub');
           try {
-            materialize(copyRel, got.bytes);
+            materialize(copyRel, got);
             ledger.setState(rel, 'conflict', {
               reason: 'the hub’s log restarted; their copy is parked beside yours',
               conflictCopy: copyRel,
@@ -1719,8 +1991,8 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
           }
           return false;
         }
-        if (!chargeOverrun(budget, rel, got.bytes.byteLength, claim, out)) return false;
-        const landed = await ledger.adoptAfter(rel, remoteHash, () => materialize(rel, got.bytes), {
+        if (!chargeOverrun(budget, rel, got.size, claim, out)) return false;
+        const landed = await ledger.adoptAfter(rel, remoteHash, () => materialize(rel, got), {
           ...(row ? { remoteSeq: row.seq } : {}),
           state: 'on-hub',
         });
@@ -1817,8 +2089,8 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
           }
           return false;
         }
-        if (!chargeOverrun(budget, rel, got.bytes.byteLength, claim, out)) return false;
-        const landed = await ledger.adoptAfter(rel, remoteHash, () => materialize(rel, got.bytes), {
+        if (!chargeOverrun(budget, rel, got.size, claim, out)) return false;
+        const landed = await ledger.adoptAfter(rel, remoteHash, () => materialize(rel, got), {
           ...(row ? { remoteSeq: row.seq } : {}),
           state: 'conflict',
         });
@@ -1905,7 +2177,7 @@ export function createFilePlane(opts: FilePlaneOptions): FilePlane {
     }
   }
 
-  function statLocal(rel: string): LocalFile | null {
+  function statLocal(rel: string): SyncableLocalFile | null {
     const abs = path.join(designRoot, rel);
     try {
       const st = statSync(abs);
@@ -1953,6 +2225,17 @@ function drop(out: FilePlaneResult, rel: string, reason: string, ledger?: FileLe
   ledger?.setState(rel, 'stuck', { reason });
 }
 
+function holdOversized(out: FilePlaneResult, ledger: FileLedger, file: OversizedLocalFile): void {
+  const mb = (n: number) => Math.round(n / (1024 * 1024));
+  const reason = `too big to sync — ${mb(file.size)} MB is over the ${mb(MAX_FILE_BYTES)} MB limit`;
+  out.dropped.push({ rel: file.rel, reason });
+  const row = ledger.row(file.rel);
+  // Unchanged rows stay unwritten: this runs every pass for as long as the
+  // file stays too big, and the ledger flushes on every setState.
+  if (row?.state === 'refused' && row.blockedClass === 'too-large' && row.reason === reason) return;
+  ledger.setState(file.rel, 'refused', { reason, blockedClass: 'too-large' });
+}
+
 /** `assets/<name>` referenced anywhere in the tree — the priority front-queue. */
 function referencedAssetNames(designRoot: string): Set<string> {
   const out = new Set<string>();