@foss.global/codefeed 4.0.0 → 4.2.0

package/ts/classes.lfsbatchclient.ts

@@ -1,6 +1,7 @@
 import * as plugins from './plugins.js';
 import * as transport from './forgeinventory.transport.js';
 import * as validation from './forgeinventory.validation.js';
+import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
 import { ForgeRequestClient } from './forgeinventory.client.js';
 import type { IForgeRequestMetrics } from './forgeinventory.client.js';
 
@@ -82,8 +83,6 @@ type TBatchEntry = ILfsObjectId & { actions: Record<string, ILfsAction>; errorCo
 const lfsMediaType = 'application/vnd.git-lfs+json';
 /** Objects per batch request; the Git LFS reference server and Gitea accept 100. */
 const maxBatchSize = 100;
-/** Response bytes read from a PUT or verify answer before the rest is discarded. */
-const maxDiscardedResponseBytes = 64 * 1024;
 const maxActionHeaders = 32;
 const maxHeaderValueLength = 8 * 1024;
 const maxHrefLength = 8 * 1024;
@@ -179,44 +178,14 @@ const actionExpiry = (actionArg: Record<string, unknown>, receivedAtArg: number)
  * wrong; otherwise it throws `lfs-size-overrun`, `lfs-short-body` or
  * `lfs-digest-mismatch`.
  */
-async function* verifiedChunks(sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> {
+const verifiedChunks = (sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> => {
   const hash = plugins.crypto.createHash('sha256');
-  let bytes = 0;
-  let held: Uint8Array | null = null;
-  for await (const chunk of sourceArg) {
-    if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError('invalid-lfs-chunk');
-    if (chunk.byteLength === 0) continue;
-    bytes += chunk.byteLength;
-    if (bytes > sizeArg) throw new validation.ForgeInventoryError('lfs-size-overrun');
-    hash.update(chunk);
-    if (held) yield held;
-    held = chunk;
-  }
-  if (bytes !== sizeArg) throw new validation.ForgeInventoryError('lfs-short-body');
-  if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
-  if (held) yield held;
-}
-
-/**
- * Reads and drops at most `maxDiscardedResponseBytes` of a response body that
- * carries nothing the client uses, so its connection can be reused; a longer
- * or failing body is discarded with its connection.
- */
-const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
-  if (!bodyArg) return;
-  const chunks = bodyArg[Symbol.asyncIterator]();
-  let bytes = 0;
-  try {
-    while (bytes <= maxDiscardedResponseBytes) {
-      const result = await chunks.next();
-      if (result.done) return;
-      bytes += result.value.byteLength;
-    }
-  } catch {
-    // The status already decided the outcome.
-  } finally {
-    try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
-  }
+  return heldBackChunks(sourceArg, sizeArg, { invalidChunk: 'invalid-lfs-chunk', overrun: 'lfs-size-overrun', short: 'lfs-short-body' }, {
+    inspect: (chunkArg) => hash.update(chunkArg),
+    complete: () => {
+      if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
+    },
+  });
 };
 
 /**
@@ -390,10 +359,8 @@ export class LfsBatchClient {
           transfer.end();
         }
       };
-      const stream = plugins.stream.Readable.from(chunks(), { objectMode: false });
-      // A consumer that destroys the stream ends the download, also while a read is pending.
-      stream.once('close', () => transfer.end());
-      return { oid, size, status: 'available', stream };
+      // Destroying the stream ends the download and closes its connection at once, also while a read is pending.
+      return { oid, size, status: 'available', stream: createTransferStream(transfer, chunks()) };
     } catch (error) {
       transfer.end();
       throw transfer.failure(error, 'lfs-download-failed');
@@ -553,62 +520,12 @@ export class LfsBatchClient {
 
   /**
    * The bounds of one transfer: the caller's signal, a wall-clock limit and an
-   * idle limit that `watch` re-arms on every chunk. Whichever ends it first
-   * decides the failure code.
+   * idle limit of `requestTimeoutMs`; a passed limit fails with
+   * `lfs-transfer-timed-out`.
    */
   #transfer(signalArg: AbortSignal | undefined, limitMsArg = this.#limits.transferTimeoutMs) {
-    const controller = new AbortController();
-    let reason: string | null = null;
-    const stop = (reasonArg: string) => {
-      reason ??= reasonArg;
-      controller.abort();
-    };
-    const abort = () => stop('request-aborted');
-    if (signalArg?.aborted) abort();
-    signalArg?.addEventListener('abort', abort, { once: true });
-    const deadline = setTimeout(() => stop('lfs-transfer-timed-out'), limitMsArg);
-    let idle: ReturnType<typeof setTimeout> | undefined;
-    const rearm = () => {
-      clearTimeout(idle);
-      if (!controller.signal.aborted) idle = setTimeout(() => stop('lfs-transfer-timed-out'), this.#limits.requestTimeoutMs);
-    };
-    rearm();
-    const end = () => {
-      clearTimeout(deadline);
-      clearTimeout(idle);
-      signalArg?.removeEventListener('abort', abort);
-      stop('transfer-ended');
-    };
-    const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
-      if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
-      return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
-    };
-    return {
-      signal: controller.signal,
-      end,
-      failure,
-      /** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
-      run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
-        if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
-        let onAbort!: () => void;
-        const aborted = new Promise<never>((_resolve, reject) => {
-          onAbort = () => reject(failure(undefined, 'request-aborted'));
-          controller.signal.addEventListener('abort', onAbort, { once: true });
-        });
-        try {
-          return await Promise.race([stepArg(), aborted]);
-        } finally {
-          controller.signal.removeEventListener('abort', onAbort);
-        }
-      },
-      /** Passes chunks on and re-arms the idle limit on each one. */
-      watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
-        for await (const chunk of sourceArg) {
-          if (controller.signal.aborted) throw failure(undefined, 'lfs-transfer-failed');
-          rearm();
-          yield chunk;
-        }
-      },
-    };
+    return createTransferBounds(signalArg, {
+      limitMs: limitMsArg, idleMs: this.#limits.requestTimeoutMs, timedOutCode: 'lfs-transfer-timed-out', failedCode: 'lfs-transfer-failed',
+    });
   }
 }

package/ts/forgeinventory.transfer.ts

@@ -0,0 +1,203 @@
+import * as plugins from './plugins.js';
+import * as validation from './forgeinventory.validation.js';
+
+/** Response bytes read from an unused answer before the rest is discarded. */
+const maxDiscardedResponseBytes = 64 * 1024;
+
+/**
+ * Reads and drops at most 64 KiB of a response body that carries nothing the
+ * caller uses, so its connection can be reused; a longer or failing body is
+ * discarded with its connection.
+ */
+export const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
+  if (!bodyArg) return;
+  const chunks = bodyArg[Symbol.asyncIterator]();
+  let bytes = 0;
+  try {
+    while (bytes <= maxDiscardedResponseBytes) {
+      const result = await chunks.next();
+      if (result.done) return;
+      bytes += result.value.byteLength;
+    }
+  } catch {
+    // The status already decided the outcome.
+  } finally {
+    try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
+  }
+};
+
+export interface ITransferBoundsOptions {
+  /** Wall-clock limit of the whole transfer. */
+  limitMs: number;
+  /** Longest gap between two chunks of a watched source. */
+  idleMs: number;
+  /** The code a passed wall-clock or idle limit fails with. */
+  timedOutCode: string;
+  /** The code a watched source that goes on after the transfer ended fails with. */
+  failedCode: string;
+}
+
+export interface ITransferBounds {
+  /** Aborts once the caller aborts, a limit passes or the transfer ends. */
+  signal: AbortSignal;
+  /** Ends the transfer: clears its timers and aborts `signal`. Idempotent. */
+  end: () => void;
+  /**
+   * The error the transfer fails with: the reason that ended it (`request-aborted`
+   * or the timeout code), else `errorArg` when it is a `ForgeInventoryError`,
+   * else `fallbackArg`.
+   */
+  failure: (errorArg: unknown, fallbackArg: string) => validation.ForgeInventoryError;
+  /** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
+  run: <T>(stepArg: () => Promise<T>) => Promise<T>;
+  /** Re-arms the idle limit, before a step that waits on the network without a watched source. */
+  touch: () => void;
+  /** Passes chunks on and re-arms the idle limit on each one. */
+  watch: (sourceArg: AsyncIterable<Uint8Array>) => AsyncGenerator<Uint8Array, void, undefined>;
+}
+
+/**
+ * The bounds of one streamed transfer: the caller's signal, a wall-clock limit
+ * and an idle limit that `watch` re-arms on every chunk. Whichever ends it
+ * first decides the failure code. Every transfer must be ended with `end`.
+ */
+export const createTransferBounds = (signalArg: AbortSignal | undefined, optionsArg: ITransferBoundsOptions): ITransferBounds => {
+  const controller = new AbortController();
+  let reason: string | null = null;
+  const stop = (reasonArg: string) => {
+    reason ??= reasonArg;
+    controller.abort();
+  };
+  const abort = () => stop('request-aborted');
+  if (signalArg?.aborted) abort();
+  signalArg?.addEventListener('abort', abort, { once: true });
+  const deadline = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.limitMs);
+  let idle: ReturnType<typeof setTimeout> | undefined;
+  const rearm = () => {
+    clearTimeout(idle);
+    if (!controller.signal.aborted) idle = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.idleMs);
+  };
+  rearm();
+  const end = () => {
+    clearTimeout(deadline);
+    clearTimeout(idle);
+    signalArg?.removeEventListener('abort', abort);
+    stop('transfer-ended');
+  };
+  const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
+    if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
+    return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
+  };
+  return {
+    signal: controller.signal,
+    end,
+    failure,
+    touch: rearm,
+    run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
+      if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
+      let onAbort!: () => void;
+      const aborted = new Promise<never>((_resolve, reject) => {
+        onAbort = () => reject(failure(undefined, 'request-aborted'));
+        controller.signal.addEventListener('abort', onAbort, { once: true });
+      });
+      try {
+        return await Promise.race([stepArg(), aborted]);
+      } finally {
+        controller.signal.removeEventListener('abort', onAbort);
+      }
+    },
+    watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
+      for await (const chunk of sourceArg) {
+        if (controller.signal.aborted) throw failure(undefined, optionsArg.failedCode);
+        rearm();
+        yield chunk;
+      }
+    },
+  };
+};
+
+export interface IHeldBackCodes {
+  /** A chunk that is not a `Uint8Array`. */
+  invalidChunk: string;
+  /** More bytes than the declared size. */
+  overrun: string;
+  /** Fewer bytes than the declared size. */
+  short: string;
+}
+
+export interface IHeldBackChecks {
+  /** Sees every non-empty chunk in order, before it is passed on. */
+  inspect?: (chunkArg: Uint8Array) => void;
+  /** Runs once exactly `size` bytes have arrived, before the last chunk is passed on; a rejection keeps it back. */
+  complete?: () => void | Promise<void>;
+}
+
+/**
+ * Passes the bytes of `sourceArg` on while it counts them. The last chunk is
+ * held back until the source has ended with exactly `sizeArg` bytes and
+ * `checksArg.complete` has passed, so a consumer never receives a complete
+ * body that is wrong; otherwise it throws `codesArg.overrun` as soon as the
+ * size is exceeded, `codesArg.short` when the source ends early, or what
+ * `complete` threw.
+ */
+export async function* heldBackChunks(
+  sourceArg: AsyncIterable<Uint8Array>, sizeArg: number, codesArg: IHeldBackCodes, checksArg: IHeldBackChecks = {},
+): AsyncGenerator<Uint8Array, void, undefined> {
+  let bytes = 0;
+  let held: Uint8Array | null = null;
+  for await (const chunk of sourceArg) {
+    if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError(codesArg.invalidChunk);
+    if (chunk.byteLength === 0) continue;
+    bytes += chunk.byteLength;
+    if (bytes > sizeArg) throw new validation.ForgeInventoryError(codesArg.overrun);
+    checksArg.inspect?.(chunk);
+    if (held) yield held;
+    held = chunk;
+  }
+  if (bytes !== sizeArg) throw new validation.ForgeInventoryError(codesArg.short);
+  await checksArg.complete?.();
+  if (held) yield held;
+}
+
+/**
+ * The readable stream of one transfer's bytes. It pulls `chunksArg` with
+ * backpressure and ends `boundsArg` and calls `onEndArg` as soon as it is
+ * destroyed, also while a pull is pending: ending the bounds aborts the
+ * request, which closes its connection, and the pending pull's result is
+ * dropped. (`Readable.from` waits for a pending pull before it finishes a
+ * destroy, so a stalled body would keep its connection until the idle limit.)
+ * The stream errors with what `chunksArg` throws.
+ */
+export const createTransferStream = (
+  boundsArg: ITransferBounds, chunksArg: AsyncGenerator<Uint8Array, void, undefined>, onEndArg?: () => void,
+): InstanceType<typeof plugins.stream.Readable> => {
+  let pulling = false;
+  const stream: InstanceType<typeof plugins.stream.Readable> = new plugins.stream.Readable({
+    read() {
+      if (pulling) return;
+      pulling = true;
+      void (async () => {
+        try {
+          while (!stream.destroyed) {
+            const result = await chunksArg.next();
+            if (stream.destroyed) return;
+            if (result.done) return void stream.push(null);
+            if (!stream.push(result.value)) return;
+          }
+        } catch (error) {
+          stream.destroy(error as Error);
+        } finally {
+          pulling = false;
+        }
+      })();
+    },
+    destroy(errorArg, callbackArg) {
+      boundsArg.end();
+      onEndArg?.();
+      // Runs the generator's cleanup once a pending pull has settled; nothing waits for it.
+      chunksArg.return(undefined).catch(() => undefined);
+      callbackArg(errorArg);
+    },
+  });
+  return stream;
+};