@foss.global/codefeed 4.1.0 → 4.3.0

package/ts/forgeinventory.transfer.ts

@@ -0,0 +1,203 @@
+import * as plugins from './plugins.js';
+import * as validation from './forgeinventory.validation.js';
+
+/** Response bytes read from an unused answer before the rest is discarded. */
+const maxDiscardedResponseBytes = 64 * 1024;
+
+/**
+ * Reads and drops at most 64 KiB of a response body that carries nothing the
+ * caller uses, so its connection can be reused; a longer or failing body is
+ * discarded with its connection.
+ */
+export const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
+  if (!bodyArg) return;
+  const chunks = bodyArg[Symbol.asyncIterator]();
+  let bytes = 0;
+  try {
+    while (bytes <= maxDiscardedResponseBytes) {
+      const result = await chunks.next();
+      if (result.done) return;
+      bytes += result.value.byteLength;
+    }
+  } catch {
+    // The status already decided the outcome.
+  } finally {
+    try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
+  }
+};
+
+export interface ITransferBoundsOptions {
+  /** Wall-clock limit of the whole transfer. */
+  limitMs: number;
+  /** Longest gap between two chunks of a watched source. */
+  idleMs: number;
+  /** The code a passed wall-clock or idle limit fails with. */
+  timedOutCode: string;
+  /** The code a watched source that goes on after the transfer ended fails with. */
+  failedCode: string;
+}
+
+export interface ITransferBounds {
+  /** Aborts once the caller aborts, a limit passes or the transfer ends. */
+  signal: AbortSignal;
+  /** Ends the transfer: clears its timers and aborts `signal`. Idempotent. */
+  end: () => void;
+  /**
+   * The error the transfer fails with: the reason that ended it (`request-aborted`
+   * or the timeout code), else `errorArg` when it is a `ForgeInventoryError`,
+   * else `fallbackArg`.
+   */
+  failure: (errorArg: unknown, fallbackArg: string) => validation.ForgeInventoryError;
+  /** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
+  run: <T>(stepArg: () => Promise<T>) => Promise<T>;
+  /** Re-arms the idle limit, before a step that waits on the network without a watched source. */
+  touch: () => void;
+  /** Passes chunks on and re-arms the idle limit on each one. */
+  watch: (sourceArg: AsyncIterable<Uint8Array>) => AsyncGenerator<Uint8Array, void, undefined>;
+}
+
+/**
+ * The bounds of one streamed transfer: the caller's signal, a wall-clock limit
+ * and an idle limit that `watch` re-arms on every chunk. Whichever ends it
+ * first decides the failure code. Every transfer must be ended with `end`.
+ */
+export const createTransferBounds = (signalArg: AbortSignal | undefined, optionsArg: ITransferBoundsOptions): ITransferBounds => {
+  const controller = new AbortController();
+  let reason: string | null = null;
+  const stop = (reasonArg: string) => {
+    reason ??= reasonArg;
+    controller.abort();
+  };
+  const abort = () => stop('request-aborted');
+  if (signalArg?.aborted) abort();
+  signalArg?.addEventListener('abort', abort, { once: true });
+  const deadline = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.limitMs);
+  let idle: ReturnType<typeof setTimeout> | undefined;
+  const rearm = () => {
+    clearTimeout(idle);
+    if (!controller.signal.aborted) idle = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.idleMs);
+  };
+  rearm();
+  const end = () => {
+    clearTimeout(deadline);
+    clearTimeout(idle);
+    signalArg?.removeEventListener('abort', abort);
+    stop('transfer-ended');
+  };
+  const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
+    if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
+    return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
+  };
+  return {
+    signal: controller.signal,
+    end,
+    failure,
+    touch: rearm,
+    run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
+      if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
+      let onAbort!: () => void;
+      const aborted = new Promise<never>((_resolve, reject) => {
+        onAbort = () => reject(failure(undefined, 'request-aborted'));
+        controller.signal.addEventListener('abort', onAbort, { once: true });
+      });
+      try {
+        return await Promise.race([stepArg(), aborted]);
+      } finally {
+        controller.signal.removeEventListener('abort', onAbort);
+      }
+    },
+    watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
+      for await (const chunk of sourceArg) {
+        if (controller.signal.aborted) throw failure(undefined, optionsArg.failedCode);
+        rearm();
+        yield chunk;
+      }
+    },
+  };
+};
+
+export interface IHeldBackCodes {
+  /** A chunk that is not a `Uint8Array`. */
+  invalidChunk: string;
+  /** More bytes than the declared size. */
+  overrun: string;
+  /** Fewer bytes than the declared size. */
+  short: string;
+}
+
+export interface IHeldBackChecks {
+  /** Sees every non-empty chunk in order, before it is passed on. */
+  inspect?: (chunkArg: Uint8Array) => void;
+  /** Runs once exactly `size` bytes have arrived, before the last chunk is passed on; a rejection keeps it back. */
+  complete?: () => void | Promise<void>;
+}
+
+/**
+ * Passes the bytes of `sourceArg` on while it counts them. The last chunk is
+ * held back until the source has ended with exactly `sizeArg` bytes and
+ * `checksArg.complete` has passed, so a consumer never receives a complete
+ * body that is wrong; otherwise it throws `codesArg.overrun` as soon as the
+ * size is exceeded, `codesArg.short` when the source ends early, or what
+ * `complete` threw.
+ */
+export async function* heldBackChunks(
+  sourceArg: AsyncIterable<Uint8Array>, sizeArg: number, codesArg: IHeldBackCodes, checksArg: IHeldBackChecks = {},
+): AsyncGenerator<Uint8Array, void, undefined> {
+  let bytes = 0;
+  let held: Uint8Array | null = null;
+  for await (const chunk of sourceArg) {
+    if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError(codesArg.invalidChunk);
+    if (chunk.byteLength === 0) continue;
+    bytes += chunk.byteLength;
+    if (bytes > sizeArg) throw new validation.ForgeInventoryError(codesArg.overrun);
+    checksArg.inspect?.(chunk);
+    if (held) yield held;
+    held = chunk;
+  }
+  if (bytes !== sizeArg) throw new validation.ForgeInventoryError(codesArg.short);
+  await checksArg.complete?.();
+  if (held) yield held;
+}
+
+/**
+ * The readable stream of one transfer's bytes. It pulls `chunksArg` with
+ * backpressure and ends `boundsArg` and calls `onEndArg` as soon as it is
+ * destroyed, also while a pull is pending: ending the bounds aborts the
+ * request, which closes its connection, and the pending pull's result is
+ * dropped. (`Readable.from` waits for a pending pull before it finishes a
+ * destroy, so a stalled body would keep its connection until the idle limit.)
+ * The stream errors with what `chunksArg` throws.
+ */
+export const createTransferStream = (
+  boundsArg: ITransferBounds, chunksArg: AsyncGenerator<Uint8Array, void, undefined>, onEndArg?: () => void,
+): InstanceType<typeof plugins.stream.Readable> => {
+  let pulling = false;
+  const stream: InstanceType<typeof plugins.stream.Readable> = new plugins.stream.Readable({
+    read() {
+      if (pulling) return;
+      pulling = true;
+      void (async () => {
+        try {
+          while (!stream.destroyed) {
+            const result = await chunksArg.next();
+            if (stream.destroyed) return;
+            if (result.done) return void stream.push(null);
+            if (!stream.push(result.value)) return;
+          }
+        } catch (error) {
+          stream.destroy(error as Error);
+        } finally {
+          pulling = false;
+        }
+      })();
+    },
+    destroy(errorArg, callbackArg) {
+      boundsArg.end();
+      onEndArg?.();
+      // Runs the generator's cleanup once a pending pull has settled; nothing waits for it.
+      chunksArg.return(undefined).catch(() => undefined);
+      callbackArg(errorArg);
+    },
+  });
+  return stream;
+};

package/ts/giteasourceinventory.git.ts

@@ -6,12 +6,17 @@ type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
 type IGitCounts = Omit<plugins.fossInterfaces.data.IForgeSourceGitCounts, 'reportedSizeKiB'>;
 type ILfsCounts = plugins.fossInterfaces.data.IForgeSourceLfsCounts;
 type IHead = plugins.fossInterfaces.data.IForgeSourceRepositoryHead;
+type IRefDigests = plugins.fossInterfaces.data.IForgeSourceRefDigests;
 
-/** The Git level measure of one repository; `head` is null when the mirror's refs could not be read. */
+/**
+ * The Git level measure of one repository; `head` and `refDigests` are null
+ * when the mirror's refs could not be read.
+ */
 export interface IGiteaMirrorMeasure {
   git: IGitCounts;
   lfs: ILfsCounts;
   head: IHead | null;
+  refDigests: IRefDigests | null;
 }
 
 export interface IGiteaMirrorSource {
@@ -207,7 +212,7 @@ export const measureGiteaMirror = async (sourceArg: IGiteaMirrorSource): Promise
     release = await reserveMirrorScratch(sourceArg.workDir, sourceArg.reportedSizeKiB, signal);
   } catch (error) {
     const failure = stepFailure(error, 'git-workdir-failed', signal);
-    return { git: failedGit(failure), lfs: failedLfs(failure), head: null };
+    return { git: failedGit(failure), lfs: failedLfs(failure), head: null, refDigests: null };
   }
   try {
     return await measureReservedMirror(sourceArg);
@@ -223,7 +228,7 @@ const measureReservedMirror = async (sourceArg: IGiteaMirrorSource): Promise<IGi
   try {
     directory = await plugins.fs.mkdtemp(plugins.path.join(sourceArg.workDir, 'mirror-'));
   } catch {
-    return { git: failedGit('git-workdir-failed'), lfs: failedLfs('git-workdir-failed'), head: null };
+    return { git: failedGit('git-workdir-failed'), lfs: failedLfs('git-workdir-failed'), head: null, refDigests: null };
   }
   const manager = new GitManager({ dataDir: directory });
   // The engine serves this measurement alone; stopping it ends the ref, object
@@ -245,16 +250,16 @@ const measureReservedMirror = async (sourceArg: IGiteaMirrorSource): Promise<IGi
       }, { signal });
     } catch (error) {
       const failure = stepFailure(error, 'git-mirror-failed', signal);
-      return { git: failedGit(failure), lfs: failedLfs(failure), head: null };
+      return { git: failedGit(failure), lfs: failedLfs(failure), head: null, refDigests: null };
     }
-    const { git, head } = await measureRefsAndObjects(manager, selector, signal);
+    const { git, head, refDigests } = await measureRefsAndObjects(manager, selector, signal);
     let pointers: Array<{ oid: string; sizeBytes: number }>;
     try {
       pointers = await manager.listRepositoryLfsPointers(selector);
     } catch (error) {
-      return { git, lfs: failedLfs(stepFailure(error, 'lfs-pointer-scan-failed', signal)), head };
+      return { git, lfs: failedLfs(stepFailure(error, 'lfs-pointer-scan-failed', signal)), head, refDigests };
     }
-    return { git, lfs: await giteaLfsPresence(sourceArg, pointers), head };
+    return { git, lfs: await giteaLfsPresence(sourceArg, pointers), head, refDigests };
   } finally {
     signal?.removeEventListener('abort', abort);
     await manager.stop().catch(() => undefined);
@@ -262,11 +267,39 @@ const measureReservedMirror = async (sourceArg: IGiteaMirrorSource): Promise<IGi
   }
 };
 
+/** The ref namespaces an import carries, each recorded with its own digest. */
+const refDigestPrefixes = { heads: 'refs/heads/', tags: 'refs/tags/', notes: 'refs/notes/' } as const;
+
+/**
+ * Reads GitManager's ref inventory digest and ref count of each namespace in
+ * `refDigestPrefixes`: one `listRefs` page of a single ref per prefix, whose
+ * digest and count cover the complete selection. Resolves with null when any
+ * read fails, never with a partial set.
+ */
+export const readRefDigests = async (
+  managerArg: Pick<plugins.TGitManager, 'listRefs'>,
+  selectorArg: { org: string; repo: string },
+): Promise<IRefDigests | null> => {
+  const read = async (prefixArg: string): Promise<IRefDigests['heads'] | null> => {
+    const page = await managerArg.listRefs({ ...selectorArg, prefixes: [prefixArg], limit: 1 });
+    // Only a cursor can be stale, and a first page sends none.
+    return page.status === 'page' ? { inventoryDigest: page.inventoryDigest, refCount: page.refCount } : null;
+  };
+  try {
+    const heads = await read(refDigestPrefixes.heads);
+    const tags = await read(refDigestPrefixes.tags);
+    const notes = await read(refDigestPrefixes.notes);
+    return heads && tags && notes ? { heads, tags, notes } : null;
+  } catch {
+    return null;
+  }
+};
+
 const measureRefsAndObjects = async (
   managerArg: plugins.TGitManager,
   selectorArg: { org: string; repo: string },
   signalArg?: AbortSignal,
-): Promise<Pick<IGiteaMirrorMeasure, 'git' | 'head'>> => {
+): Promise<Pick<IGiteaMirrorMeasure, 'git' | 'head' | 'refDigests'>> => {
   const counts = { branches: 0, tags: 0, pullRequestRefs: 0, otherRefs: 0 };
   let head: IHead | null = null;
   let refsFailure: string | null = null;
@@ -291,6 +324,8 @@ const measureRefsAndObjects = async (
     refsFailure = stepFailure(error, 'git-refs-failed', signalArg);
   }
   const refs = (valueArg: number) => refsFailure === null ? complete(valueArg) : partial(0, refsFailure);
+  // Nothing writes to the private mirror, so the digests describe the refs just counted.
+  const refDigests = refsFailure === null ? await readRefDigests(managerArg, selectorArg) : null;
   let objects: TMeasure;
   let objectBytes: TMeasure;
   try {
@@ -306,6 +341,7 @@ const measureRefsAndObjects = async (
       objects, objectBytes,
     },
     head,
+    refDigests,
   };
 };