@foss.global/codefeed 3.0.0 → 3.2.0

package/ts/forgeinventory.transport.ts

@@ -32,43 +32,125 @@ export const globalFetchTransport: TForgeTransport = async (url, headers, signal
   return { status: response.status, headers: response.headers, body: response.body };
 };
 
+/** The agents a node request may use; a URL whose protocol has no agent is refused. */
+export interface INodeRequestAgents {
+  http?: InstanceType<typeof plugins.http.Agent>;
+  https?: InstanceType<typeof plugins.https.Agent>;
+}
+
+export interface INodeRequest {
+  method: string;
+  headers: Record<string, string>;
+  /** Sent in full, or streamed with backpressure; a streamed body needs its own `content-length` header. */
+  body?: Uint8Array | AsyncIterable<Uint8Array>;
+  signal: AbortSignal;
+  agents: INodeRequestAgents;
+}
+
+export interface INodeResponse extends IForgeTransportResponse {
+  /**
+   * Settles once the request body has been sent, or rejects with the error
+   * that ended it, such as the body source's own error. Always handled.
+   */
+  sent: Promise<void>;
+}
+
+/**
+ * Idle keep-alive agents that unreference their pooled sockets, so pooled
+ * connections never keep the process alive.
+ */
+export const createNodeAgents = (caCertificatesArg?: string[], allowHttpArg = false): INodeRequestAgents => ({
+  https: new plugins.https.Agent({ ...(caCertificatesArg ? { ca: caCertificatesArg } : {}), keepAlive: true, timeout: idleSocketTimeoutMs }),
+  ...(allowHttpArg ? { http: new plugins.http.Agent({ keepAlive: true, timeout: idleSocketTimeoutMs }) } : {}),
+});
+
+/** Waits until the request can take more body bytes; rejects once it has closed. */
+const drained = (requestArg: InstanceType<typeof plugins.http.ClientRequest>): Promise<void> => new Promise((resolve, reject) => {
+  const done = () => { requestArg.off('drain', done); requestArg.off('close', closed); resolve(); };
+  const closed = () => { requestArg.off('drain', done); requestArg.off('close', closed); reject(new Error('Request closed.')); };
+  requestArg.on('drain', done);
+  requestArg.on('close', closed);
+});
+
+/**
+ * Sends one request through `node:http` or `node:https` and resolves once
+ * response headers arrive. A redirect status rejects and is never followed;
+ * aborting `signal` discards the connection, also while a body is still being
+ * sent or read. A streamed body that fails destroys the request and rejects.
+ */
+export const sendNodeRequest = (url: URL, requestArg: INodeRequest): Promise<INodeResponse> => new Promise<INodeResponse>((resolve, reject) => {
+  const { signal } = requestArg;
+  if (signal.aborted) return reject(signal.reason);
+  const agent = url.protocol === 'https:' ? requestArg.agents.https : url.protocol === 'http:' ? requestArg.agents.http : undefined;
+  if (!agent) return reject(new Error('Refused protocol.'));
+  const module = url.protocol === 'https:' ? plugins.https : plugins.http;
+  // Bodies are read as sent, so ask for them without a content coding.
+  const request = module.request(url, { method: requestArg.method, headers: { ...requestArg.headers, 'accept-encoding': 'identity' }, agent });
+  let settled = false;
+  const fail = (errorArg: unknown) => {
+    if (!settled) {
+      settled = true;
+      reject(errorArg);
+    }
+  };
+  // Stays attached for the request's lifetime, so a connection failure while
+  // the body is read never becomes an unhandled 'error' event.
+  request.on('error', fail);
+  // Destroyed without an error on purpose: an error would be re-emitted on the
+  // socket, which no longer has a listener once a keep-alive response completed.
+  const abort = () => {
+    fail(signal.reason);
+    request.destroy();
+  };
+  signal.addEventListener('abort', abort, { once: true });
+  request.once('close', () => signal.removeEventListener('abort', abort));
+  const sent = (async () => {
+    const { body } = requestArg;
+    if (body === undefined || body instanceof Uint8Array) {
+      request.end(body);
+      return;
+    }
+    try {
+      for await (const chunk of body) {
+        if (request.destroyed) throw new Error('Request closed.');
+        if (!request.write(chunk)) await drained(request);
+      }
+      request.end();
+    } catch (error) {
+      fail(error);
+      request.destroy();
+      throw error;
+    }
+  })();
+  sent.catch(() => undefined);
+  request.on('response', (response) => {
+    try {
+      const status = response.statusCode ?? 0;
+      if (redirectStatuses.has(status)) throw new Error('Refused redirect.');
+      // Same combination rules as fetch: repeated fields join with ", ".
+      const responseHeaders = new Headers();
+      for (let index = 0; index < response.rawHeaders.length; index += 2) {
+        responseHeaders.append(response.rawHeaders[index], response.rawHeaders[index + 1]);
+      }
+      if (settled) return void request.destroy();
+      settled = true;
+      resolve({ status, headers: responseHeaders, body: response, sent });
+    } catch (error) {
+      fail(error);
+      request.destroy();
+    }
+  });
+});
+
 /**
  * `node:https` with an owned keep-alive agent whose `ca` list replaces the
  * default trust store for these connections only. The agent unreferences idle
  * sockets, so pooled connections never keep the process alive.
  */
 export const createHttpsTransport = (caCertificates: string[]): TForgeTransport => {
-  const agent = new plugins.https.Agent({ ca: caCertificates, keepAlive: true, timeout: idleSocketTimeoutMs });
-  return (url, headers, signal, body) => new Promise<IForgeTransportResponse>((resolve, reject) => {
-    if (signal.aborted) return reject(signal.reason);
-    // Bodies are read as sent, so ask for them without a content coding.
-    const request = plugins.https.request(url, { method: body ? 'POST' : 'GET', headers: { ...headers, 'accept-encoding': 'identity' }, agent });
-    // Stays attached for the request's lifetime, so a connection failure while
-    // the body is read never becomes an unhandled 'error' event.
-    request.on('error', reject);
-    // Destroyed without an error on purpose: an error would be re-emitted on the
-    // socket, which no longer has a listener once a keep-alive response completed.
-    const abort = () => {
-      reject(signal.reason);
-      request.destroy();
-    };
-    signal.addEventListener('abort', abort, { once: true });
-    request.once('close', () => signal.removeEventListener('abort', abort));
-    request.on('response', (response) => {
-      try {
-        const status = response.statusCode ?? 0;
-        if (redirectStatuses.has(status)) throw new Error('Refused redirect.');
-        // Same combination rules as fetch: repeated fields join with ", ".
-        const responseHeaders = new Headers();
-        for (let index = 0; index < response.rawHeaders.length; index += 2) {
-          responseHeaders.append(response.rawHeaders[index], response.rawHeaders[index + 1]);
-        }
-        resolve({ status, headers: responseHeaders, body: response });
-      } catch (error) {
-        reject(error);
-        request.destroy();
-      }
-    });
-    request.end(body);
-  });
+  const agents = createNodeAgents(caCertificates);
+  return async (url, headers, signal, body) => {
+    const { sent: _sent, ...response } = await sendNodeRequest(url, { method: body ? 'POST' : 'GET', headers, body, signal, agents });
+    return response;
+  };
 };

package/ts/giteasourceinventory.git.ts

@@ -5,6 +5,14 @@ import type { ForgeRequestClient } from './forgeinventory.client.js';
 type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
 type IGitCounts = Omit<plugins.fossInterfaces.data.IForgeSourceGitCounts, 'reportedSizeKiB'>;
 type ILfsCounts = plugins.fossInterfaces.data.IForgeSourceLfsCounts;
+type IHead = plugins.fossInterfaces.data.IForgeSourceRepositoryHead;
+
+/** The Git level measure of one repository; `head` is null when the mirror's refs could not be read. */
+export interface IGiteaMirrorMeasure {
+  git: IGitCounts;
+  lfs: ILfsCounts;
+  head: IHead | null;
+}
 
 export interface IGiteaMirrorSource {
   /** Instance root without a trailing slash. */
@@ -98,6 +106,21 @@ export class GitScratchBudget {
     }
   }
 
+  /**
+   * The process-wide budget of the file system that holds `directoryArg`,
+   * keyed by its device; every mirror clone and every scratch GitManager in
+   * the process reserves through it.
+   */
+  static async forDirectory(directoryArg: string): Promise<GitScratchBudget> {
+    const device = (await plugins.fs.stat(directoryArg, { bigint: true })).dev;
+    let budget = scratchBudgets.get(device);
+    if (!budget) {
+      budget = new GitScratchBudget();
+      scratchBudgets.set(device, budget);
+    }
+    return budget;
+  }
+
   #releaser(bytesArg: bigint): () => void {
     let held = true;
     return () => {
@@ -127,16 +150,28 @@ export class GitScratchBudget {
 /** One budget per file system, shared by every reader in the process. */
 const scratchBudgets = new Map<bigint, GitScratchBudget>();
 
-const scratchBudget = async (workDirArg: string): Promise<GitScratchBudget> => {
-  const device = (await plugins.fs.stat(workDirArg, { bigint: true })).dev;
-  let budget = scratchBudgets.get(device);
-  if (!budget) {
-    budget = new GitScratchBudget();
-    scratchBudgets.set(device, budget);
+/**
+ * Reserves the scratch of one mirror clone into `directoryArg` in the
+ * process-wide budget of its file system and resolves with the function that
+ * releases it. Rejects with `git-workdir-too-small` when the clone does not
+ * fit even alone, `git-workdir-failed` when the directory or its free space
+ * cannot be read and `request-aborted` when `signalArg` aborts first.
+ */
+export const reserveMirrorScratch = async (directoryArg: string, reportedSizeKiBArg: bigint, signalArg?: AbortSignal): Promise<() => void> => {
+  let release: (() => void) | null;
+  try {
+    release = await (await GitScratchBudget.forDirectory(directoryArg)).reserve(directoryArg, mirrorScratchBytes(reportedSizeKiBArg), signalArg);
+  } catch {
+    throw new validation.ForgeInventoryError(signalArg?.aborted ? 'request-aborted' : 'git-workdir-failed');
   }
-  return budget;
+  if (release === null) throw new validation.ForgeInventoryError('git-workdir-too-small');
+  return release;
 };
 
+/** The HTTPS Git URL of a Gitea repository; it never carries credentials. */
+export const giteaCloneUrl = (baseUrlArg: string, ownerArg: string, nameArg: string): string =>
+  `${baseUrlArg}/${encodeURIComponent(ownerArg)}/${encodeURIComponent(nameArg)}.git`;
+
 const complete = (valueArg: bigint | number): TMeasure => ({ status: 'complete', value: BigInt(valueArg).toString() });
 const partial = (observedArg: bigint | number, reasonArg: string): TMeasure => ({ status: 'partial', observedValue: BigInt(observedArg).toString(), reason: reasonArg });
 
@@ -158,23 +193,22 @@ const stepFailure = (errorArg: unknown, stepArg: string, signalArg?: AbortSignal
 
 /**
  * Measures one repository at the Git level: clones every ref Gitea advertises
- * into a temporary mirror, counts its refs and objects, finds the LFS pointers
+ * into a temporary mirror, records its HEAD, counts its refs and objects, finds the LFS pointers
  * of every reachable revision and checks through the LFS batch API which of
  * their objects Gitea holds. The mirror is removed afterwards. The clone starts
  * only once its scratch reservation fits the work directory's free space; a
  * repository too large for it fails with `git-workdir-too-small`.
  */
-export const measureGiteaMirror = async (sourceArg: IGiteaMirrorSource): Promise<{ git: IGitCounts; lfs: ILfsCounts }> => {
+export const measureGiteaMirror = async (sourceArg: IGiteaMirrorSource): Promise<IGiteaMirrorMeasure> => {
   const { signal } = sourceArg;
   if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
-  let release: (() => void) | null;
+  let release: () => void;
   try {
-    release = await (await scratchBudget(sourceArg.workDir)).reserve(sourceArg.workDir, mirrorScratchBytes(sourceArg.reportedSizeKiB), signal);
-  } catch {
-    const failure = signal?.aborted ? 'request-aborted' : 'git-workdir-failed';
-    return { git: failedGit(failure), lfs: failedLfs(failure) };
+    release = await reserveMirrorScratch(sourceArg.workDir, sourceArg.reportedSizeKiB, signal);
+  } catch (error) {
+    const failure = stepFailure(error, 'git-workdir-failed', signal);
+    return { git: failedGit(failure), lfs: failedLfs(failure), head: null };
   }
-  if (release === null) return { git: failedGit('git-workdir-too-small'), lfs: failedLfs('git-workdir-too-small') };
   try {
     return await measureReservedMirror(sourceArg);
   } finally {
@@ -182,17 +216,18 @@ export const measureGiteaMirror = async (sourceArg: IGiteaMirrorSource): Promise
   }
 };
 
-const measureReservedMirror = async (sourceArg: IGiteaMirrorSource): Promise<{ git: IGitCounts; lfs: ILfsCounts }> => {
+const measureReservedMirror = async (sourceArg: IGiteaMirrorSource): Promise<IGiteaMirrorMeasure> => {
   const { signal } = sourceArg;
   const { GitManager } = await plugins.getGitManager();
   let directory: string;
   try {
     directory = await plugins.fs.mkdtemp(plugins.path.join(sourceArg.workDir, 'mirror-'));
   } catch {
-    return { git: failedGit('git-workdir-failed'), lfs: failedLfs('git-workdir-failed') };
+    return { git: failedGit('git-workdir-failed'), lfs: failedLfs('git-workdir-failed'), head: null };
   }
   const manager = new GitManager({ dataDir: directory });
-  // Stopping the engine ends the operation in flight.
+  // The engine serves this measurement alone; stopping it ends the ref, object
+  // and LFS scans in flight, which take no signal. The clone takes the signal.
   const abort = () => { void manager.stop().catch(() => undefined); };
   signal?.addEventListener('abort', abort, { once: true });
   const selector = { org: 'source', repo: 'mirror' };
@@ -203,23 +238,23 @@ const measureReservedMirror = async (sourceArg: IGiteaMirrorSource): Promise<{ g
       await manager.start();
       await manager.importMirror({
         ...selector,
-        remoteUrl: `${sourceArg.baseUrl}/${encodeURIComponent(sourceArg.owner)}/${encodeURIComponent(sourceArg.name)}.git`,
+        remoteUrl: giteaCloneUrl(sourceArg.baseUrl, sourceArg.owner, sourceArg.name),
         credential: { kind: 'basic', username: sourceArg.username, password: sourceArg.token },
         caCertificates: sourceArg.caCertificates,
         timeoutMs: sourceArg.timeoutMs,
-      });
+      }, { signal });
     } catch (error) {
       const failure = stepFailure(error, 'git-mirror-failed', signal);
-      return { git: failedGit(failure), lfs: failedLfs(failure) };
+      return { git: failedGit(failure), lfs: failedLfs(failure), head: null };
     }
-    const git = await measureRefsAndObjects(manager, selector, signal);
+    const { git, head } = await measureRefsAndObjects(manager, selector, signal);
     let pointers: Array<{ oid: string; sizeBytes: number }>;
     try {
       pointers = await manager.listRepositoryLfsPointers(selector);
     } catch (error) {
-      return { git, lfs: failedLfs(stepFailure(error, 'lfs-pointer-scan-failed', signal)) };
+      return { git, lfs: failedLfs(stepFailure(error, 'lfs-pointer-scan-failed', signal)), head };
     }
-    return { git, lfs: await giteaLfsPresence(sourceArg, pointers) };
+    return { git, lfs: await giteaLfsPresence(sourceArg, pointers), head };
   } finally {
     signal?.removeEventListener('abort', abort);
     await manager.stop().catch(() => undefined);
@@ -231,8 +266,9 @@ const measureRefsAndObjects = async (
   managerArg: plugins.TGitManager,
   selectorArg: { org: string; repo: string },
   signalArg?: AbortSignal,
-): Promise<IGitCounts> => {
+): Promise<Pick<IGiteaMirrorMeasure, 'git' | 'head'>> => {
   const counts = { branches: 0, tags: 0, pullRequestRefs: 0, otherRefs: 0 };
+  let head: IHead | null = null;
   let refsFailure: string | null = null;
   try {
     let cursor: string | undefined;
@@ -240,6 +276,8 @@ const measureRefsAndObjects = async (
       const page = await managerArg.listRefs({ ...selectorArg, cursor });
       // Nothing writes to the private mirror, so a changed inventory is a failure.
       if (page.status === 'stale') throw new validation.ForgeInventoryError('git-refs-changed');
+      // Every page carries HEAD; its digest binds it, so all pages agree. An unborn HEAD has no object.
+      head = { target: page.head.symbolicTarget, resolves: page.head.objectId !== null };
       for (const ref of page.refs) {
         if (ref.name.startsWith('refs/heads/')) counts.branches++;
         else if (ref.name.startsWith('refs/tags/')) counts.tags++;
@@ -249,6 +287,7 @@ const measureRefsAndObjects = async (
       cursor = page.pagination.status === 'more' ? page.pagination.cursor : undefined;
     } while (cursor !== undefined);
   } catch (error) {
+    head = null;
     refsFailure = stepFailure(error, 'git-refs-failed', signalArg);
   }
   const refs = (valueArg: number) => refsFailure === null ? complete(valueArg) : partial(0, refsFailure);
@@ -262,8 +301,11 @@ const measureRefsAndObjects = async (
     objects = objectBytes = partial(0, stepFailure(error, 'git-size-failed', signalArg));
   }
   return {
-    branches: refs(counts.branches), tags: refs(counts.tags), pullRequestRefs: refs(counts.pullRequestRefs), otherRefs: refs(counts.otherRefs),
-    objects, objectBytes,
+    git: {
+      branches: refs(counts.branches), tags: refs(counts.tags), pullRequestRefs: refs(counts.pullRequestRefs), otherRefs: refs(counts.otherRefs),
+      objects, objectBytes,
+    },
+    head,
   };
 };
 

package/ts/index.ts

@@ -488,3 +488,6 @@ export class CodeFeed {
 export * from './classes.forgeinventory.js';
 export * from './classes.forgeinventoryscanner.js';
 export * from './classes.giteasourceinventory.js';
+export * from './classes.giteasourcetransfer.js';
+export * from './classes.lfsbatchclient.js';
+export { GitScratchBudget, mirrorScratchBytes } from './giteasourceinventory.git.js';

package/ts/plugins.ts

@@ -2,9 +2,12 @@ import type * as fossInterfaces from '@foss.global/interfaces';
 
 import * as crypto from 'node:crypto';
 import * as fs from 'node:fs/promises';
+import * as http from 'node:http';
 import * as https from 'node:https';
+import * as net from 'node:net';
 import * as path from 'node:path';
-export { crypto, fs, https, path };
+import * as stream from 'node:stream';
+export { crypto, fs, http, https, net, path, stream };
 
 // module
 import * as interfaces from './interfaces/index.js';