@foss.global/codefeed 5.2.0 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,9 +3,11 @@ import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
4
  import { ForgeRequestClient } from './forgeinventory.client.js';
5
5
  import type { IForgeRequestMetrics, IForgeRequestOptions } from './forgeinventory.client.js';
6
+ import { verifiedPackageDownload } from './forgeinventory.transfer.js';
6
7
  import { measureGiteaMirror } from './giteasourceinventory.git.js';
7
8
  import {
8
9
  cargoIndexPath, comparePackages, composeGiteaPackageContent, containerIndexMediaTypes, npmPackagePath, packageKey, packageTypePattern,
10
+ readSwiftManifest, swiftManifestCandidates, swiftPackageParts,
9
11
  } from './giteasourceinventory.packages.js';
10
12
 
11
13
  type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
@@ -32,6 +34,7 @@ type IPackageRef = plugins.fossInterfaces.data.IForgeSourcePackageRef;
32
34
  type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
33
35
  type IPackageVersion = plugins.fossInterfaces.data.IForgeSourcePackageVersion;
34
36
  type IPackageBlob = plugins.fossInterfaces.data.IForgeSourcePackageBlob;
37
+ type ISwiftManifest = plugins.fossInterfaces.data.IForgeSourceSwiftManifest;
35
38
 
36
39
  export interface IGiteaSourceInventoryOptions {
37
40
  /** Operator-assigned identity of the source instance. */
@@ -58,8 +61,18 @@ export interface IGiteaSourceInventoryOptions {
58
61
  pageSize?: number;
59
62
  /** Pages read per listing before the listing counts as partial. */
60
63
  maxPages?: number;
64
+ /** Deadline of one REST, registry or LFS batch request, and the idle limit of a Swift archive download; defaults to 60 s, at most 300 s. */
61
65
  requestTimeoutMs?: number;
62
66
  maxResponseBytes?: number;
67
+ /**
68
+ * The largest Swift source archive, in bytes, that the inventory reads for
69
+ * the version's `manifests`; defaults to 128 MiB, at most 1 GiB. A version
70
+ * whose recorded archive is larger is not read and records `manifests:
71
+ * null`. A consumer passes the cap its own store puts on an archive.
72
+ */
73
+ maxSwiftArchiveBytes?: number;
74
+ /** Wall-clock limit of one Swift archive download; defaults to 30 minutes, at most six hours. */
75
+ packageTransferTimeoutMs?: number;
63
76
  }
64
77
 
65
78
  /** A class of source data a run measures only when its discovery selects it. */
@@ -88,6 +101,9 @@ export type TGiteaSourceRepositoryLookup =
88
101
  | { status: 'present'; id: string; owner: string; name: string }
89
102
  | { status: 'absent'; id: string };
90
103
 
104
+ /** A file of a version as its listing reports it: the name, and the content's SHA-256 and size as a canonical decimal. */
105
+ interface IVersionFile { name: string; sha256: string; size: string }
106
+
91
107
  interface IListing {
92
108
  items: unknown[];
93
109
  /** A sanitized reason when the listing did not finish. */
@@ -415,11 +431,13 @@ const compareText = (aArg: string, bArg: string): number => aArg < bArg ? -1 : a
415
431
  export class GiteaSourceInventory {
416
432
  readonly #options: {
417
433
  instanceId: string; baseUrl: string; pageSize: number; maxPages: number; requestTimeoutMs: number; maxResponseBytes: number;
418
- token: string; caCertificates?: string[]; gitWorkDir: string; gitTimeoutMs: number;
434
+ token: string; caCertificates?: string[]; gitWorkDir: string; gitTimeoutMs: number; maxSwiftArchiveBytes: number; packageTransferTimeoutMs: number;
419
435
  };
420
436
  readonly #transport: transport.TForgeTransport;
421
437
  readonly #client: ForgeRequestClient;
422
438
  #lfsClient: ForgeRequestClient | null = null;
439
+ /** The Swift archive GETs, which go through `verifiedPackageDownload` instead of a request client. */
440
+ readonly #archiveMetrics: IForgeRequestMetrics = { requests: 0, responseBytes: 0, rateLimitedRequests: 0 };
423
441
  #readerLogin: Promise<string> | null = null;
424
442
 
425
443
  constructor(optionsArg: IGiteaSourceInventoryOptions) {
@@ -435,9 +453,12 @@ export class GiteaSourceInventory {
435
453
  requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
436
454
  maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
437
455
  gitTimeoutMs: optionsArg.gitTimeoutMs ?? 30 * 60 * 1000,
456
+ maxSwiftArchiveBytes: optionsArg.maxSwiftArchiveBytes ?? 128 * 1024 * 1024,
457
+ packageTransferTimeoutMs: optionsArg.packageTransferTimeoutMs ?? 30 * 60 * 1000,
438
458
  };
439
459
  for (const [name, maximum] of [
440
460
  ['pageSize', 100], ['maxPages', 1_000_000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['gitTimeoutMs', 6 * 60 * 60 * 1000],
461
+ ['maxSwiftArchiveBytes', 1024 * 1024 * 1024], ['packageTransferTimeoutMs', 6 * 60 * 60 * 1000],
441
462
  ] as const) {
442
463
  const value = limits[name];
443
464
  if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
@@ -459,14 +480,15 @@ export class GiteaSourceInventory {
459
480
  /** Validated, normalized instance root; contains no credentials or API suffix. */
460
481
  public get baseUrl(): string { return this.#options.baseUrl; }
461
482
 
462
- /** REST and LFS batch requests together. */
483
+ /** REST, registry and LFS batch requests and Swift archive GETs together. */
463
484
  public get metrics(): IForgeRequestMetrics {
464
485
  const rest = this.#client.metrics;
465
486
  const lfs = this.#lfsClient?.metrics ?? { requests: 0, responseBytes: 0, rateLimitedRequests: 0 };
487
+ const archives = this.#archiveMetrics;
466
488
  return {
467
- requests: rest.requests + lfs.requests,
468
- responseBytes: rest.responseBytes + lfs.responseBytes,
469
- rateLimitedRequests: rest.rateLimitedRequests + lfs.rateLimitedRequests,
489
+ requests: rest.requests + lfs.requests + archives.requests,
490
+ responseBytes: rest.responseBytes + lfs.responseBytes + archives.responseBytes,
491
+ rateLimitedRequests: rest.rateLimitedRequests + lfs.rateLimitedRequests + archives.rateLimitedRequests,
470
492
  };
471
493
  }
472
494
 
@@ -805,9 +827,11 @@ export class GiteaSourceInventory {
805
827
  * whose only file is `manifest.json`, are read by digest for their
806
828
  * references; no other manifest is read. npm packages add the dist-tags Gitea
807
829
  * serves (a derived `latest` included) and each version's deprecation,
808
- * Cargo crates each version's yank from the sparse index. Every read is a
809
- * GET; reading an index counts a download of its version, an accepted read
810
- * stamp (S10 ruling, Addendum 1).
830
+ * Cargo crates each version's yank from the sparse index, Swift packages
831
+ * each version's manifests, read from its archive's file names and Gitea's
832
+ * manifest route. Every read is a GET; reading an index or a Swift archive
833
+ * counts a download of its version, an accepted read stamp (S10 ruling,
834
+ * Addendum 1; S10 C5 ruling, Addendum 2).
811
835
  */
812
836
  async #package(refArg: IPackageRef, signalArg?: AbortSignal): Promise<IPackageRecord> {
813
837
  const packagePath = `/packages/${encodeURIComponent(refArg.owner)}/${encodeURIComponent(refArg.type)}/${encodeURIComponent(refArg.name)}`;
@@ -838,11 +862,11 @@ export class GiteaSourceInventory {
838
862
  }
839
863
  versions.sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id));
840
864
  const blobs = new Map<string, IPackageBlob>();
841
- const filesByVersion = new Map<string, Array<{ name: string; sha256: string }>>();
865
+ const filesByVersion = new Map<string, IVersionFile[]>();
842
866
  for (const version of versions) {
843
867
  const files = (await this.#get(`${packagePath}/${encodeURIComponent(version.version)}/files`, { signal: signalArg })).body;
844
868
  if (!Array.isArray(files)) throw new validation.ForgeInventoryError('invalid-page');
845
- const read: Array<{ name: string; sha256: string }> = [];
869
+ const read: IVersionFile[] = [];
846
870
  for (const fileArg of files) {
847
871
  const file = validation.record(fileArg);
848
872
  const sha256 = requiredString(file.sha256);
@@ -853,7 +877,7 @@ export class GiteaSourceInventory {
853
877
  const name = requiredString(file.name);
854
878
  blob.files.push({ versionId: version.id, id: validation.decimal(file.id), name });
855
879
  blobs.set(sha256, blob);
856
- read.push({ name, sha256 });
880
+ read.push({ name, sha256, size });
857
881
  }
858
882
  filesByVersion.set(version.id, read);
859
883
  }
@@ -866,6 +890,7 @@ export class GiteaSourceInventory {
866
890
  if (record.type === 'container') await this.#containerReferences(record, filesByVersion, signalArg);
867
891
  if (record.type === 'npm') await this.#npmState(record, signalArg);
868
892
  if (record.type === 'cargo') await this.#cargoYanks(record, signalArg);
893
+ if (record.type === 'swift') await this.#swiftManifests(record, filesByVersion, signalArg);
869
894
  return record;
870
895
  }
871
896
 
@@ -876,9 +901,7 @@ export class GiteaSourceInventory {
876
901
  * bytes must hash to the listed SHA-256 and the response must be an index or
877
902
  * a manifest list, else the unit fails with `unexpected-manifest-shape`.
878
903
  */
879
- async #containerReferences(
880
- recordArg: IPackageRecord, filesArg: ReadonlyMap<string, Array<{ name: string; sha256: string }>>, signalArg?: AbortSignal,
881
- ): Promise<void> {
904
+ async #containerReferences(recordArg: IPackageRecord, filesArg: ReadonlyMap<string, IVersionFile[]>, signalArg?: AbortSignal): Promise<void> {
882
905
  if (!/^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*$/.test(recordArg.name)) {
883
906
  throw new validation.ForgeInventoryError('invalid-image-name');
884
907
  }
@@ -911,6 +934,69 @@ export class GiteaSourceInventory {
911
934
  }
912
935
  }
913
936
 
937
+ /**
938
+ * Each Swift version's `manifests`. Gitea stores one file per version, its
939
+ * source archive, and anything else fails the unit with
940
+ * `swift-version-files-unexpected`. An archive over `maxSwiftArchiveBytes`
941
+ * by its recorded size is not read and records `null`. Otherwise the
942
+ * archive is read once and verified against its recorded size and SHA-256,
943
+ * its central directory gives the candidate keys (`swiftManifestCandidates`),
944
+ * and the manifest route is asked for each: a 200 is recorded with the size
945
+ * and SHA-256 of the bytes, the 303 to the release's `Package.swift` means
946
+ * Gitea holds no such manifest, and `''` must answer 200. A failed read
947
+ * fails the unit; it never records `null`.
948
+ */
949
+ async #swiftManifests(recordArg: IPackageRecord, filesArg: ReadonlyMap<string, IVersionFile[]>, signalArg?: AbortSignal): Promise<void> {
950
+ const parts = swiftPackageParts(recordArg.name);
951
+ for (const version of recordArg.versions) {
952
+ const files = filesArg.get(version.id)!;
953
+ if (files.length !== 1) throw new validation.ForgeInventoryError('swift-version-files-unexpected');
954
+ // The native store's comparison: an archive of exactly the bound is held.
955
+ if (BigInt(files[0].size) > BigInt(this.#options.maxSwiftArchiveBytes)) {
956
+ version.manifests = null;
957
+ continue;
958
+ }
959
+ const keys = swiftManifestCandidates(await this.#swiftArchive(recordArg, parts, version, files[0], signalArg));
960
+ const manifests: ISwiftManifest[] = [];
961
+ for (const key of keys) {
962
+ const manifest = await readSwiftManifest(this.#client, this.#options.baseUrl, recordArg.owner, parts, version.version, key === '' ? null : key, signalArg);
963
+ if (manifest === null) continue;
964
+ manifests.push({ key, size: manifest.bytes.byteLength, sha256: plugins.crypto.createHash('sha256').update(manifest.bytes).digest('hex') });
965
+ }
966
+ version.manifests = manifests;
967
+ }
968
+ }
969
+
970
+ /**
971
+ * A Swift version's source archive, read once with the reader's `token`
972
+ * authorization through `verifiedPackageDownload` (exact size, SHA-256
973
+ * before the last chunk) into one buffer of the recorded size. The read
974
+ * counts a download of the version, an accepted read stamp (S10 ruling
975
+ * item 8; S10 C5 ruling, Addendum 2).
976
+ */
977
+ async #swiftArchive(
978
+ recordArg: IPackageRecord, partsArg: { scope: string; name: string }, versionArg: IPackageVersion, fileArg: IVersionFile, signalArg?: AbortSignal,
979
+ ): Promise<Buffer> {
980
+ const url = new URL(`${this.#options.baseUrl}/api/packages/${encodeURIComponent(recordArg.owner)}/swift/${encodeURIComponent(partsArg.scope)}/${
981
+ encodeURIComponent(partsArg.name)}/${encodeURIComponent(versionArg.version)}.zip`);
982
+ const bytes = Number(fileArg.size);
983
+ const stream = await verifiedPackageDownload(
984
+ {
985
+ caCertificates: this.#options.caCertificates, packageTransferTimeoutMs: this.#options.packageTransferTimeoutMs,
986
+ requestTimeoutMs: this.#options.requestTimeoutMs, metrics: this.#archiveMetrics,
987
+ },
988
+ url, { authorization: `token ${this.#options.token}`, accept: 'application/vnd.swift.registry.v1+zip' },
989
+ { sha256: fileArg.sha256, size: fileArg.size, bytes }, signalArg, null,
990
+ );
991
+ const archive = Buffer.alloc(bytes);
992
+ let offset = 0;
993
+ for await (const chunk of stream as AsyncIterable<Uint8Array>) {
994
+ archive.set(chunk, offset);
995
+ offset += chunk.byteLength;
996
+ }
997
+ return archive;
998
+ }
999
+
914
1000
  /** Every dist-tag Gitea serves, a `latest` it derives included, and each version's deprecation from the package document. */
915
1001
  async #npmState(recordArg: IPackageRecord, signalArg?: AbortSignal): Promise<void> {
916
1002
  const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(recordArg.owner)}/npm`;
@@ -1,12 +1,16 @@
1
1
  import * as plugins from './plugins.js';
2
2
  import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
- import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
4
+ import { createTransferBounds, createTransferStream, discardBody, heldBackChunks, verifiedPackageDownload } from './forgeinventory.transfer.js';
5
5
  import { ForgeRequestClient } from './forgeinventory.client.js';
6
6
  import { LfsBatchClient } from './classes.lfsbatchclient.js';
7
7
  import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
8
8
  import { giteaCloneUrl, reserveMirrorScratch } from './giteasourceinventory.git.js';
9
- import { cargoIndexPath, containerManifestMediaTypes, npmPackagePath, packageTypePattern, swiftPackageParts } from './giteasourceinventory.packages.js';
9
+ import {
10
+ cargoIndexPath, containerManifestMediaTypes, npmPackagePath, packageTypePattern, readSwiftManifest, swiftPackageParts,
11
+ } from './giteasourceinventory.packages.js';
12
+ import { canonicalPackageDocument, isSwiftManifestVersion } from './giteasourcetransfer.documents.js';
13
+ import type { TGiteaCanonicalPackageDocumentKind } from './giteasourcetransfer.documents.js';
10
14
 
11
15
  type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
12
16
  type IPackageVersion = plugins.fossInterfaces.data.IForgeSourcePackageVersion;
@@ -99,17 +103,29 @@ export interface IGiteaPackageFileSelector extends TGiteaPackageContentSelector
99
103
  }
100
104
 
101
105
  /**
102
- * Metadata a package version carries beside its files, read as Gitea serves
103
- * it: an npm version document, Swift release metadata, a Swift release's
104
- * `Package.swift`, or a crate version's line of the Cargo sparse index.
106
+ * Metadata a package version carries beside its files: an npm version
107
+ * document, Swift release metadata, a Swift release's manifest
108
+ * (`Package.swift`, or an alternate `Package@swift-<v>.swift`), a crate
109
+ * version's line of the Cargo sparse index, or the description the Cargo
110
+ * search serves for a crate's latest version.
105
111
  */
106
- export type TGiteaPackageDocumentKind = 'npm-version' | 'swift-release' | 'swift-manifest' | 'cargo-index-entry';
112
+ export type TGiteaPackageDocumentKind = TGiteaCanonicalPackageDocumentKind | 'swift-manifest';
107
113
 
108
114
  export interface IGiteaPackageReadOptions {
109
115
  /** Aborts the read with `request-aborted`, also while a stream is read. */
110
116
  signal?: AbortSignal;
111
117
  }
112
118
 
119
+ /** Reads an alternate Swift manifest. */
120
+ export interface IGiteaSwiftManifestReadOptions extends IGiteaPackageReadOptions {
121
+ /**
122
+ * The Swift version Gitea keys the alternate manifest by, trimmed as its
123
+ * `TrimmedVersionString` writes it (`5.8`, `5.9.1`; the patch only when it
124
+ * is not zero).
125
+ */
126
+ swiftVersion: string;
127
+ }
128
+
113
129
  /** A container manifest with the exact bytes and `Content-Type` Gitea serves. */
114
130
  export interface IGiteaContainerManifest {
115
131
  /** `sha256:` and the hex digest the bytes were checked against. */
@@ -119,12 +135,19 @@ export interface IGiteaContainerManifest {
119
135
  bytes: Buffer;
120
136
  }
121
137
 
122
- /** A version's metadata document with the exact bytes Gitea serves and their digest. */
138
+ /**
139
+ * A version's metadata document and its digest: a JSON document in its
140
+ * canonical form (`canonicalPackageDocument`), a Swift manifest verbatim.
141
+ */
123
142
  export interface IGiteaPackageDocument {
124
143
  kind: TGiteaPackageDocumentKind;
125
144
  /** The id of the version the identity reads before and after the read confirmed. */
126
145
  versionId: string;
127
- /** For `cargo-index-entry`, the version's line of the sparse index without its newline. */
146
+ /**
147
+ * The canonical JSON for every kind but `swift-manifest`, whose bytes are
148
+ * the stored manifest as Gitea serves it. The bytes Gitea served for a JSON
149
+ * document are not returned: their key order differs from read to read.
150
+ */
128
151
  bytes: Buffer;
129
152
  /** Lowercase hex SHA-256 of `bytes`. */
130
153
  sha256: string;
@@ -143,12 +166,13 @@ export interface IGiteaPackageDownload {
143
166
  stream: InstanceType<typeof plugins.stream.Readable>;
144
167
  }
145
168
 
146
- const packageContentCodes = {
147
- invalidChunk: 'package-content-invalid-chunk', overrun: 'package-content-size-overrun', short: 'package-content-short-body',
148
- };
149
-
150
169
  const sha256Hex = (bytesArg: Uint8Array): string => plugins.crypto.createHash('sha256').update(bytesArg).digest('hex');
151
170
 
171
+ /** Entries asked of one Cargo search page; Gitea serves at most its `MAX_RESPONSE_ITEMS` (50 by default). */
172
+ const cargoSearchPageSize = 50;
173
+ /** Search pages read for one crate's description before `cargo-search-over-bound`: 10,000 entries at 50 a page. */
174
+ const cargoSearchMaxPages = 200;
175
+
152
176
  /** Gitea names attachment content by a random UUID in canonical lowercase form. */
153
177
  const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
154
178
 
@@ -475,7 +499,8 @@ export class GiteaSourceTransfer {
475
499
  ): Promise<IGiteaPackageDownload> {
476
500
  const selected = this.#containerPackage(packageArg);
477
501
  const content = this.#content(contentArg);
478
- const stream = await this.#verifiedDownload(
502
+ const stream = await verifiedPackageDownload(
503
+ this.#options,
479
504
  new URL(`/v2/${selected.owner.toLowerCase()}/${selected.name}/blobs/sha256:${content.sha256}`, this.#options.baseUrl),
480
505
  { authorization: this.#basicAuthorization() }, content, optionsArg.signal, null,
481
506
  );
@@ -528,7 +553,7 @@ export class GiteaSourceTransfer {
528
553
  await this.#versionIdentified(selected, version, signal);
529
554
  let stream: InstanceType<typeof plugins.stream.Readable>;
530
555
  try {
531
- stream = await this.#verifiedDownload(url, headers, content, signal, (signalArg) => this.#versionIdentified(selected, version, signalArg));
556
+ stream = await verifiedPackageDownload(this.#options, url, headers, content, signal, (signalArg) => this.#versionIdentified(selected, version, signalArg));
532
557
  } catch (error) {
533
558
  if (signal?.aborted) throw error;
534
559
  return this.#versionExplained(selected, version, error, signal);
@@ -539,119 +564,137 @@ export class GiteaSourceTransfer {
539
564
  /**
540
565
  * Reads one metadata document of a package version with the reader's
541
566
  * `token` authorization, bracketed by identity reads of the version as for
542
- * `downloadPackageFile`: `npm-version` (the version document of an npm
543
- * package), `swift-release` (Swift release metadata), `swift-manifest` (the
544
- * release's `Package.swift`) or `cargo-index-entry` (the version's line of
545
- * the crate's sparse index). The bytes are returned as Gitea serves them,
546
- * with their SHA-256, which a consumer records beside the version id. None
547
- * of these reads counts a download. A kind that does not belong to the
548
- * package's type rejects with `package-document-kind-invalid`, a document
549
- * of another media type with `invalid-content-type`, and a crate version
550
- * missing from the index with `package-document-missing`.
567
+ * `downloadPackageFile`. None of these reads counts a download.
568
+ *
569
+ * - `npm-version`: the npm version document.
570
+ * - `swift-release`: Swift release metadata.
571
+ * - `swift-manifest`: the release's `Package.swift`, which must answer 200;
572
+ * with `swiftVersion`, the alternate manifest Gitea keys by that version,
573
+ * read without following a redirect. It resolves null only when Gitea
574
+ * answers 303 to the release's own `Package.swift` URL, its way of saying
575
+ * it holds no such manifest; a redirect anywhere else rejects with
576
+ * `swift-manifest-redirect-unexpected`, any other status with
577
+ * `http-<status>`.
578
+ * - `cargo-index-entry`: the version's line of the crate's sparse index;
579
+ * `package-document-missing` when the index has none.
580
+ * - `cargo-description`: `{"description":…}` of the crate's entry in the
581
+ * Cargo search (`q=<name>`, its exact name), which describes only the
582
+ * version with the highest id: its `max_version` must be this version,
583
+ * else `package-document-missing`, as when no entry has the name.
584
+ *
585
+ * A manifest comes back verbatim, every other document in its canonical
586
+ * form (`canonicalPackageDocument`, `document-not-canonical` when it has
587
+ * none), each with the SHA-256 of the returned bytes, which a consumer
588
+ * records beside the version id. A kind that does not belong to the
589
+ * package's type, or `swiftVersion` with another kind, rejects with
590
+ * `package-document-kind-invalid`, a `swiftVersion` Gitea would not key a
591
+ * manifest by with `swift-version-invalid`, both before any request; a
592
+ * document of another media type rejects with `invalid-content-type`.
551
593
  */
552
594
  public async readPackageDocument(
553
- packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind, optionsArg: IGiteaPackageReadOptions = {},
554
- ): Promise<IGiteaPackageDocument> {
595
+ packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: 'swift-manifest', optionsArg: IGiteaSwiftManifestReadOptions,
596
+ ): Promise<IGiteaPackageDocument | null>;
597
+ public async readPackageDocument(
598
+ packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind, optionsArg?: IGiteaPackageReadOptions,
599
+ ): Promise<IGiteaPackageDocument>;
600
+ public async readPackageDocument(
601
+ packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind,
602
+ optionsArg: IGiteaPackageReadOptions & { swiftVersion?: string } = {},
603
+ ): Promise<IGiteaPackageDocument | null> {
555
604
  const selected = this.#packageSelector(packageArg);
556
605
  const version = this.#versionSelector(versionArg);
606
+ const { signal, swiftVersion } = optionsArg;
607
+ if (swiftVersion !== undefined) {
608
+ if (kindArg !== 'swift-manifest') throw new validation.ForgeInventoryError('package-document-kind-invalid');
609
+ if (!isSwiftManifestVersion(swiftVersion)) throw new validation.ForgeInventoryError('swift-version-invalid');
610
+ }
557
611
  const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(selected.owner)}`;
558
612
  const encodedVersion = encodeURIComponent(version.version);
559
- let url: URL;
560
- let accept: string;
561
- let mediaType: string;
613
+ let read: (signalArg: AbortSignal | undefined) => Promise<{ bytes: Buffer; mediaType: string } | null>;
562
614
  if (kindArg === 'npm-version' && selected.type === 'npm') {
563
- [url, accept, mediaType] = [new URL(`${registry}/npm/${npmPackagePath(selected.name)}/${encodedVersion}`), 'application/json', 'application/json'];
564
- } else if ((kindArg === 'swift-release' || kindArg === 'swift-manifest') && selected.type === 'swift') {
615
+ read = (signalArg) => this.#documentBytes(new URL(`${registry}/npm/${npmPackagePath(selected.name)}/${encodedVersion}`), 'application/json', 'application/json', signalArg);
616
+ } else if (kindArg === 'swift-release' && selected.type === 'swift') {
617
+ const parts = swiftPackageParts(selected.name);
618
+ const release = new URL(`${registry}/swift/${encodeURIComponent(parts.scope)}/${encodeURIComponent(parts.name)}/${encodedVersion}`);
619
+ read = (signalArg) => this.#documentBytes(release, 'application/vnd.swift.registry.v1+json', 'application/json', signalArg);
620
+ } else if (kindArg === 'swift-manifest' && selected.type === 'swift') {
565
621
  const parts = swiftPackageParts(selected.name);
566
- const release = `${registry}/swift/${encodeURIComponent(parts.scope)}/${encodeURIComponent(parts.name)}/${encodedVersion}`;
567
- [url, accept, mediaType] = kindArg === 'swift-release'
568
- ? [new URL(release), 'application/vnd.swift.registry.v1+json', 'application/json']
569
- : [new URL(`${release}/Package.swift`), 'application/vnd.swift.registry.v1+swift', 'text/x-swift'];
622
+ read = (signalArg) => readSwiftManifest(this.#client, this.#options.baseUrl, selected.owner, parts, version.version, swiftVersion ?? null, signalArg);
570
623
  } else if (kindArg === 'cargo-index-entry' && selected.type === 'cargo') {
571
- [url, accept, mediaType] = [new URL(`${registry}/cargo/${cargoIndexPath(selected.name)}`), 'text/plain', 'text/plain'];
624
+ const url = new URL(`${registry}/cargo/${cargoIndexPath(selected.name)}`);
625
+ read = async (signalArg) => {
626
+ const index = await this.#documentBytes(url, 'text/plain', 'text/plain', signalArg);
627
+ let line: string | undefined;
628
+ try {
629
+ line = new TextDecoder('utf-8', { fatal: true }).decode(index.bytes).split('\n')
630
+ .find((lineArg) => lineArg !== '' && validation.record(JSON.parse(lineArg) as unknown).vers === version.version);
631
+ } catch {
632
+ throw new validation.ForgeInventoryError('invalid-cargo-index');
633
+ }
634
+ if (line === undefined) throw new validation.ForgeInventoryError('package-document-missing');
635
+ return { bytes: Buffer.from(line, 'utf8'), mediaType: index.mediaType };
636
+ };
637
+ } else if (kindArg === 'cargo-description' && selected.type === 'cargo') {
638
+ // Gitea's crate name rule, checked before any request.
639
+ void cargoIndexPath(selected.name);
640
+ read = (signalArg) => this.#cargoDescription(registry, selected.name, version.version, signalArg);
572
641
  } else {
573
642
  throw new validation.ForgeInventoryError('package-document-kind-invalid');
574
643
  }
575
- const { signal } = optionsArg;
576
644
  await this.#versionIdentified(selected, version, signal);
577
- let response: Awaited<ReturnType<ForgeRequestClient['requestBytes']>>;
645
+ let document: { bytes: Buffer; mediaType: string } | null;
578
646
  try {
579
- response = await this.#client.requestBytes(url, { accept, signal });
647
+ document = await read(signal);
580
648
  } catch (error) {
581
649
  if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
582
650
  return this.#versionExplained(selected, version, error, signal);
583
651
  }
584
- if (response.mediaType !== mediaType) throw new validation.ForgeInventoryError('invalid-content-type');
585
- let bytes = response.bytes;
586
- if (kindArg === 'cargo-index-entry') {
587
- let line: string | undefined;
588
- try {
589
- line = new TextDecoder('utf-8', { fatal: true }).decode(bytes).split('\n')
590
- .find((lineArg) => lineArg !== '' && validation.record(JSON.parse(lineArg) as unknown).vers === version.version);
591
- } catch {
592
- throw new validation.ForgeInventoryError('invalid-cargo-index');
593
- }
594
- if (line === undefined) throw new validation.ForgeInventoryError('package-document-missing');
595
- bytes = Buffer.from(line, 'utf8');
596
- }
597
652
  await this.#versionIdentified(selected, version, signal);
598
- return { kind: kindArg, versionId: version.id, bytes, sha256: sha256Hex(bytes), mediaType };
653
+ if (document === null) return null;
654
+ if (kindArg === 'swift-manifest') {
655
+ return { kind: kindArg, versionId: version.id, bytes: document.bytes, sha256: sha256Hex(document.bytes), mediaType: document.mediaType };
656
+ }
657
+ const canonical = canonicalPackageDocument(kindArg, document.bytes);
658
+ return { kind: kindArg, versionId: version.id, bytes: canonical.bytes, sha256: canonical.sha256, mediaType: document.mediaType };
659
+ }
660
+
661
+ /** A document's body, which must carry `mediaTypeArg`. */
662
+ async #documentBytes(urlArg: URL, acceptArg: string, mediaTypeArg: string, signalArg: AbortSignal | undefined): Promise<{ bytes: Buffer; mediaType: string }> {
663
+ const response = await this.#client.requestBytes(urlArg, { accept: acceptArg, signal: signalArg });
664
+ if (response.mediaType !== mediaTypeArg) throw new validation.ForgeInventoryError('invalid-content-type');
665
+ return { bytes: response.bytes, mediaType: mediaTypeArg };
599
666
  }
600
667
 
601
668
  /**
602
- * Streams one download verified against `contentArg`: the response must be a
603
- * 200 whose `content-length`, when sent, is the size; the last chunk is held
604
- * back until the size and SHA-256 match and `identifiedArg`, when given, has
605
- * passed once more.
669
+ * `{"description":…}` of the crate's entry in the Cargo search, read page by
670
+ * page until the entry with exactly `nameArg` turns up. Gitea's search
671
+ * matches names by substring and serves one entry per crate, describing its
672
+ * version with the highest id, whose version `max_version` names.
606
673
  */
607
- async #verifiedDownload(
608
- urlArg: URL, headersArg: Record<string, string>, contentArg: { sha256: string; size: string; bytes: number }, signalArg: AbortSignal | undefined,
609
- identifiedArg: ((signalArg: AbortSignal) => Promise<void>) | null,
610
- ): Promise<InstanceType<typeof plugins.stream.Readable>> {
611
- if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
612
- const agents = transport.createNodeAgents(this.#options.caCertificates);
613
- const close = () => agents.https?.destroy();
614
- const transfer = createTransferBounds(signalArg, {
615
- limitMs: this.#options.packageTransferTimeoutMs, idleMs: this.#options.requestTimeoutMs,
616
- timedOutCode: 'package-content-transfer-timed-out', failedCode: 'package-content-transfer-failed',
617
- });
618
- let body: AsyncIterable<Uint8Array>;
619
- try {
620
- const response = await transfer.run(() => transport.sendNodeRequest(urlArg, { method: 'GET', headers: headersArg, signal: transfer.signal, agents }));
621
- if (response.status !== 200 || !response.body) {
622
- await transfer.run(() => discardBody(response.body));
623
- throw new validation.ForgeInventoryError(`http-${response.status}`);
624
- }
625
- const length = response.headers.get('content-length');
626
- if (length !== null && length.trim() !== contentArg.size) {
627
- await transfer.run(() => discardBody(response.body));
628
- throw new validation.ForgeInventoryError('package-content-size-mismatch');
674
+ async #cargoDescription(registryArg: string, nameArg: string, versionArg: string, signalArg: AbortSignal | undefined): Promise<{ bytes: Buffer; mediaType: string }> {
675
+ let seen = 0;
676
+ for (let page = 1; page <= cargoSearchMaxPages; page++) {
677
+ const url = new URL(`${registryArg}/cargo/api/v1/crates`);
678
+ url.searchParams.set('q', nameArg);
679
+ url.searchParams.set('page', String(page));
680
+ url.searchParams.set('per_page', String(cargoSearchPageSize));
681
+ const body = validation.record((await this.#client.request(url, { signal: signalArg })).body);
682
+ if (!Array.isArray(body.crates)) throw new validation.ForgeInventoryError('invalid-cargo-search');
683
+ const total = validation.record(body.meta).total;
684
+ if (typeof total !== 'number' || !Number.isSafeInteger(total) || total < 0) throw new validation.ForgeInventoryError('invalid-cargo-search');
685
+ for (const entry of body.crates) {
686
+ const crate = validation.record(entry);
687
+ if (typeof crate.name !== 'string' || typeof crate.max_version !== 'string' || typeof crate.description !== 'string') {
688
+ throw new validation.ForgeInventoryError('invalid-cargo-search');
689
+ }
690
+ if (crate.name !== nameArg) continue;
691
+ if (crate.max_version !== versionArg) throw new validation.ForgeInventoryError('package-document-missing');
692
+ return { bytes: Buffer.from(JSON.stringify({ description: crate.description }), 'utf8'), mediaType: 'application/json' };
629
693
  }
630
- body = response.body;
631
- } catch (error) {
632
- const failure = transfer.failure(error, 'package-content-download-failed');
633
- transfer.end();
634
- close();
635
- throw failure;
694
+ seen += body.crates.length;
695
+ if (body.crates.length === 0 || seen >= total) throw new validation.ForgeInventoryError('package-document-missing');
636
696
  }
637
- const hash = plugins.crypto.createHash('sha256');
638
- const complete = async () => {
639
- if (hash.digest('hex') !== contentArg.sha256) throw new validation.ForgeInventoryError('package-content-digest-mismatch');
640
- if (identifiedArg) {
641
- transfer.touch();
642
- await transfer.run(() => identifiedArg(transfer.signal));
643
- }
644
- };
645
- const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
646
- try {
647
- yield* heldBackChunks(transfer.watch(body), contentArg.bytes, packageContentCodes, { inspect: (chunkArg) => hash.update(chunkArg), complete });
648
- } catch (error) {
649
- throw transfer.failure(error, 'package-content-download-failed');
650
- } finally {
651
- transfer.end();
652
- }
653
- };
654
- return createTransferStream(transfer, chunks(), close);
697
+ throw new validation.ForgeInventoryError('cargo-search-over-bound');
655
698
  }
656
699
 
657
700
  /**