@foss.global/codefeed 5.2.0 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +46 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.d.ts +11 -1
- package/dist_ts/classes.giteasourceinventory.js +77 -10
- package/dist_ts/classes.giteasourcetransfer.d.ts +52 -15
- package/dist_ts/classes.giteasourcetransfer.js +93 -103
- package/dist_ts/forgeinventory.client.d.ts +21 -1
- package/dist_ts/forgeinventory.client.js +30 -5
- package/dist_ts/forgeinventory.transfer.d.ts +24 -0
- package/dist_ts/forgeinventory.transfer.js +77 -1
- package/dist_ts/forgeinventory.transport.d.ts +21 -7
- package/dist_ts/forgeinventory.transport.js +15 -10
- package/dist_ts/giteasourceinventory.packages.d.ts +49 -0
- package/dist_ts/giteasourceinventory.packages.js +180 -1
- package/dist_ts/giteasourcetransfer.documents.d.ts +50 -0
- package/dist_ts/giteasourcetransfer.documents.js +130 -0
- package/dist_ts/index.d.ts +2 -0
- package/dist_ts/index.js +2 -1
- package/package.json +3 -3
- package/readme.md +10 -1
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +100 -14
- package/ts/classes.giteasourcetransfer.ts +145 -102
- package/ts/forgeinventory.client.ts +48 -6
- package/ts/forgeinventory.transfer.ts +88 -0
- package/ts/forgeinventory.transport.ts +33 -13
- package/ts/giteasourceinventory.packages.ts +168 -0
- package/ts/giteasourcetransfer.documents.ts +140 -0
- package/ts/index.ts +2 -0
|
@@ -3,9 +3,11 @@ import * as transport from './forgeinventory.transport.js';
|
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
4
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
5
|
import type { IForgeRequestMetrics, IForgeRequestOptions } from './forgeinventory.client.js';
|
|
6
|
+
import { verifiedPackageDownload } from './forgeinventory.transfer.js';
|
|
6
7
|
import { measureGiteaMirror } from './giteasourceinventory.git.js';
|
|
7
8
|
import {
|
|
8
9
|
cargoIndexPath, comparePackages, composeGiteaPackageContent, containerIndexMediaTypes, npmPackagePath, packageKey, packageTypePattern,
|
|
10
|
+
readSwiftManifest, swiftManifestCandidates, swiftPackageParts,
|
|
9
11
|
} from './giteasourceinventory.packages.js';
|
|
10
12
|
|
|
11
13
|
type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
|
|
@@ -32,6 +34,7 @@ type IPackageRef = plugins.fossInterfaces.data.IForgeSourcePackageRef;
|
|
|
32
34
|
type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
|
|
33
35
|
type IPackageVersion = plugins.fossInterfaces.data.IForgeSourcePackageVersion;
|
|
34
36
|
type IPackageBlob = plugins.fossInterfaces.data.IForgeSourcePackageBlob;
|
|
37
|
+
type ISwiftManifest = plugins.fossInterfaces.data.IForgeSourceSwiftManifest;
|
|
35
38
|
|
|
36
39
|
export interface IGiteaSourceInventoryOptions {
|
|
37
40
|
/** Operator-assigned identity of the source instance. */
|
|
@@ -58,8 +61,18 @@ export interface IGiteaSourceInventoryOptions {
|
|
|
58
61
|
pageSize?: number;
|
|
59
62
|
/** Pages read per listing before the listing counts as partial. */
|
|
60
63
|
maxPages?: number;
|
|
64
|
+
/** Deadline of one REST, registry or LFS batch request, and the idle limit of a Swift archive download; defaults to 60 s, at most 300 s. */
|
|
61
65
|
requestTimeoutMs?: number;
|
|
62
66
|
maxResponseBytes?: number;
|
|
67
|
+
/**
|
|
68
|
+
* The largest Swift source archive, in bytes, that the inventory reads for
|
|
69
|
+
* the version's `manifests`; defaults to 128 MiB, at most 1 GiB. A version
|
|
70
|
+
* whose recorded archive is larger is not read and records `manifests:
|
|
71
|
+
* null`. A consumer passes the cap its own store puts on an archive.
|
|
72
|
+
*/
|
|
73
|
+
maxSwiftArchiveBytes?: number;
|
|
74
|
+
/** Wall-clock limit of one Swift archive download; defaults to 30 minutes, at most six hours. */
|
|
75
|
+
packageTransferTimeoutMs?: number;
|
|
63
76
|
}
|
|
64
77
|
|
|
65
78
|
/** A class of source data a run measures only when its discovery selects it. */
|
|
@@ -88,6 +101,9 @@ export type TGiteaSourceRepositoryLookup =
|
|
|
88
101
|
| { status: 'present'; id: string; owner: string; name: string }
|
|
89
102
|
| { status: 'absent'; id: string };
|
|
90
103
|
|
|
104
|
+
/** A file of a version as its listing reports it: the name, and the content's SHA-256 and size as a canonical decimal. */
|
|
105
|
+
interface IVersionFile { name: string; sha256: string; size: string }
|
|
106
|
+
|
|
91
107
|
interface IListing {
|
|
92
108
|
items: unknown[];
|
|
93
109
|
/** A sanitized reason when the listing did not finish. */
|
|
@@ -415,11 +431,13 @@ const compareText = (aArg: string, bArg: string): number => aArg < bArg ? -1 : a
|
|
|
415
431
|
export class GiteaSourceInventory {
|
|
416
432
|
readonly #options: {
|
|
417
433
|
instanceId: string; baseUrl: string; pageSize: number; maxPages: number; requestTimeoutMs: number; maxResponseBytes: number;
|
|
418
|
-
token: string; caCertificates?: string[]; gitWorkDir: string; gitTimeoutMs: number;
|
|
434
|
+
token: string; caCertificates?: string[]; gitWorkDir: string; gitTimeoutMs: number; maxSwiftArchiveBytes: number; packageTransferTimeoutMs: number;
|
|
419
435
|
};
|
|
420
436
|
readonly #transport: transport.TForgeTransport;
|
|
421
437
|
readonly #client: ForgeRequestClient;
|
|
422
438
|
#lfsClient: ForgeRequestClient | null = null;
|
|
439
|
+
/** The Swift archive GETs, which go through `verifiedPackageDownload` instead of a request client. */
|
|
440
|
+
readonly #archiveMetrics: IForgeRequestMetrics = { requests: 0, responseBytes: 0, rateLimitedRequests: 0 };
|
|
423
441
|
#readerLogin: Promise<string> | null = null;
|
|
424
442
|
|
|
425
443
|
constructor(optionsArg: IGiteaSourceInventoryOptions) {
|
|
@@ -435,9 +453,12 @@ export class GiteaSourceInventory {
|
|
|
435
453
|
requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
|
|
436
454
|
maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
|
|
437
455
|
gitTimeoutMs: optionsArg.gitTimeoutMs ?? 30 * 60 * 1000,
|
|
456
|
+
maxSwiftArchiveBytes: optionsArg.maxSwiftArchiveBytes ?? 128 * 1024 * 1024,
|
|
457
|
+
packageTransferTimeoutMs: optionsArg.packageTransferTimeoutMs ?? 30 * 60 * 1000,
|
|
438
458
|
};
|
|
439
459
|
for (const [name, maximum] of [
|
|
440
460
|
['pageSize', 100], ['maxPages', 1_000_000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['gitTimeoutMs', 6 * 60 * 60 * 1000],
|
|
461
|
+
['maxSwiftArchiveBytes', 1024 * 1024 * 1024], ['packageTransferTimeoutMs', 6 * 60 * 60 * 1000],
|
|
441
462
|
] as const) {
|
|
442
463
|
const value = limits[name];
|
|
443
464
|
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
|
|
@@ -459,14 +480,15 @@ export class GiteaSourceInventory {
|
|
|
459
480
|
/** Validated, normalized instance root; contains no credentials or API suffix. */
|
|
460
481
|
public get baseUrl(): string { return this.#options.baseUrl; }
|
|
461
482
|
|
|
462
|
-
/** REST and LFS batch requests together. */
|
|
483
|
+
/** REST, registry and LFS batch requests and Swift archive GETs together. */
|
|
463
484
|
public get metrics(): IForgeRequestMetrics {
|
|
464
485
|
const rest = this.#client.metrics;
|
|
465
486
|
const lfs = this.#lfsClient?.metrics ?? { requests: 0, responseBytes: 0, rateLimitedRequests: 0 };
|
|
487
|
+
const archives = this.#archiveMetrics;
|
|
466
488
|
return {
|
|
467
|
-
requests: rest.requests + lfs.requests,
|
|
468
|
-
responseBytes: rest.responseBytes + lfs.responseBytes,
|
|
469
|
-
rateLimitedRequests: rest.rateLimitedRequests + lfs.rateLimitedRequests,
|
|
489
|
+
requests: rest.requests + lfs.requests + archives.requests,
|
|
490
|
+
responseBytes: rest.responseBytes + lfs.responseBytes + archives.responseBytes,
|
|
491
|
+
rateLimitedRequests: rest.rateLimitedRequests + lfs.rateLimitedRequests + archives.rateLimitedRequests,
|
|
470
492
|
};
|
|
471
493
|
}
|
|
472
494
|
|
|
@@ -805,9 +827,11 @@ export class GiteaSourceInventory {
|
|
|
805
827
|
* whose only file is `manifest.json`, are read by digest for their
|
|
806
828
|
* references; no other manifest is read. npm packages add the dist-tags Gitea
|
|
807
829
|
* serves (a derived `latest` included) and each version's deprecation,
|
|
808
|
-
* Cargo crates each version's yank from the sparse index
|
|
809
|
-
*
|
|
810
|
-
*
|
|
830
|
+
* Cargo crates each version's yank from the sparse index, Swift packages
|
|
831
|
+
* each version's manifests, read from its archive's file names and Gitea's
|
|
832
|
+
* manifest route. Every read is a GET; reading an index or a Swift archive
|
|
833
|
+
* counts a download of its version, an accepted read stamp (S10 ruling,
|
|
834
|
+
* Addendum 1; S10 C5 ruling, Addendum 2).
|
|
811
835
|
*/
|
|
812
836
|
async #package(refArg: IPackageRef, signalArg?: AbortSignal): Promise<IPackageRecord> {
|
|
813
837
|
const packagePath = `/packages/${encodeURIComponent(refArg.owner)}/${encodeURIComponent(refArg.type)}/${encodeURIComponent(refArg.name)}`;
|
|
@@ -838,11 +862,11 @@ export class GiteaSourceInventory {
|
|
|
838
862
|
}
|
|
839
863
|
versions.sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id));
|
|
840
864
|
const blobs = new Map<string, IPackageBlob>();
|
|
841
|
-
const filesByVersion = new Map<string,
|
|
865
|
+
const filesByVersion = new Map<string, IVersionFile[]>();
|
|
842
866
|
for (const version of versions) {
|
|
843
867
|
const files = (await this.#get(`${packagePath}/${encodeURIComponent(version.version)}/files`, { signal: signalArg })).body;
|
|
844
868
|
if (!Array.isArray(files)) throw new validation.ForgeInventoryError('invalid-page');
|
|
845
|
-
const read:
|
|
869
|
+
const read: IVersionFile[] = [];
|
|
846
870
|
for (const fileArg of files) {
|
|
847
871
|
const file = validation.record(fileArg);
|
|
848
872
|
const sha256 = requiredString(file.sha256);
|
|
@@ -853,7 +877,7 @@ export class GiteaSourceInventory {
|
|
|
853
877
|
const name = requiredString(file.name);
|
|
854
878
|
blob.files.push({ versionId: version.id, id: validation.decimal(file.id), name });
|
|
855
879
|
blobs.set(sha256, blob);
|
|
856
|
-
read.push({ name, sha256 });
|
|
880
|
+
read.push({ name, sha256, size });
|
|
857
881
|
}
|
|
858
882
|
filesByVersion.set(version.id, read);
|
|
859
883
|
}
|
|
@@ -866,6 +890,7 @@ export class GiteaSourceInventory {
|
|
|
866
890
|
if (record.type === 'container') await this.#containerReferences(record, filesByVersion, signalArg);
|
|
867
891
|
if (record.type === 'npm') await this.#npmState(record, signalArg);
|
|
868
892
|
if (record.type === 'cargo') await this.#cargoYanks(record, signalArg);
|
|
893
|
+
if (record.type === 'swift') await this.#swiftManifests(record, filesByVersion, signalArg);
|
|
869
894
|
return record;
|
|
870
895
|
}
|
|
871
896
|
|
|
@@ -876,9 +901,7 @@ export class GiteaSourceInventory {
|
|
|
876
901
|
* bytes must hash to the listed SHA-256 and the response must be an index or
|
|
877
902
|
* a manifest list, else the unit fails with `unexpected-manifest-shape`.
|
|
878
903
|
*/
|
|
879
|
-
async #containerReferences(
|
|
880
|
-
recordArg: IPackageRecord, filesArg: ReadonlyMap<string, Array<{ name: string; sha256: string }>>, signalArg?: AbortSignal,
|
|
881
|
-
): Promise<void> {
|
|
904
|
+
async #containerReferences(recordArg: IPackageRecord, filesArg: ReadonlyMap<string, IVersionFile[]>, signalArg?: AbortSignal): Promise<void> {
|
|
882
905
|
if (!/^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*$/.test(recordArg.name)) {
|
|
883
906
|
throw new validation.ForgeInventoryError('invalid-image-name');
|
|
884
907
|
}
|
|
@@ -911,6 +934,69 @@ export class GiteaSourceInventory {
|
|
|
911
934
|
}
|
|
912
935
|
}
|
|
913
936
|
|
|
937
|
+
/**
|
|
938
|
+
* Each Swift version's `manifests`. Gitea stores one file per version, its
|
|
939
|
+
* source archive, and anything else fails the unit with
|
|
940
|
+
* `swift-version-files-unexpected`. An archive over `maxSwiftArchiveBytes`
|
|
941
|
+
* by its recorded size is not read and records `null`. Otherwise the
|
|
942
|
+
* archive is read once and verified against its recorded size and SHA-256,
|
|
943
|
+
* its central directory gives the candidate keys (`swiftManifestCandidates`),
|
|
944
|
+
* and the manifest route is asked for each: a 200 is recorded with the size
|
|
945
|
+
* and SHA-256 of the bytes, the 303 to the release's `Package.swift` means
|
|
946
|
+
* Gitea holds no such manifest, and `''` must answer 200. A failed read
|
|
947
|
+
* fails the unit; it never records `null`.
|
|
948
|
+
*/
|
|
949
|
+
async #swiftManifests(recordArg: IPackageRecord, filesArg: ReadonlyMap<string, IVersionFile[]>, signalArg?: AbortSignal): Promise<void> {
|
|
950
|
+
const parts = swiftPackageParts(recordArg.name);
|
|
951
|
+
for (const version of recordArg.versions) {
|
|
952
|
+
const files = filesArg.get(version.id)!;
|
|
953
|
+
if (files.length !== 1) throw new validation.ForgeInventoryError('swift-version-files-unexpected');
|
|
954
|
+
// The native store's comparison: an archive of exactly the bound is held.
|
|
955
|
+
if (BigInt(files[0].size) > BigInt(this.#options.maxSwiftArchiveBytes)) {
|
|
956
|
+
version.manifests = null;
|
|
957
|
+
continue;
|
|
958
|
+
}
|
|
959
|
+
const keys = swiftManifestCandidates(await this.#swiftArchive(recordArg, parts, version, files[0], signalArg));
|
|
960
|
+
const manifests: ISwiftManifest[] = [];
|
|
961
|
+
for (const key of keys) {
|
|
962
|
+
const manifest = await readSwiftManifest(this.#client, this.#options.baseUrl, recordArg.owner, parts, version.version, key === '' ? null : key, signalArg);
|
|
963
|
+
if (manifest === null) continue;
|
|
964
|
+
manifests.push({ key, size: manifest.bytes.byteLength, sha256: plugins.crypto.createHash('sha256').update(manifest.bytes).digest('hex') });
|
|
965
|
+
}
|
|
966
|
+
version.manifests = manifests;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
/**
|
|
971
|
+
* A Swift version's source archive, read once with the reader's `token`
|
|
972
|
+
* authorization through `verifiedPackageDownload` (exact size, SHA-256
|
|
973
|
+
* before the last chunk) into one buffer of the recorded size. The read
|
|
974
|
+
* counts a download of the version, an accepted read stamp (S10 ruling
|
|
975
|
+
* item 8; S10 C5 ruling, Addendum 2).
|
|
976
|
+
*/
|
|
977
|
+
async #swiftArchive(
|
|
978
|
+
recordArg: IPackageRecord, partsArg: { scope: string; name: string }, versionArg: IPackageVersion, fileArg: IVersionFile, signalArg?: AbortSignal,
|
|
979
|
+
): Promise<Buffer> {
|
|
980
|
+
const url = new URL(`${this.#options.baseUrl}/api/packages/${encodeURIComponent(recordArg.owner)}/swift/${encodeURIComponent(partsArg.scope)}/${
|
|
981
|
+
encodeURIComponent(partsArg.name)}/${encodeURIComponent(versionArg.version)}.zip`);
|
|
982
|
+
const bytes = Number(fileArg.size);
|
|
983
|
+
const stream = await verifiedPackageDownload(
|
|
984
|
+
{
|
|
985
|
+
caCertificates: this.#options.caCertificates, packageTransferTimeoutMs: this.#options.packageTransferTimeoutMs,
|
|
986
|
+
requestTimeoutMs: this.#options.requestTimeoutMs, metrics: this.#archiveMetrics,
|
|
987
|
+
},
|
|
988
|
+
url, { authorization: `token ${this.#options.token}`, accept: 'application/vnd.swift.registry.v1+zip' },
|
|
989
|
+
{ sha256: fileArg.sha256, size: fileArg.size, bytes }, signalArg, null,
|
|
990
|
+
);
|
|
991
|
+
const archive = Buffer.alloc(bytes);
|
|
992
|
+
let offset = 0;
|
|
993
|
+
for await (const chunk of stream as AsyncIterable<Uint8Array>) {
|
|
994
|
+
archive.set(chunk, offset);
|
|
995
|
+
offset += chunk.byteLength;
|
|
996
|
+
}
|
|
997
|
+
return archive;
|
|
998
|
+
}
|
|
999
|
+
|
|
914
1000
|
/** Every dist-tag Gitea serves, a `latest` it derives included, and each version's deprecation from the package document. */
|
|
915
1001
|
async #npmState(recordArg: IPackageRecord, signalArg?: AbortSignal): Promise<void> {
|
|
916
1002
|
const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(recordArg.owner)}/npm`;
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
-
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks, verifiedPackageDownload } from './forgeinventory.transfer.js';
|
|
5
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
6
6
|
import { LfsBatchClient } from './classes.lfsbatchclient.js';
|
|
7
7
|
import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
|
|
8
8
|
import { giteaCloneUrl, reserveMirrorScratch } from './giteasourceinventory.git.js';
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
cargoIndexPath, containerManifestMediaTypes, npmPackagePath, packageTypePattern, readSwiftManifest, swiftPackageParts,
|
|
11
|
+
} from './giteasourceinventory.packages.js';
|
|
12
|
+
import { canonicalPackageDocument, isSwiftManifestVersion } from './giteasourcetransfer.documents.js';
|
|
13
|
+
import type { TGiteaCanonicalPackageDocumentKind } from './giteasourcetransfer.documents.js';
|
|
10
14
|
|
|
11
15
|
type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
|
|
12
16
|
type IPackageVersion = plugins.fossInterfaces.data.IForgeSourcePackageVersion;
|
|
@@ -99,17 +103,29 @@ export interface IGiteaPackageFileSelector extends TGiteaPackageContentSelector
|
|
|
99
103
|
}
|
|
100
104
|
|
|
101
105
|
/**
|
|
102
|
-
* Metadata a package version carries beside its files
|
|
103
|
-
*
|
|
104
|
-
* `Package.swift`, or
|
|
106
|
+
* Metadata a package version carries beside its files: an npm version
|
|
107
|
+
* document, Swift release metadata, a Swift release's manifest
|
|
108
|
+
* (`Package.swift`, or an alternate `Package@swift-<v>.swift`), a crate
|
|
109
|
+
* version's line of the Cargo sparse index, or the description the Cargo
|
|
110
|
+
* search serves for a crate's latest version.
|
|
105
111
|
*/
|
|
106
|
-
export type TGiteaPackageDocumentKind =
|
|
112
|
+
export type TGiteaPackageDocumentKind = TGiteaCanonicalPackageDocumentKind | 'swift-manifest';
|
|
107
113
|
|
|
108
114
|
export interface IGiteaPackageReadOptions {
|
|
109
115
|
/** Aborts the read with `request-aborted`, also while a stream is read. */
|
|
110
116
|
signal?: AbortSignal;
|
|
111
117
|
}
|
|
112
118
|
|
|
119
|
+
/** Reads an alternate Swift manifest. */
|
|
120
|
+
export interface IGiteaSwiftManifestReadOptions extends IGiteaPackageReadOptions {
|
|
121
|
+
/**
|
|
122
|
+
* The Swift version Gitea keys the alternate manifest by, trimmed as its
|
|
123
|
+
* `TrimmedVersionString` writes it (`5.8`, `5.9.1`; the patch only when it
|
|
124
|
+
* is not zero).
|
|
125
|
+
*/
|
|
126
|
+
swiftVersion: string;
|
|
127
|
+
}
|
|
128
|
+
|
|
113
129
|
/** A container manifest with the exact bytes and `Content-Type` Gitea serves. */
|
|
114
130
|
export interface IGiteaContainerManifest {
|
|
115
131
|
/** `sha256:` and the hex digest the bytes were checked against. */
|
|
@@ -119,12 +135,19 @@ export interface IGiteaContainerManifest {
|
|
|
119
135
|
bytes: Buffer;
|
|
120
136
|
}
|
|
121
137
|
|
|
122
|
-
/**
|
|
138
|
+
/**
|
|
139
|
+
* A version's metadata document and its digest: a JSON document in its
|
|
140
|
+
* canonical form (`canonicalPackageDocument`), a Swift manifest verbatim.
|
|
141
|
+
*/
|
|
123
142
|
export interface IGiteaPackageDocument {
|
|
124
143
|
kind: TGiteaPackageDocumentKind;
|
|
125
144
|
/** The id of the version the identity reads before and after the read confirmed. */
|
|
126
145
|
versionId: string;
|
|
127
|
-
/**
|
|
146
|
+
/**
|
|
147
|
+
* The canonical JSON for every kind but `swift-manifest`, whose bytes are
|
|
148
|
+
* the stored manifest as Gitea serves it. The bytes Gitea served for a JSON
|
|
149
|
+
* document are not returned: their key order differs from read to read.
|
|
150
|
+
*/
|
|
128
151
|
bytes: Buffer;
|
|
129
152
|
/** Lowercase hex SHA-256 of `bytes`. */
|
|
130
153
|
sha256: string;
|
|
@@ -143,12 +166,13 @@ export interface IGiteaPackageDownload {
|
|
|
143
166
|
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
144
167
|
}
|
|
145
168
|
|
|
146
|
-
const packageContentCodes = {
|
|
147
|
-
invalidChunk: 'package-content-invalid-chunk', overrun: 'package-content-size-overrun', short: 'package-content-short-body',
|
|
148
|
-
};
|
|
149
|
-
|
|
150
169
|
const sha256Hex = (bytesArg: Uint8Array): string => plugins.crypto.createHash('sha256').update(bytesArg).digest('hex');
|
|
151
170
|
|
|
171
|
+
/** Entries asked of one Cargo search page; Gitea serves at most its `MAX_RESPONSE_ITEMS` (50 by default). */
|
|
172
|
+
const cargoSearchPageSize = 50;
|
|
173
|
+
/** Search pages read for one crate's description before `cargo-search-over-bound`: 10,000 entries at 50 a page. */
|
|
174
|
+
const cargoSearchMaxPages = 200;
|
|
175
|
+
|
|
152
176
|
/** Gitea names attachment content by a random UUID in canonical lowercase form. */
|
|
153
177
|
const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
154
178
|
|
|
@@ -475,7 +499,8 @@ export class GiteaSourceTransfer {
|
|
|
475
499
|
): Promise<IGiteaPackageDownload> {
|
|
476
500
|
const selected = this.#containerPackage(packageArg);
|
|
477
501
|
const content = this.#content(contentArg);
|
|
478
|
-
const stream = await
|
|
502
|
+
const stream = await verifiedPackageDownload(
|
|
503
|
+
this.#options,
|
|
479
504
|
new URL(`/v2/${selected.owner.toLowerCase()}/${selected.name}/blobs/sha256:${content.sha256}`, this.#options.baseUrl),
|
|
480
505
|
{ authorization: this.#basicAuthorization() }, content, optionsArg.signal, null,
|
|
481
506
|
);
|
|
@@ -528,7 +553,7 @@ export class GiteaSourceTransfer {
|
|
|
528
553
|
await this.#versionIdentified(selected, version, signal);
|
|
529
554
|
let stream: InstanceType<typeof plugins.stream.Readable>;
|
|
530
555
|
try {
|
|
531
|
-
stream = await this.#
|
|
556
|
+
stream = await verifiedPackageDownload(this.#options, url, headers, content, signal, (signalArg) => this.#versionIdentified(selected, version, signalArg));
|
|
532
557
|
} catch (error) {
|
|
533
558
|
if (signal?.aborted) throw error;
|
|
534
559
|
return this.#versionExplained(selected, version, error, signal);
|
|
@@ -539,119 +564,137 @@ export class GiteaSourceTransfer {
|
|
|
539
564
|
/**
|
|
540
565
|
* Reads one metadata document of a package version with the reader's
|
|
541
566
|
* `token` authorization, bracketed by identity reads of the version as for
|
|
542
|
-
* `downloadPackageFile
|
|
543
|
-
*
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
*
|
|
547
|
-
*
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
567
|
+
* `downloadPackageFile`. None of these reads counts a download.
|
|
568
|
+
*
|
|
569
|
+
* - `npm-version`: the npm version document.
|
|
570
|
+
* - `swift-release`: Swift release metadata.
|
|
571
|
+
* - `swift-manifest`: the release's `Package.swift`, which must answer 200;
|
|
572
|
+
* with `swiftVersion`, the alternate manifest Gitea keys by that version,
|
|
573
|
+
* read without following a redirect. It resolves null only when Gitea
|
|
574
|
+
* answers 303 to the release's own `Package.swift` URL, its way of saying
|
|
575
|
+
* it holds no such manifest; a redirect anywhere else rejects with
|
|
576
|
+
* `swift-manifest-redirect-unexpected`, any other status with
|
|
577
|
+
* `http-<status>`.
|
|
578
|
+
* - `cargo-index-entry`: the version's line of the crate's sparse index;
|
|
579
|
+
* `package-document-missing` when the index has none.
|
|
580
|
+
* - `cargo-description`: `{"description":…}` of the crate's entry in the
|
|
581
|
+
* Cargo search (`q=<name>`, its exact name), which describes only the
|
|
582
|
+
* version with the highest id: its `max_version` must be this version,
|
|
583
|
+
* else `package-document-missing`, as when no entry has the name.
|
|
584
|
+
*
|
|
585
|
+
* A manifest comes back verbatim, every other document in its canonical
|
|
586
|
+
* form (`canonicalPackageDocument`, `document-not-canonical` when it has
|
|
587
|
+
* none), each with the SHA-256 of the returned bytes, which a consumer
|
|
588
|
+
* records beside the version id. A kind that does not belong to the
|
|
589
|
+
* package's type, or `swiftVersion` with another kind, rejects with
|
|
590
|
+
* `package-document-kind-invalid`, a `swiftVersion` Gitea would not key a
|
|
591
|
+
* manifest by with `swift-version-invalid`, both before any request; a
|
|
592
|
+
* document of another media type rejects with `invalid-content-type`.
|
|
551
593
|
*/
|
|
552
594
|
public async readPackageDocument(
|
|
553
|
-
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg:
|
|
554
|
-
): Promise<IGiteaPackageDocument
|
|
595
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: 'swift-manifest', optionsArg: IGiteaSwiftManifestReadOptions,
|
|
596
|
+
): Promise<IGiteaPackageDocument | null>;
|
|
597
|
+
public async readPackageDocument(
|
|
598
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind, optionsArg?: IGiteaPackageReadOptions,
|
|
599
|
+
): Promise<IGiteaPackageDocument>;
|
|
600
|
+
public async readPackageDocument(
|
|
601
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind,
|
|
602
|
+
optionsArg: IGiteaPackageReadOptions & { swiftVersion?: string } = {},
|
|
603
|
+
): Promise<IGiteaPackageDocument | null> {
|
|
555
604
|
const selected = this.#packageSelector(packageArg);
|
|
556
605
|
const version = this.#versionSelector(versionArg);
|
|
606
|
+
const { signal, swiftVersion } = optionsArg;
|
|
607
|
+
if (swiftVersion !== undefined) {
|
|
608
|
+
if (kindArg !== 'swift-manifest') throw new validation.ForgeInventoryError('package-document-kind-invalid');
|
|
609
|
+
if (!isSwiftManifestVersion(swiftVersion)) throw new validation.ForgeInventoryError('swift-version-invalid');
|
|
610
|
+
}
|
|
557
611
|
const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(selected.owner)}`;
|
|
558
612
|
const encodedVersion = encodeURIComponent(version.version);
|
|
559
|
-
let
|
|
560
|
-
let accept: string;
|
|
561
|
-
let mediaType: string;
|
|
613
|
+
let read: (signalArg: AbortSignal | undefined) => Promise<{ bytes: Buffer; mediaType: string } | null>;
|
|
562
614
|
if (kindArg === 'npm-version' && selected.type === 'npm') {
|
|
563
|
-
|
|
564
|
-
} else if (
|
|
615
|
+
read = (signalArg) => this.#documentBytes(new URL(`${registry}/npm/${npmPackagePath(selected.name)}/${encodedVersion}`), 'application/json', 'application/json', signalArg);
|
|
616
|
+
} else if (kindArg === 'swift-release' && selected.type === 'swift') {
|
|
617
|
+
const parts = swiftPackageParts(selected.name);
|
|
618
|
+
const release = new URL(`${registry}/swift/${encodeURIComponent(parts.scope)}/${encodeURIComponent(parts.name)}/${encodedVersion}`);
|
|
619
|
+
read = (signalArg) => this.#documentBytes(release, 'application/vnd.swift.registry.v1+json', 'application/json', signalArg);
|
|
620
|
+
} else if (kindArg === 'swift-manifest' && selected.type === 'swift') {
|
|
565
621
|
const parts = swiftPackageParts(selected.name);
|
|
566
|
-
|
|
567
|
-
[url, accept, mediaType] = kindArg === 'swift-release'
|
|
568
|
-
? [new URL(release), 'application/vnd.swift.registry.v1+json', 'application/json']
|
|
569
|
-
: [new URL(`${release}/Package.swift`), 'application/vnd.swift.registry.v1+swift', 'text/x-swift'];
|
|
622
|
+
read = (signalArg) => readSwiftManifest(this.#client, this.#options.baseUrl, selected.owner, parts, version.version, swiftVersion ?? null, signalArg);
|
|
570
623
|
} else if (kindArg === 'cargo-index-entry' && selected.type === 'cargo') {
|
|
571
|
-
|
|
624
|
+
const url = new URL(`${registry}/cargo/${cargoIndexPath(selected.name)}`);
|
|
625
|
+
read = async (signalArg) => {
|
|
626
|
+
const index = await this.#documentBytes(url, 'text/plain', 'text/plain', signalArg);
|
|
627
|
+
let line: string | undefined;
|
|
628
|
+
try {
|
|
629
|
+
line = new TextDecoder('utf-8', { fatal: true }).decode(index.bytes).split('\n')
|
|
630
|
+
.find((lineArg) => lineArg !== '' && validation.record(JSON.parse(lineArg) as unknown).vers === version.version);
|
|
631
|
+
} catch {
|
|
632
|
+
throw new validation.ForgeInventoryError('invalid-cargo-index');
|
|
633
|
+
}
|
|
634
|
+
if (line === undefined) throw new validation.ForgeInventoryError('package-document-missing');
|
|
635
|
+
return { bytes: Buffer.from(line, 'utf8'), mediaType: index.mediaType };
|
|
636
|
+
};
|
|
637
|
+
} else if (kindArg === 'cargo-description' && selected.type === 'cargo') {
|
|
638
|
+
// Gitea's crate name rule, checked before any request.
|
|
639
|
+
void cargoIndexPath(selected.name);
|
|
640
|
+
read = (signalArg) => this.#cargoDescription(registry, selected.name, version.version, signalArg);
|
|
572
641
|
} else {
|
|
573
642
|
throw new validation.ForgeInventoryError('package-document-kind-invalid');
|
|
574
643
|
}
|
|
575
|
-
const { signal } = optionsArg;
|
|
576
644
|
await this.#versionIdentified(selected, version, signal);
|
|
577
|
-
let
|
|
645
|
+
let document: { bytes: Buffer; mediaType: string } | null;
|
|
578
646
|
try {
|
|
579
|
-
|
|
647
|
+
document = await read(signal);
|
|
580
648
|
} catch (error) {
|
|
581
649
|
if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
582
650
|
return this.#versionExplained(selected, version, error, signal);
|
|
583
651
|
}
|
|
584
|
-
if (response.mediaType !== mediaType) throw new validation.ForgeInventoryError('invalid-content-type');
|
|
585
|
-
let bytes = response.bytes;
|
|
586
|
-
if (kindArg === 'cargo-index-entry') {
|
|
587
|
-
let line: string | undefined;
|
|
588
|
-
try {
|
|
589
|
-
line = new TextDecoder('utf-8', { fatal: true }).decode(bytes).split('\n')
|
|
590
|
-
.find((lineArg) => lineArg !== '' && validation.record(JSON.parse(lineArg) as unknown).vers === version.version);
|
|
591
|
-
} catch {
|
|
592
|
-
throw new validation.ForgeInventoryError('invalid-cargo-index');
|
|
593
|
-
}
|
|
594
|
-
if (line === undefined) throw new validation.ForgeInventoryError('package-document-missing');
|
|
595
|
-
bytes = Buffer.from(line, 'utf8');
|
|
596
|
-
}
|
|
597
652
|
await this.#versionIdentified(selected, version, signal);
|
|
598
|
-
|
|
653
|
+
if (document === null) return null;
|
|
654
|
+
if (kindArg === 'swift-manifest') {
|
|
655
|
+
return { kind: kindArg, versionId: version.id, bytes: document.bytes, sha256: sha256Hex(document.bytes), mediaType: document.mediaType };
|
|
656
|
+
}
|
|
657
|
+
const canonical = canonicalPackageDocument(kindArg, document.bytes);
|
|
658
|
+
return { kind: kindArg, versionId: version.id, bytes: canonical.bytes, sha256: canonical.sha256, mediaType: document.mediaType };
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** A document's body, which must carry `mediaTypeArg`. */
|
|
662
|
+
async #documentBytes(urlArg: URL, acceptArg: string, mediaTypeArg: string, signalArg: AbortSignal | undefined): Promise<{ bytes: Buffer; mediaType: string }> {
|
|
663
|
+
const response = await this.#client.requestBytes(urlArg, { accept: acceptArg, signal: signalArg });
|
|
664
|
+
if (response.mediaType !== mediaTypeArg) throw new validation.ForgeInventoryError('invalid-content-type');
|
|
665
|
+
return { bytes: response.bytes, mediaType: mediaTypeArg };
|
|
599
666
|
}
|
|
600
667
|
|
|
601
668
|
/**
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
669
|
+
* `{"description":…}` of the crate's entry in the Cargo search, read page by
|
|
670
|
+
* page until the entry with exactly `nameArg` turns up. Gitea's search
|
|
671
|
+
* matches names by substring and serves one entry per crate, describing its
|
|
672
|
+
* version with the highest id, whose version `max_version` names.
|
|
606
673
|
*/
|
|
607
|
-
async #
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
if (length !== null && length.trim() !== contentArg.size) {
|
|
627
|
-
await transfer.run(() => discardBody(response.body));
|
|
628
|
-
throw new validation.ForgeInventoryError('package-content-size-mismatch');
|
|
674
|
+
async #cargoDescription(registryArg: string, nameArg: string, versionArg: string, signalArg: AbortSignal | undefined): Promise<{ bytes: Buffer; mediaType: string }> {
|
|
675
|
+
let seen = 0;
|
|
676
|
+
for (let page = 1; page <= cargoSearchMaxPages; page++) {
|
|
677
|
+
const url = new URL(`${registryArg}/cargo/api/v1/crates`);
|
|
678
|
+
url.searchParams.set('q', nameArg);
|
|
679
|
+
url.searchParams.set('page', String(page));
|
|
680
|
+
url.searchParams.set('per_page', String(cargoSearchPageSize));
|
|
681
|
+
const body = validation.record((await this.#client.request(url, { signal: signalArg })).body);
|
|
682
|
+
if (!Array.isArray(body.crates)) throw new validation.ForgeInventoryError('invalid-cargo-search');
|
|
683
|
+
const total = validation.record(body.meta).total;
|
|
684
|
+
if (typeof total !== 'number' || !Number.isSafeInteger(total) || total < 0) throw new validation.ForgeInventoryError('invalid-cargo-search');
|
|
685
|
+
for (const entry of body.crates) {
|
|
686
|
+
const crate = validation.record(entry);
|
|
687
|
+
if (typeof crate.name !== 'string' || typeof crate.max_version !== 'string' || typeof crate.description !== 'string') {
|
|
688
|
+
throw new validation.ForgeInventoryError('invalid-cargo-search');
|
|
689
|
+
}
|
|
690
|
+
if (crate.name !== nameArg) continue;
|
|
691
|
+
if (crate.max_version !== versionArg) throw new validation.ForgeInventoryError('package-document-missing');
|
|
692
|
+
return { bytes: Buffer.from(JSON.stringify({ description: crate.description }), 'utf8'), mediaType: 'application/json' };
|
|
629
693
|
}
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
const failure = transfer.failure(error, 'package-content-download-failed');
|
|
633
|
-
transfer.end();
|
|
634
|
-
close();
|
|
635
|
-
throw failure;
|
|
694
|
+
seen += body.crates.length;
|
|
695
|
+
if (body.crates.length === 0 || seen >= total) throw new validation.ForgeInventoryError('package-document-missing');
|
|
636
696
|
}
|
|
637
|
-
|
|
638
|
-
const complete = async () => {
|
|
639
|
-
if (hash.digest('hex') !== contentArg.sha256) throw new validation.ForgeInventoryError('package-content-digest-mismatch');
|
|
640
|
-
if (identifiedArg) {
|
|
641
|
-
transfer.touch();
|
|
642
|
-
await transfer.run(() => identifiedArg(transfer.signal));
|
|
643
|
-
}
|
|
644
|
-
};
|
|
645
|
-
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
646
|
-
try {
|
|
647
|
-
yield* heldBackChunks(transfer.watch(body), contentArg.bytes, packageContentCodes, { inspect: (chunkArg) => hash.update(chunkArg), complete });
|
|
648
|
-
} catch (error) {
|
|
649
|
-
throw transfer.failure(error, 'package-content-download-failed');
|
|
650
|
-
} finally {
|
|
651
|
-
transfer.end();
|
|
652
|
-
}
|
|
653
|
-
};
|
|
654
|
-
return createTransferStream(transfer, chunks(), close);
|
|
697
|
+
throw new validation.ForgeInventoryError('cargo-search-over-bound');
|
|
655
698
|
}
|
|
656
699
|
|
|
657
700
|
/**
|