@foss.global/codefeed 5.1.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +45 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.d.ts +35 -9
- package/dist_ts/classes.giteasourceinventory.js +284 -26
- package/dist_ts/classes.giteasourcetransfer.d.ts +151 -6
- package/dist_ts/classes.giteasourcetransfer.js +411 -9
- package/dist_ts/forgeinventory.client.d.ts +43 -5
- package/dist_ts/forgeinventory.client.js +57 -13
- package/dist_ts/forgeinventory.transport.d.ts +21 -7
- package/dist_ts/forgeinventory.transport.js +15 -10
- package/dist_ts/giteasourceinventory.packages.d.ts +46 -0
- package/dist_ts/giteasourceinventory.packages.js +139 -0
- package/dist_ts/giteasourcetransfer.documents.d.ts +35 -0
- package/dist_ts/giteasourcetransfer.documents.js +86 -0
- package/dist_ts/index.d.ts +3 -0
- package/dist_ts/index.js +3 -1
- package/dist_ts/plugins.d.ts +2 -2
- package/dist_ts/plugins.js +3 -1
- package/package.json +3 -3
- package/readme.md +37 -2
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +293 -24
- package/ts/classes.giteasourcetransfer.ts +532 -8
- package/ts/forgeinventory.client.ts +106 -13
- package/ts/forgeinventory.transport.ts +33 -13
- package/ts/giteasourceinventory.packages.ts +160 -0
- package/ts/giteasourcetransfer.documents.ts +97 -0
- package/ts/index.ts +3 -0
- package/ts/plugins.ts +2 -2
|
@@ -6,6 +6,13 @@ import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
|
6
6
|
import { LfsBatchClient } from './classes.lfsbatchclient.js';
|
|
7
7
|
import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
|
|
8
8
|
import { giteaCloneUrl, reserveMirrorScratch } from './giteasourceinventory.git.js';
|
|
9
|
+
import { cargoIndexPath, containerManifestMediaTypes, npmPackagePath, packageTypePattern, swiftPackageParts } from './giteasourceinventory.packages.js';
|
|
10
|
+
import { canonicalPackageDocument, swiftManifestVersionPattern } from './giteasourcetransfer.documents.js';
|
|
11
|
+
import type { TGiteaCanonicalPackageDocumentKind } from './giteasourcetransfer.documents.js';
|
|
12
|
+
|
|
13
|
+
type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
|
|
14
|
+
type IPackageVersion = plugins.fossInterfaces.data.IForgeSourcePackageVersion;
|
|
15
|
+
type IPackageBlob = plugins.fossInterfaces.data.IForgeSourcePackageBlob;
|
|
9
16
|
|
|
10
17
|
export interface IGiteaSourceTransferOptions {
|
|
11
18
|
/** Instance root, including an optional deployment subpath. HTTPS is required. */
|
|
@@ -30,6 +37,8 @@ export interface IGiteaSourceTransferOptions {
|
|
|
30
37
|
lfsTransferTimeoutMs?: number;
|
|
31
38
|
/** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
|
|
32
39
|
releaseAssetTimeoutMs?: number;
|
|
40
|
+
/** Wall-clock limit of one package blob or file download, its closing identity read included; defaults to 30 minutes, at most six hours. */
|
|
41
|
+
packageTransferTimeoutMs?: number;
|
|
33
42
|
}
|
|
34
43
|
|
|
35
44
|
/**
|
|
@@ -77,6 +86,95 @@ export interface IGiteaReleaseAssetDownload {
|
|
|
77
86
|
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
78
87
|
}
|
|
79
88
|
|
|
89
|
+
/** A package as the source inventory recorded it: what a read names, and the owner id it checks Gitea still holds. */
|
|
90
|
+
export type TGiteaPackageSelector = Pick<IPackageRecord, 'owner' | 'ownerId' | 'type' | 'name'>;
|
|
91
|
+
|
|
92
|
+
/** A version as the source inventory recorded it; its id identifies it, so a tag pushed again is another version. */
|
|
93
|
+
export type TGiteaPackageVersionSelector = Pick<IPackageVersion, 'id' | 'version'>;
|
|
94
|
+
|
|
95
|
+
/** Content as the source inventory recorded it: the SHA-256 and size of a blob. */
|
|
96
|
+
export type TGiteaPackageContentSelector = Pick<IPackageBlob, 'sha256' | 'size'>;
|
|
97
|
+
|
|
98
|
+
/** A file of a version as the source inventory recorded it: its name and the content it holds. */
|
|
99
|
+
export interface IGiteaPackageFileSelector extends TGiteaPackageContentSelector {
|
|
100
|
+
name: string;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Metadata a package version carries beside its files: an npm version
|
|
105
|
+
* document, Swift release metadata, a Swift release's manifest
|
|
106
|
+
* (`Package.swift`, or an alternate `Package@swift-<v>.swift`), a crate
|
|
107
|
+
* version's line of the Cargo sparse index, or the description the Cargo
|
|
108
|
+
* search serves for a crate's latest version.
|
|
109
|
+
*/
|
|
110
|
+
export type TGiteaPackageDocumentKind = TGiteaCanonicalPackageDocumentKind | 'swift-manifest';
|
|
111
|
+
|
|
112
|
+
export interface IGiteaPackageReadOptions {
|
|
113
|
+
/** Aborts the read with `request-aborted`, also while a stream is read. */
|
|
114
|
+
signal?: AbortSignal;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Reads an alternate Swift manifest. */
|
|
118
|
+
export interface IGiteaSwiftManifestReadOptions extends IGiteaPackageReadOptions {
|
|
119
|
+
/**
|
|
120
|
+
* The Swift version Gitea keys the alternate manifest by, trimmed as its
|
|
121
|
+
* `TrimmedVersionString` writes it (`5.8`, `5.9.1`; the patch only when it
|
|
122
|
+
* is not zero).
|
|
123
|
+
*/
|
|
124
|
+
swiftVersion: string;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** A container manifest with the exact bytes and `Content-Type` Gitea serves. */
|
|
128
|
+
export interface IGiteaContainerManifest {
|
|
129
|
+
/** `sha256:` and the hex digest the bytes were checked against. */
|
|
130
|
+
digest: string;
|
|
131
|
+
/** The `Content-Type` header verbatim: the media type the manifest was pushed with. */
|
|
132
|
+
mediaType: string;
|
|
133
|
+
bytes: Buffer;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* A version's metadata document and its digest: a JSON document in its
|
|
138
|
+
* canonical form (`canonicalPackageDocument`), a Swift manifest verbatim.
|
|
139
|
+
*/
|
|
140
|
+
export interface IGiteaPackageDocument {
|
|
141
|
+
kind: TGiteaPackageDocumentKind;
|
|
142
|
+
/** The id of the version the identity reads before and after the read confirmed. */
|
|
143
|
+
versionId: string;
|
|
144
|
+
/**
|
|
145
|
+
* The canonical JSON for every kind but `swift-manifest`, whose bytes are
|
|
146
|
+
* the stored manifest as Gitea serves it. The bytes Gitea served for a JSON
|
|
147
|
+
* document are not returned: their key order differs from read to read.
|
|
148
|
+
*/
|
|
149
|
+
bytes: Buffer;
|
|
150
|
+
/** Lowercase hex SHA-256 of `bytes`. */
|
|
151
|
+
sha256: string;
|
|
152
|
+
/** The media type Gitea declared, without parameters. */
|
|
153
|
+
mediaType: string;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface IGiteaPackageDownload {
|
|
157
|
+
content: TGiteaPackageContentSelector;
|
|
158
|
+
/**
|
|
159
|
+
* The content's bytes, never held whole. The last chunk is held back until
|
|
160
|
+
* exactly `content.size` bytes have arrived, their SHA-256 equals
|
|
161
|
+
* `content.sha256` and, for a version's file, the closing identity read has
|
|
162
|
+
* passed. Destroying the stream ends the download.
|
|
163
|
+
*/
|
|
164
|
+
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const packageContentCodes = {
|
|
168
|
+
invalidChunk: 'package-content-invalid-chunk', overrun: 'package-content-size-overrun', short: 'package-content-short-body',
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
const sha256Hex = (bytesArg: Uint8Array): string => plugins.crypto.createHash('sha256').update(bytesArg).digest('hex');
|
|
172
|
+
|
|
173
|
+
/** Entries asked of one Cargo search page; Gitea serves at most its `MAX_RESPONSE_ITEMS` (50 by default). */
|
|
174
|
+
const cargoSearchPageSize = 50;
|
|
175
|
+
/** Search pages read for one crate's description before `cargo-search-over-bound`: 10,000 entries at 50 a page. */
|
|
176
|
+
const cargoSearchMaxPages = 200;
|
|
177
|
+
|
|
80
178
|
/** Gitea names attachment content by a random UUID in canonical lowercase form. */
|
|
81
179
|
const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
82
180
|
|
|
@@ -87,19 +185,25 @@ export interface IGiteaMirrorImport {
|
|
|
87
185
|
}
|
|
88
186
|
|
|
89
187
|
/**
|
|
90
|
-
* Reads a Gitea repository's Git data and LFS objects
|
|
91
|
-
* migration, with the reader's login and token: a mirror
|
|
92
|
-
* GitManager the caller supplies, verified LFS object downloads
|
|
93
|
-
* asset downloads
|
|
94
|
-
*
|
|
95
|
-
*
|
|
188
|
+
* Reads a Gitea repository's Git data and LFS objects, and package content,
|
|
189
|
+
* for its one-time migration, with the reader's login and token: a mirror
|
|
190
|
+
* clone into a GitManager the caller supplies, verified LFS object downloads,
|
|
191
|
+
* release asset downloads, container manifests and blobs, package files and
|
|
192
|
+
* package metadata documents. It never writes to Gitea: Git is read only
|
|
193
|
+
* through `git-upload-pack`, LFS only through batch `download` requests and
|
|
194
|
+
* their download actions, release assets only through GETs of their
|
|
195
|
+
* attachment, packages only through GETs of the registries and the v1 package
|
|
196
|
+
* API.
|
|
96
197
|
*/
|
|
97
198
|
export class GiteaSourceTransfer {
|
|
98
199
|
readonly #options: {
|
|
99
200
|
baseUrl: string; login: string; token: string; caCertificates?: string[];
|
|
100
201
|
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
|
|
202
|
+
packageTransferTimeoutMs: number;
|
|
101
203
|
};
|
|
102
204
|
readonly #client: ForgeRequestClient;
|
|
205
|
+
/** The registry (`/v2`) takes the token as a Basic password with the reader's login. */
|
|
206
|
+
readonly #registryClient: ForgeRequestClient;
|
|
103
207
|
|
|
104
208
|
constructor(optionsArg: IGiteaSourceTransferOptions) {
|
|
105
209
|
const baseUrl = validation.httpUrl(optionsArg.baseUrl);
|
|
@@ -113,22 +217,34 @@ export class GiteaSourceTransfer {
|
|
|
113
217
|
maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
|
|
114
218
|
lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
|
|
115
219
|
releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
|
|
220
|
+
packageTransferTimeoutMs: optionsArg.packageTransferTimeoutMs ?? 30 * 60 * 1000,
|
|
116
221
|
};
|
|
117
222
|
for (const [name, maximum] of [
|
|
118
223
|
['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
|
|
119
|
-
['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
|
|
224
|
+
['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000], ['packageTransferTimeoutMs', 6 * 60 * 60 * 1000],
|
|
120
225
|
] as const) {
|
|
121
226
|
const value = limits[name];
|
|
122
227
|
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
|
|
123
228
|
}
|
|
124
229
|
const caCertificates = optionsArg.caCertificates === undefined ? undefined : validation.caCertificates(optionsArg.caCertificates);
|
|
125
230
|
this.#options = { baseUrl: baseUrl.replace(/\/+$/, ''), login: optionsArg.login, token: optionsArg.token, caCertificates, ...limits };
|
|
231
|
+
const requestTransport = caCertificates === undefined ? transport.globalFetchTransport : transport.createHttpsTransport(caCertificates);
|
|
126
232
|
this.#client = new ForgeRequestClient({
|
|
127
|
-
transport:
|
|
233
|
+
transport: requestTransport,
|
|
128
234
|
headers: { authorization: `token ${optionsArg.token}` },
|
|
129
235
|
requestTimeoutMs: limits.requestTimeoutMs,
|
|
130
236
|
maxResponseBytes: limits.maxResponseBytes,
|
|
131
237
|
});
|
|
238
|
+
this.#registryClient = new ForgeRequestClient({
|
|
239
|
+
transport: requestTransport,
|
|
240
|
+
headers: { authorization: this.#basicAuthorization() },
|
|
241
|
+
requestTimeoutMs: limits.requestTimeoutMs,
|
|
242
|
+
maxResponseBytes: limits.maxResponseBytes,
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
#basicAuthorization(): string {
|
|
247
|
+
return `Basic ${Buffer.from(`${this.#options.login}:${this.#options.token}`).toString('base64')}`;
|
|
132
248
|
}
|
|
133
249
|
|
|
134
250
|
/** Validated, normalized instance root; contains no credentials or API suffix. */
|
|
@@ -342,6 +458,414 @@ export class GiteaSourceTransfer {
|
|
|
342
458
|
return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
|
|
343
459
|
}
|
|
344
460
|
|
|
461
|
+
/**
|
|
462
|
+
* Reads one container manifest by digest from the registry with the
|
|
463
|
+
* reader's Basic credentials: its exact bytes, which must hash to `digest`
|
|
464
|
+
* (`package-manifest-digest-mismatch`), and its `Content-Type` verbatim,
|
|
465
|
+
* which must be an OCI or Docker image manifest, index or manifest list
|
|
466
|
+
* (`package-manifest-media-type-invalid`). A manifest is content-addressed,
|
|
467
|
+
* so no identity read is needed. Reading it counts a download of the version
|
|
468
|
+
* holding it, an accepted read stamp (S10 ruling, item 8).
|
|
469
|
+
*/
|
|
470
|
+
public async readContainerManifest(
|
|
471
|
+
packageArg: TGiteaPackageSelector, digestArg: string, optionsArg: IGiteaPackageReadOptions = {},
|
|
472
|
+
): Promise<IGiteaContainerManifest> {
|
|
473
|
+
const selected = this.#containerPackage(packageArg);
|
|
474
|
+
if (typeof digestArg !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(digestArg)) throw new validation.ForgeInventoryError('package-content-invalid');
|
|
475
|
+
const response = await this.#registryClient.requestBytes(
|
|
476
|
+
new URL(`/v2/${selected.owner.toLowerCase()}/${selected.name}/manifests/${digestArg}`, this.#options.baseUrl),
|
|
477
|
+
{ accept: containerManifestMediaTypes.join(', '), signal: optionsArg.signal },
|
|
478
|
+
);
|
|
479
|
+
const contentDigest = response.headers.get('docker-content-digest');
|
|
480
|
+
if (`sha256:${sha256Hex(response.bytes)}` !== digestArg || (contentDigest !== null && contentDigest.trim() !== digestArg)) {
|
|
481
|
+
throw new validation.ForgeInventoryError('package-manifest-digest-mismatch');
|
|
482
|
+
}
|
|
483
|
+
const mediaType = response.headers.get('content-type')?.trim() ?? '';
|
|
484
|
+
if (!containerManifestMediaTypes.includes(mediaType.toLowerCase())) throw new validation.ForgeInventoryError('package-manifest-media-type-invalid');
|
|
485
|
+
return { digest: digestArg, mediaType, bytes: response.bytes };
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Downloads one container blob, a config or a layer, by digest from the
|
|
490
|
+
* registry with the reader's Basic credentials. The bytes stream verified:
|
|
491
|
+
* the last chunk is held back until exactly `content.size` bytes have
|
|
492
|
+
* arrived and their SHA-256 equals `content.sha256`. A blob is
|
|
493
|
+
* content-addressed, so no identity read is needed, and reading one counts
|
|
494
|
+
* no download. Rejects, or errors the stream, with a fixed code such as
|
|
495
|
+
* `http-<status>`, `package-content-size-mismatch` (a `content-length` that
|
|
496
|
+
* is not the size), `package-content-size-overrun`, `package-content-short-body`,
|
|
497
|
+
* `package-content-digest-mismatch` or `package-content-transfer-timed-out`.
|
|
498
|
+
*/
|
|
499
|
+
public async downloadContainerBlob(
|
|
500
|
+
packageArg: TGiteaPackageSelector, contentArg: TGiteaPackageContentSelector, optionsArg: IGiteaPackageReadOptions = {},
|
|
501
|
+
): Promise<IGiteaPackageDownload> {
|
|
502
|
+
const selected = this.#containerPackage(packageArg);
|
|
503
|
+
const content = this.#content(contentArg);
|
|
504
|
+
const stream = await this.#verifiedDownload(
|
|
505
|
+
new URL(`/v2/${selected.owner.toLowerCase()}/${selected.name}/blobs/sha256:${content.sha256}`, this.#options.baseUrl),
|
|
506
|
+
{ authorization: this.#basicAuthorization() }, content, optionsArg.signal, null,
|
|
507
|
+
);
|
|
508
|
+
return { content: { sha256: content.sha256, size: content.size }, stream };
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Downloads one file of a package version: a generic file, an npm tarball
|
|
513
|
+
* (`{package}/-/{version}/{file}`), a Swift source archive or a crate, by the
|
|
514
|
+
* route of the package's type with the reader's `token` authorization;
|
|
515
|
+
* container content is read with `readContainerManifest` and
|
|
516
|
+
* `downloadContainerBlob` (`package-type-unsupported`). The version is read
|
|
517
|
+
* by its id before the download and again after the last byte: it must
|
|
518
|
+
* still be the recorded version of the recorded owner, else
|
|
519
|
+
* `package-version-identity-changed`, or `http-404` once deleted. The bytes
|
|
520
|
+
* stream verified as for `downloadContainerBlob`. Reading a file counts a
|
|
521
|
+
* download of its version, an accepted read stamp (S10 ruling, item 8).
|
|
522
|
+
*/
|
|
523
|
+
public async downloadPackageFile(
|
|
524
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, fileArg: IGiteaPackageFileSelector, optionsArg: IGiteaPackageReadOptions = {},
|
|
525
|
+
): Promise<IGiteaPackageDownload> {
|
|
526
|
+
const selected = this.#packageSelector(packageArg);
|
|
527
|
+
const version = this.#versionSelector(versionArg);
|
|
528
|
+
const content = this.#content(fileArg);
|
|
529
|
+
if (typeof fileArg.name !== 'string' || !fileArg.name || /[\u0000-\u001f\u007f/]/.test(fileArg.name)) throw new validation.ForgeInventoryError('package-content-invalid');
|
|
530
|
+
const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(selected.owner)}`;
|
|
531
|
+
const encodedVersion = encodeURIComponent(version.version);
|
|
532
|
+
let url: URL;
|
|
533
|
+
const headers: Record<string, string> = { authorization: `token ${this.#options.token}` };
|
|
534
|
+
switch (selected.type) {
|
|
535
|
+
case 'generic':
|
|
536
|
+
url = new URL(`${registry}/generic/${encodeURIComponent(selected.name)}/${encodedVersion}/${encodeURIComponent(fileArg.name)}`);
|
|
537
|
+
break;
|
|
538
|
+
case 'npm':
|
|
539
|
+
url = new URL(`${registry}/npm/${npmPackagePath(selected.name)}/-/${encodedVersion}/${encodeURIComponent(fileArg.name)}`);
|
|
540
|
+
break;
|
|
541
|
+
case 'swift': {
|
|
542
|
+
const parts = swiftPackageParts(selected.name);
|
|
543
|
+
url = new URL(`${registry}/swift/${encodeURIComponent(parts.scope)}/${encodeURIComponent(parts.name)}/${encodedVersion}.zip`);
|
|
544
|
+
headers.accept = 'application/vnd.swift.registry.v1+zip';
|
|
545
|
+
break;
|
|
546
|
+
}
|
|
547
|
+
case 'cargo':
|
|
548
|
+
url = new URL(`${registry}/cargo/api/v1/crates/${encodeURIComponent(selected.name)}/${encodedVersion}/download`);
|
|
549
|
+
break;
|
|
550
|
+
default:
|
|
551
|
+
throw new validation.ForgeInventoryError('package-type-unsupported');
|
|
552
|
+
}
|
|
553
|
+
const { signal } = optionsArg;
|
|
554
|
+
await this.#versionIdentified(selected, version, signal);
|
|
555
|
+
let stream: InstanceType<typeof plugins.stream.Readable>;
|
|
556
|
+
try {
|
|
557
|
+
stream = await this.#verifiedDownload(url, headers, content, signal, (signalArg) => this.#versionIdentified(selected, version, signalArg));
|
|
558
|
+
} catch (error) {
|
|
559
|
+
if (signal?.aborted) throw error;
|
|
560
|
+
return this.#versionExplained(selected, version, error, signal);
|
|
561
|
+
}
|
|
562
|
+
return { content: { sha256: content.sha256, size: content.size }, stream };
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Reads one metadata document of a package version with the reader's
|
|
567
|
+
* `token` authorization, bracketed by identity reads of the version as for
|
|
568
|
+
* `downloadPackageFile`. None of these reads counts a download.
|
|
569
|
+
*
|
|
570
|
+
* - `npm-version`: the npm version document.
|
|
571
|
+
* - `swift-release`: Swift release metadata.
|
|
572
|
+
* - `swift-manifest`: the release's `Package.swift`, which must answer 200;
|
|
573
|
+
* with `swiftVersion`, the alternate manifest Gitea keys by that version,
|
|
574
|
+
* read without following a redirect. It resolves null only when Gitea
|
|
575
|
+
* answers 303 to the release's own `Package.swift` URL, its way of saying
|
|
576
|
+
* it holds no such manifest; a redirect anywhere else rejects with
|
|
577
|
+
* `swift-manifest-redirect-unexpected`, any other status with
|
|
578
|
+
* `http-<status>`.
|
|
579
|
+
* - `cargo-index-entry`: the version's line of the crate's sparse index;
|
|
580
|
+
* `package-document-missing` when the index has none.
|
|
581
|
+
* - `cargo-description`: `{"description":…}` of the crate's entry in the
|
|
582
|
+
* Cargo search (`q=<name>`, its exact name), which describes only the
|
|
583
|
+
* version with the highest id: its `max_version` must be this version,
|
|
584
|
+
* else `package-document-missing`, as when no entry has the name.
|
|
585
|
+
*
|
|
586
|
+
* A manifest comes back verbatim, every other document in its canonical
|
|
587
|
+
* form (`canonicalPackageDocument`, `document-not-canonical` when it has
|
|
588
|
+
* none), each with the SHA-256 of the returned bytes, which a consumer
|
|
589
|
+
* records beside the version id. A kind that does not belong to the
|
|
590
|
+
* package's type, or `swiftVersion` with another kind, rejects with
|
|
591
|
+
* `package-document-kind-invalid`, a `swiftVersion` Gitea would not key a
|
|
592
|
+
* manifest by with `swift-version-invalid`, both before any request; a
|
|
593
|
+
* document of another media type rejects with `invalid-content-type`.
|
|
594
|
+
*/
|
|
595
|
+
public async readPackageDocument(
|
|
596
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: 'swift-manifest', optionsArg: IGiteaSwiftManifestReadOptions,
|
|
597
|
+
): Promise<IGiteaPackageDocument | null>;
|
|
598
|
+
public async readPackageDocument(
|
|
599
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind, optionsArg?: IGiteaPackageReadOptions,
|
|
600
|
+
): Promise<IGiteaPackageDocument>;
|
|
601
|
+
public async readPackageDocument(
|
|
602
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, kindArg: TGiteaPackageDocumentKind,
|
|
603
|
+
optionsArg: IGiteaPackageReadOptions & { swiftVersion?: string } = {},
|
|
604
|
+
): Promise<IGiteaPackageDocument | null> {
|
|
605
|
+
const selected = this.#packageSelector(packageArg);
|
|
606
|
+
const version = this.#versionSelector(versionArg);
|
|
607
|
+
const { signal, swiftVersion } = optionsArg;
|
|
608
|
+
if (swiftVersion !== undefined) {
|
|
609
|
+
if (kindArg !== 'swift-manifest') throw new validation.ForgeInventoryError('package-document-kind-invalid');
|
|
610
|
+
if (typeof swiftVersion !== 'string' || !swiftManifestVersionPattern.test(swiftVersion)) throw new validation.ForgeInventoryError('swift-version-invalid');
|
|
611
|
+
}
|
|
612
|
+
const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(selected.owner)}`;
|
|
613
|
+
const encodedVersion = encodeURIComponent(version.version);
|
|
614
|
+
let read: (signalArg: AbortSignal | undefined) => Promise<{ bytes: Buffer; mediaType: string } | null>;
|
|
615
|
+
if (kindArg === 'npm-version' && selected.type === 'npm') {
|
|
616
|
+
read = (signalArg) => this.#documentBytes(new URL(`${registry}/npm/${npmPackagePath(selected.name)}/${encodedVersion}`), 'application/json', 'application/json', signalArg);
|
|
617
|
+
} else if (kindArg === 'swift-release' && selected.type === 'swift') {
|
|
618
|
+
const parts = swiftPackageParts(selected.name);
|
|
619
|
+
const release = new URL(`${registry}/swift/${encodeURIComponent(parts.scope)}/${encodeURIComponent(parts.name)}/${encodedVersion}`);
|
|
620
|
+
read = (signalArg) => this.#documentBytes(release, 'application/vnd.swift.registry.v1+json', 'application/json', signalArg);
|
|
621
|
+
} else if (kindArg === 'swift-manifest' && selected.type === 'swift') {
|
|
622
|
+
const parts = swiftPackageParts(selected.name);
|
|
623
|
+
read = (signalArg) => this.#swiftManifest(selected, parts, version, swiftVersion ?? null, signalArg);
|
|
624
|
+
} else if (kindArg === 'cargo-index-entry' && selected.type === 'cargo') {
|
|
625
|
+
const url = new URL(`${registry}/cargo/${cargoIndexPath(selected.name)}`);
|
|
626
|
+
read = async (signalArg) => {
|
|
627
|
+
const index = await this.#documentBytes(url, 'text/plain', 'text/plain', signalArg);
|
|
628
|
+
let line: string | undefined;
|
|
629
|
+
try {
|
|
630
|
+
line = new TextDecoder('utf-8', { fatal: true }).decode(index.bytes).split('\n')
|
|
631
|
+
.find((lineArg) => lineArg !== '' && validation.record(JSON.parse(lineArg) as unknown).vers === version.version);
|
|
632
|
+
} catch {
|
|
633
|
+
throw new validation.ForgeInventoryError('invalid-cargo-index');
|
|
634
|
+
}
|
|
635
|
+
if (line === undefined) throw new validation.ForgeInventoryError('package-document-missing');
|
|
636
|
+
return { bytes: Buffer.from(line, 'utf8'), mediaType: index.mediaType };
|
|
637
|
+
};
|
|
638
|
+
} else if (kindArg === 'cargo-description' && selected.type === 'cargo') {
|
|
639
|
+
// Gitea's crate name rule, checked before any request.
|
|
640
|
+
void cargoIndexPath(selected.name);
|
|
641
|
+
read = (signalArg) => this.#cargoDescription(registry, selected.name, version.version, signalArg);
|
|
642
|
+
} else {
|
|
643
|
+
throw new validation.ForgeInventoryError('package-document-kind-invalid');
|
|
644
|
+
}
|
|
645
|
+
await this.#versionIdentified(selected, version, signal);
|
|
646
|
+
let document: { bytes: Buffer; mediaType: string } | null;
|
|
647
|
+
try {
|
|
648
|
+
document = await read(signal);
|
|
649
|
+
} catch (error) {
|
|
650
|
+
if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
651
|
+
return this.#versionExplained(selected, version, error, signal);
|
|
652
|
+
}
|
|
653
|
+
await this.#versionIdentified(selected, version, signal);
|
|
654
|
+
if (document === null) return null;
|
|
655
|
+
if (kindArg === 'swift-manifest') {
|
|
656
|
+
return { kind: kindArg, versionId: version.id, bytes: document.bytes, sha256: sha256Hex(document.bytes), mediaType: document.mediaType };
|
|
657
|
+
}
|
|
658
|
+
const canonical = canonicalPackageDocument(kindArg, document.bytes);
|
|
659
|
+
return { kind: kindArg, versionId: version.id, bytes: canonical.bytes, sha256: canonical.sha256, mediaType: document.mediaType };
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/** A document's body, which must carry `mediaTypeArg`. */
|
|
663
|
+
async #documentBytes(urlArg: URL, acceptArg: string, mediaTypeArg: string, signalArg: AbortSignal | undefined): Promise<{ bytes: Buffer; mediaType: string }> {
|
|
664
|
+
const response = await this.#client.requestBytes(urlArg, { accept: acceptArg, signal: signalArg });
|
|
665
|
+
if (response.mediaType !== mediaTypeArg) throw new validation.ForgeInventoryError('invalid-content-type');
|
|
666
|
+
return { bytes: response.bytes, mediaType: mediaTypeArg };
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/**
|
|
670
|
+
* A Swift release's `Package.swift`, or with `swiftVersionArg` its
|
|
671
|
+
* alternate manifest; null when Gitea redirects the alternate one to the
|
|
672
|
+
* release's own `Package.swift` (303), which is how it answers a version it
|
|
673
|
+
* holds no manifest for. No redirect is followed.
|
|
674
|
+
*/
|
|
675
|
+
async #swiftManifest(
|
|
676
|
+
packageArg: TGiteaPackageSelector, partsArg: { scope: string; name: string }, versionArg: TGiteaPackageVersionSelector, swiftVersionArg: string | null,
|
|
677
|
+
signalArg: AbortSignal | undefined,
|
|
678
|
+
): Promise<{ bytes: Buffer; mediaType: string } | null> {
|
|
679
|
+
const release = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(packageArg.owner)}/swift/${encodeURIComponent(partsArg.scope)}/${
|
|
680
|
+
encodeURIComponent(partsArg.name)}/${encodeURIComponent(versionArg.version)}/Package.swift`;
|
|
681
|
+
const url = new URL(release);
|
|
682
|
+
if (swiftVersionArg !== null) url.searchParams.set('swift-version', swiftVersionArg);
|
|
683
|
+
const response = await this.#client.requestBytesOrRedirect(url, { accept: 'application/vnd.swift.registry.v1+swift', signal: signalArg });
|
|
684
|
+
if (response.kind === 'redirect') {
|
|
685
|
+
if (swiftVersionArg === null) throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
686
|
+
// Gitea's answer for a missing alternate manifest: 303 to the release's
|
|
687
|
+
// `Package.swift` under its configured root, with the owner lowercased
|
|
688
|
+
// and scope, name and version as the request named them.
|
|
689
|
+
const expected = new URL(`${this.#options.baseUrl}/api/packages/${packageArg.owner.toLowerCase()}/swift/${partsArg.scope}/${partsArg.name}/${
|
|
690
|
+
versionArg.version}/Package.swift`);
|
|
691
|
+
let location: URL | null = null;
|
|
692
|
+
try {
|
|
693
|
+
location = response.location === null ? null : new URL(response.location, url);
|
|
694
|
+
} catch {
|
|
695
|
+
location = null;
|
|
696
|
+
}
|
|
697
|
+
if (response.status !== 303 || location === null || location.href !== expected.href) {
|
|
698
|
+
throw new validation.ForgeInventoryError('swift-manifest-redirect-unexpected');
|
|
699
|
+
}
|
|
700
|
+
return null;
|
|
701
|
+
}
|
|
702
|
+
if (response.mediaType !== 'text/x-swift') throw new validation.ForgeInventoryError('invalid-content-type');
|
|
703
|
+
return { bytes: response.bytes, mediaType: 'text/x-swift' };
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* `{"description":…}` of the crate's entry in the Cargo search, read page by
|
|
708
|
+
* page until the entry with exactly `nameArg` turns up. Gitea's search
|
|
709
|
+
* matches names by substring and serves one entry per crate, describing its
|
|
710
|
+
* version with the highest id, whose version `max_version` names.
|
|
711
|
+
*/
|
|
712
|
+
async #cargoDescription(registryArg: string, nameArg: string, versionArg: string, signalArg: AbortSignal | undefined): Promise<{ bytes: Buffer; mediaType: string }> {
|
|
713
|
+
let seen = 0;
|
|
714
|
+
for (let page = 1; page <= cargoSearchMaxPages; page++) {
|
|
715
|
+
const url = new URL(`${registryArg}/cargo/api/v1/crates`);
|
|
716
|
+
url.searchParams.set('q', nameArg);
|
|
717
|
+
url.searchParams.set('page', String(page));
|
|
718
|
+
url.searchParams.set('per_page', String(cargoSearchPageSize));
|
|
719
|
+
const body = validation.record((await this.#client.request(url, { signal: signalArg })).body);
|
|
720
|
+
if (!Array.isArray(body.crates)) throw new validation.ForgeInventoryError('invalid-cargo-search');
|
|
721
|
+
const total = validation.record(body.meta).total;
|
|
722
|
+
if (typeof total !== 'number' || !Number.isSafeInteger(total) || total < 0) throw new validation.ForgeInventoryError('invalid-cargo-search');
|
|
723
|
+
for (const entry of body.crates) {
|
|
724
|
+
const crate = validation.record(entry);
|
|
725
|
+
if (typeof crate.name !== 'string' || typeof crate.max_version !== 'string' || typeof crate.description !== 'string') {
|
|
726
|
+
throw new validation.ForgeInventoryError('invalid-cargo-search');
|
|
727
|
+
}
|
|
728
|
+
if (crate.name !== nameArg) continue;
|
|
729
|
+
if (crate.max_version !== versionArg) throw new validation.ForgeInventoryError('package-document-missing');
|
|
730
|
+
return { bytes: Buffer.from(JSON.stringify({ description: crate.description }), 'utf8'), mediaType: 'application/json' };
|
|
731
|
+
}
|
|
732
|
+
seen += body.crates.length;
|
|
733
|
+
if (body.crates.length === 0 || seen >= total) throw new validation.ForgeInventoryError('package-document-missing');
|
|
734
|
+
}
|
|
735
|
+
throw new validation.ForgeInventoryError('cargo-search-over-bound');
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* Streams one download verified against `contentArg`: the response must be a
|
|
740
|
+
* 200 whose `content-length`, when sent, is the size; the last chunk is held
|
|
741
|
+
* back until the size and SHA-256 match and `identifiedArg`, when given, has
|
|
742
|
+
* passed once more.
|
|
743
|
+
*/
|
|
744
|
+
async #verifiedDownload(
|
|
745
|
+
urlArg: URL, headersArg: Record<string, string>, contentArg: { sha256: string; size: string; bytes: number }, signalArg: AbortSignal | undefined,
|
|
746
|
+
identifiedArg: ((signalArg: AbortSignal) => Promise<void>) | null,
|
|
747
|
+
): Promise<InstanceType<typeof plugins.stream.Readable>> {
|
|
748
|
+
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
749
|
+
const agents = transport.createNodeAgents(this.#options.caCertificates);
|
|
750
|
+
const close = () => agents.https?.destroy();
|
|
751
|
+
const transfer = createTransferBounds(signalArg, {
|
|
752
|
+
limitMs: this.#options.packageTransferTimeoutMs, idleMs: this.#options.requestTimeoutMs,
|
|
753
|
+
timedOutCode: 'package-content-transfer-timed-out', failedCode: 'package-content-transfer-failed',
|
|
754
|
+
});
|
|
755
|
+
let body: AsyncIterable<Uint8Array>;
|
|
756
|
+
try {
|
|
757
|
+
const response = await transfer.run(() => transport.sendNodeRequest(urlArg, { method: 'GET', headers: headersArg, signal: transfer.signal, agents }));
|
|
758
|
+
if (response.status !== 200 || !response.body) {
|
|
759
|
+
await transfer.run(() => discardBody(response.body));
|
|
760
|
+
throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
761
|
+
}
|
|
762
|
+
const length = response.headers.get('content-length');
|
|
763
|
+
if (length !== null && length.trim() !== contentArg.size) {
|
|
764
|
+
await transfer.run(() => discardBody(response.body));
|
|
765
|
+
throw new validation.ForgeInventoryError('package-content-size-mismatch');
|
|
766
|
+
}
|
|
767
|
+
body = response.body;
|
|
768
|
+
} catch (error) {
|
|
769
|
+
const failure = transfer.failure(error, 'package-content-download-failed');
|
|
770
|
+
transfer.end();
|
|
771
|
+
close();
|
|
772
|
+
throw failure;
|
|
773
|
+
}
|
|
774
|
+
const hash = plugins.crypto.createHash('sha256');
|
|
775
|
+
const complete = async () => {
|
|
776
|
+
if (hash.digest('hex') !== contentArg.sha256) throw new validation.ForgeInventoryError('package-content-digest-mismatch');
|
|
777
|
+
if (identifiedArg) {
|
|
778
|
+
transfer.touch();
|
|
779
|
+
await transfer.run(() => identifiedArg(transfer.signal));
|
|
780
|
+
}
|
|
781
|
+
};
|
|
782
|
+
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
783
|
+
try {
|
|
784
|
+
yield* heldBackChunks(transfer.watch(body), contentArg.bytes, packageContentCodes, { inspect: (chunkArg) => hash.update(chunkArg), complete });
|
|
785
|
+
} catch (error) {
|
|
786
|
+
throw transfer.failure(error, 'package-content-download-failed');
|
|
787
|
+
} finally {
|
|
788
|
+
transfer.end();
|
|
789
|
+
}
|
|
790
|
+
};
|
|
791
|
+
return createTransferStream(transfer, chunks(), close);
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
/**
|
|
795
|
+
* Reads the version by Gitea's v1 package API: it must still carry the
|
|
796
|
+
* recorded version id, owner id, type and name, else
|
|
797
|
+
* `package-version-identity-changed`; a deleted version answers `http-404`.
|
|
798
|
+
*/
|
|
799
|
+
async #versionIdentified(packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, signalArg: AbortSignal | undefined): Promise<void> {
|
|
800
|
+
let record: Record<string, unknown>;
|
|
801
|
+
try {
|
|
802
|
+
record = validation.record((await this.#client.request(new URL(`${this.#options.baseUrl}/api/v1/packages/${encodeURIComponent(packageArg.owner)}/${
|
|
803
|
+
encodeURIComponent(packageArg.type)}/${encodeURIComponent(packageArg.name)}/${encodeURIComponent(versionArg.version)}`), { signal: signalArg })).body);
|
|
804
|
+
} catch (error) {
|
|
805
|
+
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
806
|
+
throw error;
|
|
807
|
+
}
|
|
808
|
+
if (validation.decimal(record.id) !== versionArg.id || validation.decimal(validation.record(record.owner).id) !== packageArg.ownerId
|
|
809
|
+
|| record.type !== packageArg.type || record.name !== packageArg.name || record.version !== versionArg.version) {
|
|
810
|
+
throw new validation.ForgeInventoryError('package-version-identity-changed');
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
/** Rejects with `errorArg` unless the version's identity read explains it: then with `package-version-identity-changed` or `http-404`. */
|
|
815
|
+
async #versionExplained(
|
|
816
|
+
packageArg: TGiteaPackageSelector, versionArg: TGiteaPackageVersionSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
|
|
817
|
+
): Promise<never> {
|
|
818
|
+
try {
|
|
819
|
+
await this.#versionIdentified(packageArg, versionArg, signalArg);
|
|
820
|
+
} catch (error) {
|
|
821
|
+
if (error instanceof validation.ForgeInventoryError && (error.code === 'package-version-identity-changed' || error.code === 'http-404')) throw error;
|
|
822
|
+
}
|
|
823
|
+
throw errorArg;
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
#packageSelector(packageArg: TGiteaPackageSelector): TGiteaPackageSelector {
|
|
827
|
+
try {
|
|
828
|
+
const segments = validation.segments(packageArg?.owner);
|
|
829
|
+
if (segments.length !== 1 || typeof packageArg.type !== 'string' || !packageTypePattern.test(packageArg.type)
|
|
830
|
+
|| typeof packageArg.name !== 'string' || !packageArg.name || packageArg.name.length > 512 || /[\u0000-\u001f\u007f]/.test(packageArg.name)) throw new Error();
|
|
831
|
+
return { owner: segments[0], ownerId: validation.decimal(packageArg.ownerId), type: packageArg.type, name: packageArg.name };
|
|
832
|
+
} catch {
|
|
833
|
+
throw new validation.ForgeInventoryError('package-invalid');
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
/** A container package selector whose name is a registry image name, which `/v2` paths carry unencoded. */
|
|
838
|
+
#containerPackage(packageArg: TGiteaPackageSelector): TGiteaPackageSelector {
|
|
839
|
+
const selected = this.#packageSelector(packageArg);
|
|
840
|
+
if (selected.type !== 'container') throw new validation.ForgeInventoryError('package-type-unsupported');
|
|
841
|
+
if (!/^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*$/.test(selected.name)) {
|
|
842
|
+
throw new validation.ForgeInventoryError('package-invalid');
|
|
843
|
+
}
|
|
844
|
+
return selected;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
#versionSelector(versionArg: TGiteaPackageVersionSelector): TGiteaPackageVersionSelector {
|
|
848
|
+
try {
|
|
849
|
+
if (typeof versionArg?.version !== 'string' || !versionArg.version || /[\u0000-\u001f\u007f]/.test(versionArg.version)) throw new Error();
|
|
850
|
+
return { id: validation.decimal(versionArg.id), version: versionArg.version };
|
|
851
|
+
} catch {
|
|
852
|
+
throw new validation.ForgeInventoryError('package-version-invalid');
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/** Content as recorded; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
|
|
857
|
+
#content(contentArg: TGiteaPackageContentSelector): { sha256: string; size: string; bytes: number } {
|
|
858
|
+
try {
|
|
859
|
+
if (typeof contentArg?.sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(contentArg.sha256)) throw new Error();
|
|
860
|
+
const size = validation.decimal(contentArg.size);
|
|
861
|
+
const bytes = Number(size);
|
|
862
|
+
if (!Number.isSafeInteger(bytes)) throw new Error();
|
|
863
|
+
return { sha256: contentArg.sha256, size, bytes };
|
|
864
|
+
} catch {
|
|
865
|
+
throw new validation.ForgeInventoryError('package-content-invalid');
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
|
|
345
869
|
/**
|
|
346
870
|
* Reads the repository by its id and checks that Gitea still names it
|
|
347
871
|
* `owner/name`, without case as Gitea looks names up: other names reject
|