@foss.global/codefeed 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +41 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.d.ts +38 -16
- package/dist_ts/classes.giteasourceinventory.js +537 -79
- package/dist_ts/classes.giteasourcetransfer.js +8 -103
- package/dist_ts/forgeinventory.transfer.d.ts +24 -0
- package/dist_ts/forgeinventory.transfer.js +77 -1
- package/dist_ts/giteasourceinventory.packages.d.ts +49 -0
- package/dist_ts/giteasourceinventory.packages.js +180 -1
- package/dist_ts/giteasourcetransfer.documents.d.ts +19 -4
- package/dist_ts/giteasourcetransfer.documents.js +49 -5
- package/package.json +4 -4
- package/readme.md +28 -4
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +598 -83
- package/ts/classes.giteasourcetransfer.ts +10 -104
- package/ts/forgeinventory.transfer.ts +88 -0
- package/ts/giteasourceinventory.packages.ts +168 -0
- package/ts/giteasourcetransfer.documents.ts +47 -4
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
-
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks, verifiedPackageDownload } from './forgeinventory.transfer.js';
|
|
5
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
6
6
|
import { LfsBatchClient } from './classes.lfsbatchclient.js';
|
|
7
7
|
import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
|
|
8
8
|
import { giteaCloneUrl, reserveMirrorScratch } from './giteasourceinventory.git.js';
|
|
9
|
-
import {
|
|
10
|
-
|
|
9
|
+
import {
|
|
10
|
+
cargoIndexPath, containerManifestMediaTypes, npmPackagePath, packageTypePattern, readSwiftManifest, swiftPackageParts,
|
|
11
|
+
} from './giteasourceinventory.packages.js';
|
|
12
|
+
import { canonicalPackageDocument, isSwiftManifestVersion } from './giteasourcetransfer.documents.js';
|
|
11
13
|
import type { TGiteaCanonicalPackageDocumentKind } from './giteasourcetransfer.documents.js';
|
|
12
14
|
|
|
13
15
|
type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
|
|
@@ -164,10 +166,6 @@ export interface IGiteaPackageDownload {
|
|
|
164
166
|
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
165
167
|
}
|
|
166
168
|
|
|
167
|
-
const packageContentCodes = {
|
|
168
|
-
invalidChunk: 'package-content-invalid-chunk', overrun: 'package-content-size-overrun', short: 'package-content-short-body',
|
|
169
|
-
};
|
|
170
|
-
|
|
171
169
|
const sha256Hex = (bytesArg: Uint8Array): string => plugins.crypto.createHash('sha256').update(bytesArg).digest('hex');
|
|
172
170
|
|
|
173
171
|
/** Entries asked of one Cargo search page; Gitea serves at most its `MAX_RESPONSE_ITEMS` (50 by default). */
|
|
@@ -501,7 +499,8 @@ export class GiteaSourceTransfer {
|
|
|
501
499
|
): Promise<IGiteaPackageDownload> {
|
|
502
500
|
const selected = this.#containerPackage(packageArg);
|
|
503
501
|
const content = this.#content(contentArg);
|
|
504
|
-
const stream = await
|
|
502
|
+
const stream = await verifiedPackageDownload(
|
|
503
|
+
this.#options,
|
|
505
504
|
new URL(`/v2/${selected.owner.toLowerCase()}/${selected.name}/blobs/sha256:${content.sha256}`, this.#options.baseUrl),
|
|
506
505
|
{ authorization: this.#basicAuthorization() }, content, optionsArg.signal, null,
|
|
507
506
|
);
|
|
@@ -554,7 +553,7 @@ export class GiteaSourceTransfer {
|
|
|
554
553
|
await this.#versionIdentified(selected, version, signal);
|
|
555
554
|
let stream: InstanceType<typeof plugins.stream.Readable>;
|
|
556
555
|
try {
|
|
557
|
-
stream = await this.#
|
|
556
|
+
stream = await verifiedPackageDownload(this.#options, url, headers, content, signal, (signalArg) => this.#versionIdentified(selected, version, signalArg));
|
|
558
557
|
} catch (error) {
|
|
559
558
|
if (signal?.aborted) throw error;
|
|
560
559
|
return this.#versionExplained(selected, version, error, signal);
|
|
@@ -607,7 +606,7 @@ export class GiteaSourceTransfer {
|
|
|
607
606
|
const { signal, swiftVersion } = optionsArg;
|
|
608
607
|
if (swiftVersion !== undefined) {
|
|
609
608
|
if (kindArg !== 'swift-manifest') throw new validation.ForgeInventoryError('package-document-kind-invalid');
|
|
610
|
-
if (
|
|
609
|
+
if (!isSwiftManifestVersion(swiftVersion)) throw new validation.ForgeInventoryError('swift-version-invalid');
|
|
611
610
|
}
|
|
612
611
|
const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(selected.owner)}`;
|
|
613
612
|
const encodedVersion = encodeURIComponent(version.version);
|
|
@@ -620,7 +619,7 @@ export class GiteaSourceTransfer {
|
|
|
620
619
|
read = (signalArg) => this.#documentBytes(release, 'application/vnd.swift.registry.v1+json', 'application/json', signalArg);
|
|
621
620
|
} else if (kindArg === 'swift-manifest' && selected.type === 'swift') {
|
|
622
621
|
const parts = swiftPackageParts(selected.name);
|
|
623
|
-
read = (signalArg) => this.#
|
|
622
|
+
read = (signalArg) => readSwiftManifest(this.#client, this.#options.baseUrl, selected.owner, parts, version.version, swiftVersion ?? null, signalArg);
|
|
624
623
|
} else if (kindArg === 'cargo-index-entry' && selected.type === 'cargo') {
|
|
625
624
|
const url = new URL(`${registry}/cargo/${cargoIndexPath(selected.name)}`);
|
|
626
625
|
read = async (signalArg) => {
|
|
@@ -666,43 +665,6 @@ export class GiteaSourceTransfer {
|
|
|
666
665
|
return { bytes: response.bytes, mediaType: mediaTypeArg };
|
|
667
666
|
}
|
|
668
667
|
|
|
669
|
-
/**
|
|
670
|
-
* A Swift release's `Package.swift`, or with `swiftVersionArg` its
|
|
671
|
-
* alternate manifest; null when Gitea redirects the alternate one to the
|
|
672
|
-
* release's own `Package.swift` (303), which is how it answers a version it
|
|
673
|
-
* holds no manifest for. No redirect is followed.
|
|
674
|
-
*/
|
|
675
|
-
async #swiftManifest(
|
|
676
|
-
packageArg: TGiteaPackageSelector, partsArg: { scope: string; name: string }, versionArg: TGiteaPackageVersionSelector, swiftVersionArg: string | null,
|
|
677
|
-
signalArg: AbortSignal | undefined,
|
|
678
|
-
): Promise<{ bytes: Buffer; mediaType: string } | null> {
|
|
679
|
-
const release = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(packageArg.owner)}/swift/${encodeURIComponent(partsArg.scope)}/${
|
|
680
|
-
encodeURIComponent(partsArg.name)}/${encodeURIComponent(versionArg.version)}/Package.swift`;
|
|
681
|
-
const url = new URL(release);
|
|
682
|
-
if (swiftVersionArg !== null) url.searchParams.set('swift-version', swiftVersionArg);
|
|
683
|
-
const response = await this.#client.requestBytesOrRedirect(url, { accept: 'application/vnd.swift.registry.v1+swift', signal: signalArg });
|
|
684
|
-
if (response.kind === 'redirect') {
|
|
685
|
-
if (swiftVersionArg === null) throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
686
|
-
// Gitea's answer for a missing alternate manifest: 303 to the release's
|
|
687
|
-
// `Package.swift` under its configured root, with the owner lowercased
|
|
688
|
-
// and scope, name and version as the request named them.
|
|
689
|
-
const expected = new URL(`${this.#options.baseUrl}/api/packages/${packageArg.owner.toLowerCase()}/swift/${partsArg.scope}/${partsArg.name}/${
|
|
690
|
-
versionArg.version}/Package.swift`);
|
|
691
|
-
let location: URL | null = null;
|
|
692
|
-
try {
|
|
693
|
-
location = response.location === null ? null : new URL(response.location, url);
|
|
694
|
-
} catch {
|
|
695
|
-
location = null;
|
|
696
|
-
}
|
|
697
|
-
if (response.status !== 303 || location === null || location.href !== expected.href) {
|
|
698
|
-
throw new validation.ForgeInventoryError('swift-manifest-redirect-unexpected');
|
|
699
|
-
}
|
|
700
|
-
return null;
|
|
701
|
-
}
|
|
702
|
-
if (response.mediaType !== 'text/x-swift') throw new validation.ForgeInventoryError('invalid-content-type');
|
|
703
|
-
return { bytes: response.bytes, mediaType: 'text/x-swift' };
|
|
704
|
-
}
|
|
705
|
-
|
|
706
668
|
/**
|
|
707
669
|
* `{"description":…}` of the crate's entry in the Cargo search, read page by
|
|
708
670
|
* page until the entry with exactly `nameArg` turns up. Gitea's search
|
|
@@ -735,62 +697,6 @@ export class GiteaSourceTransfer {
|
|
|
735
697
|
throw new validation.ForgeInventoryError('cargo-search-over-bound');
|
|
736
698
|
}
|
|
737
699
|
|
|
738
|
-
/**
|
|
739
|
-
* Streams one download verified against `contentArg`: the response must be a
|
|
740
|
-
* 200 whose `content-length`, when sent, is the size; the last chunk is held
|
|
741
|
-
* back until the size and SHA-256 match and `identifiedArg`, when given, has
|
|
742
|
-
* passed once more.
|
|
743
|
-
*/
|
|
744
|
-
async #verifiedDownload(
|
|
745
|
-
urlArg: URL, headersArg: Record<string, string>, contentArg: { sha256: string; size: string; bytes: number }, signalArg: AbortSignal | undefined,
|
|
746
|
-
identifiedArg: ((signalArg: AbortSignal) => Promise<void>) | null,
|
|
747
|
-
): Promise<InstanceType<typeof plugins.stream.Readable>> {
|
|
748
|
-
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
749
|
-
const agents = transport.createNodeAgents(this.#options.caCertificates);
|
|
750
|
-
const close = () => agents.https?.destroy();
|
|
751
|
-
const transfer = createTransferBounds(signalArg, {
|
|
752
|
-
limitMs: this.#options.packageTransferTimeoutMs, idleMs: this.#options.requestTimeoutMs,
|
|
753
|
-
timedOutCode: 'package-content-transfer-timed-out', failedCode: 'package-content-transfer-failed',
|
|
754
|
-
});
|
|
755
|
-
let body: AsyncIterable<Uint8Array>;
|
|
756
|
-
try {
|
|
757
|
-
const response = await transfer.run(() => transport.sendNodeRequest(urlArg, { method: 'GET', headers: headersArg, signal: transfer.signal, agents }));
|
|
758
|
-
if (response.status !== 200 || !response.body) {
|
|
759
|
-
await transfer.run(() => discardBody(response.body));
|
|
760
|
-
throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
761
|
-
}
|
|
762
|
-
const length = response.headers.get('content-length');
|
|
763
|
-
if (length !== null && length.trim() !== contentArg.size) {
|
|
764
|
-
await transfer.run(() => discardBody(response.body));
|
|
765
|
-
throw new validation.ForgeInventoryError('package-content-size-mismatch');
|
|
766
|
-
}
|
|
767
|
-
body = response.body;
|
|
768
|
-
} catch (error) {
|
|
769
|
-
const failure = transfer.failure(error, 'package-content-download-failed');
|
|
770
|
-
transfer.end();
|
|
771
|
-
close();
|
|
772
|
-
throw failure;
|
|
773
|
-
}
|
|
774
|
-
const hash = plugins.crypto.createHash('sha256');
|
|
775
|
-
const complete = async () => {
|
|
776
|
-
if (hash.digest('hex') !== contentArg.sha256) throw new validation.ForgeInventoryError('package-content-digest-mismatch');
|
|
777
|
-
if (identifiedArg) {
|
|
778
|
-
transfer.touch();
|
|
779
|
-
await transfer.run(() => identifiedArg(transfer.signal));
|
|
780
|
-
}
|
|
781
|
-
};
|
|
782
|
-
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
783
|
-
try {
|
|
784
|
-
yield* heldBackChunks(transfer.watch(body), contentArg.bytes, packageContentCodes, { inspect: (chunkArg) => hash.update(chunkArg), complete });
|
|
785
|
-
} catch (error) {
|
|
786
|
-
throw transfer.failure(error, 'package-content-download-failed');
|
|
787
|
-
} finally {
|
|
788
|
-
transfer.end();
|
|
789
|
-
}
|
|
790
|
-
};
|
|
791
|
-
return createTransferStream(transfer, chunks(), close);
|
|
792
|
-
}
|
|
793
|
-
|
|
794
700
|
/**
|
|
795
701
|
* Reads the version by Gitea's v1 package API: it must still carry the
|
|
796
702
|
* recorded version id, owner id, type and name, else
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
|
+
import * as transport from './forgeinventory.transport.js';
|
|
2
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
+
import type { IForgeRequestMetrics } from './forgeinventory.client.js';
|
|
3
5
|
|
|
4
6
|
/** Response bytes read from an unused answer before the rest is discarded. */
|
|
5
7
|
const maxDiscardedResponseBytes = 64 * 1024;
|
|
@@ -201,3 +203,89 @@ export const createTransferStream = (
|
|
|
201
203
|
});
|
|
202
204
|
return stream;
|
|
203
205
|
};
|
|
206
|
+
|
|
207
|
+
/** Passes `sourceArg`'s chunks on, adding each one's bytes to `metricsArg` as it arrives. */
|
|
208
|
+
async function* countedBytes(sourceArg: AsyncIterable<Uint8Array>, metricsArg: IForgeRequestMetrics): AsyncGenerator<Uint8Array, void, undefined> {
|
|
209
|
+
for await (const chunk of sourceArg) {
|
|
210
|
+
if (chunk instanceof Uint8Array) metricsArg.responseBytes += chunk.byteLength;
|
|
211
|
+
yield chunk;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** The `heldBackChunks` codes of a package content download. */
|
|
216
|
+
const packageContentCodes = {
|
|
217
|
+
invalidChunk: 'package-content-invalid-chunk', overrun: 'package-content-size-overrun', short: 'package-content-short-body',
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
/** The reader settings a verified package content download runs with. */
|
|
221
|
+
export interface IVerifiedPackageDownloadOptions {
|
|
222
|
+
/** PEM certificates that replace the default trust store for the download's connection. */
|
|
223
|
+
caCertificates?: string[];
|
|
224
|
+
/** Wall-clock limit of the whole download, the closing `identifiedArg` read included. */
|
|
225
|
+
packageTransferTimeoutMs: number;
|
|
226
|
+
/** Idle limit of the download, re-armed on every chunk and before the closing `identifiedArg` read. */
|
|
227
|
+
requestTimeoutMs: number;
|
|
228
|
+
/** Counts the request, a 429 answer and every body byte that arrives, as `ForgeRequestClient` counts its reads. */
|
|
229
|
+
metrics?: IForgeRequestMetrics;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Streams one package content download verified against `contentArg`, sent
|
|
234
|
+
* over node:https with the caller's CA certificates: the response must be a
|
|
235
|
+
* 200 whose `content-length`, when sent, is the size; the last chunk is held
|
|
236
|
+
* back until the size and SHA-256 match and `identifiedArg`, when given, has
|
|
237
|
+
* passed once more.
|
|
238
|
+
*/
|
|
239
|
+
export const verifiedPackageDownload = async (
|
|
240
|
+
optionsArg: IVerifiedPackageDownloadOptions, urlArg: URL, headersArg: Record<string, string>,
|
|
241
|
+
contentArg: { sha256: string; size: string; bytes: number }, signalArg: AbortSignal | undefined,
|
|
242
|
+
identifiedArg: ((signalArg: AbortSignal) => Promise<void>) | null,
|
|
243
|
+
): Promise<InstanceType<typeof plugins.stream.Readable>> => {
|
|
244
|
+
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
245
|
+
const agents = transport.createNodeAgents(optionsArg.caCertificates);
|
|
246
|
+
const close = () => agents.https?.destroy();
|
|
247
|
+
const transfer = createTransferBounds(signalArg, {
|
|
248
|
+
limitMs: optionsArg.packageTransferTimeoutMs, idleMs: optionsArg.requestTimeoutMs,
|
|
249
|
+
timedOutCode: 'package-content-transfer-timed-out', failedCode: 'package-content-transfer-failed',
|
|
250
|
+
});
|
|
251
|
+
const { metrics } = optionsArg;
|
|
252
|
+
let body: AsyncIterable<Uint8Array>;
|
|
253
|
+
try {
|
|
254
|
+
if (metrics) metrics.requests++;
|
|
255
|
+
const response = await transfer.run(() => transport.sendNodeRequest(urlArg, { method: 'GET', headers: headersArg, signal: transfer.signal, agents }));
|
|
256
|
+
if (metrics && response.status === 429) metrics.rateLimitedRequests++;
|
|
257
|
+
if (response.status !== 200 || !response.body) {
|
|
258
|
+
await transfer.run(() => discardBody(response.body));
|
|
259
|
+
throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
260
|
+
}
|
|
261
|
+
const length = response.headers.get('content-length');
|
|
262
|
+
if (length !== null && length.trim() !== contentArg.size) {
|
|
263
|
+
await transfer.run(() => discardBody(response.body));
|
|
264
|
+
throw new validation.ForgeInventoryError('package-content-size-mismatch');
|
|
265
|
+
}
|
|
266
|
+
body = metrics ? countedBytes(response.body, metrics) : response.body;
|
|
267
|
+
} catch (error) {
|
|
268
|
+
const failure = transfer.failure(error, 'package-content-download-failed');
|
|
269
|
+
transfer.end();
|
|
270
|
+
close();
|
|
271
|
+
throw failure;
|
|
272
|
+
}
|
|
273
|
+
const hash = plugins.crypto.createHash('sha256');
|
|
274
|
+
const complete = async () => {
|
|
275
|
+
if (hash.digest('hex') !== contentArg.sha256) throw new validation.ForgeInventoryError('package-content-digest-mismatch');
|
|
276
|
+
if (identifiedArg) {
|
|
277
|
+
transfer.touch();
|
|
278
|
+
await transfer.run(() => identifiedArg(transfer.signal));
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
282
|
+
try {
|
|
283
|
+
yield* heldBackChunks(transfer.watch(body), contentArg.bytes, packageContentCodes, { inspect: (chunkArg) => hash.update(chunkArg), complete });
|
|
284
|
+
} catch (error) {
|
|
285
|
+
throw transfer.failure(error, 'package-content-download-failed');
|
|
286
|
+
} finally {
|
|
287
|
+
transfer.end();
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
return createTransferStream(transfer, chunks(), close);
|
|
291
|
+
};
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as validation from './forgeinventory.validation.js';
|
|
3
|
+
import type { ForgeRequestClient } from './forgeinventory.client.js';
|
|
4
|
+
import { swiftManifestKey } from './giteasourcetransfer.documents.js';
|
|
3
5
|
|
|
4
6
|
type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
|
|
5
7
|
type IPackageRef = plugins.fossInterfaces.data.IForgeSourcePackageRef;
|
|
@@ -64,6 +66,172 @@ export const swiftPackageParts = (nameArg: string): { scope: string; name: strin
|
|
|
64
66
|
return { scope: match[1], name: match[2] };
|
|
65
67
|
};
|
|
66
68
|
|
|
69
|
+
/**
|
|
70
|
+
* A Swift release's `Package.swift`, or with `swiftVersionArg` its alternate
|
|
71
|
+
* manifest, read with `clientArg`'s `token` authorization; null when Gitea
|
|
72
|
+
* redirects the alternate one to the release's own `Package.swift` (303),
|
|
73
|
+
* which is how it answers a version it holds no manifest for. No redirect is
|
|
74
|
+
* followed: any other redirect rejects with `swift-manifest-redirect-unexpected`,
|
|
75
|
+
* a redirect of `Package.swift` itself with `http-<status>`, and a body that
|
|
76
|
+
* is not `text/x-swift` with `invalid-content-type`. The read counts no
|
|
77
|
+
* download.
|
|
78
|
+
*/
|
|
79
|
+
export const readSwiftManifest = async (
|
|
80
|
+
clientArg: ForgeRequestClient, baseUrlArg: string, ownerArg: string, partsArg: { scope: string; name: string }, versionArg: string,
|
|
81
|
+
swiftVersionArg: string | null, signalArg: AbortSignal | undefined,
|
|
82
|
+
): Promise<{ bytes: Buffer; mediaType: string } | null> => {
|
|
83
|
+
const url = new URL(`${baseUrlArg}/api/packages/${encodeURIComponent(ownerArg)}/swift/${encodeURIComponent(partsArg.scope)}/${
|
|
84
|
+
encodeURIComponent(partsArg.name)}/${encodeURIComponent(versionArg)}/Package.swift`);
|
|
85
|
+
if (swiftVersionArg !== null) url.searchParams.set('swift-version', swiftVersionArg);
|
|
86
|
+
const response = await clientArg.requestBytesOrRedirect(url, { accept: 'application/vnd.swift.registry.v1+swift', signal: signalArg });
|
|
87
|
+
if (response.kind === 'redirect') {
|
|
88
|
+
if (swiftVersionArg === null) throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
89
|
+
// Gitea's answer for a missing alternate manifest: 303 to the release's
|
|
90
|
+
// `Package.swift` under its configured root, with the owner lowercased
|
|
91
|
+
// and scope, name and version as the request named them.
|
|
92
|
+
const expected = new URL(`${baseUrlArg}/api/packages/${ownerArg.toLowerCase()}/swift/${partsArg.scope}/${partsArg.name}/${versionArg}/Package.swift`);
|
|
93
|
+
let location: URL | null = null;
|
|
94
|
+
try {
|
|
95
|
+
location = response.location === null ? null : new URL(response.location, url);
|
|
96
|
+
} catch {
|
|
97
|
+
location = null;
|
|
98
|
+
}
|
|
99
|
+
if (response.status !== 303 || location === null || location.href !== expected.href) {
|
|
100
|
+
throw new validation.ForgeInventoryError('swift-manifest-redirect-unexpected');
|
|
101
|
+
}
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
if (response.mediaType !== 'text/x-swift') throw new validation.ForgeInventoryError('invalid-content-type');
|
|
105
|
+
return { bytes: response.bytes, mediaType: 'text/x-swift' };
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
/** Distinct manifest keys read per Swift version, `''` included; Gitea keeps at most 64 manifests (`maxManifestFiles`). */
|
|
109
|
+
const maxSwiftManifestCandidates = 64;
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* The name of every central-directory entry of a zip archive, in directory
|
|
113
|
+
* order, each byte one code unit (Go strings are bytes, and only ASCII names
|
|
114
|
+
* can match `manifestPattern`). It reads the directory as Go 1.27.2's
|
|
115
|
+
* `archive/zip` `NewReader` does, the archive/zip Gitea 28.2.0 builds with:
|
|
116
|
+
* - the end record is the last signature in the final 65 KiB whose comment
|
|
117
|
+
* fits in the archive;
|
|
118
|
+
* - its zip64 locator and end record are used when a count, size or offset
|
|
119
|
+
* is saturated and a locator for disk 0 of 1 sits right before it;
|
|
120
|
+
* - the base offset of data prepended to the archive is the end record's
|
|
121
|
+
* position minus the directory's size and offset, unless a directory
|
|
122
|
+
* header sits at the unadjusted offset;
|
|
123
|
+
* - headers are read one after another until one is malformed or cut off
|
|
124
|
+
* (a zip64 field too short for the saturated sizes and offset it stands
|
|
125
|
+
* for is malformed), and their count must agree with the end record's in
|
|
126
|
+
* its low 16 bits.
|
|
127
|
+
* Anything Go refuses rejects with `swift-archive-invalid`.
|
|
128
|
+
*/
|
|
129
|
+
export const zipEntryNames = (archiveArg: Buffer): string[] => {
|
|
130
|
+
const invalid = () => new validation.ForgeInventoryError('swift-archive-invalid');
|
|
131
|
+
const size = archiveArg.byteLength;
|
|
132
|
+
const sizeBig = BigInt(size);
|
|
133
|
+
let endOffset = -1;
|
|
134
|
+
for (let index = size - 22; index >= Math.max(0, size - 65 * 1024); index--) {
|
|
135
|
+
if (archiveArg.readUInt32LE(index) === 0x06054b50) {
|
|
136
|
+
endOffset = index;
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (endOffset < 0 || endOffset + 22 + archiveArg.readUInt16LE(endOffset + 20) > size) throw invalid();
|
|
141
|
+
let directoryEnd = BigInt(endOffset);
|
|
142
|
+
let records = BigInt(archiveArg.readUInt16LE(endOffset + 10));
|
|
143
|
+
let directorySize = BigInt(archiveArg.readUInt32LE(endOffset + 12));
|
|
144
|
+
let directoryOffset = BigInt(archiveArg.readUInt32LE(endOffset + 16));
|
|
145
|
+
if (records === 0xffffn || directorySize === 0xffffffffn || directoryOffset === 0xffffffffn) {
|
|
146
|
+
const locator = endOffset - 20;
|
|
147
|
+
if (locator >= 0 && archiveArg.readUInt32LE(locator) === 0x07064b50 && archiveArg.readUInt32LE(locator + 4) === 0
|
|
148
|
+
&& archiveArg.readUInt32LE(locator + 16) === 1) {
|
|
149
|
+
// Go reads the offset as an int64: one past its range is negative and leaves the archive as zip32.
|
|
150
|
+
const record = archiveArg.readBigInt64LE(locator + 8);
|
|
151
|
+
if (record >= 0n) {
|
|
152
|
+
if (record + 56n > sizeBig || archiveArg.readUInt32LE(Number(record)) !== 0x06064b50) throw invalid();
|
|
153
|
+
directoryEnd = record;
|
|
154
|
+
records = archiveArg.readBigUInt64LE(Number(record) + 32);
|
|
155
|
+
directorySize = archiveArg.readBigUInt64LE(Number(record) + 40);
|
|
156
|
+
directoryOffset = archiveArg.readBigUInt64LE(Number(record) + 48);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (directorySize > 0x7fffffffffffffffn || directoryOffset > 0x7fffffffffffffffn) throw invalid();
|
|
161
|
+
let baseOffset = directoryEnd - directorySize - directoryOffset;
|
|
162
|
+
if (baseOffset + directoryOffset < 0n || baseOffset + directoryOffset >= sizeBig) throw invalid();
|
|
163
|
+
if (baseOffset > 0n && directoryHeader(archiveArg, Number(directoryOffset)).kind === 'entry') baseOffset = 0n;
|
|
164
|
+
const names: string[] = [];
|
|
165
|
+
let offset = Number(baseOffset + directoryOffset);
|
|
166
|
+
while (true) {
|
|
167
|
+
const header = directoryHeader(archiveArg, offset);
|
|
168
|
+
if (header.kind === 'end') break;
|
|
169
|
+
// Go's `io.ReadFull` reports a read that gets no byte at all as `io.EOF`, which fails the archive.
|
|
170
|
+
if (header.kind === 'eof') throw invalid();
|
|
171
|
+
names.push(header.name);
|
|
172
|
+
offset = header.next;
|
|
173
|
+
}
|
|
174
|
+
if ((BigInt(names.length) & 0xffffn) !== (records & 0xffffn)) throw invalid();
|
|
175
|
+
return names;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The central-directory header at `offsetArg`, as Go 1.27.2's
|
|
180
|
+
* `readDirectoryHeader` reads it: `entry` with its name and the offset of the
|
|
181
|
+
* next header; `end` where Go stops reading headers (a header cut off by the
|
|
182
|
+
* archive's end, another signature, or a zip64 field shorter than 8 bytes for
|
|
183
|
+
* each saturated size and offset); `eof` where a read gets no byte at all.
|
|
184
|
+
*/
|
|
185
|
+
const directoryHeader = (archiveArg: Buffer, offsetArg: number):
|
|
186
|
+
{ kind: 'entry'; name: string; next: number } | { kind: 'end' } | { kind: 'eof' } => {
|
|
187
|
+
const available = archiveArg.byteLength - offsetArg;
|
|
188
|
+
if (available <= 0) return { kind: 'eof' };
|
|
189
|
+
if (available < 46) return { kind: 'end' };
|
|
190
|
+
if (archiveArg.readUInt32LE(offsetArg) !== 0x02014b50) return { kind: 'end' };
|
|
191
|
+
const nameLength = archiveArg.readUInt16LE(offsetArg + 28);
|
|
192
|
+
const extraLength = archiveArg.readUInt16LE(offsetArg + 30);
|
|
193
|
+
const commentLength = archiveArg.readUInt16LE(offsetArg + 32);
|
|
194
|
+
const variable = nameLength + extraLength + commentLength;
|
|
195
|
+
if (variable > 0 && available === 46) return { kind: 'eof' };
|
|
196
|
+
if (available - 46 < variable) return { kind: 'end' };
|
|
197
|
+
const saturated = [20, 24, 42].filter((fieldArg) => archiveArg.readUInt32LE(offsetArg + fieldArg) === 0xffffffff).length;
|
|
198
|
+
let extra = offsetArg + 46 + nameLength;
|
|
199
|
+
const extraEnd = extra + extraLength;
|
|
200
|
+
while (extraEnd - extra >= 4) {
|
|
201
|
+
const tag = archiveArg.readUInt16LE(extra);
|
|
202
|
+
const fieldSize = archiveArg.readUInt16LE(extra + 2);
|
|
203
|
+
extra += 4;
|
|
204
|
+
if (extraEnd - extra < fieldSize) break;
|
|
205
|
+
// The zip64 field holds 8 bytes for each saturated field, in order.
|
|
206
|
+
if (tag === 0x0001 && fieldSize < 8 * saturated) return { kind: 'end' };
|
|
207
|
+
extra += fieldSize;
|
|
208
|
+
}
|
|
209
|
+
return { kind: 'entry', name: archiveArg.toString('latin1', offsetArg + 46, offsetArg + 46 + nameLength), next: offsetArg + 46 + variable };
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* The keys of the manifests Gitea 28.2.0 may hold for a Swift source
|
|
214
|
+
* archive, sorted in code-unit order: `''`, and the key of every
|
|
215
|
+
* central-directory entry, in any directory, whose name does not end in `/`
|
|
216
|
+
* and whose base name matches Gitea's `manifestPattern` and parses
|
|
217
|
+
* (`swiftManifestKey`). Gitea keeps only the manifests of the shallowest
|
|
218
|
+
* directory, so the keys are a superset of what it holds, and its answer for
|
|
219
|
+
* each key decides. More than `maxSwiftManifestCandidates` distinct keys
|
|
220
|
+
* reject with `swift-manifest-candidates-over-bound`; an archive Go's
|
|
221
|
+
* `archive/zip` refuses with `swift-archive-invalid`.
|
|
222
|
+
*/
|
|
223
|
+
export const swiftManifestCandidates = (archiveArg: Buffer): string[] => {
|
|
224
|
+
const keys = new Set<string>(['']);
|
|
225
|
+
for (const name of zipEntryNames(archiveArg)) {
|
|
226
|
+
if (name.endsWith('/')) continue;
|
|
227
|
+
const key = swiftManifestKey(name.slice(name.lastIndexOf('/') + 1));
|
|
228
|
+
if (key === null) continue;
|
|
229
|
+
keys.add(key);
|
|
230
|
+
if (keys.size > maxSwiftManifestCandidates) throw new validation.ForgeInventoryError('swift-manifest-candidates-over-bound');
|
|
231
|
+
}
|
|
232
|
+
return [...keys].sort(compareText);
|
|
233
|
+
};
|
|
234
|
+
|
|
67
235
|
/**
|
|
68
236
|
* The digest of the manifest a container version holds: `sha256:` and the
|
|
69
237
|
* SHA-256 of the blob holding the version's `manifest.json`; null for a
|
|
@@ -90,8 +90,51 @@ export const canonicalPackageDocument = (kindArg: TGiteaCanonicalPackageDocument
|
|
|
90
90
|
};
|
|
91
91
|
|
|
92
92
|
/**
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
93
|
+
* The largest segment Gitea's version parser reads: hashicorp/go-version
|
|
94
|
+
* (v1.9.0 in Gitea 28.2.0) parses every segment as a signed 64-bit integer
|
|
95
|
+
* and refuses a larger one.
|
|
96
96
|
*/
|
|
97
|
-
|
|
97
|
+
const maxSwiftVersionSegment = 9223372036854775807n;
|
|
98
|
+
|
|
99
|
+
/** Major and minor, and an optional patch, in decimal without leading zeros. */
|
|
100
|
+
const trimmedSwiftVersionShape = /^(0|[1-9]\d*)\.(0|[1-9]\d*)(?:\.([1-9]\d*))?$/;
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Whether `valueArg` is a Swift version Gitea 28.2.0 can key an alternate
|
|
104
|
+
* manifest (`Package@swift-<v>.swift`) by. Gitea parses the file name's
|
|
105
|
+
* version with go-version's `NewSemver`, refuses an archive whose version
|
|
106
|
+
* does not parse (`ErrInvalidManifestVersion`), and keys the manifest by
|
|
107
|
+
* `TrimmedVersionString`: major and minor, and the patch only when it is not
|
|
108
|
+
* zero, each in decimal without leading zeros (`modules/packages/swift/metadata.go`).
|
|
109
|
+
* So a key has two or three segments, no leading zeros, no zero patch, and
|
|
110
|
+
* every segment at most 9223372036854775807, compared by value.
|
|
111
|
+
*/
|
|
112
|
+
export const isSwiftManifestVersion = (valueArg: unknown): valueArg is string => {
|
|
113
|
+
if (typeof valueArg !== 'string') return false;
|
|
114
|
+
const match = trimmedSwiftVersionShape.exec(valueArg);
|
|
115
|
+
if (!match) return false;
|
|
116
|
+
return match.slice(1).every((segmentArg) => segmentArg === undefined || BigInt(segmentArg) <= maxSwiftVersionSegment);
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
/** Gitea 28.2.0's `manifestPattern` (`modules/packages/swift/metadata.go:27`), matched against a file's base name. */
|
|
120
|
+
const swiftManifestNamePattern = /^Package(?:@swift-(\d+(?:\.\d+)?(?:\.\d+)?))?\.swift$/;
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The key Gitea 28.2.0 would hold a manifest with base name `baseNameArg`
|
|
124
|
+
* by: `''` for `Package.swift`; for `Package@swift-<v>.swift`, `<v>` parsed
|
|
125
|
+
* as go-version's `NewSemver` parses it (leading zeros allowed, every segment
|
|
126
|
+
* a signed 64-bit integer, missing segments zero) and written as
|
|
127
|
+
* `TrimmedVersionString` writes it, a key `isSwiftManifestVersion` accepts.
|
|
128
|
+
* Null when the name is not a manifest name, or when a segment is over int64,
|
|
129
|
+
* which `NewSemver` refuses, so Gitea can never hold the key.
|
|
130
|
+
*/
|
|
131
|
+
export const swiftManifestKey = (baseNameArg: string): string | null => {
|
|
132
|
+
const match = swiftManifestNamePattern.exec(baseNameArg);
|
|
133
|
+
if (!match) return null;
|
|
134
|
+
if (match[1] === undefined) return '';
|
|
135
|
+
const segments = match[1].split('.').map((segmentArg) => BigInt(segmentArg));
|
|
136
|
+
if (segments.some((segmentArg) => segmentArg > maxSwiftVersionSegment)) return null;
|
|
137
|
+
const [major, minor = 0n, patch = 0n] = segments;
|
|
138
|
+
const key = patch === 0n ? `${major}.${minor}` : `${major}.${minor}.${patch}`;
|
|
139
|
+
return isSwiftManifestVersion(key) ? key : null;
|
|
140
|
+
};
|