@foss.global/codefeed 5.1.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +21 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.d.ts +35 -9
- package/dist_ts/classes.giteasourceinventory.js +284 -26
- package/dist_ts/classes.giteasourcetransfer.d.ts +114 -6
- package/dist_ts/classes.giteasourcetransfer.js +326 -9
- package/dist_ts/forgeinventory.client.d.ts +22 -4
- package/dist_ts/forgeinventory.client.js +32 -13
- package/dist_ts/giteasourceinventory.packages.d.ts +46 -0
- package/dist_ts/giteasourceinventory.packages.js +139 -0
- package/dist_ts/index.d.ts +1 -0
- package/dist_ts/index.js +2 -1
- package/dist_ts/plugins.d.ts +2 -2
- package/dist_ts/plugins.js +3 -1
- package/package.json +3 -3
- package/readme.md +28 -2
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +293 -24
- package/ts/classes.giteasourcetransfer.ts +395 -8
- package/ts/forgeinventory.client.ts +63 -12
- package/ts/giteasourceinventory.packages.ts +160 -0
- package/ts/index.ts +1 -0
- package/ts/plugins.ts +2 -2
|
@@ -4,6 +4,9 @@ import * as validation from './forgeinventory.validation.js';
|
|
|
4
4
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
5
|
import type { IForgeRequestMetrics, IForgeRequestOptions } from './forgeinventory.client.js';
|
|
6
6
|
import { measureGiteaMirror } from './giteasourceinventory.git.js';
|
|
7
|
+
import {
|
|
8
|
+
cargoIndexPath, comparePackages, composeGiteaPackageContent, containerIndexMediaTypes, npmPackagePath, packageKey, packageTypePattern,
|
|
9
|
+
} from './giteasourceinventory.packages.js';
|
|
7
10
|
|
|
8
11
|
type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
|
|
9
12
|
type TUnit = plugins.fossInterfaces.data.TForgeSourceInventoryUnit;
|
|
@@ -25,6 +28,10 @@ type ICollaborator = plugins.fossInterfaces.data.IForgeSourceCollaborator;
|
|
|
25
28
|
type IDeployKey = plugins.fossInterfaces.data.IForgeSourceDeployKey;
|
|
26
29
|
type IDeployToken = plugins.fossInterfaces.data.IForgeSourceDeployToken;
|
|
27
30
|
type IAccessCounts = plugins.fossInterfaces.data.IForgeSourceAccessCounts;
|
|
31
|
+
type IPackageRef = plugins.fossInterfaces.data.IForgeSourcePackageRef;
|
|
32
|
+
type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
|
|
33
|
+
type IPackageVersion = plugins.fossInterfaces.data.IForgeSourcePackageVersion;
|
|
34
|
+
type IPackageBlob = plugins.fossInterfaces.data.IForgeSourcePackageBlob;
|
|
28
35
|
|
|
29
36
|
export interface IGiteaSourceInventoryOptions {
|
|
30
37
|
/** Operator-assigned identity of the source instance. */
|
|
@@ -55,6 +62,19 @@ export interface IGiteaSourceInventoryOptions {
|
|
|
55
62
|
maxResponseBytes?: number;
|
|
56
63
|
}
|
|
57
64
|
|
|
65
|
+
/** A class of source data a run measures only when its discovery selects it. */
|
|
66
|
+
export type TGiteaSourceInventoryClass = 'packages';
|
|
67
|
+
|
|
68
|
+
export interface IGiteaSourceDiscoveryOptions {
|
|
69
|
+
/**
|
|
70
|
+
* Classes beside the instance, owners and repositories to list for
|
|
71
|
+
* measurement. `packages` lists every package, so `units` measures one unit
|
|
72
|
+
* per package; without it, discovery carries no `packages` and the package
|
|
73
|
+
* class is unread.
|
|
74
|
+
*/
|
|
75
|
+
classes?: readonly TGiteaSourceInventoryClass[];
|
|
76
|
+
}
|
|
77
|
+
|
|
58
78
|
export interface IGiteaSourceMeasureOptions {
|
|
59
79
|
signal?: AbortSignal;
|
|
60
80
|
/** Units measured at the same time. Requests within one unit are sequential. */
|
|
@@ -339,16 +359,56 @@ const countedEnvelope = (bodyArg: unknown, rowsKeyArg: string): { rows: unknown[
|
|
|
339
359
|
return { rows, failure: validation.decimal(body.total_count) === String(rows.length) ? null : 'count-mismatch' };
|
|
340
360
|
};
|
|
341
361
|
|
|
342
|
-
|
|
343
|
-
|
|
362
|
+
/** The unit a result answers. */
|
|
363
|
+
const resultUnit = (resultArg: TUnitResult): TUnit => {
|
|
364
|
+
switch (resultArg.kind) {
|
|
365
|
+
case 'failed': return resultArg.unit;
|
|
366
|
+
case 'instance': return { kind: 'instance' };
|
|
367
|
+
case 'owner': return { kind: 'owner', ownerKind: resultArg.owner.kind, name: resultArg.owner.name };
|
|
368
|
+
case 'repository': return { kind: 'repository', owner: resultArg.repository.owner, name: resultArg.repository.name };
|
|
369
|
+
case 'package': return { kind: 'package', owner: resultArg.package.owner, type: resultArg.package.type, name: resultArg.package.name };
|
|
370
|
+
default: return unknownUnit(resultArg);
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
|
|
374
|
+
/** Refuses a unit kind this generator does not know, which a newer contract may add. */
|
|
375
|
+
const unknownUnit = (unitArg: never): never => {
|
|
376
|
+
void unitArg;
|
|
377
|
+
throw new validation.ForgeInventoryError('unit-kind-unsupported');
|
|
378
|
+
};
|
|
379
|
+
|
|
380
|
+
const unitKey = (unitArg: TUnit): string => {
|
|
381
|
+
switch (unitArg.kind) {
|
|
382
|
+
case 'instance': return 'instance';
|
|
383
|
+
case 'owner': return `owner:${unitArg.ownerKind}:${unitArg.name}`;
|
|
384
|
+
case 'repository': return `repository:${unitArg.owner}/${unitArg.name}`;
|
|
385
|
+
case 'package': return `package:${packageKey(unitArg)}`;
|
|
386
|
+
default: return unknownUnit(unitArg);
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
/** A package type from a listing. */
|
|
391
|
+
const packageType = (valueArg: unknown): string => {
|
|
392
|
+
const type = requiredString(valueArg);
|
|
393
|
+
if (!packageTypePattern.test(type)) throw new validation.ForgeInventoryError('invalid-package-type');
|
|
394
|
+
return type;
|
|
395
|
+
};
|
|
396
|
+
|
|
397
|
+
/** A package name as Gitea stores it; container and npm names may hold `/`, so it is no single path segment. */
|
|
398
|
+
const packageName = (valueArg: unknown): string => {
|
|
399
|
+
const name = requiredString(valueArg);
|
|
400
|
+
if (name.length > 512 || /[\u0000-\u001f\u007f]/.test(name)) throw new validation.ForgeInventoryError('invalid-package-name');
|
|
401
|
+
return name;
|
|
402
|
+
};
|
|
344
403
|
|
|
345
404
|
const compareText = (aArg: string, bArg: string): number => aArg < bArg ? -1 : aArg > bArg ? 1 : 0;
|
|
346
405
|
|
|
347
406
|
/**
|
|
348
407
|
* Authenticated, read-only inventory of a Gitea instance, read with a site
|
|
349
|
-
* administrator's token: GET requests to its REST API
|
|
350
|
-
* repository for refs, objects and LFS pointers, and
|
|
351
|
-
* requests that check which referenced objects Gitea
|
|
408
|
+
* administrator's token: GET requests to its REST API and package registries,
|
|
409
|
+
* a mirror clone of every repository for refs, objects and LFS pointers, and
|
|
410
|
+
* LFS batch `download` requests that check which referenced objects Gitea
|
|
411
|
+
* holds. It counts secrets
|
|
352
412
|
* and variables without retaining their values, and reports every read that
|
|
353
413
|
* did not finish as a partial or unmeasured value instead of a count.
|
|
354
414
|
*/
|
|
@@ -412,11 +472,15 @@ export class GiteaSourceInventory {
|
|
|
412
472
|
|
|
413
473
|
/**
|
|
414
474
|
* Identifies the source and its reader and lists every owner and repository
|
|
415
|
-
* with its id
|
|
416
|
-
*
|
|
417
|
-
*
|
|
475
|
+
* with its id, and with `classes: ['packages']` every package from the
|
|
476
|
+
* administrator package listing (`packages`, absent otherwise). Rejects
|
|
477
|
+
* unless the token belongs to a site administrator, and rejects an unknown
|
|
478
|
+
* class with `invalid-options`; a listing that does not finish, or that
|
|
479
|
+
* lists a repository id twice, is reported in `incompleteReasons`.
|
|
418
480
|
*/
|
|
419
|
-
public async discover(signalArg?: AbortSignal): Promise<IDiscovery> {
|
|
481
|
+
public async discover(signalArg?: AbortSignal, optionsArg: IGiteaSourceDiscoveryOptions = {}): Promise<IDiscovery> {
|
|
482
|
+
const classes = optionsArg.classes ?? [];
|
|
483
|
+
if (!Array.isArray(classes) || classes.some((classArg) => classArg !== 'packages')) throw new validation.ForgeInventoryError('invalid-options');
|
|
420
484
|
const reader = validation.record((await this.#get('/user', { signal: signalArg })).body);
|
|
421
485
|
if (reader.is_admin !== true) throw new validation.ForgeInventoryError('reader-not-site-admin');
|
|
422
486
|
const version = validation.record((await this.#get('/version', { signal: signalArg })).body).version;
|
|
@@ -453,16 +517,34 @@ export class GiteaSourceInventory {
|
|
|
453
517
|
}
|
|
454
518
|
if (ids.size !== repositories.items.length) incompleteReasons.push('discovery: repositories: duplicate-id');
|
|
455
519
|
discovered.sort((aArg, bArg) => compareText(aArg.owner, bArg.owner) || compareText(aArg.name, bArg.name));
|
|
520
|
+
let packages: IPackageRef[] | undefined;
|
|
521
|
+
if (classes.includes('packages')) {
|
|
522
|
+
// One row per package version, of every owner, user-owned packages included.
|
|
523
|
+
const versions = await this.#list('/admin/packages', { signal: signalArg });
|
|
524
|
+
if (versions.failure !== null) incompleteReasons.push(`discovery: packages: ${versions.failure}`);
|
|
525
|
+
const refs = new Map<string, IPackageRef>();
|
|
526
|
+
for (const itemArg of versions.items) {
|
|
527
|
+
const item = validation.record(itemArg);
|
|
528
|
+
const ref = { owner: segment(validation.record(item.owner).login), type: packageType(item.type), name: packageName(item.name) };
|
|
529
|
+
refs.set(packageKey(ref), ref);
|
|
530
|
+
}
|
|
531
|
+
packages = [...refs.values()].sort(comparePackages);
|
|
532
|
+
}
|
|
456
533
|
return {
|
|
457
534
|
source: { instanceId: this.#options.instanceId, kind: 'gitea', baseUrl: this.#options.baseUrl, version },
|
|
458
535
|
reader: { login: segment(reader.login), siteAdmin: true },
|
|
459
536
|
owners,
|
|
460
537
|
repositories: discovered,
|
|
538
|
+
...(packages === undefined ? {} : { packages }),
|
|
461
539
|
incompleteReasons: incompleteReasons.sort(compareText),
|
|
462
540
|
};
|
|
463
541
|
}
|
|
464
542
|
|
|
465
|
-
/**
|
|
543
|
+
/**
|
|
544
|
+
* Every unit a run over this discovery measures: the instance, then owners,
|
|
545
|
+
* then repositories, then the packages discovery lists when it selected
|
|
546
|
+
* them.
|
|
547
|
+
*/
|
|
466
548
|
public units(discoveryArg: IDiscovery): TUnit[] {
|
|
467
549
|
return [
|
|
468
550
|
{ kind: 'instance' },
|
|
@@ -470,6 +552,7 @@ export class GiteaSourceInventory {
|
|
|
470
552
|
...discoveryArg.repositories.map((repositoryArg): TUnit => ({
|
|
471
553
|
kind: 'repository', owner: repositoryArg.owner, name: repositoryArg.name, ...repositoryArg.id === undefined ? {} : { id: repositoryArg.id },
|
|
472
554
|
})),
|
|
555
|
+
...(discoveryArg.packages ?? []).map((packageArg): TUnit => ({ kind: 'package', owner: packageArg.owner, type: packageArg.type, name: packageArg.name })),
|
|
473
556
|
];
|
|
474
557
|
}
|
|
475
558
|
|
|
@@ -497,15 +580,31 @@ export class GiteaSourceInventory {
|
|
|
497
580
|
/**
|
|
498
581
|
* Measures one unit. Never rejects: a unit whose primary record cannot be
|
|
499
582
|
* read is `failed`, and so is a repository unit carrying an id that its name
|
|
500
|
-
* no longer answers with (`repository-identity-changed`).
|
|
583
|
+
* no longer answers with (`repository-identity-changed`). A package unit is
|
|
584
|
+
* read whole or fails, never partly: a listing that does not finish, an
|
|
585
|
+
* index that is not what its file list promises (`unexpected-manifest-shape`)
|
|
586
|
+
* and a result over `forgeSourceInventoryUnitMaxBytes` of UTF-8 JSON
|
|
587
|
+
* (`unit-over-bound`) fail it, and its record is never truncated.
|
|
501
588
|
*/
|
|
502
589
|
public async measureUnit(unitArg: TUnit, signalArg?: AbortSignal): Promise<TUnitResult> {
|
|
503
590
|
try {
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
591
|
+
switch (unitArg.kind) {
|
|
592
|
+
case 'instance':
|
|
593
|
+
return { kind: 'instance', instance: await this.#instance(signalArg) };
|
|
594
|
+
case 'owner':
|
|
595
|
+
return { kind: 'owner', owner: unitArg.ownerKind === 'organization' ? await this.#organization(unitArg.name, signalArg) : await this.#user(unitArg.name, signalArg) };
|
|
596
|
+
case 'repository':
|
|
597
|
+
return { kind: 'repository', repository: await this.#repository(unitArg.owner, unitArg.name, unitArg.id, signalArg) };
|
|
598
|
+
case 'package': {
|
|
599
|
+
const result: TUnitResult = { kind: 'package', package: await this.#package(unitArg, signalArg) };
|
|
600
|
+
if (Buffer.byteLength(JSON.stringify(result), 'utf8') > plugins.fossInterfaces.data.forgeSourceInventoryUnitMaxBytes) {
|
|
601
|
+
return { kind: 'failed', unit: unitArg, reason: 'unit-over-bound' };
|
|
602
|
+
}
|
|
603
|
+
return result;
|
|
604
|
+
}
|
|
605
|
+
default:
|
|
606
|
+
return unknownUnit(unitArg);
|
|
507
607
|
}
|
|
508
|
-
return { kind: 'repository', repository: await this.#repository(unitArg.owner, unitArg.name, unitArg.id, signalArg) };
|
|
509
608
|
} catch (error) {
|
|
510
609
|
return { kind: 'failed', unit: unitArg, reason: reason(error) };
|
|
511
610
|
}
|
|
@@ -700,6 +799,164 @@ export class GiteaSourceInventory {
|
|
|
700
799
|
return counts;
|
|
701
800
|
}
|
|
702
801
|
|
|
802
|
+
/**
|
|
803
|
+
* One package as Gitea's v1 package API lists it: every version with its
|
|
804
|
+
* files, each distinct content once. Container index candidates, versions
|
|
805
|
+
* whose only file is `manifest.json`, are read by digest for their
|
|
806
|
+
* references; no other manifest is read. npm packages add the dist-tags Gitea
|
|
807
|
+
* serves (a derived `latest` included) and each version's deprecation,
|
|
808
|
+
* Cargo crates each version's yank from the sparse index. Every read is a
|
|
809
|
+
* GET; reading an index counts a download of its version, an accepted read
|
|
810
|
+
* stamp (S10 ruling, Addendum 1).
|
|
811
|
+
*/
|
|
812
|
+
async #package(refArg: IPackageRef, signalArg?: AbortSignal): Promise<IPackageRecord> {
|
|
813
|
+
const packagePath = `/packages/${encodeURIComponent(refArg.owner)}/${encodeURIComponent(refArg.type)}/${encodeURIComponent(refArg.name)}`;
|
|
814
|
+
const listing = await this.#list(packagePath, { signal: signalArg });
|
|
815
|
+
if (listing.failure !== null) throw new validation.ForgeInventoryError(listing.failure);
|
|
816
|
+
if (listing.items.length === 0) throw new validation.ForgeInventoryError('package-without-versions');
|
|
817
|
+
let ownerId: string | null = null;
|
|
818
|
+
let repositoryId: string | null | undefined;
|
|
819
|
+
const versions: IPackageVersion[] = [];
|
|
820
|
+
const ids = new Set<string>();
|
|
821
|
+
for (const itemArg of listing.items) {
|
|
822
|
+
const item = validation.record(itemArg);
|
|
823
|
+
const owner = validation.record(item.owner);
|
|
824
|
+
const repository = item.repository === null || item.repository === undefined ? null : validation.decimal(validation.record(item.repository).id);
|
|
825
|
+
if (segment(owner.login) !== refArg.owner || packageType(item.type) !== refArg.type || packageName(item.name) !== refArg.name) {
|
|
826
|
+
throw new validation.ForgeInventoryError('package-identity-changed');
|
|
827
|
+
}
|
|
828
|
+
// Every version names the package's owner and its one repository link.
|
|
829
|
+
if ((ownerId ?? validation.decimal(owner.id)) !== validation.decimal(owner.id) || (repositoryId !== undefined && repositoryId !== repository)) {
|
|
830
|
+
throw new validation.ForgeInventoryError('package-identity-changed');
|
|
831
|
+
}
|
|
832
|
+
ownerId = validation.decimal(owner.id);
|
|
833
|
+
repositoryId = repository;
|
|
834
|
+
const id = validation.decimal(item.id);
|
|
835
|
+
if (ids.has(id)) throw new validation.ForgeInventoryError('duplicate-id');
|
|
836
|
+
ids.add(id);
|
|
837
|
+
versions.push({ id, version: requiredString(item.version), createdAt: requiredTimestamp(item.created_at) });
|
|
838
|
+
}
|
|
839
|
+
versions.sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id));
|
|
840
|
+
const blobs = new Map<string, IPackageBlob>();
|
|
841
|
+
const filesByVersion = new Map<string, Array<{ name: string; sha256: string }>>();
|
|
842
|
+
for (const version of versions) {
|
|
843
|
+
const files = (await this.#get(`${packagePath}/${encodeURIComponent(version.version)}/files`, { signal: signalArg })).body;
|
|
844
|
+
if (!Array.isArray(files)) throw new validation.ForgeInventoryError('invalid-page');
|
|
845
|
+
const read: Array<{ name: string; sha256: string }> = [];
|
|
846
|
+
for (const fileArg of files) {
|
|
847
|
+
const file = validation.record(fileArg);
|
|
848
|
+
const sha256 = requiredString(file.sha256);
|
|
849
|
+
if (!/^[0-9a-f]{64}$/.test(sha256)) throw new validation.ForgeInventoryError('invalid-sha256');
|
|
850
|
+
const size = validation.decimal(file.size);
|
|
851
|
+
const blob = blobs.get(sha256) ?? { sha256, size, files: [] };
|
|
852
|
+
if (blob.size !== size) throw new validation.ForgeInventoryError('package-blob-size-conflict');
|
|
853
|
+
const name = requiredString(file.name);
|
|
854
|
+
blob.files.push({ versionId: version.id, id: validation.decimal(file.id), name });
|
|
855
|
+
blobs.set(sha256, blob);
|
|
856
|
+
read.push({ name, sha256 });
|
|
857
|
+
}
|
|
858
|
+
filesByVersion.set(version.id, read);
|
|
859
|
+
}
|
|
860
|
+
const record: IPackageRecord = {
|
|
861
|
+
owner: refArg.owner, ownerId: ownerId!, type: refArg.type, name: refArg.name, repositoryId: repositoryId ?? null, versions,
|
|
862
|
+
blobs: [...blobs.values()].sort((aArg, bArg) => compareText(aArg.sha256, bArg.sha256)).map((blobArg) => ({
|
|
863
|
+
...blobArg, files: blobArg.files.sort((aArg, bArg) => compareDecimal(aArg.versionId, bArg.versionId) || compareDecimal(aArg.id, bArg.id)),
|
|
864
|
+
})),
|
|
865
|
+
};
|
|
866
|
+
if (record.type === 'container') await this.#containerReferences(record, filesByVersion, signalArg);
|
|
867
|
+
if (record.type === 'npm') await this.#npmState(record, signalArg);
|
|
868
|
+
if (record.type === 'cargo') await this.#cargoYanks(record, signalArg);
|
|
869
|
+
return record;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
/**
|
|
873
|
+
* Reads the references of every index candidate, a version whose file list
|
|
874
|
+
* is exactly `manifest.json` (Gitea stores an index's manifest and nothing
|
|
875
|
+
* else), by digest from the registry with the reader's Basic credentials. The
|
|
876
|
+
* bytes must hash to the listed SHA-256 and the response must be an index or
|
|
877
|
+
* a manifest list, else the unit fails with `unexpected-manifest-shape`.
|
|
878
|
+
*/
|
|
879
|
+
async #containerReferences(
|
|
880
|
+
recordArg: IPackageRecord, filesArg: ReadonlyMap<string, Array<{ name: string; sha256: string }>>, signalArg?: AbortSignal,
|
|
881
|
+
): Promise<void> {
|
|
882
|
+
if (!/^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*$/.test(recordArg.name)) {
|
|
883
|
+
throw new validation.ForgeInventoryError('invalid-image-name');
|
|
884
|
+
}
|
|
885
|
+
for (const version of recordArg.versions) {
|
|
886
|
+
const files = filesArg.get(version.id)!;
|
|
887
|
+
if (files.length !== 1 || files[0].name !== 'manifest.json') continue;
|
|
888
|
+
const digest = `sha256:${files[0].sha256}`;
|
|
889
|
+
const client = this.#basic(await this.#login(signalArg));
|
|
890
|
+
const response = await client.requestBytes(
|
|
891
|
+
new URL(`/v2/${recordArg.owner.toLowerCase()}/${recordArg.name}/manifests/${digest}`, this.#options.baseUrl),
|
|
892
|
+
{ accept: containerIndexMediaTypes.join(', '), signal: signalArg },
|
|
893
|
+
);
|
|
894
|
+
if (plugins.crypto.createHash('sha256').update(response.bytes).digest('hex') !== files[0].sha256
|
|
895
|
+
|| response.mediaType === null || !containerIndexMediaTypes.includes(response.mediaType)) {
|
|
896
|
+
throw new validation.ForgeInventoryError('unexpected-manifest-shape');
|
|
897
|
+
}
|
|
898
|
+
let entries: unknown;
|
|
899
|
+
try {
|
|
900
|
+
entries = validation.record(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(response.bytes)) as unknown).manifests;
|
|
901
|
+
} catch {
|
|
902
|
+
throw new validation.ForgeInventoryError('unexpected-manifest-shape');
|
|
903
|
+
}
|
|
904
|
+
if (!Array.isArray(entries)) throw new validation.ForgeInventoryError('unexpected-manifest-shape');
|
|
905
|
+
const references = entries.map((entryArg: unknown) => {
|
|
906
|
+
const entry = entryArg !== null && typeof entryArg === 'object' && !Array.isArray(entryArg) ? (entryArg as Record<string, unknown>).digest : undefined;
|
|
907
|
+
if (typeof entry !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(entry)) throw new validation.ForgeInventoryError('unexpected-manifest-shape');
|
|
908
|
+
return entry;
|
|
909
|
+
});
|
|
910
|
+
version.references = [...new Set(references)].sort(compareText);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/** Every dist-tag Gitea serves, a `latest` it derives included, and each version's deprecation from the package document. */
|
|
915
|
+
async #npmState(recordArg: IPackageRecord, signalArg?: AbortSignal): Promise<void> {
|
|
916
|
+
const registry = `${this.#options.baseUrl}/api/packages/${encodeURIComponent(recordArg.owner)}/npm`;
|
|
917
|
+
const name = npmPackagePath(recordArg.name);
|
|
918
|
+
const tags = validation.record((await this.#client.request(new URL(`${registry}/-/package/${name}/dist-tags`), { signal: signalArg })).body);
|
|
919
|
+
recordArg.distTags = Object.entries(tags).map(([tag, version]) => ({ tag, version: requiredString(version) }))
|
|
920
|
+
.sort((aArg, bArg) => compareText(aArg.tag, bArg.tag));
|
|
921
|
+
const versions = validation.record(validation.record((await this.#client.request(new URL(`${registry}/${name}`), { signal: signalArg })).body).versions);
|
|
922
|
+
for (const version of recordArg.versions) {
|
|
923
|
+
const entry = versions[version.version];
|
|
924
|
+
if (entry === undefined) throw new validation.ForgeInventoryError('npm-version-missing');
|
|
925
|
+
const deprecated = validation.record(entry).deprecated;
|
|
926
|
+
version.deprecated = deprecated === undefined ? '' : text(deprecated);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/** Each version's yank from the crate's line in the Cargo sparse index. */
|
|
931
|
+
async #cargoYanks(recordArg: IPackageRecord, signalArg?: AbortSignal): Promise<void> {
|
|
932
|
+
const response = await this.#client.requestBytes(
|
|
933
|
+
new URL(`${this.#options.baseUrl}/api/packages/${encodeURIComponent(recordArg.owner)}/cargo/${cargoIndexPath(recordArg.name)}`),
|
|
934
|
+
{ accept: 'text/plain', signal: signalArg },
|
|
935
|
+
);
|
|
936
|
+
if (response.mediaType !== 'text/plain') throw new validation.ForgeInventoryError('invalid-content-type');
|
|
937
|
+
const yanked = new Map<string, boolean>();
|
|
938
|
+
let lines: string[];
|
|
939
|
+
try {
|
|
940
|
+
lines = new TextDecoder('utf-8', { fatal: true }).decode(response.bytes).split('\n').filter((lineArg) => lineArg !== '');
|
|
941
|
+
} catch {
|
|
942
|
+
throw new validation.ForgeInventoryError('invalid-cargo-index');
|
|
943
|
+
}
|
|
944
|
+
for (const line of lines) {
|
|
945
|
+
let entry: Record<string, unknown>;
|
|
946
|
+
try {
|
|
947
|
+
entry = validation.record(JSON.parse(line) as unknown);
|
|
948
|
+
} catch {
|
|
949
|
+
throw new validation.ForgeInventoryError('invalid-cargo-index');
|
|
950
|
+
}
|
|
951
|
+
yanked.set(requiredString(entry.vers), requiredBoolean(entry.yanked));
|
|
952
|
+
}
|
|
953
|
+
for (const version of recordArg.versions) {
|
|
954
|
+
const value = yanked.get(version.version);
|
|
955
|
+
if (value === undefined) throw new validation.ForgeInventoryError('cargo-index-incomplete');
|
|
956
|
+
version.yanked = value;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
|
|
703
960
|
async #repository(ownerArg: string, nameArg: string, idArg: string | undefined, signalArg?: AbortSignal): Promise<IRepository> {
|
|
704
961
|
const path = `/repos/${encodeURIComponent(ownerArg)}/${encodeURIComponent(nameArg)}`;
|
|
705
962
|
const item = validation.record((await this.#get(path, { signal: signalArg })).body);
|
|
@@ -837,7 +1094,7 @@ export class GiteaSourceInventory {
|
|
|
837
1094
|
baseUrl: this.#options.baseUrl, owner: ownerArg, name: nameArg, username, token: this.#options.token,
|
|
838
1095
|
caCertificates: this.#options.caCertificates, workDir: this.#options.gitWorkDir, reportedSizeKiB: sizeKiB,
|
|
839
1096
|
timeoutMs: this.#options.gitTimeoutMs,
|
|
840
|
-
lfsClient: this.#
|
|
1097
|
+
lfsClient: this.#basic(username), signal: signalArg,
|
|
841
1098
|
});
|
|
842
1099
|
return { git: { reportedSizeKiB: complete(sizeKiB), ...measured.git }, lfs: measured.lfs, head: measured.head, refDigests: measured.refDigests };
|
|
843
1100
|
}
|
|
@@ -853,8 +1110,8 @@ export class GiteaSourceInventory {
|
|
|
853
1110
|
return this.#readerLogin;
|
|
854
1111
|
}
|
|
855
1112
|
|
|
856
|
-
/** LFS routes accept the token as a Basic password, not as a `token` authorization. */
|
|
857
|
-
#
|
|
1113
|
+
/** LFS and registry (`/v2`) routes accept the token as a Basic password, not as a `token` authorization. */
|
|
1114
|
+
#basic(usernameArg: string): ForgeRequestClient {
|
|
858
1115
|
this.#lfsClient ??= new ForgeRequestClient({
|
|
859
1116
|
transport: this.#transport,
|
|
860
1117
|
headers: { authorization: `Basic ${Buffer.from(`${usernameArg}:${this.#options.token}`).toString('base64')}` },
|
|
@@ -1119,6 +1376,10 @@ const measuresOf = (valueArg: unknown, pathArg: string, outArg: Array<{ path: st
|
|
|
1119
1376
|
* repositories measured with one id, owner repository counts that disagree
|
|
1120
1377
|
* with discovery, and every partial or unmeasured value make the inventory
|
|
1121
1378
|
* incomplete, with reasons aggregated per subject kind, path and reason.
|
|
1379
|
+
* `packages` and `totals.packageContent` (`composeGiteaPackageContent`) are
|
|
1380
|
+
* present exactly when discovery lists packages, an empty list included; a
|
|
1381
|
+
* package without a record is left out, never truncated, and makes its
|
|
1382
|
+
* content measures partial.
|
|
1122
1383
|
*/
|
|
1123
1384
|
export const composeGiteaSourceInventory = (
|
|
1124
1385
|
discoveryArg: IDiscovery,
|
|
@@ -1126,12 +1387,7 @@ export const composeGiteaSourceInventory = (
|
|
|
1126
1387
|
generatorsArg: plugins.fossInterfaces.data.IForgeSourceInventoryGenerator[],
|
|
1127
1388
|
): IInventory => {
|
|
1128
1389
|
const results = new Map<string, TUnitResult>();
|
|
1129
|
-
for (const result of resultsArg)
|
|
1130
|
-
const unit: TUnit = result.kind === 'failed' ? result.unit : result.kind === 'instance' ? { kind: 'instance' }
|
|
1131
|
-
: result.kind === 'owner' ? { kind: 'owner', ownerKind: result.owner.kind, name: result.owner.name }
|
|
1132
|
-
: { kind: 'repository', owner: result.repository.owner, name: result.repository.name };
|
|
1133
|
-
results.set(unitKey(unit), result);
|
|
1134
|
-
}
|
|
1390
|
+
for (const result of resultsArg) results.set(unitKey(resultUnit(result)), result);
|
|
1135
1391
|
const reasons = new Map<string, number>();
|
|
1136
1392
|
const addReason = (textArg: string) => reasons.set(textArg, (reasons.get(textArg) ?? 0) + 1);
|
|
1137
1393
|
for (const text of discoveryArg.incompleteReasons) addReason(text);
|
|
@@ -1169,6 +1425,17 @@ export const composeGiteaSourceInventory = (
|
|
|
1169
1425
|
addReason(`repository: ${result?.kind === 'failed' ? `unit failed: ${result.reason}` : result?.kind === 'repository' ? 'unit failed: repository-identity-changed' : 'unit not measured'}`);
|
|
1170
1426
|
}
|
|
1171
1427
|
}
|
|
1428
|
+
// Packages only when discovery listed them; a package without a record leaves the class incomplete, never truncated.
|
|
1429
|
+
let packages: IPackageRecord[] | undefined;
|
|
1430
|
+
if (discoveryArg.packages !== undefined) {
|
|
1431
|
+
packages = [];
|
|
1432
|
+
for (const ref of discoveryArg.packages) {
|
|
1433
|
+
const result = results.get(unitKey({ kind: 'package', ...ref }));
|
|
1434
|
+
if (result?.kind === 'package') packages.push(result.package);
|
|
1435
|
+
else addReason(`package: ${result?.kind === 'failed' ? `unit failed: ${result.reason}` : 'unit not measured'}`);
|
|
1436
|
+
}
|
|
1437
|
+
packages.sort(comparePackages);
|
|
1438
|
+
}
|
|
1172
1439
|
// Without discovered ids, two names measured as one id show a rename between their units.
|
|
1173
1440
|
if (new Set(repositories.map((repositoryArg) => repositoryArg.id)).size !== repositories.length) addReason('repository: duplicate-id');
|
|
1174
1441
|
owners.sort((aArg, bArg) => compareText(aArg.kind, bArg.kind) || compareText(aArg.name, bArg.name));
|
|
@@ -1205,6 +1472,7 @@ export const composeGiteaSourceInventory = (
|
|
|
1205
1472
|
instance,
|
|
1206
1473
|
owners,
|
|
1207
1474
|
repositories,
|
|
1475
|
+
...(packages === undefined ? {} : { packages }),
|
|
1208
1476
|
totals: {
|
|
1209
1477
|
organizations: instance.organizations,
|
|
1210
1478
|
users: instance.users.total,
|
|
@@ -1248,6 +1516,7 @@ export const composeGiteaSourceInventory = (
|
|
|
1248
1516
|
type,
|
|
1249
1517
|
...sumGroup(countsArg.map(({ type: _type, ...measures }) => measures), ['packages', 'versions', 'files', 'bytes'], missingOwners),
|
|
1250
1518
|
})),
|
|
1519
|
+
...(packages === undefined ? {} : { packageContent: composeGiteaPackageContent(discoveryArg.packages!, packages) }),
|
|
1251
1520
|
},
|
|
1252
1521
|
};
|
|
1253
1522
|
|