@foss.global/codefeed 5.1.0 → 5.2.0

package/ts/giteasourceinventory.packages.ts

@@ -0,0 +1,160 @@
+import * as plugins from './plugins.js';
+import * as validation from './forgeinventory.validation.js';
+
+type TMeasure = plugins.fossInterfaces.data.TForgeSourceMeasure;
+type IPackageRef = plugins.fossInterfaces.data.IForgeSourcePackageRef;
+type IPackageRecord = plugins.fossInterfaces.data.IForgeSourcePackageInventory;
+type IPackageContent = plugins.fossInterfaces.data.IForgeSourcePackageContent;
+type IPackageContentSelection = plugins.fossInterfaces.data.IForgeSourcePackageContentSelection;
+type IPackageContentGroup = plugins.fossInterfaces.data.IForgeSourcePackageContentGroup;
+type IPackageContentTotals = plugins.fossInterfaces.data.IForgeSourcePackageContentTotals;
+
+/** Media types Gitea serves an index or a manifest list as; an index candidate must be one of them. */
+export const containerIndexMediaTypes: readonly string[] = Object.freeze([
+  'application/vnd.oci.image.index.v1+json',
+  'application/vnd.docker.distribution.manifest.list.v2+json',
+]);
+
+/** Every manifest media type Gitea serves: images, indexes and manifest lists, OCI and Docker. */
+export const containerManifestMediaTypes: readonly string[] = Object.freeze([
+  ...containerIndexMediaTypes,
+  'application/vnd.oci.image.manifest.v1+json',
+  'application/vnd.docker.distribution.manifest.v2+json',
+]);
+
+/** A package type as Gitea names it; it names reason paths, so it must be one of Gitea's plain identifiers. */
+export const packageTypePattern = /^[a-z][a-z0-9_-]{0,31}$/;
+
+/** Gitea's npm name rule (`modules/packages/npm/creator.go`): `name` or `@scope/name`, URL-safe as it stands. */
+const npmNamePattern = /^(?:@[a-z0-9-][a-z0-9-._]*\/)?[a-z0-9-][a-z0-9-._]*$/;
+/** Gitea's Cargo crate name rule (`modules/packages/cargo/parser.go`). */
+const cargoNamePattern = /^[A-Za-z][A-Za-z0-9-_]{0,63}$/;
+/** Gitea's Swift scope and name rules (`routers/api/packages/swift/swift.go`); it stores the package as `scope.name`. */
+const swiftPackagePattern = /^([A-Za-z0-9][A-Za-z0-9-]{0,38})\.([A-Za-z0-9][A-Za-z0-9-_]{0,99})$/;
+
+const compareText = (aArg: string, bArg: string): number => (aArg < bArg ? -1 : aArg > bArg ? 1 : 0);
+
+/** The key of a package within one instance: owner, type and name, which Gitea keeps unique (the name without case). */
+export const packageKey = (refArg: IPackageRef): string => `${refArg.owner}/${refArg.type}/${refArg.name}`;
+
+/** Orders packages by owner, type and name in code-unit order. */
+export const comparePackages = (aArg: IPackageRef, bArg: IPackageRef): number =>
+  compareText(aArg.owner, bArg.owner) || compareText(aArg.type, bArg.type) || compareText(aArg.name, bArg.name);
+
+/** The path segment(s) of an npm package in Gitea's npm registry: the name unencoded, as Gitea's npm routes match it. */
+export const npmPackagePath = (nameArg: string): string => {
+  if (!npmNamePattern.test(nameArg) || nameArg.length > 214) throw new validation.ForgeInventoryError('invalid-npm-name');
+  return nameArg;
+};
+
+/** The path of a crate's entry in the Cargo sparse index, lowercased, per the Cargo registry index layout. */
+export const cargoIndexPath = (nameArg: string): string => {
+  if (!cargoNamePattern.test(nameArg)) throw new validation.ForgeInventoryError('invalid-cargo-name');
+  const name = nameArg.toLowerCase();
+  if (name.length === 1) return `1/${name}`;
+  if (name.length === 2) return `2/${name}`;
+  if (name.length === 3) return `3/${name[0]}/${name}`;
+  return `${name.slice(0, 2)}/${name.slice(2, 4)}/${name}`;
+};
+
+/** The scope and name of a Swift package stored as `scope.name`. */
+export const swiftPackageParts = (nameArg: string): { scope: string; name: string } => {
+  const match = swiftPackagePattern.exec(nameArg);
+  if (!match) throw new validation.ForgeInventoryError('invalid-swift-name');
+  return { scope: match[1], name: match[2] };
+};
+
+/**
+ * The digest of the manifest a container version holds: `sha256:` and the
+ * SHA-256 of the blob holding the version's `manifest.json`; null for a
+ * version without one.
+ */
+export const containerManifestDigest = (packageArg: IPackageRecord, versionIdArg: string): string | null => {
+  const blob = packageArg.blobs.find((blobArg) => blobArg.files.some((fileArg) => fileArg.versionId === versionIdArg && fileArg.name === 'manifest.json'));
+  return blob ? `sha256:${blob.sha256}` : null;
+};
+
+/**
+ * The version ids of a package's tagged-and-referenced selection: for a
+ * container package the tagged versions and the versions holding a manifest
+ * a tagged index references; every version of any other type.
+ */
+export const taggedAndReferencedVersionIds = (packageArg: IPackageRecord): Set<string> => {
+  if (packageArg.type !== 'container') return new Set(packageArg.versions.map((versionArg) => versionArg.id));
+  const tagged = packageArg.versions.filter((versionArg) => !versionArg.version.startsWith('sha256:'));
+  const referenced = new Set(tagged.flatMap((versionArg) => versionArg.references ?? []));
+  const selected = new Set(tagged.map((versionArg) => versionArg.id));
+  for (const version of packageArg.versions) {
+    const digest = containerManifestDigest(packageArg, version.id);
+    if (digest !== null && referenced.has(digest)) selected.add(version.id);
+  }
+  return selected;
+};
+
+interface IContentSum {
+  blobs: Map<string, bigint>;
+}
+
+const contentMeasure = (sumArg: IContentSum, missingArg: number): IPackageContent => {
+  const blobs = BigInt(sumArg.blobs.size);
+  const bytes = [...sumArg.blobs.values()].reduce((totalArg, sizeArg) => totalArg + sizeArg, 0n);
+  if (missingArg === 0) return { blobs: { status: 'complete', value: blobs.toString() }, bytes: { status: 'complete', value: bytes.toString() } };
+  const reason = `Package units not measured: ${missingArg}.`;
+  const partial = (valueArg: bigint): TMeasure => ({ status: 'partial', observedValue: valueArg.toString(), reason });
+  return { blobs: partial(blobs), bytes: partial(bytes) };
+};
+
+/** Distinct content of a set of package records over both selections; `missingArg` packages of the set were not read. */
+const selection = (packagesArg: readonly IPackageRecord[], missingArg: number): IPackageContentSelection => {
+  const all: IContentSum = { blobs: new Map() };
+  const tagged: IContentSum = { blobs: new Map() };
+  for (const record of packagesArg) {
+    const selected = taggedAndReferencedVersionIds(record);
+    for (const blob of record.blobs) {
+      const size = BigInt(blob.size);
+      all.blobs.set(blob.sha256, size);
+      if (blob.files.some((fileArg) => selected.has(fileArg.versionId))) tagged.blobs.set(blob.sha256, size);
+    }
+  }
+  return { allVersions: contentMeasure(all, missingArg), taggedAndReferenced: contentMeasure(tagged, missingArg) };
+};
+
+/** A group of packages with its content by type; `refsArg` are the group's discovered packages. */
+const group = (refsArg: readonly IPackageRef[], recordsArg: ReadonlyMap<string, IPackageRecord>): IPackageContentGroup => {
+  const read = (refs: readonly IPackageRef[]) => refs.flatMap((refArg) => {
+    const record = recordsArg.get(packageKey(refArg));
+    return record ? [record] : [];
+  });
+  const missing = (refs: readonly IPackageRef[]) => refs.filter((refArg) => !recordsArg.has(packageKey(refArg))).length;
+  const types = [...new Set(refsArg.map((refArg) => refArg.type))].sort(compareText);
+  return {
+    ...selection(read(refsArg), missing(refsArg)),
+    types: types.map((typeArg) => {
+      const refs = refsArg.filter((refArg) => refArg.type === typeArg);
+      return { type: typeArg, ...selection(read(refs), missing(refs)) };
+    }),
+  };
+};
+
+/**
+ * The distinct content of an inventory's packages, `totals.packageContent`:
+ * a pure function of the discovered packages and the records their units
+ * returned. Every SHA-256 counts once per scope, globally and per owner, each
+ * by type, over all versions and over the tagged versions with the manifests
+ * their indexes reference. A measure is complete only when every discovered
+ * package of its scope has a record; otherwise it is partial with the sum of
+ * the records read. Records of packages discovery does not list are ignored.
+ */
+export const composeGiteaPackageContent = (
+  discoveredArg: readonly IPackageRef[],
+  packagesArg: readonly IPackageRecord[],
+): IPackageContentTotals => {
+  const discovered = new Set(discoveredArg.map(packageKey));
+  const records = new Map(packagesArg.filter((recordArg) => discovered.has(packageKey(recordArg))).map((recordArg) => [packageKey(recordArg), recordArg]));
+  const refs = [...new Map(discoveredArg.map((refArg) => [packageKey(refArg), { owner: refArg.owner, type: refArg.type, name: refArg.name }])).values()];
+  const owners = [...new Set(refs.map((refArg) => refArg.owner))].sort(compareText);
+  return {
+    global: group(refs, records),
+    owners: owners.map((ownerArg) => ({ owner: ownerArg, ...group(refs.filter((refArg) => refArg.owner === ownerArg), records) })),
+  };
+};

package/ts/index.ts

@@ -491,3 +491,4 @@ export * from './classes.giteasourceinventory.js';
 export * from './classes.giteasourcetransfer.js';
 export * from './classes.lfsbatchclient.js';
 export { GitScratchBudget, mirrorScratchBytes } from './giteasourceinventory.git.js';
+export { composeGiteaPackageContent } from './giteasourceinventory.packages.js';

package/ts/plugins.ts

@@ -1,4 +1,4 @@
-import type * as fossInterfaces from '@foss.global/interfaces';
+import * as fossInterfaces from '@foss.global/interfaces';
 
 import * as crypto from 'node:crypto';
 import * as fs from 'node:fs/promises';
@@ -20,7 +20,7 @@ export {
 import * as lik from '@push.rocks/lik';
 import * as smarttime from '@push.rocks/smarttime';
 
-export type { fossInterfaces };
+export { fossInterfaces };
 export type TNpmRegistry = import('@push.rocks/smartnpm').NpmRegistry;
 export type TGitManager = import('@foss.global/gitmanager').GitManager;
 export type TGitManagerCommitSummary = import('@foss.global/gitmanager').IGitManagerCommitSummary;