ghcr-manager 0.0.4 → 0.9.0

package/dist/execute/_types.d.ts

@@ -0,0 +1,51 @@
+import type { DeletePlan } from "../db/index.js";
+export interface DeletePackageVersionOperation {
+    versionId: number;
+    digest: string;
+}
+export interface UnsupportedUntagRoot {
+    versionId: number;
+    digest: string;
+    reason: string;
+}
+export interface UntagTagOperation {
+    tag: string;
+    sourceVersionId: number;
+    sourceDigest: string;
+    detachedVersionId: number;
+    detachedDigest: string;
+}
+export interface DeleteExecutionSummary {
+    owner: string;
+    packageName: string;
+    scanCompletedAt: string;
+    plannerInputs: DeletePlan["plannerInputs"];
+    deletedPackageVersions: DeletePackageVersionOperation[];
+    untaggedTags: UntagTagOperation[];
+    blockedRoots: DeletePlan["blockedRoots"];
+    unsupportedUntagRoots: UnsupportedUntagRoot[];
+}
+export interface DeleteExecutionOptions {
+    token: string;
+    logger: DeleteExecutionLogger;
+    fetchImpl?: GitHubPackageFetch;
+    listRootTags?: (root: {
+        owner: string;
+        packageName: string;
+        versionId: number;
+        digest: string;
+    }) => string[];
+}
+export interface DeleteExecutionLogger {
+    debug(message: string): void;
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+}
+export interface GitHubPackageFetchResponse {
+    ok: boolean;
+    status: number;
+    headers: Headers;
+    json(): Promise<unknown>;
+}
+export type GitHubPackageFetch = (input: string, init?: RequestInit) => Promise<GitHubPackageFetchResponse>;

package/dist/execute/_types.js

@@ -0,0 +1 @@
+export {};

package/dist/execute/_untag-client.d.ts

@@ -0,0 +1,2 @@
+import type { DeleteExecutionOptions, UntagTagOperation } from "./_types.js";
+export declare function untagRootTags(owner: string, packageName: string, sourceVersionId: number, sourceDigest: string, tags: string[], options: DeleteExecutionOptions): Promise<UntagTagOperation[]>;

package/dist/execute/_untag-client.js

@@ -0,0 +1,71 @@
+import { buildDetachedManifestClone } from "./_manifest-detach.js";
+import { findPackageVersionByDigestAndTag } from "./_package-version-page-client.js";
+import { listPackageVersionTagSources, listPresentPackageVersionIds } from "./_package-version-tag-source-client.js";
+import { deletePackageVersion } from "./_package-version-delete-client.js";
+import { loadRegistryManifestByDigest, putRegistryManifestForTag } from "./_registry-manifest-client.js";
+import { loadRegistryPushToken } from "./_registry-token-client.js";
+export async function untagRootTags(owner, packageName, sourceVersionId, sourceDigest, tags, options) {
+    const registryToken = await loadRegistryPushToken(owner, packageName, options.token, options.logger, {
+        fetchImpl: options.fetchImpl
+    });
+    const sourceManifest = await loadRegistryManifestByDigest(owner, packageName, sourceDigest, registryToken, options.logger, {
+        fetchImpl: options.fetchImpl
+    });
+    const operations = [];
+    for (const tag of tags) {
+        options.logger.info(`Detaching tag ${owner}/${packageName}:${tag} from ${sourceDigest}`);
+        const detachedManifestJson = buildDetachedManifestClone(sourceManifest.rawJson, sourceManifest.mediaType, {
+            detachedTag: tag,
+            sourceDigest
+        });
+        const detachedDigest = await putRegistryManifestForTag(owner, packageName, tag, sourceManifest.mediaType, detachedManifestJson, registryToken, options.logger, {
+            fetchImpl: options.fetchImpl
+        });
+        const detachedVersionId = await findPackageVersionByDigestAndTag(owner, packageName, detachedDigest, tag, options.token, options.logger, {
+            fetchImpl: options.fetchImpl
+        });
+        await deletePackageVersion(owner, packageName, detachedVersionId, options.token, options.logger, {
+            fetchImpl: options.fetchImpl
+        });
+        await _assertTagRemoved(owner, packageName, tag, options);
+        await _assertVersionRemoved(owner, packageName, detachedVersionId, options);
+        operations.push({
+            tag,
+            sourceVersionId,
+            sourceDigest,
+            detachedVersionId,
+            detachedDigest
+        });
+    }
+    return operations;
+}
+async function _assertTagRemoved(owner, packageName, tag, options) {
+    for (let attempt = 1; attempt <= 5; attempt += 1) {
+        const remaining = await listPackageVersionTagSources(owner, packageName, [tag], options.token, options.logger, {
+            fetchImpl: options.fetchImpl
+        });
+        if (remaining.length === 0) {
+            return;
+        }
+        if (attempt < 5) {
+            options.logger.warn(`Tag ${owner}/${packageName}:${tag} is still visible after untag; retrying check ${attempt}/5`);
+            await new Promise((resolve) => setTimeout(resolve, 1000));
+        }
+    }
+    throw new Error(`tag ${owner}/${packageName}:${tag} is still visible after untag`);
+}
+async function _assertVersionRemoved(owner, packageName, versionId, options) {
+    for (let attempt = 1; attempt <= 5; attempt += 1) {
+        const presentVersionIds = await listPresentPackageVersionIds(owner, packageName, [versionId], options.token, options.logger, {
+            fetchImpl: options.fetchImpl
+        });
+        if (presentVersionIds.length === 0) {
+            return;
+        }
+        if (attempt < 5) {
+            options.logger.warn(`Temporary package version ${owner}/${packageName}#${versionId} is still visible after untag; retrying check ${attempt}/5`);
+            await new Promise((resolve) => setTimeout(resolve, 1000));
+        }
+    }
+    throw new Error(`temporary package version ${owner}/${packageName}#${versionId} is still visible after untag`);
+}

package/dist/execute/index.d.ts

@@ -0,0 +1,5 @@
+export { executeDeletePlan } from "./_plan-executor.js";
+export { listPackageVersionTagSources, listPresentPackageVersionIds } from "./_package-version-tag-source-client.js";
+export { untagRootTags } from "./_untag-client.js";
+export type { GitHubPackageVersionTagSource } from "./_package-version-tag-source-client.js";
+export type { DeleteExecutionSummary, UntagTagOperation } from "./_types.js";

package/dist/execute/index.js

@@ -0,0 +1,3 @@
+export { executeDeletePlan } from "./_plan-executor.js";
+export { listPackageVersionTagSources, listPresentPackageVersionIds } from "./_package-version-tag-source-client.js";
+export { untagRootTags } from "./_untag-client.js";

package/dist/ingest/github/_manifest-client.d.ts

@@ -1,8 +1,14 @@
 import type { ManifestDescriptorRecord, ManifestEdgeRecord, ManifestRecord } from "../../core/index.js";
 import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+type _LoadedManifestRecord = Omit<ManifestRecord, "versionId">;
 export declare function loadManifestGraph(fetchImpl: FetchLike, registryBaseUrl: string, digest: string, registryToken: string, options: GitHubScanOptions): Promise<{
-    record: ManifestRecord;
+    record: _LoadedManifestRecord;
     descriptorRecords: ManifestDescriptorRecord[];
     edgeRecords: ManifestEdgeRecord[];
     rawJson: string;
 }>;
+export declare function buildManifestRelations(digest: string, rawJson: string): {
+    descriptorRecords: ManifestDescriptorRecord[];
+    edgeRecords: ManifestEdgeRecord[];
+};
+export {};

package/dist/ingest/github/_manifest-client.js

@@ -1,3 +1,4 @@
+import { classifyManifestKind } from "./_manifest-kind.js";
 import { acceptedManifestMediaTypes, buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
 export async function loadManifestGraph(fetchImpl, registryBaseUrl, digest, registryToken, options) {
     const startTime = Date.now();
@@ -42,12 +43,19 @@ export async function loadManifestGraph(fetchImpl, registryBaseUrl, digest, regi
         rawJson,
         record: {
             digest,
+            manifestKind: classifyManifestKind(document),
             mediaType,
             artifactType: document.artifactType,
             configMediaType: document.config?.mediaType,
             subjectDigest: document.subject?.digest,
             annotations: document.annotations
         },
+        ...buildManifestRelations(digest, rawJson)
+    };
+}
+export function buildManifestRelations(digest, rawJson) {
+    const document = JSON.parse(rawJson);
+    return {
         descriptorRecords: _buildDescriptorRecords(digest, document),
         edgeRecords: _buildEdges(digest, document)
     };

package/dist/ingest/github/_manifest-ingest.js

@@ -1,53 +1,37 @@
-import { manifestFetchConcurrency, manifestIngestProgressStepRatio } from "../../tuning/index.js";
-import { loadManifestGraph } from "./_manifest-client.js";
+import { manifestFetchConcurrency, manifestIngestProgressStepRatio } from "../../config/index.js";
+import { buildManifestRelations, loadManifestGraph } from "./_manifest-client.js";
 import { loadRegistryPullToken } from "./_registry-token-client.js";
 export async function ingestManifests(fetchImpl, registryBaseUrl, options, writer, repository, scanId) {
-    const pendingDigests = repository.listPackageVersionDigests(scanId);
-    const initialDigestCount = pendingDigests.length;
-    const progressStep = Math.max(1, Math.ceil(initialDigestCount * manifestIngestProgressStepRatio));
-    const queuedDigests = new Set(pendingDigests);
-    const fetchedDigests = new Set();
-    const persistedDigests = new Set();
+    const manifests = repository.listPackageVersionManifestRefs(scanId);
+    const totalDigestCount = manifests.length;
+    const progressStep = Math.max(1, Math.ceil(totalDigestCount * manifestIngestProgressStepRatio));
     const registryPullTokenState = {};
-    options.logger.info(`Fetching manifests for ${pendingDigests.length} package versions`);
+    options.logger.info(`Fetching manifests for ${totalDigestCount} package versions`);
     let completed = 0;
-    const edgeRecords = [];
+    let nextManifestIndex = 0;
     const activeLoads = new Set();
-    while (pendingDigests.length > 0 || activeLoads.size > 0) {
-        while (pendingDigests.length > 0 && activeLoads.size < manifestFetchConcurrency) {
-            const digest = pendingDigests.shift();
-            if (!digest || fetchedDigests.has(digest)) {
-                continue;
-            }
-            const load = _loadQueuedManifest(digest, fetchImpl, registryBaseUrl, options, writer, pendingDigests, queuedDigests, fetchedDigests, persistedDigests, edgeRecords, completed, async () => (await _getRegistryPullToken(fetchImpl, registryBaseUrl, options, registryPullTokenState)).token, () => {
+    while (nextManifestIndex < manifests.length || activeLoads.size > 0) {
+        while (nextManifestIndex < manifests.length && activeLoads.size < manifestFetchConcurrency) {
+            const load = _fetchManifest(manifests[nextManifestIndex], fetchImpl, registryBaseUrl, options, writer, completed, async () => (await _getRegistryPullToken(fetchImpl, registryBaseUrl, options, registryPullTokenState)).token, () => {
                 completed += 1;
-                if (completed % progressStep === 0 || pendingDigests.length === 0) {
-                    options.logger.info(`Fetched manifests ${completed}/${queuedDigests.size}`);
+                if (completed % progressStep === 0 || completed === totalDigestCount) {
+                    options.logger.info(`Fetched manifests ${completed}/${totalDigestCount}`);
                 }
             }).finally(() => {
                 activeLoads.delete(load);
             });
+            nextManifestIndex += 1;
             activeLoads.add(load);
         }
         if (activeLoads.size > 0) {
             await Promise.race(activeLoads);
         }
     }
-    options.logger.info(`Starting manifest graph processing for ${edgeRecords.length} edges`);
-    let persistedEdgeCount = 0;
-    for (const edge of edgeRecords) {
-        if (!persistedDigests.has(edge.parentDigest) || !persistedDigests.has(edge.childDigest)) {
-            continue;
-        }
-        writer.insertManifestEdge(edge);
-        persistedEdgeCount += 1;
-    }
-    options.logger.info(`Inserted ${persistedEdgeCount} manifest edges; rebuilding reachability`);
-    writer.rebuildManifestReachability();
-    options.logger.info("Completed manifest graph processing");
+    _processManifestPayloads(options, writer, repository, scanId);
 }
-async function _loadQueuedManifest(digest, fetchImpl, registryBaseUrl, options, writer, pendingDigests, queuedDigests, fetchedDigests, persistedDigests, edgeRecords, completed, getRegistryToken, onComplete) {
-    options.logger.debug(`Fetching manifest ${completed + 1}/${queuedDigests.size}: ${digest}`);
+async function _fetchManifest(manifestRef, fetchImpl, registryBaseUrl, options, writer, completed, getRegistryToken, onComplete) {
+    const { versionId, digest } = manifestRef;
+    options.logger.debug(`Fetching manifest ${completed + 1}: ${digest}`);
     let manifest;
     try {
         manifest = await loadManifestGraph(fetchImpl, registryBaseUrl, digest, await getRegistryToken(), options);
@@ -55,27 +39,36 @@ async function _loadQueuedManifest(digest, fetchImpl, registryBaseUrl, options,
     catch (error) {
         if (_isMissingManifestError(error)) {
             options.logger.warn(`Skipping missing GHCR manifest ${digest}`);
-            fetchedDigests.add(digest);
             onComplete();
             return;
         }
         throw error;
     }
-    writer.insertManifest(manifest.record);
-    persistedDigests.add(manifest.record.digest);
+    writer.insertManifest({ versionId, ...manifest.record });
     writer.insertManifestPayload(manifest.record.digest, manifest.rawJson);
-    for (const descriptor of manifest.descriptorRecords) {
-        writer.insertManifestDescriptor(descriptor);
-        _enqueueDigest(descriptor.childDigest, pendingDigests, queuedDigests, fetchedDigests);
-    }
-    edgeRecords.push(...manifest.edgeRecords);
-    for (const edge of manifest.edgeRecords) {
-        _enqueueDigest(edge.parentDigest, pendingDigests, queuedDigests, fetchedDigests);
-        _enqueueDigest(edge.childDigest, pendingDigests, queuedDigests, fetchedDigests);
-    }
-    fetchedDigests.add(digest);
     onComplete();
 }
+function _processManifestPayloads(options, writer, repository, scanId) {
+    const digests = new Set(repository.listManifestDigests(scanId));
+    const payloads = repository.listManifestPayloads(scanId);
+    options.logger.info(`Starting manifest graph processing for ${payloads.length} manifest payloads`);
+    let persistedEdgeCount = 0;
+    for (const payload of payloads) {
+        const relations = buildManifestRelations(payload.digest, payload.rawJson);
+        for (const descriptor of relations.descriptorRecords) {
+            writer.insertManifestDescriptor(descriptor);
+        }
+        for (const edge of relations.edgeRecords) {
+            if (digests.has(edge.parentDigest) && digests.has(edge.childDigest)) {
+                writer.insertManifestEdge(edge);
+                persistedEdgeCount += 1;
+            }
+        }
+    }
+    options.logger.info(`Inserted ${persistedEdgeCount} manifest edges; rebuilding reachability`);
+    writer.rebuildManifestReachability();
+    options.logger.info("Completed manifest graph processing");
+}
 function _isMissingManifestError(error) {
     if (!(error instanceof Error)) {
         return false;
@@ -95,10 +88,3 @@ async function _getRegistryPullToken(fetchImpl, registryBaseUrl, options, regist
     registryPullTokenState.token = registryPullToken;
     return registryPullToken;
 }
-function _enqueueDigest(digest, pendingDigests, queuedDigests, fetchedDigests) {
-    if (queuedDigests.has(digest) || fetchedDigests.has(digest)) {
-        return;
-    }
-    pendingDigests.push(digest);
-    queuedDigests.add(digest);
-}

package/dist/ingest/github/_manifest-kind.d.ts

@@ -0,0 +1,20 @@
+import type { ManifestKind } from "../../core/index.js";
+interface _RegistryLayer {
+    mediaType?: string;
+    annotations?: Record<string, unknown>;
+}
+interface _RegistryManifestDocument {
+    mediaType?: string;
+    artifactType?: string;
+    annotations?: Record<string, unknown>;
+    config?: {
+        mediaType?: string;
+        artifactType?: string;
+    };
+    layers?: _RegistryLayer[];
+    subject?: {
+        digest?: string;
+    };
+}
+export declare function classifyManifestKind(document: _RegistryManifestDocument): ManifestKind | undefined;
+export {};

package/dist/ingest/github/_manifest-kind.js

@@ -0,0 +1,50 @@
+export function classifyManifestKind(document) {
+    if (document.mediaType === "application/vnd.oci.image.index.v1+json" ||
+        document.mediaType === "application/vnd.docker.distribution.manifest.list.v2+json") {
+        return "image_index";
+    }
+    if (_isSignatureManifest(document)) {
+        return "signature_manifest";
+    }
+    if (_isAttestationManifest(document)) {
+        return "attestation_manifest";
+    }
+    if (document.mediaType === "application/vnd.oci.image.manifest.v1+json") {
+        return "image_manifest";
+    }
+    if (document.mediaType === "application/vnd.oci.artifact.manifest.v1+json") {
+        return "artifact_manifest";
+    }
+    return undefined;
+}
+function _isSignatureManifest(document) {
+    const candidates = [
+        document.artifactType,
+        document.config?.artifactType,
+        document.config?.mediaType,
+        ...(document.layers?.map((layer) => layer.mediaType) ?? [])
+    ];
+    if (candidates.some((value) => typeof value === "string" && value.includes("application/vnd.dev.sigstore"))) {
+        return true;
+    }
+    return (typeof document.annotations?.["dev.sigstore.bundle.predicateType"] === "string" &&
+        document.annotations["dev.sigstore.bundle.predicateType"] === "https://sigstore.dev/cosign/sign/v1");
+}
+function _isAttestationManifest(document) {
+    const candidates = [
+        document.artifactType,
+        document.config?.artifactType,
+        document.config?.mediaType,
+        ...(document.layers?.map((layer) => layer.mediaType) ?? [])
+    ];
+    if (candidates.some((value) => typeof value === "string" && value.includes("application/vnd.in-toto"))) {
+        return true;
+    }
+    if (typeof document.annotations?.["vnd.docker.reference.type"] === "string") {
+        return document.annotations["vnd.docker.reference.type"] === "attestation-manifest";
+    }
+    if (typeof document.annotations?.["dev.sigstore.bundle.predicateType"] === "string") {
+        return document.annotations["dev.sigstore.bundle.predicateType"] !== "https://sigstore.dev/cosign/sign/v1";
+    }
+    return (document.layers?.some((layer) => typeof layer.annotations?.["in-toto.io/predicate-type"] === "string") ?? false);
+}

package/dist/ingest/github/_package-metadata-load.d.ts

@@ -0,0 +1,5 @@
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export interface GitHubPackageMetadata {
+    rawJson: string;
+}
+export declare function loadPackageMetadata(fetchImpl: FetchLike, options: GitHubScanOptions): Promise<GitHubPackageMetadata>;

package/dist/ingest/github/_package-metadata-load.js

@@ -0,0 +1,45 @@
+import { githubApiBaseUrl, githubApiVersion } from "../../config/index.js";
+import { getOwnerURIComponent } from "../../core/index.js";
+import { buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
+export async function loadPackageMetadata(fetchImpl, options) {
+    const ownerURIComponent = await getOwnerURIComponent(fetchImpl, options.owner, options.token, options.logger);
+    const url = new URL(`/${ownerURIComponent}/packages/container/${encodeURIComponent(options.packageName)}`, githubApiBaseUrl).toString();
+    let response;
+    try {
+        response = await withFetchRetry(async () => {
+            const packageResponse = await fetchImpl(url, {
+                headers: {
+                    Accept: "application/vnd.github+json",
+                    Authorization: `Bearer ${options.token}`,
+                    "User-Agent": "ghcr-manager",
+                    "X-GitHub-Api-Version": githubApiVersion
+                }
+            });
+            if (!packageResponse.ok && _shouldRetryStatus(packageResponse.status)) {
+                throw new Error(await buildHttpErrorMessage(packageResponse, "GitHub package metadata request failed"));
+            }
+            return packageResponse;
+        }, {
+            logger: options.logger,
+            label: "GitHub package metadata request",
+            shouldRetry: (error) => _shouldRetryError(error)
+        });
+    }
+    catch (error) {
+        throw new Error(buildFetchTransportErrorMessage(error, "GitHub package metadata request failed"), { cause: error });
+    }
+    if (!response.ok) {
+        throw new Error(await buildHttpErrorMessage(response, "GitHub package metadata request failed"));
+    }
+    const payload = (await response.json());
+    return { rawJson: JSON.stringify(payload) };
+}
+function _shouldRetryStatus(status) {
+    return status === 429 || status === 502 || status === 503 || status === 504;
+}
+function _shouldRetryError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /fetch failed|status 429|status 502|status 503|status 504/.test(error.message);
+}

package/dist/ingest/github/_package-version-page-load.d.ts

@@ -10,4 +10,4 @@ export interface GitHubPackageVersionPageItem {
         };
     };
 }
-export declare function loadPackageVersionPage(fetchImpl: FetchLike, githubApiBaseUrl: string, options: GitHubScanOptions, page: number): Promise<GitHubPackageVersionPageItem[]>;
+export declare function loadPackageVersionPage(fetchImpl: FetchLike, options: GitHubScanOptions, page: number): Promise<GitHubPackageVersionPageItem[]>;

package/dist/ingest/github/_package-version-page-load.js

@@ -1,7 +1,9 @@
+import { githubApiBaseUrl, githubApiVersion } from "../../config/index.js";
+import { getOwnerURIComponent } from "../../core/index.js";
 import { buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
-export async function loadPackageVersionPage(fetchImpl, githubApiBaseUrl, options, page) {
+export async function loadPackageVersionPage(fetchImpl, options, page) {
     const startTime = Date.now();
-    const url = buildPackageVersionPageUrl(githubApiBaseUrl, options, page);
+    const url = await buildPackageVersionPageUrl(fetchImpl, options, page);
     let response;
     try {
         response = await withFetchRetry(async () => {
@@ -10,7 +12,7 @@ export async function loadPackageVersionPage(fetchImpl, githubApiBaseUrl, option
                     Accept: "application/vnd.github+json",
                     Authorization: `Bearer ${options.token}`,
                     "User-Agent": "ghcr-manager",
-                    "X-GitHub-Api-Version": "2022-11-28"
+                    "X-GitHub-Api-Version": githubApiVersion
                 }
             });
             if (!pageResponse.ok && _shouldRetryStatus(pageResponse.status)) {
@@ -35,8 +37,9 @@ export async function loadPackageVersionPage(fetchImpl, githubApiBaseUrl, option
     options.logger.debug(`Loaded GitHub package-version page ${page} in ${Date.now() - startTime}ms (${pageItems.length} items)`);
     return pageItems;
 }
-function buildPackageVersionPageUrl(githubApiBaseUrl, options, page) {
-    const url = new URL(`/orgs/${encodeURIComponent(options.owner)}/packages/container/${encodeURIComponent(options.packageName)}/versions`, githubApiBaseUrl);
+async function buildPackageVersionPageUrl(fetchImpl, options, page) {
+    const ownerURIComponent = await getOwnerURIComponent(fetchImpl, options.owner, options.token, options.logger);
+    const url = new URL(`/${ownerURIComponent}/packages/container/${encodeURIComponent(options.packageName)}/versions`, githubApiBaseUrl);
     url.searchParams.set("per_page", "100");
     url.searchParams.set("page", String(page));
     return url.toString();

package/dist/ingest/github/_packages-client.d.ts

@@ -2,7 +2,7 @@ import type { PackageVersionRecord, TagRecord } from "../../core/index.js";
 import type { ScanWriter } from "../../db/index.js";
 import { type GitHubPackageVersionPageItem } from "./_package-version-page-load.js";
 import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
-export declare function ingestPackageVersions(fetchImpl: FetchLike, githubApiBaseUrl: string, options: GitHubScanOptions, writer: ScanWriter): Promise<{
+export declare function ingestPackageVersions(fetchImpl: FetchLike, options: GitHubScanOptions, writer: ScanWriter): Promise<{
     packageVersions: number;
     tags: number;
 }>;

package/dist/ingest/github/_packages-client.js

@@ -1,17 +1,20 @@
 import { loadPackageVersionPage } from "./_package-version-page-load.js";
 import { ingestParallelPaginated } from "./_parallel-paginated-ingest.js";
-export async function ingestPackageVersions(fetchImpl, githubApiBaseUrl, options, writer) {
+export async function ingestPackageVersions(fetchImpl, options, writer) {
     let tagCount = 0;
+    const firstPageItems = await loadPackageVersionPage(fetchImpl, options, 1);
     const result = await ingestParallelPaginated({
         logger: options.logger,
+        firstPageItems,
         loadPage(page) {
-            return loadPackageVersionPage(fetchImpl, githubApiBaseUrl, options, page);
+            return loadPackageVersionPage(fetchImpl, options, page);
         },
         writePage(pageItems) {
             _writePage(writer, pageItems);
             tagCount += _countTags(pageItems);
         }
     });
+    await _assertStableFirstPage(fetchImpl, options, firstPageItems);
     return { packageVersions: result.items, tags: tagCount };
 }
 export function buildTags(packageVersions) {
@@ -24,7 +27,6 @@ export function buildTags(packageVersions) {
         for (const tagName of tagNames) {
             tags.push({
                 tag: tagName,
-                digest: version.digest,
                 versionId: version.versionId
             });
         }
@@ -35,7 +37,6 @@ export function normalizePackageVersions(packageVersions) {
     return packageVersions
         .map((version) => ({
         versionId: version.id,
-        digest: version.name,
         createdAt: version.created_at,
         updatedAt: version.updated_at,
         metadata: version.metadata
@@ -57,3 +58,19 @@ function _writePage(writer, pageItems) {
 function _countTags(pageItems) {
     return buildTags(normalizePackageVersions(pageItems)).length;
 }
+async function _assertStableFirstPage(fetchImpl, options, initialPageItems) {
+    const reloadedPageItems = await loadPackageVersionPage(fetchImpl, options, 1);
+    if (_buildPageSignature(initialPageItems) === _buildPageSignature(reloadedPageItems)) {
+        return;
+    }
+    throw new Error(`GitHub package-version page 1 changed while scanning ${options.owner}/${options.packageName}; aborting scan`);
+}
+function _buildPageSignature(pageItems) {
+    return JSON.stringify(pageItems.map((pageItem) => ({
+        id: pageItem.id,
+        name: pageItem.name,
+        createdAt: pageItem.created_at,
+        updatedAt: pageItem.updated_at,
+        tags: Array.isArray(pageItem.metadata?.container?.tags) ? [...pageItem.metadata.container.tags] : []
+    })));
+}

package/dist/ingest/github/_parallel-paginated-ingest.d.ts

@@ -3,6 +3,7 @@ export interface ParallelPaginatedIngestOptions<T> {
     loadPage(page: number): Promise<T[]>;
     writePage(pageItems: T[], page: number): Promise<void> | void;
     logger: GitHubScanLogger;
+    firstPageItems?: T[];
 }
 export interface ParallelPaginatedIngestResult {
     pages: number;

package/dist/ingest/github/_parallel-paginated-ingest.js

@@ -1,8 +1,8 @@
-import { packageVersionPageFetchConcurrency, paginatedIngestProgressIntervalPages } from "../../tuning/index.js";
+import { packageVersionPageFetchConcurrency, paginatedIngestProgressIntervalPages } from "../../config/index.js";
 const _DEFAULT_PAGE_SIZE = 100;
 const _PROGRESS_LABEL = "GitHub package-version pages";
 export async function ingestParallelPaginated(options) {
-    const firstPageItems = await options.loadPage(1);
+    const firstPageItems = options.firstPageItems ?? (await options.loadPage(1));
     let pages = 0;
     let items = 0;
     let lastLoggedPage = 0;

package/dist/ingest/github/_shared.d.ts

@@ -1,3 +1,4 @@
+export { buildHttpErrorMessage } from "../../core/index.js";
 export interface GitHubScanOptions {
     owner: string;
     packageName: string;
@@ -20,7 +21,6 @@ export type FetchLike = (input: string, init?: RequestInit) => Promise<FetchLike
 export declare const acceptedManifestMediaTypes: string;
 export declare function defaultFetch(input: string, init?: RequestInit): Promise<FetchLikeResponse>;
 export declare function buildFetchTransportErrorMessage(error: unknown, fallback: string): string;
-export declare function buildHttpErrorMessage(response: FetchLikeResponse, fallback: string): Promise<string>;
 export declare function withFetchRetry<T>(run: () => Promise<T>, options: {
     logger: GitHubScanLogger;
     label: string;

package/dist/ingest/github/_shared.js

@@ -1,4 +1,5 @@
-import { ingestRequestRetryCount, ingestRequestRetryDelayMs } from "../../tuning/index.js";
+import { ingestRequestRetryCount, ingestRequestRetryDelayMs } from "../../config/index.js";
+export { buildHttpErrorMessage } from "../../core/index.js";
 export const acceptedManifestMediaTypes = [
     "application/vnd.oci.image.index.v1+json",
     "application/vnd.oci.image.manifest.v1+json",
@@ -14,23 +15,6 @@ export function buildFetchTransportErrorMessage(error, fallback) {
     details.push(..._collectErrorDetails(error));
     return details.join(" - ");
 }
-export async function buildHttpErrorMessage(response, fallback) {
-    const details = [fallback, `status ${response.status}`];
-    const body = await _readJsonErrorBody(response);
-    const message = typeof body?.message === "string" ? body.message : undefined;
-    const documentationUrl = typeof body?.documentation_url === "string" ? body.documentation_url : undefined;
-    const authenticateHeader = response.headers.get("www-authenticate") ?? undefined;
-    if (message) {
-        details.push(message);
-    }
-    if (documentationUrl) {
-        details.push(documentationUrl);
-    }
-    if (authenticateHeader) {
-        details.push(`www-authenticate: ${authenticateHeader}`);
-    }
-    return details.join(" - ");
-}
 export async function withFetchRetry(run, options) {
     let attempt = 0;
     for (;;) {
@@ -49,22 +33,6 @@ export async function withFetchRetry(run, options) {
         }
     }
 }
-async function _readJsonErrorBody(response) {
-    const contentType = response.headers.get("content-type")?.split(";")[0];
-    if (contentType && contentType !== "application/json" && !contentType.endsWith("+json")) {
-        return undefined;
-    }
-    try {
-        const body = await response.json();
-        if (body && typeof body === "object") {
-            return body;
-        }
-    }
-    catch {
-        return undefined;
-    }
-    return undefined;
-}
 function _sleep(delayMs) {
     return new Promise((resolve) => setTimeout(resolve, delayMs));
 }

package/dist/ingest/github/index.d.ts

@@ -1,7 +1,11 @@
 import { ScanWriter, SnapshotRepository } from "../../db/index.js";
+import { type GitHubPackageMetadata } from "./_package-metadata-load.js";
 import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
 export { type GitHubScanOptions } from "./_shared.js";
+export { loadPackageMetadata, type GitHubPackageMetadata } from "./_package-metadata-load.js";
+export { defaultFetch, type FetchLike, type GitHubScanLogger } from "./_shared.js";
 interface _GitHubScanRuntime {
     fetchImpl?: FetchLike;
+    packageMetadata?: GitHubPackageMetadata;
 }
 export declare function importGitHubScan(options: GitHubScanOptions, writer: ScanWriter, repository: SnapshotRepository, runtime?: _GitHubScanRuntime): Promise<void>;