ghcr-cleanup-manager 1.1.0

package/dist/ingest/github/_package-version-page-load.d.ts

@@ -0,0 +1,13 @@
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export interface GitHubPackageVersionPageItem {
+    id: number;
+    name: string;
+    created_at: string;
+    updated_at: string;
+    metadata?: {
+        container?: {
+            tags?: string[];
+        };
+    };
+}
+export declare function loadPackageVersionPage(fetchImpl: FetchLike, options: GitHubScanOptions, page: number): Promise<GitHubPackageVersionPageItem[]>;

package/dist/ingest/github/_package-version-page-load.js

@@ -0,0 +1,55 @@
+import { githubApiBaseUrl, githubApiVersion } from "../../config/index.js";
+import { getOwnerURIComponent } from "../../core/index.js";
+import { buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
+export async function loadPackageVersionPage(fetchImpl, options, page) {
+    const startTime = Date.now();
+    const url = await buildPackageVersionPageUrl(fetchImpl, options, page);
+    let response;
+    try {
+        response = await withFetchRetry(async () => {
+            const pageResponse = await fetchImpl(url, {
+                headers: {
+                    Accept: "application/vnd.github+json",
+                    Authorization: `Bearer ${options.token}`,
+                    "User-Agent": "ghcr-cleanup-manager",
+                    "X-GitHub-Api-Version": githubApiVersion
+                }
+            });
+            if (!pageResponse.ok && _shouldRetryStatus(pageResponse.status)) {
+                throw new Error(await buildHttpErrorMessage(pageResponse, `GitHub Packages request for page ${page} failed`));
+            }
+            return pageResponse;
+        }, {
+            logger: options.logger,
+            label: `GitHub Packages request for page ${page}`,
+            shouldRetry: (error) => _shouldRetryError(error)
+        });
+    }
+    catch (error) {
+        throw new Error(buildFetchTransportErrorMessage(error, `GitHub Packages request for page ${page} failed`), {
+            cause: error
+        });
+    }
+    if (!response.ok) {
+        throw new Error(await buildHttpErrorMessage(response, "GitHub Packages request failed"));
+    }
+    const pageItems = (await response.json());
+    options.logger.debug(`Loaded GitHub package-version page ${page} in ${Date.now() - startTime}ms (${pageItems.length} items)`);
+    return pageItems;
+}
+async function buildPackageVersionPageUrl(fetchImpl, options, page) {
+    const ownerURIComponent = await getOwnerURIComponent(fetchImpl, options.owner, options.token, options.logger);
+    const url = new URL(`/${ownerURIComponent}/packages/container/${encodeURIComponent(options.packageName)}/versions`, githubApiBaseUrl);
+    url.searchParams.set("per_page", "100");
+    url.searchParams.set("page", String(page));
+    return url.toString();
+}
+function _shouldRetryStatus(status) {
+    return status === 429 || status === 502 || status === 503 || status === 504;
+}
+function _shouldRetryError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /fetch failed|status 429|status 502|status 503|status 504/.test(error.message);
+}

package/dist/ingest/github/_packages-client.d.ts

@@ -0,0 +1,10 @@
+import type { PackageVersionRecord, TagRecord } from "../../core/index.js";
+import type { ScanWriter } from "../../db/index.js";
+import { type GitHubPackageVersionPageItem } from "./_package-version-page-load.js";
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export declare function ingestPackageVersions(fetchImpl: FetchLike, options: GitHubScanOptions, writer: ScanWriter): Promise<{
+    packageVersions: number;
+    tags: number;
+}>;
+export declare function buildTags(packageVersions: PackageVersionRecord[]): TagRecord[];
+export declare function normalizePackageVersions(packageVersions: GitHubPackageVersionPageItem[]): PackageVersionRecord[];

package/dist/ingest/github/_packages-client.js

@@ -0,0 +1,76 @@
+import { loadPackageVersionPage } from "./_package-version-page-load.js";
+import { ingestParallelPaginated } from "./_parallel-paginated-ingest.js";
+export async function ingestPackageVersions(fetchImpl, options, writer) {
+    let tagCount = 0;
+    const firstPageItems = await loadPackageVersionPage(fetchImpl, options, 1);
+    const result = await ingestParallelPaginated({
+        logger: options.logger,
+        firstPageItems,
+        loadPage(page) {
+            return loadPackageVersionPage(fetchImpl, options, page);
+        },
+        writePage(pageItems) {
+            _writePage(writer, pageItems);
+            tagCount += _countTags(pageItems);
+        }
+    });
+    await _assertStableFirstPage(fetchImpl, options, firstPageItems);
+    return { packageVersions: result.items, tags: tagCount };
+}
+export function buildTags(packageVersions) {
+    const tags = [];
+    for (const version of packageVersions) {
+        const metadata = version.metadata?.container;
+        const tagNames = Array.isArray(metadata?.tags)
+            ? metadata.tags.filter((tag) => typeof tag === "string")
+            : [];
+        for (const tagName of tagNames) {
+            tags.push({
+                tag: tagName,
+                versionId: version.versionId
+            });
+        }
+    }
+    return tags.sort((left, right) => left.tag.localeCompare(right.tag));
+}
+export function normalizePackageVersions(packageVersions) {
+    return packageVersions
+        .map((version) => ({
+        versionId: version.id,
+        createdAt: version.created_at,
+        updatedAt: version.updated_at,
+        metadata: version.metadata
+    }))
+        .sort((left, right) => left.versionId - right.versionId);
+}
+function _writePage(writer, pageItems) {
+    const versions = normalizePackageVersions(pageItems);
+    const rawItemsByVersionId = new Map(pageItems.map((pageItem) => [pageItem.id, pageItem]));
+    for (const version of versions) {
+        writer.insertPackageVersion(version);
+        writer.insertPackageVersionPayload(version.versionId, JSON.stringify(rawItemsByVersionId.get(version.versionId)));
+    }
+    const tags = buildTags(versions);
+    for (const tag of tags) {
+        writer.insertTag(tag);
+    }
+}
+function _countTags(pageItems) {
+    return buildTags(normalizePackageVersions(pageItems)).length;
+}
+async function _assertStableFirstPage(fetchImpl, options, initialPageItems) {
+    const reloadedPageItems = await loadPackageVersionPage(fetchImpl, options, 1);
+    if (_buildPageSignature(initialPageItems) === _buildPageSignature(reloadedPageItems)) {
+        return;
+    }
+    throw new Error(`GitHub package-version page 1 changed while scanning ${options.owner}/${options.packageName}; aborting scan`);
+}
+function _buildPageSignature(pageItems) {
+    return JSON.stringify(pageItems.map((pageItem) => ({
+        id: pageItem.id,
+        name: pageItem.name,
+        createdAt: pageItem.created_at,
+        updatedAt: pageItem.updated_at,
+        tags: Array.isArray(pageItem.metadata?.container?.tags) ? [...pageItem.metadata.container.tags] : []
+    })));
+}

package/dist/ingest/github/_parallel-paginated-ingest.d.ts

@@ -0,0 +1,12 @@
+import type { GitHubScanLogger } from "./_shared.js";
+export interface ParallelPaginatedIngestOptions<T> {
+    loadPage(page: number): Promise<T[]>;
+    writePage(pageItems: T[], page: number): Promise<void> | void;
+    logger: GitHubScanLogger;
+    firstPageItems?: T[];
+}
+export interface ParallelPaginatedIngestResult {
+    pages: number;
+    items: number;
+}
+export declare function ingestParallelPaginated<T>(options: ParallelPaginatedIngestOptions<T>): Promise<ParallelPaginatedIngestResult>;

package/dist/ingest/github/_parallel-paginated-ingest.js

@@ -0,0 +1,49 @@
+import { packageVersionPageFetchConcurrency, paginatedIngestProgressIntervalPages } from "../../config/index.js";
+const _DEFAULT_PAGE_SIZE = 100;
+const _PROGRESS_LABEL = "GitHub package-version pages";
+export async function ingestParallelPaginated(options) {
+    const firstPageItems = options.firstPageItems ?? (await options.loadPage(1));
+    let pages = 0;
+    let items = 0;
+    let lastLoggedPage = 0;
+    if (firstPageItems.length === 0) {
+        return { pages: 0, items: 0 };
+    }
+    await options.writePage(firstPageItems, 1);
+    pages = 1;
+    items = firstPageItems.length;
+    options.logger.info(`Loaded ${_PROGRESS_LABEL} 1 (${items} items total)`);
+    lastLoggedPage = 1;
+    if (firstPageItems.length < _DEFAULT_PAGE_SIZE) {
+        return { pages, items };
+    }
+    let nextPage = 2;
+    let stopPageExclusive = Number.POSITIVE_INFINITY;
+    const workers = Array.from({ length: packageVersionPageFetchConcurrency }, async () => {
+        while (nextPage < stopPageExclusive) {
+            const page = nextPage;
+            nextPage += 1;
+            const pageItems = await options.loadPage(page);
+            if (pageItems.length === 0) {
+                stopPageExclusive = Math.min(stopPageExclusive, page);
+                return;
+            }
+            await options.writePage(pageItems, page);
+            pages = Math.max(pages, page);
+            items += pageItems.length;
+            if (page % paginatedIngestProgressIntervalPages === 0 || pageItems.length < _DEFAULT_PAGE_SIZE) {
+                options.logger.info(`Loaded ${_PROGRESS_LABEL} ${page} (${items} items total)`);
+                lastLoggedPage = Math.max(lastLoggedPage, page);
+            }
+            if (pageItems.length < _DEFAULT_PAGE_SIZE) {
+                stopPageExclusive = Math.min(stopPageExclusive, page + 1);
+                return;
+            }
+        }
+    });
+    await Promise.all(workers);
+    if (pages > 0 && lastLoggedPage !== pages) {
+        options.logger.info(`Loaded ${_PROGRESS_LABEL} ${pages} (${items} items total)`);
+    }
+    return { pages, items };
+}

package/dist/ingest/github/_registry-token-client.d.ts

@@ -0,0 +1,6 @@
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export interface RegistryPullToken {
+    token: string;
+    expiresAt: number;
+}
+export declare function loadRegistryPullToken(fetchImpl: FetchLike, registryBaseUrl: string, options: GitHubScanOptions): Promise<RegistryPullToken>;

package/dist/ingest/github/_registry-token-client.js

@@ -0,0 +1,67 @@
+import { buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
+export async function loadRegistryPullToken(fetchImpl, registryBaseUrl, options) {
+    const startTime = Date.now();
+    const url = _buildRegistryTokenUrl(registryBaseUrl, options);
+    let response;
+    try {
+        response = await withFetchRetry(async () => {
+            const tokenResponse = await fetchImpl(url, {
+                headers: _buildTokenHeaders(options)
+            });
+            if (!tokenResponse.ok && _shouldRetryStatus(tokenResponse.status)) {
+                throw new Error(await buildHttpErrorMessage(tokenResponse, "GHCR token request failed"));
+            }
+            return tokenResponse;
+        }, {
+            logger: options.logger,
+            label: "GHCR token request",
+            shouldRetry: (error) => _shouldRetryError(error)
+        });
+    }
+    catch (error) {
+        throw new Error(buildFetchTransportErrorMessage(error, "GHCR token request failed"), {
+            cause: error
+        });
+    }
+    if (!response.ok) {
+        throw new Error(await buildHttpErrorMessage(response, "GHCR token request failed"));
+    }
+    const body = (await response.json());
+    if (!body.token) {
+        throw new Error("GHCR token response did not include a token");
+    }
+    const registryPullToken = {
+        token: body.token,
+        expiresAt: _getExpiresAt(body.expires_in, body.issued_at)
+    };
+    options.logger.debug(`Loaded GHCR pull token in ${Date.now() - startTime}ms (expires in ${Math.max(0, Math.round((registryPullToken.expiresAt - Date.now()) / 1000))}s)`);
+    return registryPullToken;
+}
+function _buildRegistryTokenUrl(registryBaseUrl, options) {
+    const registryUrl = new URL(registryBaseUrl);
+    const tokenUrl = new URL("/token", registryUrl);
+    tokenUrl.searchParams.set("service", registryUrl.host);
+    tokenUrl.searchParams.set("scope", `repository:${options.owner}/${options.packageName}:pull`);
+    return tokenUrl.toString();
+}
+function _buildTokenHeaders(options) {
+    const basicAuth = Buffer.from(`${options.owner}:${options.token}`).toString("base64");
+    return {
+        "User-Agent": "ghcr-cleanup-manager",
+        Authorization: `Basic ${basicAuth}`
+    };
+}
+function _getExpiresAt(expiresIn, issuedAt) {
+    const expiresInSeconds = typeof expiresIn === "number" && expiresIn > 0 ? expiresIn : 60;
+    const issuedAtMilliseconds = typeof issuedAt === "string" && !Number.isNaN(Date.parse(issuedAt)) ? Date.parse(issuedAt) : Date.now();
+    return issuedAtMilliseconds + expiresInSeconds * 1000;
+}
+function _shouldRetryStatus(status) {
+    return status === 429 || status === 502 || status === 503 || status === 504;
+}
+function _shouldRetryError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /fetch failed|status 429|status 502|status 503|status 504/.test(error.message);
+}

package/dist/ingest/github/_shared.d.ts

@@ -0,0 +1,28 @@
+export { buildHttpErrorMessage } from "../../core/index.js";
+export interface GitHubScanOptions {
+    owner: string;
+    packageName: string;
+    token: string;
+    logger: GitHubScanLogger;
+}
+export interface GitHubScanLogger {
+    debug(message: string): void;
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+}
+export interface FetchLikeResponse {
+    ok: boolean;
+    status: number;
+    headers: Headers;
+    json(): Promise<unknown>;
+}
+export type FetchLike = (input: string, init?: RequestInit) => Promise<FetchLikeResponse>;
+export declare const acceptedManifestMediaTypes: string;
+export declare function defaultFetch(input: string, init?: RequestInit): Promise<FetchLikeResponse>;
+export declare function buildFetchTransportErrorMessage(error: unknown, fallback: string): string;
+export declare function withFetchRetry<T>(run: () => Promise<T>, options: {
+    logger: GitHubScanLogger;
+    label: string;
+    shouldRetry?: (error: unknown) => boolean;
+}): Promise<T>;

package/dist/ingest/github/_shared.js

@@ -0,0 +1,70 @@
+import { ingestRequestRetryCount, ingestRequestRetryDelayMs } from "../../config/index.js";
+export { buildHttpErrorMessage } from "../../core/index.js";
+export const acceptedManifestMediaTypes = [
+    "application/vnd.oci.image.index.v1+json",
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.oci.artifact.manifest.v1+json"
+].join(", ");
+export async function defaultFetch(input, init) {
+    return fetch(input, init);
+}
+export function buildFetchTransportErrorMessage(error, fallback) {
+    const details = [fallback];
+    details.push(..._collectErrorDetails(error));
+    return details.join(" - ");
+}
+export async function withFetchRetry(run, options) {
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await run();
+        }
+        catch (error) {
+            attempt += 1;
+            const shouldRetry = options.shouldRetry ? options.shouldRetry(error) : true;
+            if (!shouldRetry || attempt > ingestRequestRetryCount) {
+                throw error;
+            }
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            options.logger.warn(`${options.label} failed on attempt ${attempt}/${ingestRequestRetryCount + 1}; retrying in ${ingestRequestRetryDelayMs}ms - ${errorMessage}`);
+            await _sleep(ingestRequestRetryDelayMs);
+        }
+    }
+}
+function _sleep(delayMs) {
+    return new Promise((resolve) => setTimeout(resolve, delayMs));
+}
+function _collectErrorDetails(error) {
+    if (!(error instanceof Error)) {
+        return [String(error)];
+    }
+    const details = [];
+    const seen = new Set();
+    let current = error;
+    while (current instanceof Error && !seen.has(current)) {
+        seen.add(current);
+        const message = _formatErrorMessage(current);
+        if (message && !details.includes(message)) {
+            details.push(message);
+        }
+        current = current.cause;
+    }
+    return details.length > 0 ? details : [String(error)];
+}
+function _formatErrorMessage(error) {
+    const code = "code" in error && typeof error.code === "string"
+        ? error.code
+        : undefined;
+    if (error.message && code) {
+        return `${error.message} (${code})`;
+    }
+    if (error.message) {
+        return error.message;
+    }
+    if (code) {
+        return code;
+    }
+    return undefined;
+}

package/dist/ingest/github/index.d.ts

@@ -0,0 +1,11 @@
+import { ScanWriter, SnapshotRepository } from "../../db/index.js";
+import { type GitHubPackageMetadata } from "./_package-metadata-load.js";
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export { type GitHubScanOptions } from "./_shared.js";
+export { loadPackageMetadata, type GitHubPackageMetadata } from "./_package-metadata-load.js";
+export { defaultFetch, type FetchLike, type GitHubScanLogger } from "./_shared.js";
+interface _GitHubScanRuntime {
+    fetchImpl?: FetchLike;
+    packageMetadata?: GitHubPackageMetadata;
+}
+export declare function importGitHubScan(options: GitHubScanOptions, writer: ScanWriter, repository: SnapshotRepository, runtime?: _GitHubScanRuntime): Promise<void>;

package/dist/ingest/github/index.js

@@ -0,0 +1,29 @@
+import { ghcrRegistryBaseUrl } from "../../config/index.js";
+import { ingestManifests } from "./_manifest-ingest.js";
+import { loadPackageMetadata } from "./_package-metadata-load.js";
+import { ingestPackageVersions } from "./_packages-client.js";
+import { defaultFetch } from "./_shared.js";
+export { loadPackageMetadata } from "./_package-metadata-load.js";
+export { defaultFetch } from "./_shared.js";
+export async function importGitHubScan(options, writer, repository, runtime) {
+    const fetchImpl = runtime?.fetchImpl ?? defaultFetch;
+    const scanStartedAt = new Date().toISOString();
+    const fullPackageName = `${options.owner}/${options.packageName}`;
+    const packageMetadata = runtime?.packageMetadata ?? (await loadPackageMetadata(fetchImpl, options));
+    writer.startScan(options.owner, options.packageName, scanStartedAt, packageMetadata);
+    const scanId = writer.getActiveScanId();
+    options.logger.info(`Starting GitHub package scan for ${fullPackageName}`);
+    try {
+        options.logger.info(`Starting remote data pull for ${fullPackageName}`);
+        const counts = await ingestPackageVersions(fetchImpl, options, writer);
+        options.logger.info(`Loaded ${counts.packageVersions} package versions and ${counts.tags} tags`);
+        await ingestManifests(fetchImpl, ghcrRegistryBaseUrl, options, writer, repository, scanId);
+        options.logger.info(`Completed remote data pull for ${fullPackageName}`);
+        writer.markScanCompleted(new Date().toISOString());
+        options.logger.info(`Completed GitHub package scan for ${fullPackageName}`);
+    }
+    catch (error) {
+        writer.markScanFailed(new Date().toISOString());
+        throw error;
+    }
+}

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "ghcr-cleanup-manager",
+  "description": "GHCR Cleanup Manager: GitHub Action and CLI for safe GitHub Container Registry cleanup and inspection.",
+  "homepage": "https://github.com/ghcr-manager/ghcr-cleanup-manager#readme",
+  "version": "v1.1.0",
+  "license": "MIT",
+  "type": "module",
+  "keywords": [
+    "ghcr",
+    "github-container-registry",
+    "github-packages",
+    "container-registry",
+    "docker",
+    "oci",
+    "cleanup",
+    "github-actions",
+    "sqlite"
+  ],
+  "bugs": {
+    "url": "https://github.com/ghcr-manager/ghcr-cleanup-manager/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ghcr-manager/ghcr-cleanup-manager"
+  },
+  "workspaces": [
+    "visualizer"
+  ],
+  "engines": {
+    "node": ">=24.0.0"
+  },
+  "files": [
+    "dist",
+    "resources/sql",
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "bin": {
+    "ghcr-cleanup-manager": "./dist/cli/index.js"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.json && npm run build:visualizer",
+    "build:visualizer": "npm run build --workspace visualizer",
+    "ghcr-cleanup-manager": "tsx src/cli/index.ts",
+    "ghcr-cleanup-manager:dist": "node dist/cli/index.js",
+    "visualize": "npm run start --workspace visualizer --",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "npm run check:file-names && npm run check:test-mapping && npm run typecheck && npm run typecheck --workspace visualizer && npm run lint:ts && npm run lint:yaml && npm run lint:markdown && npm run lint --workspace visualizer && npm run format:check",
+    "check:file-names": "node tools/tests/check-src-file-names.mjs",
+    "check:test-mapping": "node tools/tests/check-test-mapping.mjs",
+    "lint:ts": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "lint:yaml": "eslint \".github/**/*.yml\" \"*.yml\"",
+    "lint:markdown": "markdownlint-cli2",
+    "typecheck": "tsc --noEmit --project tsconfig.json",
+    "test": "bash -lc 'shopt -s globstar nullglob; node --import tsx --test tests/**/*.test.ts' && npm run test --workspace visualizer",
+    "coverage": "bash -lc 'shopt -s globstar nullglob; c8 --src src --reporter=text --reporter=text-summary node --import tsx --test tests/**/*.test.ts'"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.9.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^24.12.4",
+    "c8": "^11.0.0",
+    "eslint": "^10.2.1",
+    "eslint-plugin-yml": "^3.4.0",
+    "globals": "^17.5.0",
+    "markdownlint-cli2": "^0.22.1",
+    "prettier": "^3.8.3",
+    "tsx": "^4.20.6",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.46.2",
+    "yaml": "^2.9.0"
+  }
+}