ghcr-manager 0.0.4

package/dist/ingest/github/_registry-token-client.js

@@ -0,0 +1,67 @@
+import { buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
+export async function loadRegistryPullToken(fetchImpl, registryBaseUrl, options) {
+    const startTime = Date.now();
+    const url = _buildRegistryTokenUrl(registryBaseUrl, options);
+    let response;
+    try {
+        response = await withFetchRetry(async () => {
+            const tokenResponse = await fetchImpl(url, {
+                headers: _buildTokenHeaders(options)
+            });
+            if (!tokenResponse.ok && _shouldRetryStatus(tokenResponse.status)) {
+                throw new Error(await buildHttpErrorMessage(tokenResponse, "GHCR token request failed"));
+            }
+            return tokenResponse;
+        }, {
+            logger: options.logger,
+            label: "GHCR token request",
+            shouldRetry: (error) => _shouldRetryError(error)
+        });
+    }
+    catch (error) {
+        throw new Error(buildFetchTransportErrorMessage(error, "GHCR token request failed"), {
+            cause: error
+        });
+    }
+    if (!response.ok) {
+        throw new Error(await buildHttpErrorMessage(response, "GHCR token request failed"));
+    }
+    const body = (await response.json());
+    if (!body.token) {
+        throw new Error("GHCR token response did not include a token");
+    }
+    const registryPullToken = {
+        token: body.token,
+        expiresAt: _getExpiresAt(body.expires_in, body.issued_at)
+    };
+    options.logger.debug(`Loaded GHCR pull token in ${Date.now() - startTime}ms (expires in ${Math.max(0, Math.round((registryPullToken.expiresAt - Date.now()) / 1000))}s)`);
+    return registryPullToken;
+}
+function _buildRegistryTokenUrl(registryBaseUrl, options) {
+    const registryUrl = new URL(registryBaseUrl);
+    const tokenUrl = new URL("/token", registryUrl);
+    tokenUrl.searchParams.set("service", registryUrl.host);
+    tokenUrl.searchParams.set("scope", `repository:${options.owner}/${options.packageName}:pull`);
+    return tokenUrl.toString();
+}
+function _buildTokenHeaders(options) {
+    const basicAuth = Buffer.from(`${options.owner}:${options.token}`).toString("base64");
+    return {
+        "User-Agent": "ghcr-manager",
+        Authorization: `Basic ${basicAuth}`
+    };
+}
+function _getExpiresAt(expiresIn, issuedAt) {
+    const expiresInSeconds = typeof expiresIn === "number" && expiresIn > 0 ? expiresIn : 60;
+    const issuedAtMilliseconds = typeof issuedAt === "string" && !Number.isNaN(Date.parse(issuedAt)) ? Date.parse(issuedAt) : Date.now();
+    return issuedAtMilliseconds + expiresInSeconds * 1000;
+}
+function _shouldRetryStatus(status) {
+    return status === 429 || status === 502 || status === 503 || status === 504;
+}
+function _shouldRetryError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /fetch failed|status 429|status 502|status 503|status 504/.test(error.message);
+}

package/dist/ingest/github/_shared.d.ts

@@ -0,0 +1,28 @@
+export interface GitHubScanOptions {
+    owner: string;
+    packageName: string;
+    token: string;
+    logger: GitHubScanLogger;
+}
+export interface GitHubScanLogger {
+    debug(message: string): void;
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+}
+export interface FetchLikeResponse {
+    ok: boolean;
+    status: number;
+    headers: Headers;
+    json(): Promise<unknown>;
+}
+export type FetchLike = (input: string, init?: RequestInit) => Promise<FetchLikeResponse>;
+export declare const acceptedManifestMediaTypes: string;
+export declare function defaultFetch(input: string, init?: RequestInit): Promise<FetchLikeResponse>;
+export declare function buildFetchTransportErrorMessage(error: unknown, fallback: string): string;
+export declare function buildHttpErrorMessage(response: FetchLikeResponse, fallback: string): Promise<string>;
+export declare function withFetchRetry<T>(run: () => Promise<T>, options: {
+    logger: GitHubScanLogger;
+    label: string;
+    shouldRetry?: (error: unknown) => boolean;
+}): Promise<T>;

package/dist/ingest/github/_shared.js

@@ -0,0 +1,102 @@
+import { ingestRequestRetryCount, ingestRequestRetryDelayMs } from "../../tuning/index.js";
+export const acceptedManifestMediaTypes = [
+    "application/vnd.oci.image.index.v1+json",
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.oci.artifact.manifest.v1+json"
+].join(", ");
+export async function defaultFetch(input, init) {
+    return fetch(input, init);
+}
+export function buildFetchTransportErrorMessage(error, fallback) {
+    const details = [fallback];
+    details.push(..._collectErrorDetails(error));
+    return details.join(" - ");
+}
+export async function buildHttpErrorMessage(response, fallback) {
+    const details = [fallback, `status ${response.status}`];
+    const body = await _readJsonErrorBody(response);
+    const message = typeof body?.message === "string" ? body.message : undefined;
+    const documentationUrl = typeof body?.documentation_url === "string" ? body.documentation_url : undefined;
+    const authenticateHeader = response.headers.get("www-authenticate") ?? undefined;
+    if (message) {
+        details.push(message);
+    }
+    if (documentationUrl) {
+        details.push(documentationUrl);
+    }
+    if (authenticateHeader) {
+        details.push(`www-authenticate: ${authenticateHeader}`);
+    }
+    return details.join(" - ");
+}
+export async function withFetchRetry(run, options) {
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await run();
+        }
+        catch (error) {
+            attempt += 1;
+            const shouldRetry = options.shouldRetry ? options.shouldRetry(error) : true;
+            if (!shouldRetry || attempt > ingestRequestRetryCount) {
+                throw error;
+            }
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            options.logger.warn(`${options.label} failed on attempt ${attempt}/${ingestRequestRetryCount + 1}; retrying in ${ingestRequestRetryDelayMs}ms - ${errorMessage}`);
+            await _sleep(ingestRequestRetryDelayMs);
+        }
+    }
+}
+async function _readJsonErrorBody(response) {
+    const contentType = response.headers.get("content-type")?.split(";")[0];
+    if (contentType && contentType !== "application/json" && !contentType.endsWith("+json")) {
+        return undefined;
+    }
+    try {
+        const body = await response.json();
+        if (body && typeof body === "object") {
+            return body;
+        }
+    }
+    catch {
+        return undefined;
+    }
+    return undefined;
+}
+function _sleep(delayMs) {
+    return new Promise((resolve) => setTimeout(resolve, delayMs));
+}
+function _collectErrorDetails(error) {
+    if (!(error instanceof Error)) {
+        return [String(error)];
+    }
+    const details = [];
+    const seen = new Set();
+    let current = error;
+    while (current instanceof Error && !seen.has(current)) {
+        seen.add(current);
+        const message = _formatErrorMessage(current);
+        if (message && !details.includes(message)) {
+            details.push(message);
+        }
+        current = current.cause;
+    }
+    return details.length > 0 ? details : [String(error)];
+}
+function _formatErrorMessage(error) {
+    const code = "code" in error && typeof error.code === "string"
+        ? error.code
+        : undefined;
+    if (error.message && code) {
+        return `${error.message} (${code})`;
+    }
+    if (error.message) {
+        return error.message;
+    }
+    if (code) {
+        return code;
+    }
+    return undefined;
+}

package/dist/ingest/github/index.d.ts

@@ -0,0 +1,7 @@
+import { ScanWriter, SnapshotRepository } from "../../db/index.js";
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export { type GitHubScanOptions } from "./_shared.js";
+interface _GitHubScanRuntime {
+    fetchImpl?: FetchLike;
+}
+export declare function importGitHubScan(options: GitHubScanOptions, writer: ScanWriter, repository: SnapshotRepository, runtime?: _GitHubScanRuntime): Promise<void>;

package/dist/ingest/github/index.js

@@ -0,0 +1,26 @@
+import { ingestManifests } from "./_manifest-ingest.js";
+import { ingestPackageVersions } from "./_packages-client.js";
+import { defaultFetch } from "./_shared.js";
+const _GITHUB_API_BASE_URL = "https://api.github.com";
+const _REGISTRY_BASE_URL = "https://ghcr.io";
+export async function importGitHubScan(options, writer, repository, runtime) {
+    const fetchImpl = runtime?.fetchImpl ?? defaultFetch;
+    const scanStartedAt = new Date().toISOString();
+    const fullPackageName = `${options.owner}/${options.packageName}`;
+    writer.resetScan(options.owner, options.packageName, scanStartedAt);
+    const scanId = writer.getActiveScanId();
+    options.logger.info(`Starting GitHub package scan for ${fullPackageName}`);
+    try {
+        options.logger.info(`Starting remote data pull for ${fullPackageName}`);
+        const counts = await ingestPackageVersions(fetchImpl, _GITHUB_API_BASE_URL, options, writer);
+        options.logger.info(`Loaded ${counts.packageVersions} package versions and ${counts.tags} tags`);
+        await ingestManifests(fetchImpl, _REGISTRY_BASE_URL, options, writer, repository, scanId);
+        options.logger.info(`Completed remote data pull for ${fullPackageName}`);
+        writer.markScanCompleted(new Date().toISOString());
+        options.logger.info(`Completed GitHub package scan for ${fullPackageName}`);
+    }
+    catch (error) {
+        writer.markScanFailed(new Date().toISOString());
+        throw error;
+    }
+}

package/dist/tuning/index.d.ts

@@ -0,0 +1,6 @@
+export declare const packageVersionPageFetchConcurrency = 4;
+export declare const manifestFetchConcurrency = 16;
+export declare const ingestRequestRetryCount = 3;
+export declare const ingestRequestRetryDelayMs = 1000;
+export declare const paginatedIngestProgressIntervalPages = 10;
+export declare const manifestIngestProgressStepRatio = 0.05;

package/dist/tuning/index.js

@@ -0,0 +1,6 @@
+export const packageVersionPageFetchConcurrency = 4;
+export const manifestFetchConcurrency = 16;
+export const ingestRequestRetryCount = 3;
+export const ingestRequestRetryDelayMs = 1000;
+export const paginatedIngestProgressIntervalPages = 10;
+export const manifestIngestProgressStepRatio = 0.05;

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "ghcr-manager",
+  "description": "Inspect, analyze, and manage GitHub Container Registry packages.",
+  "homepage": "https://github.com/gh-workflow/ghcr-manager#readme",
+  "bugs": {
+    "url": "https://github.com/gh-workflow/ghcr-manager/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gh-workflow/ghcr-manager"
+  },
+  "version": "0.0.4",
+  "type": "module",
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "files": [
+    "dist",
+    "resources/sql",
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "bin": {
+    "ghcr-manager": "./dist/cli/index.js"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "coverage": "c8 --src src --reporter=text --reporter=text-summary node --import tsx --test",
+    "ghcr-manager": "tsx src/cli/index.ts",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "npm run check:file-names && npm run check:test-mapping && npm run typecheck && npm run lint:ts && npm run lint:yaml && npm run lint:markdown && npm run format:check",
+    "check:file-names": "node tools/check-src-file-names.mjs",
+    "check:test-mapping": "node tools/check-test-mapping.mjs",
+    "lint:ts": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "lint:yaml": "eslint \".github/**/*.yml\" \"*.yml\"",
+    "lint:markdown": "markdownlint-cli2",
+    "typecheck": "tsc --noEmit --project tsconfig.json",
+    "test": "node --import tsx --test"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.9.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.6.0",
+    "c8": "^11.0.0",
+    "eslint": "^10.2.1",
+    "eslint-plugin-yml": "^3.3.2",
+    "globals": "^17.5.0",
+    "markdownlint-cli2": "^0.22.1",
+    "prettier": "^3.6.2",
+    "tsx": "^4.20.6",
+    "typescript-eslint": "^8.46.2",
+    "typescript": "^6.0.3"
+  }
+}

package/resources/sql/schema/001_schema.sql

@@ -0,0 +1,109 @@
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE IF NOT EXISTS package_scans (
+  scan_id INTEGER PRIMARY KEY,
+  scan_uuid TEXT NOT NULL UNIQUE,
+  owner TEXT NOT NULL,
+  package_name TEXT NOT NULL,
+  scan_started_at TEXT NOT NULL,
+  scan_completed_at TEXT,
+  status TEXT NOT NULL,
+  CHECK(status IN ('running', 'completed', 'failed'))
+);
+
+CREATE TABLE IF NOT EXISTS package_versions (
+  scan_id INTEGER NOT NULL,
+  version_id INTEGER NOT NULL,
+  digest TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY(scan_id, version_id),
+  UNIQUE(scan_id, version_id, digest),
+  FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
+);
+
+CREATE TABLE IF NOT EXISTS package_version_payloads (
+  scan_id INTEGER NOT NULL,
+  version_id INTEGER NOT NULL,
+  raw_json TEXT NOT NULL,
+  PRIMARY KEY(scan_id, version_id),
+  FOREIGN KEY(scan_id, version_id) REFERENCES package_versions(scan_id, version_id)
+);
+
+CREATE TABLE IF NOT EXISTS tags (
+  scan_id INTEGER NOT NULL,
+  tag TEXT NOT NULL,
+  digest TEXT NOT NULL,
+  version_id INTEGER NOT NULL,
+  PRIMARY KEY(scan_id, tag),
+  FOREIGN KEY(scan_id, version_id, digest) REFERENCES package_versions(scan_id, version_id, digest)
+);
+
+CREATE TABLE IF NOT EXISTS manifests (
+  scan_id INTEGER NOT NULL,
+  digest TEXT NOT NULL,
+  media_type TEXT NOT NULL,
+  artifact_type TEXT,
+  config_media_type TEXT,
+  subject_digest TEXT,
+  annotations_json TEXT,
+  platform_os TEXT,
+  platform_architecture TEXT,
+  platform_variant TEXT,
+  PRIMARY KEY(scan_id, digest),
+  FOREIGN KEY(scan_id) REFERENCES package_scans(scan_id)
+);
+
+CREATE TABLE IF NOT EXISTS manifest_descriptors (
+  scan_id INTEGER NOT NULL,
+  parent_digest TEXT NOT NULL,
+  child_digest TEXT NOT NULL,
+  media_type TEXT NOT NULL,
+  artifact_type TEXT,
+  platform_os TEXT,
+  platform_architecture TEXT,
+  platform_variant TEXT,
+  PRIMARY KEY(scan_id, parent_digest, child_digest),
+  FOREIGN KEY(scan_id, parent_digest) REFERENCES manifests(scan_id, digest)
+);
+
+CREATE TABLE IF NOT EXISTS manifest_payloads (
+  scan_id INTEGER NOT NULL,
+  digest TEXT NOT NULL,
+  raw_json TEXT NOT NULL,
+  PRIMARY KEY(scan_id, digest),
+  FOREIGN KEY(scan_id, digest) REFERENCES manifests(scan_id, digest)
+);
+
+CREATE TABLE IF NOT EXISTS manifest_edges (
+  scan_id INTEGER NOT NULL,
+  parent_digest TEXT NOT NULL,
+  child_digest TEXT NOT NULL,
+  edge_kind TEXT NOT NULL,
+  PRIMARY KEY(scan_id, parent_digest, child_digest, edge_kind),
+  FOREIGN KEY(scan_id, parent_digest) REFERENCES manifests(scan_id, digest),
+  FOREIGN KEY(scan_id, child_digest) REFERENCES manifests(scan_id, digest)
+);
+
+CREATE TABLE IF NOT EXISTS manifest_reachability (
+  scan_id INTEGER NOT NULL,
+  ancestor_digest TEXT NOT NULL,
+  descendant_digest TEXT NOT NULL,
+  min_distance INTEGER NOT NULL,
+  PRIMARY KEY(scan_id, ancestor_digest, descendant_digest),
+  FOREIGN KEY(scan_id, ancestor_digest) REFERENCES manifests(scan_id, digest),
+  FOREIGN KEY(scan_id, descendant_digest) REFERENCES manifests(scan_id, digest),
+  CHECK(min_distance >= 0)
+);
+
+CREATE INDEX IF NOT EXISTS idx_package_versions_scan_created_at ON package_versions(scan_id, created_at);
+CREATE INDEX IF NOT EXISTS idx_package_versions_scan_digest ON package_versions(scan_id, digest);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_package_scans_scan_uuid ON package_scans(scan_uuid);
+CREATE INDEX IF NOT EXISTS idx_package_scans_owner_name_started_at
+  ON package_scans(owner, package_name, scan_started_at DESC);
+CREATE INDEX IF NOT EXISTS idx_tags_scan_digest ON tags(scan_id, digest);
+CREATE INDEX IF NOT EXISTS idx_manifest_descriptors_scan_child ON manifest_descriptors(scan_id, child_digest);
+CREATE INDEX IF NOT EXISTS idx_manifest_edges_scan_parent ON manifest_edges(scan_id, parent_digest);
+CREATE INDEX IF NOT EXISTS idx_manifest_edges_scan_child ON manifest_edges(scan_id, child_digest);
+CREATE INDEX IF NOT EXISTS idx_manifest_reachability_scan_descendant
+  ON manifest_reachability(scan_id, descendant_digest);

package/resources/sql/views/001_v_latest_scan_per_package.sql

@@ -0,0 +1,27 @@
+DROP VIEW IF EXISTS v_latest_scan_per_package;
+
+CREATE VIEW v_latest_scan_per_package AS
+SELECT scan_id,
+       scan_uuid,
+       owner,
+       package_name,
+       scan_started_at,
+       scan_completed_at
+FROM (
+         SELECT
+             ps.scan_id,
+             ps.scan_uuid,
+             ps.owner,
+             ps.package_name,
+             ps.scan_started_at,
+             ps.scan_completed_at,
+             ROW_NUMBER() OVER (
+                 PARTITION BY ps.owner, ps.package_name
+                 ORDER BY ps.scan_completed_at DESC
+                 ) AS rn
+         FROM package_scans ps
+         WHERE ps.scan_completed_at IS NOT NULL
+           AND ps.status = 'completed'
+     )
+WHERE rn = 1
+;

package/resources/sql/views/002_v_missing_digests.sql

@@ -0,0 +1,32 @@
+DROP VIEW IF EXISTS v_missing_digests;
+
+CREATE VIEW v_missing_digests AS
+SELECT DISTINCT
+    lsp.scan_id,
+    lsp.owner,
+    lsp.package_name,
+    d.child_digest AS missing_digest,
+    d.parent_digest AS anchor_digest
+FROM manifest_descriptors d
+         JOIN v_latest_scan_per_package lsp ON lsp.scan_id = d.scan_id
+         LEFT JOIN manifests m
+                   ON m.scan_id = d.scan_id
+                       AND m.digest = d.child_digest
+WHERE m.digest IS NULL
+
+UNION
+
+SELECT DISTINCT
+    lsp.scan_id,
+    lsp.owner,
+    lsp.package_name,
+    mf.subject_digest AS missing_digest,
+    mf.digest AS anchor_digest
+FROM manifests mf
+         JOIN v_latest_scan_per_package lsp ON lsp.scan_id = mf.scan_id
+         LEFT JOIN manifests m
+                   ON m.scan_id = mf.scan_id
+                       AND m.digest = mf.subject_digest
+WHERE mf.subject_digest IS NOT NULL
+  AND m.digest IS NULL
+;

package/resources/sql/views/003_v_missing_digests_related_manifests.sql

@@ -0,0 +1,78 @@
+DROP VIEW IF EXISTS v_missing_digests_related_manifests;
+
+CREATE VIEW v_missing_digests_related_manifests AS
+WITH
+related_manifests AS (
+  SELECT DISTINCT
+    m.scan_id,
+    md.missing_digest,
+    m.digest AS related_manifest_digest,
+    m.media_type,
+    1 AS hops_missing_to_related_manifest
+  FROM v_missing_digests md
+  JOIN manifests m
+    ON m.scan_id = md.scan_id
+   AND m.digest = md.anchor_digest
+
+  UNION
+
+  SELECT DISTINCT
+    m.scan_id,
+    md.missing_digest,
+    m.digest AS related_manifest_digest,
+    m.media_type,
+    r.min_distance + 1 AS hops_missing_to_related_manifest
+  FROM v_missing_digests md
+  JOIN manifests m
+    ON m.scan_id = md.scan_id
+  JOIN manifest_reachability r
+    ON r.scan_id = m.scan_id
+   AND r.ancestor_digest = m.digest
+   AND r.descendant_digest = md.anchor_digest
+
+  UNION
+
+  SELECT DISTINCT
+    m.scan_id,
+    md.missing_digest,
+    m.digest AS related_manifest_digest,
+    m.media_type,
+    r.min_distance + 1 AS hops_missing_to_related_manifest
+  FROM v_missing_digests md
+  JOIN manifests m
+    ON m.scan_id = md.scan_id
+  JOIN manifest_reachability r
+    ON r.scan_id = m.scan_id
+   AND r.ancestor_digest = md.anchor_digest
+   AND r.descendant_digest = m.digest
+),
+closest_related_manifests AS (
+  SELECT
+    scan_id,
+    missing_digest,
+    related_manifest_digest,
+    media_type,
+    MIN(hops_missing_to_related_manifest) AS hops_missing_to_related_manifest
+  FROM related_manifests
+  GROUP BY
+    missing_digest,
+    scan_id,
+    related_manifest_digest,
+    media_type
+)
+SELECT
+  ps.scan_id,
+  ps.owner,
+  ps.package_name,
+  crm.missing_digest,
+  crm.related_manifest_digest,
+  crm.media_type,
+  crm.hops_missing_to_related_manifest,
+  t.tag,
+  t.version_id
+FROM closest_related_manifests crm
+JOIN package_scans ps
+  ON ps.scan_id = crm.scan_id
+LEFT JOIN tags t
+  ON t.scan_id = crm.scan_id
+ AND t.digest = crm.related_manifest_digest;