ghcr-manager 0.0.4

package/dist/db/_schema.js

@@ -0,0 +1,19 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+export function initializeSchema(database) {
+    _initializeSqlDirectory(database, "schema");
+    _initializeSqlDirectory(database, "views");
+}
+function _initializeSqlDirectory(database, directoryName) {
+    const sqlDirectory = join(process.cwd(), "resources", "sql", directoryName);
+    if (!existsSync(sqlDirectory)) {
+        return;
+    }
+    const sqlFiles = readdirSync(sqlDirectory)
+        .filter((file) => file.endsWith(".sql"))
+        .sort();
+    for (const sqlFile of sqlFiles) {
+        const sql = readFileSync(join(sqlDirectory, sqlFile), "utf8");
+        database.exec(sql);
+    }
+}

package/dist/db/_snapshot-repository.d.ts

@@ -0,0 +1,24 @@
+import type Database from "better-sqlite3";
+export declare class SnapshotRepository {
+    #private;
+    constructor(database: Database.Database);
+    getPackageMetadata(scanId: number): {
+        owner: string;
+        packageName: string;
+        scanCompletedAt: string;
+    };
+    getTaggedDigests(scanId: number): Set<string>;
+    getDigestsForTags(scanId: number, tags: string[]): Set<string>;
+    getChildDigests(scanId: number, parentDigests: Iterable<string>): string[];
+    getVersionsCreatedBefore(scanId: number, cutoffTimestamp: string): Array<{
+        versionId: number;
+        digest: string;
+    }>;
+    getTaggedVersionIds(scanId: number): number[];
+    countPackageVersions(scanId: number): number;
+    countTaggedVersions(scanId: number): number;
+    countTags(scanId: number): number;
+    countManifests(scanId: number): number;
+    countManifestEdges(scanId: number): number;
+    listPackageVersionDigests(scanId: number): string[];
+}

package/dist/db/_snapshot-repository.js

@@ -0,0 +1,98 @@
+export class SnapshotRepository {
+    #database;
+    constructor(database) {
+        this.#database = database;
+    }
+    getPackageMetadata(scanId) {
+        const row = this.#database
+            .prepare(`
+          SELECT owner, package_name, scan_completed_at
+          FROM package_scans
+          WHERE scan_id = ?
+        `)
+            .get(scanId);
+        if (!row) {
+            throw new Error(`database does not contain package scan for scan_id=${scanId}`);
+        }
+        if (!row.scan_completed_at) {
+            throw new Error(`scan ${scanId} has not completed`);
+        }
+        return {
+            owner: row.owner,
+            packageName: row.package_name,
+            scanCompletedAt: row.scan_completed_at
+        };
+    }
+    getTaggedDigests(scanId) {
+        return _getDigestSet(this.#database.prepare("SELECT DISTINCT digest FROM tags WHERE scan_id = ?").all(scanId), "digest");
+    }
+    getDigestsForTags(scanId, tags) {
+        if (tags.length === 0) {
+            return new Set();
+        }
+        const placeholders = tags.map(() => "?").join(", ");
+        const rows = this.#database
+            .prepare(`SELECT DISTINCT digest FROM tags WHERE scan_id = ? AND tag IN (${placeholders})`)
+            .all(scanId, ...tags);
+        return _getDigestSet(rows, "digest");
+    }
+    getChildDigests(scanId, parentDigests) {
+        const digestList = [...parentDigests];
+        if (digestList.length === 0) {
+            return [];
+        }
+        const placeholders = digestList.map(() => "?").join(", ");
+        const rows = this.#database
+            .prepare(`SELECT child_digest FROM manifest_edges WHERE scan_id = ? AND parent_digest IN (${placeholders})`)
+            .all(scanId, ...digestList);
+        return rows.map((row) => row.child_digest);
+    }
+    getVersionsCreatedBefore(scanId, cutoffTimestamp) {
+        const rows = this.#database
+            .prepare(`
+          SELECT version_id, digest
+          FROM package_versions
+          WHERE scan_id = ? AND created_at < ?
+          ORDER BY version_id
+        `)
+            .all(scanId, cutoffTimestamp);
+        return rows.map((row) => ({
+            versionId: row.version_id,
+            digest: row.digest
+        }));
+    }
+    getTaggedVersionIds(scanId) {
+        const rows = this.#database
+            .prepare("SELECT DISTINCT version_id FROM tags WHERE scan_id = ? ORDER BY version_id")
+            .all(scanId);
+        return rows.map((row) => row.version_id);
+    }
+    countPackageVersions(scanId) {
+        return _count(this.#database, "SELECT COUNT(*) AS total FROM package_versions WHERE scan_id = ?", "total", scanId);
+    }
+    countTaggedVersions(scanId) {
+        return _count(this.#database, "SELECT COUNT(DISTINCT version_id) AS total FROM tags WHERE scan_id = ?", "total", scanId);
+    }
+    countTags(scanId) {
+        return _count(this.#database, "SELECT COUNT(*) AS total FROM tags WHERE scan_id = ?", "total", scanId);
+    }
+    countManifests(scanId) {
+        return _count(this.#database, "SELECT COUNT(*) AS total FROM manifests WHERE scan_id = ?", "total", scanId);
+    }
+    countManifestEdges(scanId) {
+        return _count(this.#database, "SELECT COUNT(*) AS total FROM manifest_edges WHERE scan_id = ?", "total", scanId);
+    }
+    listPackageVersionDigests(scanId) {
+        const rows = this.#database
+            .prepare("SELECT digest FROM package_versions WHERE scan_id = ? ORDER BY version_id")
+            .all(scanId);
+        return rows.map((row) => row.digest);
+    }
+}
+function _getDigestSet(rows, key) {
+    return new Set(rows.map((row) => row[key]));
+}
+function _count(database, sql, field, ...params) {
+    const row = database.prepare(sql).get(...params);
+    return row[field];
+}

package/dist/db/index.d.ts

@@ -0,0 +1,4 @@
+import Database from "better-sqlite3";
+export { ScanWriter } from "./_scan-writer.js";
+export { SnapshotRepository } from "./_snapshot-repository.js";
+export declare function openDatabase(databasePath: string): Database.Database;

package/dist/db/index.js

@@ -0,0 +1,9 @@
+import Database from "better-sqlite3";
+import { initializeSchema } from "./_schema.js";
+export { ScanWriter } from "./_scan-writer.js";
+export { SnapshotRepository } from "./_snapshot-repository.js";
+export function openDatabase(databasePath) {
+    const database = new Database(databasePath);
+    initializeSchema(database);
+    return database;
+}

package/dist/ingest/github/_manifest-client.d.ts

@@ -0,0 +1,8 @@
+import type { ManifestDescriptorRecord, ManifestEdgeRecord, ManifestRecord } from "../../core/index.js";
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export declare function loadManifestGraph(fetchImpl: FetchLike, registryBaseUrl: string, digest: string, registryToken: string, options: GitHubScanOptions): Promise<{
+    record: ManifestRecord;
+    descriptorRecords: ManifestDescriptorRecord[];
+    edgeRecords: ManifestEdgeRecord[];
+    rawJson: string;
+}>;

package/dist/ingest/github/_manifest-client.js

@@ -0,0 +1,100 @@
+import { acceptedManifestMediaTypes, buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
+export async function loadManifestGraph(fetchImpl, registryBaseUrl, digest, registryToken, options) {
+    const startTime = Date.now();
+    const url = new URL(`/v2/${options.owner}/${options.packageName}/manifests/${digest}`, registryBaseUrl);
+    let response;
+    try {
+        response = await withFetchRetry(async () => {
+            const manifestResponse = await fetchImpl(url.toString(), {
+                headers: {
+                    Accept: acceptedManifestMediaTypes,
+                    Authorization: `Bearer ${registryToken}`,
+                    "User-Agent": "ghcr-manager"
+                }
+            });
+            if (!manifestResponse.ok && _shouldRetryStatus(manifestResponse.status)) {
+                throw new Error(await buildHttpErrorMessage(manifestResponse, `GHCR manifest request for ${digest} failed`));
+            }
+            return manifestResponse;
+        }, {
+            logger: options.logger,
+            label: `GHCR manifest request for ${digest}`,
+            shouldRetry: (error) => _shouldRetryError(error)
+        });
+    }
+    catch (error) {
+        throw new Error(buildFetchTransportErrorMessage(error, `GHCR manifest request for ${digest} failed`), {
+            cause: error
+        });
+    }
+    if (!response.ok) {
+        throw new Error(await buildHttpErrorMessage(response, `GHCR manifest request for ${digest} failed`));
+    }
+    const mediaTypeHeader = response.headers.get("content-type")?.split(";")[0];
+    const document = (await response.json());
+    const rawJson = JSON.stringify(document);
+    const mediaType = document.mediaType ?? mediaTypeHeader;
+    if (!mediaType) {
+        throw new Error(`manifest response for ${digest} did not include a media type`);
+    }
+    options.logger.debug(`Loaded GHCR manifest ${digest} in ${Date.now() - startTime}ms (${mediaType})`);
+    return {
+        rawJson,
+        record: {
+            digest,
+            mediaType,
+            artifactType: document.artifactType,
+            configMediaType: document.config?.mediaType,
+            subjectDigest: document.subject?.digest,
+            annotations: document.annotations
+        },
+        descriptorRecords: _buildDescriptorRecords(digest, document),
+        edgeRecords: _buildEdges(digest, document)
+    };
+}
+function _buildDescriptorRecords(parentDigest, document) {
+    const records = [];
+    for (const child of document.manifests ?? []) {
+        if (!child.digest || !child.mediaType) {
+            continue;
+        }
+        records.push({
+            parentDigest,
+            childDigest: child.digest,
+            mediaType: child.mediaType,
+            artifactType: child.artifactType,
+            platform: child.platform
+        });
+    }
+    return records;
+}
+function _buildEdges(parentDigest, document) {
+    const edges = [];
+    for (const child of document.manifests ?? []) {
+        if (!child.digest || !child.mediaType) {
+            continue;
+        }
+        edges.push({
+            parentDigest,
+            childDigest: child.digest,
+            edgeKind: "image-child"
+        });
+    }
+    if (document.subject?.digest) {
+        edges.push({
+            parentDigest: document.subject.digest,
+            childDigest: parentDigest,
+            edgeKind: "referrer"
+        });
+    }
+    return edges;
+}
+function _shouldRetryStatus(status) {
+    return status === 429 || status === 502 || status === 503 || status === 504;
+}
+function _shouldRetryError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /fetch failed|status 429|status 502|status 503|status 504/.test(error.message);
+}

package/dist/ingest/github/_manifest-ingest.d.ts

@@ -0,0 +1,3 @@
+import type { ScanWriter, SnapshotRepository } from "../../db/index.js";
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export declare function ingestManifests(fetchImpl: FetchLike, registryBaseUrl: string, options: GitHubScanOptions, writer: ScanWriter, repository: SnapshotRepository, scanId: number): Promise<void>;

package/dist/ingest/github/_manifest-ingest.js

@@ -0,0 +1,104 @@
+import { manifestFetchConcurrency, manifestIngestProgressStepRatio } from "../../tuning/index.js";
+import { loadManifestGraph } from "./_manifest-client.js";
+import { loadRegistryPullToken } from "./_registry-token-client.js";
+export async function ingestManifests(fetchImpl, registryBaseUrl, options, writer, repository, scanId) {
+    const pendingDigests = repository.listPackageVersionDigests(scanId);
+    const initialDigestCount = pendingDigests.length;
+    const progressStep = Math.max(1, Math.ceil(initialDigestCount * manifestIngestProgressStepRatio));
+    const queuedDigests = new Set(pendingDigests);
+    const fetchedDigests = new Set();
+    const persistedDigests = new Set();
+    const registryPullTokenState = {};
+    options.logger.info(`Fetching manifests for ${pendingDigests.length} package versions`);
+    let completed = 0;
+    const edgeRecords = [];
+    const activeLoads = new Set();
+    while (pendingDigests.length > 0 || activeLoads.size > 0) {
+        while (pendingDigests.length > 0 && activeLoads.size < manifestFetchConcurrency) {
+            const digest = pendingDigests.shift();
+            if (!digest || fetchedDigests.has(digest)) {
+                continue;
+            }
+            const load = _loadQueuedManifest(digest, fetchImpl, registryBaseUrl, options, writer, pendingDigests, queuedDigests, fetchedDigests, persistedDigests, edgeRecords, completed, async () => (await _getRegistryPullToken(fetchImpl, registryBaseUrl, options, registryPullTokenState)).token, () => {
+                completed += 1;
+                if (completed % progressStep === 0 || pendingDigests.length === 0) {
+                    options.logger.info(`Fetched manifests ${completed}/${queuedDigests.size}`);
+                }
+            }).finally(() => {
+                activeLoads.delete(load);
+            });
+            activeLoads.add(load);
+        }
+        if (activeLoads.size > 0) {
+            await Promise.race(activeLoads);
+        }
+    }
+    options.logger.info(`Starting manifest graph processing for ${edgeRecords.length} edges`);
+    let persistedEdgeCount = 0;
+    for (const edge of edgeRecords) {
+        if (!persistedDigests.has(edge.parentDigest) || !persistedDigests.has(edge.childDigest)) {
+            continue;
+        }
+        writer.insertManifestEdge(edge);
+        persistedEdgeCount += 1;
+    }
+    options.logger.info(`Inserted ${persistedEdgeCount} manifest edges; rebuilding reachability`);
+    writer.rebuildManifestReachability();
+    options.logger.info("Completed manifest graph processing");
+}
+async function _loadQueuedManifest(digest, fetchImpl, registryBaseUrl, options, writer, pendingDigests, queuedDigests, fetchedDigests, persistedDigests, edgeRecords, completed, getRegistryToken, onComplete) {
+    options.logger.debug(`Fetching manifest ${completed + 1}/${queuedDigests.size}: ${digest}`);
+    let manifest;
+    try {
+        manifest = await loadManifestGraph(fetchImpl, registryBaseUrl, digest, await getRegistryToken(), options);
+    }
+    catch (error) {
+        if (_isMissingManifestError(error)) {
+            options.logger.warn(`Skipping missing GHCR manifest ${digest}`);
+            fetchedDigests.add(digest);
+            onComplete();
+            return;
+        }
+        throw error;
+    }
+    writer.insertManifest(manifest.record);
+    persistedDigests.add(manifest.record.digest);
+    writer.insertManifestPayload(manifest.record.digest, manifest.rawJson);
+    for (const descriptor of manifest.descriptorRecords) {
+        writer.insertManifestDescriptor(descriptor);
+        _enqueueDigest(descriptor.childDigest, pendingDigests, queuedDigests, fetchedDigests);
+    }
+    edgeRecords.push(...manifest.edgeRecords);
+    for (const edge of manifest.edgeRecords) {
+        _enqueueDigest(edge.parentDigest, pendingDigests, queuedDigests, fetchedDigests);
+        _enqueueDigest(edge.childDigest, pendingDigests, queuedDigests, fetchedDigests);
+    }
+    fetchedDigests.add(digest);
+    onComplete();
+}
+function _isMissingManifestError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /GHCR manifest request .* failed - status 404/.test(error.message);
+}
+async function _getRegistryPullToken(fetchImpl, registryBaseUrl, options, registryPullTokenState) {
+    if (registryPullTokenState.token && Date.now() < registryPullTokenState.token.expiresAt - 5000) {
+        return registryPullTokenState.token;
+    }
+    if (!registryPullTokenState.load) {
+        registryPullTokenState.load = loadRegistryPullToken(fetchImpl, registryBaseUrl, options).finally(() => {
+            registryPullTokenState.load = undefined;
+        });
+    }
+    const registryPullToken = await registryPullTokenState.load;
+    registryPullTokenState.token = registryPullToken;
+    return registryPullToken;
+}
+function _enqueueDigest(digest, pendingDigests, queuedDigests, fetchedDigests) {
+    if (queuedDigests.has(digest) || fetchedDigests.has(digest)) {
+        return;
+    }
+    pendingDigests.push(digest);
+    queuedDigests.add(digest);
+}

package/dist/ingest/github/_package-version-page-load.d.ts

@@ -0,0 +1,13 @@
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export interface GitHubPackageVersionPageItem {
+    id: number;
+    name: string;
+    created_at: string;
+    updated_at: string;
+    metadata?: {
+        container?: {
+            tags?: string[];
+        };
+    };
+}
+export declare function loadPackageVersionPage(fetchImpl: FetchLike, githubApiBaseUrl: string, options: GitHubScanOptions, page: number): Promise<GitHubPackageVersionPageItem[]>;

package/dist/ingest/github/_package-version-page-load.js

@@ -0,0 +1,52 @@
+import { buildFetchTransportErrorMessage, buildHttpErrorMessage, withFetchRetry } from "./_shared.js";
+export async function loadPackageVersionPage(fetchImpl, githubApiBaseUrl, options, page) {
+    const startTime = Date.now();
+    const url = buildPackageVersionPageUrl(githubApiBaseUrl, options, page);
+    let response;
+    try {
+        response = await withFetchRetry(async () => {
+            const pageResponse = await fetchImpl(url, {
+                headers: {
+                    Accept: "application/vnd.github+json",
+                    Authorization: `Bearer ${options.token}`,
+                    "User-Agent": "ghcr-manager",
+                    "X-GitHub-Api-Version": "2022-11-28"
+                }
+            });
+            if (!pageResponse.ok && _shouldRetryStatus(pageResponse.status)) {
+                throw new Error(await buildHttpErrorMessage(pageResponse, `GitHub Packages request for page ${page} failed`));
+            }
+            return pageResponse;
+        }, {
+            logger: options.logger,
+            label: `GitHub Packages request for page ${page}`,
+            shouldRetry: (error) => _shouldRetryError(error)
+        });
+    }
+    catch (error) {
+        throw new Error(buildFetchTransportErrorMessage(error, `GitHub Packages request for page ${page} failed`), {
+            cause: error
+        });
+    }
+    if (!response.ok) {
+        throw new Error(await buildHttpErrorMessage(response, "GitHub Packages request failed"));
+    }
+    const pageItems = (await response.json());
+    options.logger.debug(`Loaded GitHub package-version page ${page} in ${Date.now() - startTime}ms (${pageItems.length} items)`);
+    return pageItems;
+}
+function buildPackageVersionPageUrl(githubApiBaseUrl, options, page) {
+    const url = new URL(`/orgs/${encodeURIComponent(options.owner)}/packages/container/${encodeURIComponent(options.packageName)}/versions`, githubApiBaseUrl);
+    url.searchParams.set("per_page", "100");
+    url.searchParams.set("page", String(page));
+    return url.toString();
+}
+function _shouldRetryStatus(status) {
+    return status === 429 || status === 502 || status === 503 || status === 504;
+}
+function _shouldRetryError(error) {
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    return /fetch failed|status 429|status 502|status 503|status 504/.test(error.message);
+}

package/dist/ingest/github/_packages-client.d.ts

@@ -0,0 +1,10 @@
+import type { PackageVersionRecord, TagRecord } from "../../core/index.js";
+import type { ScanWriter } from "../../db/index.js";
+import { type GitHubPackageVersionPageItem } from "./_package-version-page-load.js";
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export declare function ingestPackageVersions(fetchImpl: FetchLike, githubApiBaseUrl: string, options: GitHubScanOptions, writer: ScanWriter): Promise<{
+    packageVersions: number;
+    tags: number;
+}>;
+export declare function buildTags(packageVersions: PackageVersionRecord[]): TagRecord[];
+export declare function normalizePackageVersions(packageVersions: GitHubPackageVersionPageItem[]): PackageVersionRecord[];

package/dist/ingest/github/_packages-client.js

@@ -0,0 +1,59 @@
+import { loadPackageVersionPage } from "./_package-version-page-load.js";
+import { ingestParallelPaginated } from "./_parallel-paginated-ingest.js";
+export async function ingestPackageVersions(fetchImpl, githubApiBaseUrl, options, writer) {
+    let tagCount = 0;
+    const result = await ingestParallelPaginated({
+        logger: options.logger,
+        loadPage(page) {
+            return loadPackageVersionPage(fetchImpl, githubApiBaseUrl, options, page);
+        },
+        writePage(pageItems) {
+            _writePage(writer, pageItems);
+            tagCount += _countTags(pageItems);
+        }
+    });
+    return { packageVersions: result.items, tags: tagCount };
+}
+export function buildTags(packageVersions) {
+    const tags = [];
+    for (const version of packageVersions) {
+        const metadata = version.metadata?.container;
+        const tagNames = Array.isArray(metadata?.tags)
+            ? metadata.tags.filter((tag) => typeof tag === "string")
+            : [];
+        for (const tagName of tagNames) {
+            tags.push({
+                tag: tagName,
+                digest: version.digest,
+                versionId: version.versionId
+            });
+        }
+    }
+    return tags.sort((left, right) => left.tag.localeCompare(right.tag));
+}
+export function normalizePackageVersions(packageVersions) {
+    return packageVersions
+        .map((version) => ({
+        versionId: version.id,
+        digest: version.name,
+        createdAt: version.created_at,
+        updatedAt: version.updated_at,
+        metadata: version.metadata
+    }))
+        .sort((left, right) => left.versionId - right.versionId);
+}
+function _writePage(writer, pageItems) {
+    const versions = normalizePackageVersions(pageItems);
+    const rawItemsByVersionId = new Map(pageItems.map((pageItem) => [pageItem.id, pageItem]));
+    for (const version of versions) {
+        writer.insertPackageVersion(version);
+        writer.insertPackageVersionPayload(version.versionId, JSON.stringify(rawItemsByVersionId.get(version.versionId)));
+    }
+    const tags = buildTags(versions);
+    for (const tag of tags) {
+        writer.insertTag(tag);
+    }
+}
+function _countTags(pageItems) {
+    return buildTags(normalizePackageVersions(pageItems)).length;
+}

package/dist/ingest/github/_paginated-ingest.d.ts

@@ -0,0 +1,11 @@
+import type { GitHubScanLogger } from "./_shared.js";
+export interface PaginatedIngestOptions<T> {
+    loadPage(page: number): Promise<T[]>;
+    writePage(pageItems: T[], page: number): Promise<void> | void;
+    logger: GitHubScanLogger;
+}
+export interface PaginatedIngestResult {
+    pages: number;
+    items: number;
+}
+export declare function ingestPaginated<T>(options: PaginatedIngestOptions<T>): Promise<PaginatedIngestResult>;

package/dist/ingest/github/_paginated-ingest.js

@@ -0,0 +1,28 @@
+import { paginatedIngestProgressIntervalPages } from "../../tuning/index.js";
+const _DEFAULT_PAGE_SIZE = 100;
+const _PROGRESS_LABEL = "GitHub package-version pages";
+export async function ingestPaginated(options) {
+    let pages = 0;
+    let items = 0;
+    let lastLoggedPage = 0;
+    for (let page = 1;; page += 1) {
+        const pageItems = await options.loadPage(page);
+        if (pageItems.length === 0) {
+            break;
+        }
+        await options.writePage(pageItems, page);
+        pages = page;
+        items += pageItems.length;
+        if (page === 1 || page % paginatedIngestProgressIntervalPages === 0 || pageItems.length < _DEFAULT_PAGE_SIZE) {
+            options.logger.info(`Loaded ${_PROGRESS_LABEL} ${page} (${items} items total)`);
+            lastLoggedPage = page;
+        }
+        if (pageItems.length < _DEFAULT_PAGE_SIZE) {
+            break;
+        }
+    }
+    if (pages > 0 && lastLoggedPage !== pages) {
+        options.logger.info(`Loaded ${_PROGRESS_LABEL} ${pages} (${items} items total)`);
+    }
+    return { pages, items };
+}

package/dist/ingest/github/_parallel-paginated-ingest.d.ts

@@ -0,0 +1,11 @@
+import type { GitHubScanLogger } from "./_shared.js";
+export interface ParallelPaginatedIngestOptions<T> {
+    loadPage(page: number): Promise<T[]>;
+    writePage(pageItems: T[], page: number): Promise<void> | void;
+    logger: GitHubScanLogger;
+}
+export interface ParallelPaginatedIngestResult {
+    pages: number;
+    items: number;
+}
+export declare function ingestParallelPaginated<T>(options: ParallelPaginatedIngestOptions<T>): Promise<ParallelPaginatedIngestResult>;

package/dist/ingest/github/_parallel-paginated-ingest.js

@@ -0,0 +1,49 @@
+import { packageVersionPageFetchConcurrency, paginatedIngestProgressIntervalPages } from "../../tuning/index.js";
+const _DEFAULT_PAGE_SIZE = 100;
+const _PROGRESS_LABEL = "GitHub package-version pages";
+export async function ingestParallelPaginated(options) {
+    const firstPageItems = await options.loadPage(1);
+    let pages = 0;
+    let items = 0;
+    let lastLoggedPage = 0;
+    if (firstPageItems.length === 0) {
+        return { pages: 0, items: 0 };
+    }
+    await options.writePage(firstPageItems, 1);
+    pages = 1;
+    items = firstPageItems.length;
+    options.logger.info(`Loaded ${_PROGRESS_LABEL} 1 (${items} items total)`);
+    lastLoggedPage = 1;
+    if (firstPageItems.length < _DEFAULT_PAGE_SIZE) {
+        return { pages, items };
+    }
+    let nextPage = 2;
+    let stopPageExclusive = Number.POSITIVE_INFINITY;
+    const workers = Array.from({ length: packageVersionPageFetchConcurrency }, async () => {
+        while (nextPage < stopPageExclusive) {
+            const page = nextPage;
+            nextPage += 1;
+            const pageItems = await options.loadPage(page);
+            if (pageItems.length === 0) {
+                stopPageExclusive = Math.min(stopPageExclusive, page);
+                return;
+            }
+            await options.writePage(pageItems, page);
+            pages = Math.max(pages, page);
+            items += pageItems.length;
+            if (page % paginatedIngestProgressIntervalPages === 0 || pageItems.length < _DEFAULT_PAGE_SIZE) {
+                options.logger.info(`Loaded ${_PROGRESS_LABEL} ${page} (${items} items total)`);
+                lastLoggedPage = Math.max(lastLoggedPage, page);
+            }
+            if (pageItems.length < _DEFAULT_PAGE_SIZE) {
+                stopPageExclusive = Math.min(stopPageExclusive, page + 1);
+                return;
+            }
+        }
+    });
+    await Promise.all(workers);
+    if (pages > 0 && lastLoggedPage !== pages) {
+        options.logger.info(`Loaded ${_PROGRESS_LABEL} ${pages} (${items} items total)`);
+    }
+    return { pages, items };
+}

package/dist/ingest/github/_registry-token-client.d.ts

@@ -0,0 +1,6 @@
+import { type FetchLike, type GitHubScanOptions } from "./_shared.js";
+export interface RegistryPullToken {
+    token: string;
+    expiresAt: number;
+}
+export declare function loadRegistryPullToken(fetchImpl: FetchLike, registryBaseUrl: string, options: GitHubScanOptions): Promise<RegistryPullToken>;