@package-broker/core 0.17.4 → 0.19.7

package/dist/routes/composer.js

@@ -12,6 +12,8 @@ import { nanoid } from 'nanoid';
 import { encryptCredentials } from '../utils/encryption';
 import { getLogger } from '../utils/logger';
 import { getAnalytics } from '../utils/analytics';
+import { runInBackground } from '../utils/background';
+import { revalidateStalePackage, REVALIDATION_TTL_SECONDS } from './package-revalidation';
 /** Default maximum versions per package to avoid CPU timeout on Cloudflare Workers */
 export const DEFAULT_MAX_VERSIONS = 50;
 /**
@@ -28,7 +30,7 @@ function schedulePackageStorage(c, packageName, packageData, repoId, baseUrl) {
     const logger = getLogger();
     if (workflow) {
         // Use Cloudflare Workflow for durable background processing
-        c.executionCtx.waitUntil((async () => {
+        runInBackground(c, (async () => {
             try {
                 const instance = await workflow.create({
                     id: `pkg-${packageName.replace('/', '-')}-${repoId}-${Date.now()}`,
@@ -67,7 +69,7 @@ function schedulePackageStorage(c, packageName, packageData, repoId, baseUrl) {
     }
     else {
         // Inline background processing
-        c.executionCtx.waitUntil((async () => {
+        runInBackground(c, (async () => {
             try {
                 const db = c.get('database');
                 const { storedCount, errors } = await transformPackageDistUrls(packageData, repoId, baseUrl, db);
@@ -133,10 +135,13 @@ export async function packagesJsonRoute(c) {
     }
     // No cache - build packages.json from database
     const packagesJson = await buildPackagesJson(c);
-    // Cache the result (fire-and-forget to avoid blocking on KV rate limits)
+    // Cache the result (fire-and-forget to avoid blocking on KV rate limits).
+    // Never cache while repository syncs are pending or in flight: the list is
+    // built from pre-sync data and would otherwise be pinned in KV until the
+    // next manual purge.
     const cachingEnabled = await isPackageCachingEnabled(c.env.KV);
-    if (cachingEnabled && c.env.KV) {
-        c.executionCtx.waitUntil(Promise.all([
+    if (cachingEnabled && c.env.KV && !hasPendingRepos) {
+        runInBackground(c, Promise.all([
             c.env.KV.put(kvKey, JSON.stringify(packagesJson)).catch(() => { }),
             c.env.KV.put(metadataKey, JSON.stringify({ lastModified: Date.now() })).catch(() => { })
         ]));
@@ -186,6 +191,36 @@ function matchesPackageFilter(packageName, filter) {
         return pkgLower === pattern;
     });
 }
+/**
+ * Fetch package metadata from a single repository, dispatching on vcs_type.
+ * Returns the raw upstream package data, or null when the repo does not
+ * provide the package (filter mismatch, unsupported host, not found).
+ */
+export async function fetchPackageFromRepository(repo, packageName, encryptionKey) {
+    if (!matchesPackageFilter(packageName, repo.package_filter)) {
+        return null;
+    }
+    const repoRef = {
+        id: repo.id,
+        url: repo.url,
+        vcs_type: repo.vcs_type,
+        credential_type: repo.credential_type,
+        auth_credentials: repo.auth_credentials,
+        package_filter: repo.package_filter,
+    };
+    if (repo.vcs_type === 'composer') {
+        const { fetchPackageFromUpstream } = await import('../utils/upstream-fetch');
+        return fetchPackageFromUpstream(repoRef, packageName, encryptionKey);
+    }
+    if (repo.vcs_type === 'git') {
+        const { fetchPackageFromGitHub, isGitHubUrl } = await import('../utils/upstream-fetch');
+        if (!isGitHubUrl(repo.url)) {
+            return null;
+        }
+        return fetchPackageFromGitHub(repoRef, packageName, encryptionKey);
+    }
+    return null;
+}
 /**
  * Lazy load package metadata from all available repositories
  * Returns package data and repo_id if found, null otherwise
@@ -198,35 +233,7 @@ export async function lazyLoadPackageFromRepositories(db, packageName, encryptio
     const sortedRepos = sortRepositoriesByPriority(allRepos);
     for (const repo of sortedRepos) {
         try {
-            if (!matchesPackageFilter(packageName, repo.package_filter)) {
-                continue;
-            }
-            let packageData = null;
-            if (repo.vcs_type === 'composer') {
-                const { fetchPackageFromUpstream } = await import('../utils/upstream-fetch');
-                packageData = await fetchPackageFromUpstream({
-                    id: repo.id,
-                    url: repo.url,
-                    vcs_type: repo.vcs_type,
-                    credential_type: repo.credential_type,
-                    auth_credentials: repo.auth_credentials,
-                    package_filter: repo.package_filter,
-                }, packageName, encryptionKey);
-            }
-            else if (repo.vcs_type === 'git') {
-                const { fetchPackageFromGitHub, isGitHubUrl } = await import('../utils/upstream-fetch');
-                if (!isGitHubUrl(repo.url)) {
-                    continue;
-                }
-                packageData = await fetchPackageFromGitHub({
-                    id: repo.id,
-                    url: repo.url,
-                    vcs_type: repo.vcs_type,
-                    credential_type: repo.credential_type,
-                    auth_credentials: repo.auth_credentials,
-                    package_filter: repo.package_filter,
-                }, packageName, encryptionKey);
-            }
+            const packageData = await fetchPackageFromRepository(repo, packageName, encryptionKey);
             if (packageData) {
                 return { packageData, repoId: repo.id };
             }
@@ -245,6 +252,7 @@ export async function lazyLoadPackageFromRepositories(db, packageName, encryptio
     const mirroringEnabled = await isPackagistMirroringEnabled(kv);
     if (mirroringEnabled) {
         try {
+            await ensurePackagistRepository(db, encryptionKey, kv || undefined);
             const packagistUrl = `https://repo.packagist.org/p2/${packageName}.json`;
             const response = await fetch(packagistUrl, {
                 headers: {
@@ -274,13 +282,16 @@ export async function lazyLoadPackageFromRepositories(db, packageName, encryptio
 export async function p2PackageRoute(c) {
     const vendor = c.req.param('vendor');
     const packageFile = c.req.param('package');
-    const packageName = `${vendor}/${packageFile?.replace('.json', '')}`;
+    // Strip only the trailing extension - package names may contain ".json"
+    const packageName = `${vendor}/${packageFile?.replace(/\.json$/, '')}`;
     if (!vendor || !packageFile) {
         return c.json({ error: 'Bad Request', message: 'Invalid package name' }, 400);
     }
     const kvKey = `p2:${packageName}`;
     const metadataKey = `p2:${packageName}:metadata`;
     const db = c.get('database');
+    const requestUrl = new URL(c.req.url);
+    const requestBaseUrl = `${requestUrl.protocol}//${requestUrl.host}`;
     // Check conditional request
     const ifModifiedSince = c.req.header('If-Modified-Since');
     const metadata = c.env.KV ? await c.env.KV.get(metadataKey, 'json') : null;
@@ -303,7 +314,7 @@ export async function p2PackageRoute(c) {
                 logger.warn('Invalid cache format (not an object or old format), treating as cache miss', { packageName });
                 // Fire-and-forget cache deletion
                 if (c.env.KV) {
-                    c.executionCtx.waitUntil(Promise.all([
+                    runInBackground(c, Promise.all([
                         c.env.KV.delete(kvKey).catch(() => { }),
                         c.env.KV.delete(metadataKey).catch(() => { })
                     ]));
@@ -336,7 +347,7 @@ export async function p2PackageRoute(c) {
             logger.warn('Invalid JSON in cache, treating as cache miss', { packageName, error: e instanceof Error ? e.message : String(e) });
             // Fire-and-forget cache deletion
             if (c.env.KV) {
-                c.executionCtx.waitUntil(Promise.all([
+                runInBackground(c, Promise.all([
                     c.env.KV.delete(kvKey).catch(() => { }),
                     c.env.KV.delete(metadataKey).catch(() => { })
                 ]));
@@ -351,16 +362,41 @@ export async function p2PackageRoute(c) {
     if (existingPackages.length > 0) {
         // Build response from database packages
         const maxVersions = getMaxVersions(c.env);
-        const url = new URL(c.req.url);
-        const baseUrl = `${url.protocol}//${url.host}`;
-        const packageData = buildP2Response(packageName, existingPackages, maxVersions, baseUrl);
+        const packageData = buildP2Response(packageName, existingPackages, maxVersions, requestBaseUrl);
         // Cache the result (fire-and-forget to avoid blocking on KV rate limits)
+        // TTL keeps cached responses from outliving the revalidation window
         const cachingEnabled = await isPackageCachingEnabled(c.env.KV);
         if (cachingEnabled && c.env.KV) {
-            c.executionCtx.waitUntil(Promise.all([
-                c.env.KV.put(kvKey, JSON.stringify(packageData)).catch(() => { }),
-                c.env.KV.put(metadataKey, JSON.stringify({ lastModified: Date.now() })).catch(() => { })
-            ]));
+            const kv = c.env.KV;
+            // Stale-while-revalidate: this branch is only reached when the cached
+            // response has expired (at most once per TTL), so it doubles as the
+            // trigger to re-check upstream for new releases. The cache write MUST
+            // complete before revalidation runs - otherwise revalidation's cache
+            // purge could be overwritten by this stale response.
+            // Pinned to the repos the rows came from - see package-revalidation.ts.
+            const repoIds = [...new Set(existingPackages.map((pkg) => pkg.repo_id))];
+            runInBackground(c, (async () => {
+                await Promise.all([
+                    kv.put(kvKey, JSON.stringify(packageData), { expirationTtl: REVALIDATION_TTL_SECONDS }).catch(() => { }),
+                    kv.put(metadataKey, JSON.stringify({ lastModified: Date.now() }), { expirationTtl: REVALIDATION_TTL_SECONDS }).catch(() => { })
+                ]);
+                await revalidateStalePackage({
+                    db,
+                    cache: kv,
+                    packageName,
+                    repoIds,
+                    encryptionKey: c.env.ENCRYPTION_KEY,
+                    proxyBaseUrl: requestBaseUrl,
+                    loadPackageFromRepo: fetchPackageFromRepository,
+                    storePackages: transformPackageDistUrls,
+                });
+            })().catch((error) => {
+                const logger = getLogger();
+                logger.warn('Background package revalidation failed', {
+                    packageName,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }));
         }
         // Track metadata request (cache miss, from DB)
         const analytics = getAnalytics();
@@ -379,15 +415,13 @@ export async function p2PackageRoute(c) {
         return new Response(JSON.stringify(packageData), { status: 200, headers });
     }
     // Not in database - try lazy loading from upstream repositories
-    const url = new URL(c.req.url);
-    const baseUrl = `${url.protocol}//${url.host}`;
     const maxVersions = getMaxVersions(c.env);
     const lazyLoadResult = await lazyLoadPackageFromRepositories(db, packageName, c.env.ENCRYPTION_KEY, c.env.KV);
     if (lazyLoadResult) {
         const { packageData, repoId } = lazyLoadResult;
-        const transformedData = transformDistUrlsInMemory(packageData, repoId, baseUrl, maxVersions);
+        const transformedData = transformDistUrlsInMemory(packageData, repoId, requestBaseUrl, maxVersions);
         // Store package versions in background
-        schedulePackageStorage(c, packageName, packageData, repoId, baseUrl);
+        schedulePackageStorage(c, packageName, packageData, repoId, requestBaseUrl);
         // Track metadata request (cache miss, from upstream)
         const analytics = getAnalytics();
         const requestId = c.get('requestId');
@@ -673,9 +707,61 @@ export function buildP2Response(packageName, packageVersions, maxVersions = DEFA
     };
 }
 export function deriveVersionNormalized(version) {
-    const v = version.startsWith('v') ? version.slice(1) : version;
+    const v = version.trim();
+    if (v.startsWith('dev-')) {
+        return v;
+    }
+    const versionWithoutV = v.startsWith('v') ? v.slice(1) : v;
+    const isNumericSegment = (part) => {
+        if (part.length === 0) {
+            return false;
+        }
+        for (let i = 0; i < part.length; i++) {
+            const ch = part.charCodeAt(i);
+            if (ch < 48 || ch > 57) {
+                return false;
+            }
+        }
+        return true;
+    };
+    const normalizeBranchName = (branchName) => {
+        const branchWithoutV = branchName.startsWith('v') ? branchName.slice(1) : branchName;
+        const parts = branchWithoutV.split('.');
+        if (parts.length < 1 || parts.length > 4) {
+            return undefined;
+        }
+        const normalizedParts = [];
+        for (const part of parts) {
+            if (isNumericSegment(part)) {
+                normalizedParts.push(part);
+            }
+            else if (part === 'x' || part === 'X' || part === '*') {
+                normalizedParts.push('9999999');
+            }
+            else {
+                return undefined;
+            }
+        }
+        while (normalizedParts.length < 4) {
+            normalizedParts.push('9999999');
+        }
+        return `${normalizedParts.join('.')}-dev`;
+    };
+    const lowerVersion = versionWithoutV.toLowerCase();
+    let branchName = null;
+    if (lowerVersion.endsWith('-dev') || lowerVersion.endsWith('.dev')) {
+        branchName = versionWithoutV.slice(0, -4);
+    }
+    if (branchName) {
+        if (branchName.includes('x') || branchName.includes('X') || branchName.includes('*')) {
+            const normalizedBranch = normalizeBranchName(branchName);
+            if (normalizedBranch) {
+                return normalizedBranch;
+            }
+        }
+    }
     // Handle patch versions (e.g., "103.0.7-p8" → "103.0.7.0-patch8")
-    const patchMatch = v.match(/^(\d+\.\d+\.\d+)-p(\d+)$/);
+    const patchMatch = versionWithoutV.match(/^(\d+\.\d+\.\d+)-p(\d+)$/);
     if (patchMatch) {
         const [, base, patch] = patchMatch;
         return `${base}.0-patch${patch}`;
@@ -684,13 +770,13 @@ export function deriveVersionNormalized(version) {
     // 1. Parse up to 4 numeric segments separated by dots from the start of the string.
     // 2. Treat everything after the parsed numeric part as the suffix (e.g., "-beta", "-alpha", "-dev").
     let index = 0;
-    const length = v.length;
+    const length = versionWithoutV.length;
     const segments = [];
     while (index < length && segments.length < 4) {
         const start = index;
         // Read a numeric segment
         while (index < length) {
-            const ch = v.charCodeAt(index);
+            const ch = versionWithoutV.charCodeAt(index);
             if (ch < 48 || ch > 57) { // not '0'–'9'
                 break;
             }
@@ -700,13 +786,13 @@ export function deriveVersionNormalized(version) {
             // No digits found where a segment was expected
             break;
         }
-        segments.push(v.slice(start, index));
+        segments.push(versionWithoutV.slice(start, index));
         // If we have 4 segments or we've reached the end, stop
         if (segments.length === 4 || index >= length) {
             break;
         }
         // Expect a dot between segments; if not present, stop parsing numeric part
-        if (v[index] === '.') {
+        if (versionWithoutV[index] === '.') {
             index++;
             continue;
         }
@@ -714,7 +800,7 @@ export function deriveVersionNormalized(version) {
     }
     if (segments.length > 0) {
         const [major, minor = '0', patch = '0', build = '0'] = segments;
-        const suffix = v.slice(index); // everything after the numeric part
+        const suffix = versionWithoutV.slice(index); // everything after the numeric part
         return `${major}.${minor}.${patch}.${build}${suffix}`;
     }
     // Fallback: return undefined for non-standard versions
@@ -764,21 +850,35 @@ export async function ensurePackagistRepository(db, encryptionKey, kv) {
         await kv.put(cacheKey, 'true', { expirationTtl: 3600 });
     }
 }
+/** A repo stuck in 'syncing' longer than this is treated as dead (killed worker) */
+const STALE_SYNC_THRESHOLD_SECONDS = 900;
 /**
- * Sync all pending repositories
- * Uses job processor which automatically chooses sync vs async execution
- * @returns true if any repositories were synced
+ * Sync all pending repositories in the background.
+ * Uses job processor which automatically chooses sync vs async execution.
+ * @returns true while any repository sync is pending or still running -
+ *          callers must not cache packages.json built in that window
  */
 async function syncPendingRepositories(c) {
     const db = c.get('database');
-    // Get repositories that need sync (pending status)
-    const reposToSync = await db
+    const unsettledRepos = await db
         .select()
         .from(repositories)
-        .where(eq(repositories.status, 'pending'));
+        .where(inArray(repositories.status, ['pending', 'syncing']));
+    const now = Math.floor(Date.now() / 1000);
+    const isStaleSync = (repo) => repo.status === 'syncing' && (repo.last_synced_at || 0) <= now - STALE_SYNC_THRESHOLD_SECONDS;
+    // Re-sync repos stuck in 'syncing' (worker killed mid-sync) along with
+    // pending ones - otherwise they would stay unsynced forever
+    const reposToSync = unsettledRepos.filter((repo) => repo.status === 'pending' || isStaleSync(repo));
+    const syncsInFlight = unsettledRepos.filter((repo) => repo.status === 'syncing' && !isStaleSync(repo));
     if (reposToSync.length === 0) {
-        return false;
+        return syncsInFlight.length > 0;
     }
+    // Claim the repos before backgrounding the sync so concurrent
+    // packages.json requests do not start duplicate syncs of the same repo
+    await db
+        .update(repositories)
+        .set({ status: 'syncing', last_synced_at: now })
+        .where(and(inArray(repositories.id, reposToSync.map((repo) => repo.id)), inArray(repositories.status, ['pending', 'syncing'])));
     // Determine proxy base URL from request
     const url = new URL(c.req.url);
     const proxyBaseUrl = `${url.protocol}//${url.host}`;
@@ -799,8 +899,12 @@ async function syncPendingRepositories(c) {
         type: 'sync_repository',
         repoId: repo.id,
     }));
-    // Process all jobs (parallel for sync, queued for async)
-    await jobProcessor.enqueueAll(syncJobs);
+    // Run syncs in the background: inline sync of a large repository would
+    // otherwise block the packages.json response until every repo finishes
+    runInBackground(c, jobProcessor.enqueueAll(syncJobs).catch((error) => {
+        const logger = getLogger();
+        logger.error('Background repository sync failed', {}, error instanceof Error ? error : new Error(String(error)));
+    }));
     return true;
 }
 /**
@@ -1053,7 +1157,7 @@ export async function transformPackageDistUrls(packageData, repoId, proxyBaseUrl
             const allVersions = await db
                 .select()
                 .from(packages)
-                .where(inArray(packages.name, nameChunk));
+                .where(and(eq(packages.repo_id, repoId), inArray(packages.name, nameChunk)));
             // Add all results to map
             for (const existing of allVersions) {
                 existingMap.set(`${existing.name}:${existing.version}`, existing);
@@ -1124,9 +1228,8 @@ export async function transformPackageDistUrls(packageData, repoId, proxyBaseUrl
                             .insert(packages)
                             .values(pkgData)
                             .onConflictDoUpdate({
-                            target: [packages.name, packages.version],
+                            target: [packages.repo_id, packages.name, packages.version],
                             set: {
-                                repo_id: pkgData.repo_id,
                                 dist_url: pkgData.dist_url,
                                 source_dist_url: pkgData.source_dist_url,
                                 dist_reference: pkgData.dist_reference,
@@ -1195,7 +1298,7 @@ export async function storePackageInDB(db, packageName, version, metadata, repoI
     const [existing] = await db
         .select()
         .from(packages)
-        .where(and(eq(packages.name, packageName), eq(packages.version, version)))
+        .where(and(eq(packages.repo_id, repoId), eq(packages.name, packageName), eq(packages.version, version)))
         .limit(1);
     // Use existing reference or generate simple one (no expensive crypto)
     // Most Packagist packages already have dist.reference, so this is rarely needed
@@ -1229,7 +1332,6 @@ export async function storePackageInDB(db, packageName, version, metadata, repoI
         await db
             .update(packages)
             .set({
-            repo_id: repoId, // Update repo_id in case it changed
             dist_url: proxyDistUrl,
             source_dist_url: sourceDistUrl,
             dist_reference: distReference,
@@ -1240,7 +1342,7 @@ export async function storePackageInDB(db, packageName, version, metadata, repoI
             released_at: releasedAt,
             metadata: packageData.metadata, // Update metadata
         })
-            .where(and(eq(packages.name, packageName), eq(packages.version, version)));
+            .where(and(eq(packages.repo_id, repoId), eq(packages.name, packageName), eq(packages.version, version)));
     }
     else {
         await db.insert(packages).values(packageData);