@adea-ai/plugins 3.0.10 → 4.0.1

package/dist/index.js

@@ -1,10 +1,12 @@
 import { spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { promises as fs } from 'node:fs';
-import { basename, dirname, join, relative, resolve } from 'node:path';
+import { basename, join, relative, resolve } from 'node:path';
 import { Readable } from 'node:stream';
+import { setTimeout as sleep } from 'node:timers/promises';
 import { createGunzip } from 'node:zlib';
 import { extract } from 'tar-stream';
+import { publishCatalog } from './catalog-store.js';
 import { CatalogSchema, HarnessSchema, SourcesLockSchema, PluginSchema, } from './schema.js';
 import { canonicalRepositoryUrl, parseJsonDocument, parseMarketplaceManifest, safeRelativePath, } from './sources.js';
 import { buildConsumerIndex, CATALOG_INDEX_ARTIFACT, iconCoverage, } from './consumer-index.js';
@@ -13,6 +15,9 @@ import { stageIconBytes } from './icon-mirror.js';
 import { isVendorHomepage, resolveSiteIcon, } from './site-icons.js';
 import { normalizeDisplayName } from './display-name.js';
 import { immutableAssetUrl } from './publication.js';
+import { sourceDigest as bytesDigest } from './source-digest.js';
+export { bytesDigest };
+export { catalogFingerprint, publishCatalog, withCatalogRead, writeFileAtomically, } from './catalog-store.js';
 /**
  * Marks the catalog format this compiler emits.
  *
@@ -55,14 +60,6 @@ export function canonicalJson(value) {
 export function digest(value) {
     return `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}`;
 }
-export function bytesDigest(files) {
-    const hash = createHash('sha256');
-    for (const [path, bytes] of [...files.entries()].toSorted(([left], [right]) => left.localeCompare(right))) {
-        hash.update(`${path.length}:${path}:${bytes.byteLength}:`);
-        hash.update(bytes);
-    }
-    return `sha256:${hash.digest('hex')}`;
-}
 export function stablePluginId(sourceId, upstreamName) {
     return `plugin:${slug(sourceId)}:${slug(upstreamName)}`;
 }
@@ -88,17 +85,22 @@ export async function synchronize(input) {
         sourceId: config.sourceId,
         commitSha,
     }));
+    const resolvedLock = createSourcesLock(resolved);
     const unchanged = input.fromLock === undefined &&
         input.forceRebuild !== true &&
         input.existingLock !== undefined &&
         resolved.length === input.existingLock.sources.length &&
-        resolved.every((source) => {
-            const previous = input.existingLock?.sources.find((entry) => entry.sourceId === source.config.sourceId);
-            if (!previous)
+        resolvedLock.sources.every((source) => {
+            const previous = input.existingLock?.sources.find((entry) => entry.sourceId === source.sourceId);
+            if (!previous || sourceLockChanged(previous, source))
                 return false;
-            return (previous.resolvedCommitSha === source.commitSha &&
-                previous.sourceManifestDigest === source.manifestDigest &&
-                digest(previous.pluginPins) === digest(source.pluginPins));
+            if (!input.existingCatalog)
+                return true;
+            const priorCatalog = input.existingCatalog.sources.find((entry) => entry.sourceId === source.sourceId);
+            const config = resolved.find((entry) => entry.config.sourceId === source.sourceId).config;
+            return (!!priorCatalog &&
+                priorCatalog.displayName === config.displayName &&
+                priorCatalog.trustClassification === config.trustClassification);
         });
     if (unchanged) {
         return {
@@ -135,7 +137,7 @@ export async function synchronize(input) {
         ...(input.transformRelease ? { transformRelease: input.transformRelease } : {}),
     });
     const catalog = buildResult.catalog;
-    const lock = createSourcesLock(resolved);
+    const lock = resolvedLock;
     const artifacts = createArtifacts(orderLeadingPlugins(catalog, input.leading), lock, {
         ...(input.leading ? { leading: input.leading } : {}),
         ...(input.topCount !== undefined ? { topCount: input.topCount } : {}),
@@ -179,6 +181,7 @@ export async function buildCatalog(input) {
         ...(input.productCategories ? { productCategories: input.productCategories } : {}),
         ...(input.leading ? { leading: input.leading } : {}),
         ...(input.productIconOverrides ? { productIconOverrides: input.productIconOverrides } : {}),
+        ...(input.iconAssetsDirectory ? { iconAssetsDirectory: input.iconAssetsDirectory } : {}),
     })).catalog;
 }
 async function buildCatalogInternal(input) {
@@ -583,7 +586,7 @@ async function resolvePluginSource(pluginName, spec, source, resolveExternalRefs
     const commitSha = pin?.resolvedCommitSha ??
         spec.sha ??
         (resolveExternalRefs
-            ? await resolveGitRef(spec.repositoryUrl, spec.ref?.startsWith('refs/') ? spec.ref : `refs/heads/${spec.ref ?? 'main'}`)
+            ? await resolveGitRef(spec.repositoryUrl, externalGitRef(spec.ref))
             : source.commitSha);
     return {
         repositoryUrl: canonicalUrl,
@@ -790,6 +793,15 @@ function findLicense(entry, metadata) {
         source: metadata?.license !== undefined ? 'plugin-manifest' : 'marketplace-entry',
     };
 }
+function sourceLockChanged(previous, current) {
+    return (previous.repositoryUrl !== current.repositoryUrl ||
+        previous.defaultBranch !== current.defaultBranch ||
+        previous.manifestPath !== current.manifestPath ||
+        previous.marketplaceDialect !== current.marketplaceDialect ||
+        previous.resolvedCommitSha !== current.resolvedCommitSha ||
+        previous.sourceManifestDigest !== current.sourceManifestDigest ||
+        digest(previous.pluginPins) !== digest(current.pluginPins));
+}
 function createSourcesLock(sources) {
     const entries = sources
         .map((source) => ({
@@ -997,38 +1009,12 @@ export function iconAssets(index) {
     }
     return [...assets.values()].toSorted((left, right) => left.name.localeCompare(right.name));
 }
-export async function writeArtifacts(generatedDirectory, artifacts) {
-    const parent = dirname(generatedDirectory);
-    await fs.mkdir(parent, { recursive: true });
-    const temporary = await fs.mkdtemp(join(parent, '.catalog-'));
-    try {
-        for (const [name, content] of Object.entries(artifacts))
-            await fs.writeFile(join(temporary, name), content, 'utf8');
-        const backup = `${generatedDirectory}.previous`;
-        await fs.rm(backup, { recursive: true, force: true });
-        let movedExisting = false;
-        try {
-            await fs.rename(generatedDirectory, backup);
-            movedExisting = true;
-        }
-        catch (error) {
-            if (error.code !== 'ENOENT')
-                throw error;
-        }
-        try {
-            await fs.rename(temporary, generatedDirectory);
-        }
-        catch (error) {
-            if (movedExisting)
-                await fs.rename(backup, generatedDirectory);
-            throw error;
-        }
-        if (movedExisting)
-            await fs.rm(backup, { recursive: true, force: true });
-    }
-    finally {
-        await fs.rm(temporary, { recursive: true, force: true });
-    }
+/**
+ * Publishes a verified artifact set to `generatedDirectory` under the output's
+ * coordination lock. See `catalog-store.ts` for the publication contract.
+ */
+export async function writeArtifacts(generatedDirectory, artifacts, options = {}) {
+    await publishCatalog(generatedDirectory, artifacts, options);
 }
 export function verifyArtifacts(artifacts) {
     const parsedCatalog = CatalogSchema.parse(JSON.parse(artifacts['catalog.v1.json']));
@@ -1330,10 +1316,15 @@ function createChangeReport(catalog, lock, existingLock, existingCatalog, skippe
         ? lock.sources
             .filter((source) => (() => {
             const previous = existingLock.sources.find((candidate) => candidate.sourceId === source.sourceId);
+            const priorCatalog = existingCatalog?.sources.find((entry) => entry.sourceId === source.sourceId);
+            const currentCatalog = catalog.sources.find((entry) => entry.sourceId === source.sourceId);
             return (!previous ||
-                previous?.resolvedCommitSha !== source.resolvedCommitSha ||
-                previous.sourceManifestDigest !== source.sourceManifestDigest ||
-                digest(previous.pluginPins) !== digest(source.pluginPins));
+                sourceLockChanged(previous, source) ||
+                (existingCatalog !== undefined &&
+                    (!priorCatalog ||
+                        !currentCatalog ||
+                        priorCatalog.displayName !== currentCatalog.displayName ||
+                        priorCatalog.trustClassification !== currentCatalog.trustClassification)));
         })())
             .map((source) => source.sourceId)
         : lock.sources.map((source) => source.sourceId);
@@ -1416,7 +1407,10 @@ async function resolvePluginPins(input) {
         }
         const repositoryUrl = canonicalRepositoryUrl(entry.source.repositoryUrl);
         const pluginSubdirectory = entry.source.subdirectory;
-        const locked = input.lockedPins?.find((pin) => pin.pluginName === entry.name);
+        // Existing locks describe the prior sync; only explicit replay freezes revisions.
+        const locked = input.fromLock
+            ? input.lockedPins?.find((pin) => pin.pluginName === entry.name)
+            : undefined;
         if (input.fromLock && !locked)
             throw new Error(`SOURCE_LOCK_PLUGIN_PIN_MISSING: ${input.config.sourceId}:${entry.name}`);
         if (locked) {
@@ -1428,13 +1422,16 @@ async function resolvePluginPins(input) {
         }
         const resolvedCommitSha = entry.source.sha ??
             (input.resolveExternalRefs
-                ? await resolveGitRef(repositoryUrl, entry.source.ref?.startsWith('refs/')
-                    ? entry.source.ref
-                    : `refs/heads/${entry.source.ref ?? 'main'}`)
+                ? await resolveGitRef(repositoryUrl, externalGitRef(entry.source.ref))
                 : input.sourceCommitSha);
         return { pluginName: entry.name, repositoryUrl, pluginSubdirectory, resolvedCommitSha };
     });
-    return pins.toSorted((left, right) => left.pluginName.localeCompare(right.pluginName));
+    return pins.toSorted((left, right) => left.pluginName < right.pluginName ? -1 : left.pluginName > right.pluginName ? 1 : 0);
+}
+function externalGitRef(ref) {
+    if (ref === undefined)
+        return 'HEAD';
+    return ref.startsWith('refs/') ? ref : `refs/heads/${ref}`;
 }
 async function mapWithConcurrency(values, concurrency, mapper) {
     const results = Array.from({ length: values.length });
@@ -1456,7 +1453,8 @@ async function readFixtureManifest(config, fixtureRoot) {
     return fs.readFile(join(resolve(fixtureRoot), config.fixturePath, config.manifestPath), 'utf8');
 }
 async function resolveGitRef(repositoryUrl, ref) {
-    const result = spawn('git', ['ls-remote', canonicalRepositoryUrl(repositoryUrl), ref], {
+    const peeled = ref.startsWith('refs/tags/') && !ref.endsWith('^{}') ? `${ref}^{}` : undefined;
+    const result = spawn('git', ['ls-remote', canonicalRepositoryUrl(repositoryUrl), ref, ...(peeled ? [peeled] : [])], {
         stdio: ['ignore', 'pipe', 'pipe'],
     });
     const [stdout, stderr, exitCode] = await Promise.all([
@@ -1479,7 +1477,12 @@ async function resolveGitRef(repositoryUrl, ref) {
     ]);
     if (exitCode !== 0)
         throw new Error(`GIT_REF_LOOKUP_FAILED: ${repositoryUrl}:${ref}:${stderr.trim()}`);
-    const sha = stdout.trim().split(/\s+/)[0];
+    const rows = stdout
+        .trim()
+        .split('\n')
+        .map((line) => line.split(/\s+/));
+    const sha = (peeled && rows.find((row) => row[1] === peeled)?.[0]) ??
+        rows.find((row) => row[1] === ref)?.[0];
     if (!sha || !/^[a-f0-9]{40}$/.test(sha))
         throw new Error(`GIT_REF_NOT_FOUND: ${repositoryUrl}:${ref}`);
     return sha;
@@ -1503,7 +1506,8 @@ function rawUrl(repositoryUrl, commitSha, path) {
     return `https://raw.githubusercontent.com${parsed.pathname}/${commitSha}/${path.split('/').map(encodeURIComponent).join('/')}`;
 }
 async function fetchText(url) {
-    const response = await fetch(url, { headers: upstreamRequestHeaders(url) });
+    // Manifest requests use the same bounded retry policy as the tree and archive.
+    const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) });
     if (!response.ok)
         throw new Error(`HTTP_FETCH_FAILED: ${response.status}:${url}`);
     return response.text();
@@ -1579,9 +1583,6 @@ const UPSTREAM_WEATHER_SKIP_CODES = new Set([
     'PLUGIN_FILE_UNAVAILABLE',
     'PLUGIN_PATH_NOT_FOUND',
 ]);
-function sleep(ms) {
-    return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
-}
 function upstreamRetryDelayMs(attempt, response) {
     const retryAfter = response?.headers.get('retry-after');
     if (retryAfter !== null && retryAfter !== undefined) {
@@ -1615,9 +1616,9 @@ async function fetchUpstream(url, init) {
             response = await fetch(url, init);
         }
         catch (error) {
-            if (attempt >= UPSTREAM_FETCH_MAX_ATTEMPTS - 1)
+            if (init.signal?.aborted || attempt >= UPSTREAM_FETCH_MAX_ATTEMPTS - 1)
                 throw error;
-            await sleep(upstreamRetryDelayMs(attempt));
+            await sleep(upstreamRetryDelayMs(attempt), undefined, { signal: init.signal ?? undefined });
             attempt += 1;
             continue;
         }
@@ -1625,7 +1626,10 @@ async function fetchUpstream(url, init) {
             !isRetryableUpstreamStatus(response.status, response) ||
             attempt >= UPSTREAM_FETCH_MAX_ATTEMPTS - 1)
             return response;
-        await sleep(upstreamRetryDelayMs(attempt, response));
+        await response.body?.cancel().catch(() => { });
+        await sleep(upstreamRetryDelayMs(attempt, response), undefined, {
+            signal: init.signal ?? undefined,
+        });
         attempt += 1;
     }
 }
@@ -1740,23 +1744,29 @@ export class NetworkSnapshotLoader {
         if (!owner || !repo)
             throw new Error('GIT_REPOSITORY_INVALID');
         const key = `${owner}/${repo}@${commitSha}`;
-        let tree = this.#trees.get(key);
-        if (!tree) {
-            const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitSha}?recursive=1`;
-            const response = await fetchUpstream(treeUrl, {
-                headers: upstreamRequestHeaders(treeUrl),
+        let pendingTree = this.#trees.get(key);
+        if (!pendingTree) {
+            pendingTree = (async () => {
+                const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitSha}?recursive=1`;
+                const response = await fetchUpstream(treeUrl, {
+                    headers: upstreamRequestHeaders(treeUrl),
+                });
+                if (!response.ok)
+                    throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`]);
+                const payload = (await response.json());
+                if (payload.truncated)
+                    throw new Error(`GIT_TREE_TRUNCATED: ${owner}/${repo}@${commitSha}`);
+                return (payload.tree ?? []).map((entry) => ({
+                    ...entry,
+                    path: safeRelativePath(entry.path, 'TREE_PATH_INVALID'),
+                }));
+            })().catch((error) => {
+                this.#trees.delete(key);
+                throw error;
             });
-            if (!response.ok)
-                throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`]);
-            const payload = (await response.json());
-            if (payload.truncated)
-                throw new Error(`GIT_TREE_TRUNCATED: ${owner}/${repo}@${commitSha}`);
-            tree = (payload.tree ?? []).map((entry) => ({
-                ...entry,
-                path: safeRelativePath(entry.path, 'TREE_PATH_INVALID'),
-            }));
-            this.#trees.set(key, tree);
+            this.#trees.set(key, pendingTree);
         }
+        const tree = await pendingTree;
         const prefix = pluginSubdirectory ? `${safeRelativePath(pluginSubdirectory)}/` : '';
         const submodules = tree.filter((entry) => entry.path.startsWith(prefix) && (entry.type === 'commit' || entry.mode === '160000'));
         if (submodules.length > 0)
@@ -1799,13 +1809,18 @@ export class NetworkSnapshotLoader {
         const cached = this.#archives.get(key);
         if (cached)
             return cached;
-        const url = `https://api.github.com/repos/${owner}/${repo}/tarball/${commitSha}`;
-        const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) });
-        if (!response.ok)
-            throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`]);
-        const files = await readArchiveFiles(new Uint8Array(await response.arrayBuffer()), this.#policy);
-        this.#archives.set(key, files);
-        return files;
+        const pending = (async () => {
+            const url = `https://api.github.com/repos/${owner}/${repo}/tarball/${commitSha}`;
+            const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) });
+            if (!response.ok)
+                throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`]);
+            return readArchiveFiles(new Uint8Array(await response.arrayBuffer()), this.#policy);
+        })().catch((error) => {
+            this.#archives.delete(key);
+            throw error;
+        });
+        this.#archives.set(key, pending);
+        return pending;
     }
 }
 export class FixtureSnapshotLoader {