@adea-ai/plugins 4.0.1 → 4.0.6

package/src/synchronize.ts

@@ -0,0 +1,213 @@
+import { createArtifacts, type ConsumerIndexOptions, type GeneratedArtifacts } from './artifacts.js'
+import { buildCatalogInternal, type BuildCatalogInput } from './build.js'
+import { mapWithConcurrency } from './concurrency.js'
+import { digest } from './digest.js'
+import type { SkippedPlugin } from './plugin-skips.js'
+import type { CatalogPolicy } from './policy.js'
+import type { Catalog, CatalogSource, SourceLockEntry, SourcesLock } from './schema.js'
+import { FixtureSnapshotLoader, NetworkSnapshotLoader, type SnapshotLoader } from './snapshots.js'
+import { createSourcesLock, resolveSource, sourceLockChanged } from './source-resolution.js'
+import type { SourceConfig } from './sources.js'
+import { verifyArtifacts } from './verify-artifacts.js'
+
+export interface SyncInput
+  extends
+    ConsumerIndexOptions,
+    Omit<BuildCatalogInput, 'resolvedSources' | 'snapshotLoader' | 'resolveExternalRefs'> {
+  readonly sources: readonly SourceConfig[]
+  readonly policy: CatalogPolicy
+  readonly mode?: 'live' | 'offline'
+  readonly fixtureRoot?: string
+  readonly sourceId?: string
+  readonly dryRun?: boolean
+  readonly existingLock?: SourcesLock
+  readonly existingCatalog?: Catalog
+  readonly fromLock?: SourcesLock
+  readonly snapshotLoader?: SnapshotLoader
+  /** Rebuild even when source pins are unchanged (used for explicit format migrations). */
+  readonly forceRebuild?: boolean
+}
+
+export interface SyncResult {
+  readonly changed: boolean
+  readonly dryRun: boolean
+  readonly sourceHeads: readonly { sourceId: string; commitSha: string }[]
+  readonly catalog?: Catalog
+  readonly lock?: SourcesLock
+  readonly artifacts?: GeneratedArtifacts
+  readonly changeReport: ChangeReport
+}
+
+export interface ChangeReport {
+  readonly schemaVersion: 1
+  readonly changedSources: readonly string[]
+  readonly addedPlugins: readonly string[]
+  readonly removedPlugins: readonly string[]
+  readonly changedPlugins: readonly string[]
+  readonly permissionSensitiveChanges: readonly string[]
+  readonly skippedPlugins: readonly SkippedPlugin[]
+  readonly contentResolution: 'complete' | 'complete-with-skips' | 'metadata-only'
+}
+
+export async function synchronize(input: SyncInput): Promise<SyncResult> {
+  const mode = input.mode ?? 'live'
+  const configs = input.sources.filter(
+    (source) => !input.sourceId || source.sourceId === input.sourceId
+  )
+  if (configs.length === 0) throw new Error(`SOURCE_NOT_FOUND: ${input.sourceId ?? 'none'}`)
+  const lockToUse = input.fromLock ?? input.existingLock
+  const resolved = await mapWithConcurrency(configs, 4, (config) =>
+    resolveSource(config, mode, input.fixtureRoot, lockToUse, input.fromLock !== undefined)
+  )
+  const sourceHeads = resolved.map(({ config, commitSha }) => ({
+    sourceId: config.sourceId,
+    commitSha,
+  }))
+  const lock = createSourcesLock(resolved)
+  const unchanged =
+    input.fromLock === undefined &&
+    input.forceRebuild !== true &&
+    input.existingLock !== undefined &&
+    resolved.length === input.existingLock.sources.length &&
+    lock.sources.every(
+      (source) =>
+        !sourceChanged(
+          source,
+          resolved.find((entry) => entry.config.sourceId === source.sourceId)?.config,
+          input.existingLock,
+          input.existingCatalog
+        )
+    )
+  if (unchanged) {
+    return {
+      changed: false,
+      dryRun: input.dryRun ?? false,
+      sourceHeads,
+      changeReport: {
+        schemaVersion: 1,
+        changedSources: [],
+        addedPlugins: [],
+        removedPlugins: [],
+        changedPlugins: [],
+        permissionSensitiveChanges: [],
+        skippedPlugins: [],
+        contentResolution: input.metadataOnly ? 'metadata-only' : 'complete',
+      },
+    }
+  }
+
+  const { catalog, skippedPlugins } = await buildCatalogInternal({
+    ...input,
+    resolvedSources: resolved,
+    snapshotLoader:
+      input.snapshotLoader ??
+      (mode === 'offline'
+        ? new FixtureSnapshotLoader(input.fixtureRoot ?? 'fixtures', configs)
+        : new NetworkSnapshotLoader(input.policy)),
+    resolveExternalRefs: mode === 'live',
+  })
+  // An offline build replays fixtures, so its curation findings are noise.
+  const artifacts = createArtifacts(catalog, lock, {
+    ...input,
+    curationDiagnostics: mode !== 'offline',
+  })
+  verifyArtifacts(artifacts)
+  const report = createChangeReport(
+    catalog,
+    lock,
+    input.existingLock,
+    input.existingCatalog,
+    skippedPlugins
+  )
+  if (input.policy.publishRequiresCompleteContent && input.metadataOnly) {
+    throw new Error('CATALOG_CONTENT_INCOMPLETE: publication requires complete content')
+  }
+  return {
+    changed: true,
+    dryRun: input.dryRun ?? false,
+    sourceHeads,
+    catalog,
+    lock,
+    artifacts,
+    changeReport: report,
+  }
+}
+
+/** Whether a source differs from the last published lock and catalog. */
+function sourceChanged(
+  source: SourceLockEntry,
+  current: Pick<CatalogSource, 'displayName' | 'trustClassification'> | undefined,
+  existingLock: SourcesLock | undefined,
+  existingCatalog: Catalog | undefined
+): boolean {
+  const previous = existingLock?.sources.find((entry) => entry.sourceId === source.sourceId)
+  if (!previous || sourceLockChanged(previous, source)) return true
+  if (!existingCatalog) return false
+  const prior = existingCatalog.sources.find((entry) => entry.sourceId === source.sourceId)
+  return (
+    !prior ||
+    !current ||
+    prior.displayName !== current.displayName ||
+    prior.trustClassification !== current.trustClassification
+  )
+}
+
+function createChangeReport(
+  catalog: Catalog,
+  lock: SourcesLock,
+  existingLock?: SourcesLock,
+  existingCatalog?: Catalog,
+  skippedPlugins: readonly SkippedPlugin[] = []
+): ChangeReport {
+  const changedSources = lock.sources
+    .filter(
+      (source) =>
+        existingLock === undefined ||
+        sourceChanged(
+          source,
+          catalog.sources.find((entry) => entry.sourceId === source.sourceId),
+          existingLock,
+          existingCatalog
+        )
+    )
+    .map((source) => source.sourceId)
+  const previousPlugins = new Map(
+    existingCatalog?.plugins.map((plugin) => [plugin.pluginId, plugin]) ?? []
+  )
+  const currentPlugins = new Map(catalog.plugins.map((plugin) => [plugin.pluginId, plugin]))
+  const addedPlugins = catalog.plugins
+    .filter((plugin) => !previousPlugins.has(plugin.pluginId))
+    .map((plugin) => plugin.pluginId)
+  const removedPlugins = [...previousPlugins.keys()].filter(
+    (pluginId) => !currentPlugins.has(pluginId)
+  )
+  const changedPlugins = catalog.plugins
+    .filter((plugin) => {
+      const previous = previousPlugins.get(plugin.pluginId)
+      return previous !== undefined && digest(previous) !== digest(plugin)
+    })
+    .map((plugin) => plugin.pluginId)
+  return {
+    schemaVersion: 1,
+    changedSources: changedSources.toSorted(),
+    addedPlugins: addedPlugins.toSorted(),
+    removedPlugins: removedPlugins.toSorted(),
+    changedPlugins: changedPlugins.toSorted(),
+    skippedPlugins,
+    permissionSensitiveChanges: catalog.plugins
+      .flatMap((plugin) =>
+        plugin.securityClassification.permissionSensitiveChanges.map(
+          (type) => `${plugin.pluginId}:${type}`
+        )
+      )
+      .toSorted(),
+    contentResolution:
+      skippedPlugins.length > 0
+        ? 'complete-with-skips'
+        : catalog.plugins.some(
+              (plugin) => plugin.securityClassification.contentResolution === 'metadata-only'
+            )
+          ? 'metadata-only'
+          : 'complete',
+  }
+}

package/src/taxonomy.ts

@@ -0,0 +1,87 @@
+import { contentId } from './digest.js'
+
+export interface CategoryMap {
+  readonly aliases: Readonly<Record<string, string>>
+  readonly fallback: string
+}
+
+export interface ProductAliases {
+  readonly aliases: Readonly<Record<string, string>>
+}
+
+export function stablePluginId(sourceId: string, upstreamName: string): string {
+  return `plugin:${slug(sourceId)}:${slug(upstreamName)}`
+}
+
+export function stableReleaseId(
+  repositoryUrl: string,
+  subdirectory: string,
+  commitSha: string,
+  contentDigest: string
+): string {
+  return contentId('release', { repositoryUrl, subdirectory, commitSha, contentDigest })
+}
+
+export function normalizeCategory(value: string, map: CategoryMap): string {
+  const key = value.trim().toLocaleLowerCase()
+  return map.aliases[key] ?? (slug(key) || map.fallback)
+}
+
+export function productGroupingKey(name: string, aliases: ProductAliases): string {
+  const key = name.trim().toLocaleLowerCase()
+  return aliases.aliases[key] ?? slug(key)
+}
+
+/**
+ * Orders plugins so each category's curated leading products come first, in
+ * the configured order. Consumers preview the head of each category, so this
+ * ordering decides what users see before expanding.
+ */
+export function orderLeadingPlugins<
+  T extends {
+    readonly plugins: readonly {
+      categories: readonly string[]
+      upstreamPluginName: string
+    }[]
+  },
+>(catalog: T, leading?: Record<string, readonly string[]>): T {
+  if (!leading) return catalog
+  const leadRank = new Map<string, number>()
+  for (const [category, names] of Object.entries(leading)) {
+    names.forEach((name, index) => {
+      const key = `${category}::${name}`
+      if (!leadRank.has(key)) leadRank.set(key, index)
+    })
+  }
+  const rank = (plugin: T['plugins'][number]): number => {
+    let best = Number.POSITIVE_INFINITY
+    for (const category of plugin.categories) {
+      const value = leadRank.get(`${category}::${plugin.upstreamPluginName}`)
+      if (value !== undefined && value < best) best = value
+    }
+    return best
+  }
+  return {
+    ...catalog,
+    plugins: catalog.plugins.toSorted((left, right) => rank(left) - rank(right)),
+  }
+}
+
+function trimSlugEdges(value: string): string {
+  let start = 0
+  let end = value.length
+  while (start < end && value[start] === '-') start += 1
+  while (end > start && value[end - 1] === '-') end -= 1
+  return value.slice(start, end)
+}
+
+export function slug(value: string): string {
+  const result =
+    trimSlugEdges(
+      value
+        .trim()
+        .toLocaleLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+    ).slice(0, 128) || 'unknown'
+  return result.length < 2 ? `${result}-plugin` : result
+}

package/src/upstream.ts

@@ -0,0 +1,194 @@
+import { spawn } from 'node:child_process'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { canonicalRepositoryUrl } from './sources.js'
+import type { SiteIconFetcher } from './site-icons.js'
+
+export function externalGitRef(ref: string | undefined): string {
+  if (ref === undefined) return 'HEAD'
+  return ref.startsWith('refs/') ? ref : `refs/heads/${ref}`
+}
+
+export async function resolveGitRef(repositoryUrl: string, ref: string): Promise<string> {
+  const peeled = ref.startsWith('refs/tags/') && !ref.endsWith('^{}') ? `${ref}^{}` : undefined
+  const result = spawn(
+    'git',
+    ['ls-remote', canonicalRepositoryUrl(repositoryUrl), ref, ...(peeled ? [peeled] : [])],
+    {
+      stdio: ['ignore', 'pipe', 'pipe'],
+    }
+  )
+  const [stdout, stderr, exitCode] = await Promise.all([
+    new Promise<string>((resolveOutput, reject) => {
+      const chunks: Buffer[] = []
+      result.stdout?.on('data', (chunk: Buffer) => chunks.push(chunk))
+      result.stdout?.on('error', reject)
+      result.stdout?.on('end', () => resolveOutput(Buffer.concat(chunks).toString('utf8')))
+    }),
+    new Promise<string>((resolveError) => {
+      const chunks: Buffer[] = []
+      result.stderr?.on('data', (chunk: Buffer) => chunks.push(chunk))
+      result.stderr?.on('error', () => resolveError(''))
+      result.stderr?.on('end', () => resolveError(Buffer.concat(chunks).toString('utf8')))
+    }),
+    new Promise<number | null>((resolveExit, reject) => {
+      result.on('error', reject)
+      result.on('close', (code) => resolveExit(code ?? 1))
+    }),
+  ])
+  if (exitCode !== 0)
+    throw new Error(`GIT_REF_LOOKUP_FAILED: ${repositoryUrl}:${ref}:${stderr.trim()}`)
+  const rows = stdout
+    .trim()
+    .split('\n')
+    .map((line) => line.split(/\s+/))
+  const sha =
+    (peeled && rows.find((row) => row[1] === peeled)?.[0]) ??
+    rows.find((row) => row[1] === ref)?.[0]
+  if (!sha || !/^[a-f0-9]{40}$/.test(sha))
+    throw new Error(`GIT_REF_NOT_FOUND: ${repositoryUrl}:${ref}`)
+  return sha
+}
+
+export async function fetchUpstreamBytes(url: string, signal?: AbortSignal): Promise<Uint8Array> {
+  const response = await fetchUpstream(url, {
+    headers: upstreamRequestHeaders(url),
+    ...(signal ? { signal } : {}),
+  })
+  if (!response.ok) throw new Error(`HTTP_FETCH_FAILED: ${response.status}: ${url}`)
+  return new Uint8Array(await response.arrayBuffer())
+}
+
+export function rawContentUrl(repositoryUrl: string, commitSha: string, path: string): string {
+  const parsed = new URL(canonicalRepositoryUrl(repositoryUrl))
+  return `https://raw.githubusercontent.com${parsed.pathname}/${commitSha}/${path.split('/').map(encodeURIComponent).join('/')}`
+}
+
+export async function fetchText(url: string): Promise<string> {
+  const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) })
+  if (!response.ok) throw new Error(`HTTP_FETCH_FAILED: ${response.status}:${url}`)
+  return response.text()
+}
+
+const UPSTREAM_USER_AGENT = 'agent-hq-plugin-marketplace'
+
+/**
+ * Hosts that may receive the repository credential.
+ *
+ * Everything else is fetched anonymously. The token this build runs with
+ * authorises repository writes for the duration of the job, so a vendor
+ * homepage, its icon CDN or any other third party must never be handed it.
+ */
+/** The hosts the network snapshot loader reads repositories from. */
+export const GITHUB_REPOSITORY_HOSTS: readonly string[] = ['github.com', 'www.github.com']
+
+const GITHUB_CREDENTIAL_HOSTS = /(^|\.)(github\.com|githubusercontent\.com)$/
+
+/**
+ * Request headers for one upstream URL.
+ *
+ * The credential and GitHub's media type are attached only for GitHub hosts.
+ * Sending them elsewhere is not merely a needless disclosure: `appwrite.io`
+ * answers a page carrying an `Authorization` header with HTTP 500, and a CDN
+ * asked for an image with `accept: application/vnd.github+json` can answer with
+ * bytes that are not the mark. A build that did this dropped marks it had
+ * already curated, and the curation verifier — which used its own anonymous
+ * headers — could not see it.
+ */
+export function upstreamRequestHeaders(url: string): Record<string, string> {
+  const anonymous = { 'user-agent': UPSTREAM_USER_AGENT }
+  let host: string
+  try {
+    host = new URL(url).hostname.toLowerCase()
+  } catch {
+    return anonymous
+  }
+  const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN
+  if (!token || !GITHUB_CREDENTIAL_HOSTS.test(host)) return anonymous
+  return {
+    accept: 'application/vnd.github+json',
+    authorization: `Bearer ${token}`,
+    'user-agent': UPSTREAM_USER_AGENT,
+  }
+}
+
+/**
+ * Fetches vendor-site documents exactly as a live build does.
+ *
+ * Shared with the curation verifier so both ask a vendor the same question: a
+ * verifier with its own headers reports sites healthy that the build then fails
+ * to resolve.
+ *
+ * `deadline` optionally bounds one product's whole resolution. The per-request
+ * ceiling is always created fresh inside each fetch, so a caller that passes a
+ * deadline — or shares one — can never hand later requests an already-expired
+ * signal.
+ */
+export function createSiteIconFetcher(deadline?: AbortSignal): SiteIconFetcher {
+  const bounded = (): AbortSignal => {
+    const ceiling = AbortSignal.timeout(SITE_FETCH_TIMEOUT_MS)
+    return deadline ? AbortSignal.any([deadline, ceiling]) : ceiling
+  }
+  return {
+    fetchText: async (url) => new TextDecoder().decode(await fetchUpstreamBytes(url, bounded())),
+    fetchBytes: (url) => fetchUpstreamBytes(url, bounded()),
+  }
+}
+
+export const SITE_FETCH_TIMEOUT_MS = 15_000
+
+const UPSTREAM_FETCH_MAX_ATTEMPTS = 3
+const UPSTREAM_FETCH_BASE_DELAY_MS = 1000
+const UPSTREAM_FETCH_MAX_DELAY_MS = 30000
+
+function upstreamRetryDelayMs(attempt: number, response?: Response): number {
+  const retryAfter = response?.headers.get('retry-after')
+  if (retryAfter !== null && retryAfter !== undefined) {
+    const seconds = Number(retryAfter)
+    if (Number.isFinite(seconds) && seconds >= 0)
+      return Math.min(seconds * 1000, UPSTREAM_FETCH_MAX_DELAY_MS)
+    const date = Date.parse(retryAfter)
+    if (!Number.isNaN(date))
+      return Math.min(Math.max(date - Date.now(), 0), UPSTREAM_FETCH_MAX_DELAY_MS)
+  }
+  return Math.min(UPSTREAM_FETCH_BASE_DELAY_MS * 2 ** attempt, UPSTREAM_FETCH_MAX_DELAY_MS)
+}
+
+function isRateLimitedUpstream(response: Response): boolean {
+  return (
+    response.headers.get('x-ratelimit-remaining') === '0' ||
+    response.headers.get('retry-after') !== null
+  )
+}
+
+function isRetryableUpstreamStatus(status: number, response?: Response): boolean {
+  if (status === 429) return true
+  if (status >= 500) return true
+  if (status === 403 && response !== undefined && isRateLimitedUpstream(response)) return true
+  return false
+}
+
+export async function fetchUpstream(url: string, init: RequestInit): Promise<Response> {
+  let attempt = 0
+  for (;;) {
+    let response: Response
+    try {
+      response = await fetch(url, init)
+    } catch (error) {
+      if (init.signal?.aborted || attempt >= UPSTREAM_FETCH_MAX_ATTEMPTS - 1) throw error
+      await sleep(upstreamRetryDelayMs(attempt), undefined, { signal: init.signal ?? undefined })
+      attempt += 1
+      continue
+    }
+    if (
+      response.ok ||
+      !isRetryableUpstreamStatus(response.status, response) ||
+      attempt >= UPSTREAM_FETCH_MAX_ATTEMPTS - 1
+    )
+      return response
+    await response.body?.cancel().catch(() => {})
+    await sleep(upstreamRetryDelayMs(attempt, response), undefined, {
+      signal: init.signal ?? undefined,
+    })
+    attempt += 1
+  }
+}