@aion0/forge 0.10.39 → 0.10.41

package/lib/connectors/sync.ts

@@ -1,35 +1,52 @@
 /**
- * Connector sync — pull registry + manifests from forge-connectors.
+ * Connector sync — pull registry + manifests from one or more remote
+ * forge-connectors-style repos and surface them as a single marketplace
+ * view with enterprise-over-public priority.
  *
- * Default base URL: https://raw.githubusercontent.com/aiwatching/forge-connectors/main
- * Override via settings.connectorsRepoUrl.
+ * Sources (priority order, 0 = highest):
+ *   1. Enterprise repos — listed by lib/enterprise.ts (one per configured
+ *      key). Fetched via the GitHub Contents API with the key's PAT so
+ *      private repos work.
+ *   2. Public — settings.connectorsRepoUrl or DEFAULT_REPO. Fetched via
+ *      raw.githubusercontent.com (no auth needed).
  *
- * The remote repo is expected to expose:
- *   /registry.json       — listing of all available connectors (light)
- *   /<id>/manifest.yaml  — the full YAML for one connector
- *
- * Local state:
- *   <dataDir>/connectors/registry-cache.json — last successful fetch
- *   <dataDir>/connectors/<id>/manifest.yaml  — per-connector manifest
- *     (installed copy, refreshed on update)
+ * On-disk layout:
+ *   <dataDir>/connectors/sources/<source-id>/registry-cache.json
+ *     — last successful registry fetch for that source (one file per source)
+ *   <dataDir>/connectors/<id>/manifest.yaml
+ *     — installed connector manifest (single copy regardless of source;
+ *       the source field on the in-memory marketplace entry tells the UI
+ *       where it came from)
  *
  * Sync is called non-blocking on startup and on-demand from the
- * Settings Marketplace UI. Network failures are silent — callers
- * keep the cached state. No built-in fallback: if the cache is empty
- * and the network is down, the marketplace is just empty.
+ * Settings Marketplace UI. Network failures on any one source are
+ * isolated — the marketplace falls back to whichever source caches
+ * still exist. No built-in fallback connectors.
  */
 
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { loadSettings } from '../settings';
 import { getDataDir } from '../dirs';
+import { listEnterpriseSources, type EnterpriseSource } from '../enterprise';
 import { getConnector, installConnector, listConfigOnlyIds, listInstalledConnectors } from './registry';
 import type { ConnectorMarketEntry } from './types';
 
 const DEFAULT_REPO = 'https://raw.githubusercontent.com/aiwatching/forge-connectors/main';
 const FETCH_TIMEOUT_MS = 10_000;
 
-interface RegistryFile {
+/**
+ * Cross-marketplace shape: one enterprise repo declares its full
+ * inventory here — connectors AND workflow templates (recipes /
+ * pipelines). Connector sync writes the whole object to disk; the
+ * workflow marketplace reads the recipes/pipelines arrays from the
+ * same cache (via readSourceRegistry) so we never double-fetch.
+ *
+ * Public side splits across two repos (forge-connectors, forge-workflow);
+ * each writes only its own field group. Either way, readers should
+ * treat all fields as optional.
+ */
+export interface RegistryFile {
   version?: number;
   connectors?: Array<{
     id: string;
@@ -42,185 +59,525 @@ interface RegistryFile {
     /** Override manifest path; defaults to `<id>/manifest.yaml`. */
     manifest?: string;
   }>;
+  /** Recipe summaries (when this repo ships them). */
+  recipes?: Array<{
+    name: string;
+    display_name?: string;
+    description?: string;
+    version?: string;
+    author?: string;
+    tags?: string[];
+    score?: number;
+    rating?: number;
+  }>;
+  /** Pipeline summaries (when this repo ships them). */
+  pipelines?: Array<{
+    name: string;
+    display_name?: string;
+    description?: string;
+    version?: string;
+    author?: string;
+    tags?: string[];
+    score?: number;
+    rating?: number;
+  }>;
+  /**
+   * Repo-relative path to the onboarding wizard template JSON. When set,
+   * sync also fetches that file and caches it next to the registry so
+   * lib/connectors/wizard-template.ts can resolve a per-enterprise (or
+   * public) override at wizard time. Omit to defer to the next-lower
+   * priority source / the Forge-bundled fallback.
+   */
+  wizard_template?: string;
+  /**
+   * E3: multi-department templates. When the source ships >1 template
+   * (one per department / product line), each entry lists its id,
+   * display label, and repo-relative path. Sync fetches and caches each
+   * one under `wizards/<dept>.json`. Mutually exclusive with the
+   * scalar `wizard_template`: if both are set, `wizard_templates` wins
+   * and the scalar is ignored (logged as a warning).
+   *
+   * The dept id is the slug used in URLs (`?source_id=...&dept=fortinac`),
+   * `display_name` is what the wizard's dropdown shows.
+   */
+  wizard_templates?: Array<{
+    id: string;
+    display_name: string;
+    /** Optional. When omitted, the dept is just a label — picking it in
+     *  the wizard falls through to the source/public default template,
+     *  but the dept name still flows into {dept.name} substitutions and
+     *  the installed_dept stamp on connector rows. Lets enterprises
+     *  declare departments without authoring per-dept templates. */
+    path?: string;
+  }>;
+  /**
+   * Craft summaries (when this repo ships them). Forge's craft sync
+   * subsystem (currently project-scoped via forge-crafts; multi-source
+   * wiring is Phase 2) will read these from per-source caches.
+   */
+  crafts?: Array<{
+    name: string;
+    display_name?: string;
+    description?: string;
+    version?: string;
+    author?: string;
+    icon?: string;
+    tags?: string[];
+  }>;
+  /**
+   * Skill summaries (when this repo ships them). Forge's skills sync
+   * subsystem (currently forge-skills only; multi-source wiring is
+   * Phase 2) will read these from per-source caches.
+   */
+  skills?: Array<{
+    name: string;
+    display_name?: string;
+    description?: string;
+    version?: string;
+    tags?: string[];
+  }>;
 }
 
-interface RegistryCache {
+interface SourceCache {
   fetched_at: string;
   base_url: string;
   data: RegistryFile;
 }
 
+// ─── Source model ─────────────────────────────────────────
+//
+// `public` and each `enterprise-<tenant_id>` are uniformly modelled as
+// a SourceMeta — the rest of the file iterates over an ordered list of
+// these and never cares which kind it's looking at.
+
+export interface SourceMeta {
+  id: string;                         // 'public' | 'enterprise-<tenant_id>'
+  display_name: string;
+  is_enterprise: boolean;
+  priority: number;                   // 0 = highest
+  fetch_mode: 'raw' | 'github_api';
+  base_url?: string;                  // raw mode
+  repo_url?: string;                  // github_api mode — 'github.com/owner/repo'
+  github_pat?: string;                // github_api mode
+}
+
+function publicBaseUrl(): string {
+  return (loadSettings().connectorsRepoUrl || '').trim() || DEFAULT_REPO;
+}
+
+function enterpriseToSource(es: EnterpriseSource, priority: number): SourceMeta {
+  return {
+    id: `enterprise-${es.tenant_id}`,
+    display_name: es.display_name,
+    is_enterprise: true,
+    priority,
+    fetch_mode: 'github_api',
+    repo_url: es.repo_url,
+    github_pat: es.github_pat,
+  };
+}
+
+/** Ordered list of sources, highest priority first. Always includes
+ *  `public` last; enterprise sources prepend in their configured order. */
+export function listSources(): SourceMeta[] {
+  const enterprise = listEnterpriseSources().map(enterpriseToSource);
+  const publicSrc: SourceMeta = {
+    id: 'public',
+    display_name: 'Public',
+    is_enterprise: false,
+    priority: enterprise.length,
+    fetch_mode: 'raw',
+    base_url: publicBaseUrl(),
+  };
+  return [...enterprise, publicSrc];
+}
+
 // ─── Paths ────────────────────────────────────────────────
 
 function connectorsDir(): string {
   return join(getDataDir(), 'connectors');
 }
 
-function cacheFile(): string {
-  return join(connectorsDir(), 'registry-cache.json');
+function sourceCacheFile(sourceId: string): string {
+  return join(connectorsDir(), 'sources', sourceId, 'registry-cache.json');
 }
 
-function ensureDir(p: string): void {
-  if (!existsSync(p)) mkdirSync(p, { recursive: true, mode: 0o700 });
+function sourceWizardTemplatePath(sourceId: string): string {
+  return join(connectorsDir(), 'sources', sourceId, 'wizard-template.json');
 }
 
-function baseUrl(): string {
-  return (loadSettings().connectorsRepoUrl || '').trim() || DEFAULT_REPO;
+/**
+ * E3: per-dept template cache path. Each department template lands at
+ * <sourceId>/wizards/<dept>.json next to the legacy single-template
+ * file, plus an index file `wizards/_index.json` listing the dept ids
+ * + display_names so the wizard UI can render the picker without
+ * re-reading the registry.
+ */
+export function sourceDeptTemplatePath(sourceId: string, deptId: string): string {
+  return join(connectorsDir(), 'sources', sourceId, 'wizards', `${deptId}.json`);
+}
+export function sourceDeptIndexPath(sourceId: string): string {
+  return join(connectorsDir(), 'sources', sourceId, 'wizards', '_index.json');
+}
+
+function ensureDir(p: string): void {
+  if (!existsSync(p)) mkdirSync(p, { recursive: true, mode: 0o700 });
 }
 
 // ─── Cache I/O ────────────────────────────────────────────
 
-export function readCache(): RegistryCache | null {
-  const p = cacheFile();
+function readSourceCache(sourceId: string): SourceCache | null {
+  const p = sourceCacheFile(sourceId);
   if (!existsSync(p)) return null;
   try {
-    return JSON.parse(readFileSync(p, 'utf-8')) as RegistryCache;
+    return JSON.parse(readFileSync(p, 'utf-8')) as SourceCache;
   } catch {
     return null;
   }
 }
 
-function writeCache(cache: RegistryCache): void {
-  ensureDir(connectorsDir());
-  writeFileSync(cacheFile(), JSON.stringify(cache, null, 2), { mode: 0o600 });
+function writeSourceCache(sourceId: string, cache: SourceCache): void {
+  const p = sourceCacheFile(sourceId);
+  ensureDir(join(connectorsDir(), 'sources', sourceId));
+  writeFileSync(p, JSON.stringify(cache, null, 2), { mode: 0o600 });
+}
+
+/** Internal — exposed for tests / debug only. */
+export function readCache(): SourceCache | null {
+  // Back-compat shim: callers that imported readCache previously expected
+  // a single registry view. Hand back the public source's cache (the
+  // pre-enterprise behaviour). For enterprise-aware merging, use
+  // listMarketplace instead.
+  return readSourceCache('public');
 }
 
 // ─── Fetch helpers ────────────────────────────────────────
 
-async function fetchText(url: string): Promise<string> {
+async function rawFetch(url: string, headers: Record<string, string> = {}): Promise<string> {
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT_MS);
   try {
     const r = await fetch(url, {
       signal: ctrl.signal,
       headers: {
-        // Some CDN edges (cloudflare in front of raw.githubusercontent.com)
-        // refuse default Node UA — set an explicit one.
         'User-Agent': 'forge-connectors-sync/1.0',
         'Cache-Control': 'no-cache',
+        ...headers,
       },
     });
-    if (!r.ok) throw new Error(`HTTP ${r.status} for ${url}`);
+    if (!r.ok) {
+      const err: Error & { status?: number } = new Error(`HTTP ${r.status} for ${url}`);
+      err.status = r.status;
+      throw err;
+    }
     return await r.text();
   } catch (err) {
-    // undici wraps the real failure under `err.cause`. Surface it so
-    // the user gets a useful message instead of a bare "fetch failed".
-    const e = err as Error & { cause?: unknown };
+    const e = err as Error & { cause?: unknown; status?: number };
     const cause = e.cause instanceof Error ? `: ${e.cause.message}` : e.cause ? `: ${String(e.cause)}` : '';
-    throw new Error(`${e.message}${cause} (${url})`);
+    const wrapped: Error & { status?: number } = new Error(`${e.message}${cause} (${url})`);
+    if (e.status) wrapped.status = e.status;
+    throw wrapped;
   } finally {
     clearTimeout(t);
   }
 }
 
-async function fetchJson<T>(url: string): Promise<T> {
-  const txt = await fetchText(url);
-  return JSON.parse(txt) as T;
-}
-
 function cacheBust(): string {
   return `?_t=${Date.now()}`;
 }
 
+/**
+ * Enterprise sources only, in configured priority order. Workflow-
+ * marketplace shares these — one enterprise repo serves connectors
+ * AND workflows from the same registry.json + bearer PAT.
+ */
+export function listEnterpriseSourceMetas(): SourceMeta[] {
+  return listEnterpriseSources().map(enterpriseToSource);
+}
+
+/**
+ * Read an enterprise (or `public`) source's cached registry data.
+ * Returns null when the cache file is missing. Used by other
+ * marketplaces (workflows) that piggyback on the same registry.json.
+ */
+export function readSourceRegistry(sourceId: string): RegistryFile | null {
+  const cache = readSourceCache(sourceId);
+  return cache ? cache.data : null;
+}
+
+/** Fetch a file from one source. Picks raw vs github API based on mode. */
+export async function fetchSourceFile(source: SourceMeta, relPath: string): Promise<string> {
+  if (source.fetch_mode === 'raw') {
+    return rawFetch(`${source.base_url}/${relPath}${cacheBust()}`);
+  }
+  // github_api: GET /repos/<owner>/<repo>/contents/<path>?ref=main
+  // with Accept: application/vnd.github.raw — returns the file body
+  // directly (no base64 decode needed). Works for private repos given
+  // a Fine-grained PAT with Contents: read.
+  const repo = (source.repo_url || '').replace(/^github\.com\//, '');
+  const url = `https://api.github.com/repos/${repo}/contents/${relPath}?ref=main`;
+  try {
+    return await rawFetch(url, {
+      Authorization: `Bearer ${source.github_pat}`,
+      Accept: 'application/vnd.github.raw',
+    });
+  } catch (err) {
+    // GitHub returns 404 (not 401) for private repos when the PAT
+    // lacks repository access OR the Contents:Read scope. Translate
+    // that into actionable wording so users don't chase missing files.
+    const e = err as Error & { status?: number };
+    if (e.status === 404) {
+      throw new Error(
+        `PAT cannot read github.com/${repo} — open https://github.com/settings/personal-access-tokens, ` +
+          `select your Forge PAT, and ensure (a) the repo is in "Repository access" and ` +
+          `(b) "Repository permissions → Contents" is set to Read-only. ` +
+          `Then re-sync. (original: HTTP 404 ${relPath})`,
+      );
+    }
+    if (e.status === 401 || e.status === 403) {
+      throw new Error(
+        `PAT rejected by GitHub for ${repo} — token expired or revoked. ` +
+          `Generate a new fine-grained PAT and update it in Settings → Marketplace Providers. ` +
+          `(original: HTTP ${e.status} ${relPath})`,
+      );
+    }
+    throw err;
+  }
+}
+
+// ─── Last-sync tracking ───────────────────────────────────
+// In-memory map of source_id → last attempt result. Populated by
+// syncRegistry on every run; read by /api/enterprise-keys so the UI
+// can flag stuck/404'd sources (typically: PAT lacks Contents:Read).
+// Lives on globalThis so HMR + multiple module copies share state.
+//
+type LastSync = { ok: boolean; error?: string; fetched_at: string };
+const G = globalThis as unknown as { __forgeLastSync?: Map<string, LastSync> };
+if (!G.__forgeLastSync) G.__forgeLastSync = new Map();
+const lastSyncMap = G.__forgeLastSync;
+
+export function getLastSync(sourceId: string): LastSync | null {
+  return lastSyncMap.get(sourceId) || null;
+}
+
 // ─── Sync API ─────────────────────────────────────────────
 
 export interface SyncResult {
   ok: boolean;
-  registry_count: number;
+  registry_count: number;        // unique connector ids across all sources
   manifests_refreshed: number;
   error?: string;
   fetched_at?: string;
+  /** Per-source breakdown so the UI can surface partial failures. */
+  sources?: Array<{
+    source_id: string;
+    display_name: string;
+    ok: boolean;
+    count: number;
+    error?: string;
+  }>;
 }
 
 /**
- * Pull the registry.json and refresh on-disk manifests for any
- * connector the user has installed (so install_version stays accurate
- * and scripts get bug fixes). For not-yet-installed entries we only
- * cache the registry row — the manifest is pulled on demand at install
- * time (see installFromRegistry).
+ * Pull registry.json from every configured source and refresh manifests
+ * for connectors the user has installed. Per-source errors are isolated:
+ * if enterprise-fortinet is unreachable, public still updates and vice versa.
  */
 export async function syncRegistry(
   opts: { refreshInstalled?: boolean; force?: boolean } = {},
 ): Promise<SyncResult> {
-  const base = baseUrl();
-  console.log(`[connectors] Syncing registry from ${base}`);
-  let registry: RegistryFile;
-  try {
-    registry = await fetchJson<RegistryFile>(`${base}/registry.json${cacheBust()}`);
-  } catch (err) {
-    const msg = err instanceof Error ? err.message : String(err);
-    console.warn(`[connectors] registry fetch failed: ${msg}`);
-    return { ok: false, registry_count: 0, manifests_refreshed: 0, error: msg };
+  const sources = listSources();
+  const fetched_at = new Date().toISOString();
+  const sourceResults: NonNullable<SyncResult['sources']> = [];
+  const uniqueIds = new Set<string>();
+
+  for (const source of sources) {
+    const baseLabel = source.fetch_mode === 'raw'
+      ? source.base_url
+      : `github.com/${(source.repo_url || '').replace(/^github\.com\//, '')}`;
+    console.log(`[connectors] Syncing ${source.id} from ${baseLabel}`);
+
+    try {
+      const registry = await fetchSourceFile(source, 'registry.json').then(t => JSON.parse(t) as RegistryFile);
+      const count = registry.connectors?.length || 0;
+      writeSourceCache(source.id, { fetched_at, base_url: baseLabel || '', data: registry });
+      registry.connectors?.forEach(c => uniqueIds.add(c.id));
+
+      // Wizard template(s) — multi-dept (E3) wins over the legacy scalar
+      // when both are present. Each dept JSON lands at wizards/<id>.json
+      // and a sibling _index.json lists the labels so the wizard UI can
+      // build a picker without re-reading registry.json. Failures are
+      // per-file isolated; rest of the source still counts ok.
+      const depts = registry.wizard_templates;
+      if (Array.isArray(depts) && depts.length > 0) {
+        if (registry.wizard_template) {
+          console.warn(`[connectors] ${source.id}: both wizard_template and wizard_templates set — using the latter, ignoring the scalar`);
+        }
+        ensureDir(join(connectorsDir(), 'sources', source.id, 'wizards'));
+        const index: Array<{ id: string; display_name: string; has_template: boolean }> = [];
+        for (const dept of depts) {
+          if (!dept?.id) continue;
+          let hasTemplate = false;
+          if (dept.path) {
+            try {
+              const wt = await fetchSourceFile(source, dept.path);
+              writeFileSync(sourceDeptTemplatePath(source.id, dept.id), wt, { mode: 0o600 });
+              hasTemplate = true;
+            } catch (wtErr) {
+              const m = wtErr instanceof Error ? wtErr.message : String(wtErr);
+              console.warn(`[connectors] ${source.id} dept template '${dept.id}' fetch failed: ${m}`);
+            }
+          }
+          // Even template-less depts land in the index — they're real
+          // departments, just without per-dept customization. The
+          // wizard renders them with public/bundled defaults but
+          // stamps installed_dept on connectors so the user's choice
+          // is still recorded.
+          index.push({ id: dept.id, display_name: dept.display_name || dept.id, has_template: hasTemplate });
+        }
+        writeFileSync(sourceDeptIndexPath(source.id), JSON.stringify(index, null, 2), { mode: 0o600 });
+      } else if (registry.wizard_template) {
+        try {
+          const wt = await fetchSourceFile(source, registry.wizard_template);
+          const wtPath = sourceWizardTemplatePath(source.id);
+          ensureDir(join(connectorsDir(), 'sources', source.id));
+          writeFileSync(wtPath, wt, { mode: 0o600 });
+        } catch (wtErr) {
+          const m = wtErr instanceof Error ? wtErr.message : String(wtErr);
+          console.warn(`[connectors] ${source.id} wizard template fetch failed: ${m}`);
+        }
+      }
+
+      sourceResults.push({
+        source_id: source.id,
+        display_name: source.display_name,
+        ok: true,
+        count,
+      });
+      lastSyncMap.set(source.id, { ok: true, fetched_at });
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[connectors] ${source.id} registry fetch failed: ${msg}`);
+      sourceResults.push({
+        source_id: source.id,
+        display_name: source.display_name,
+        ok: false,
+        count: 0,
+        error: msg,
+      });
+      lastSyncMap.set(source.id, { ok: false, error: msg, fetched_at });
+    }
   }
 
-  const entries = registry.connectors || [];
-  const cache: RegistryCache = {
-    fetched_at: new Date().toISOString(),
-    base_url: base,
-    data: registry,
-  };
-  writeCache(cache);
+  const anyOk = sourceResults.some(s => s.ok);
+  if (!anyOk) {
+    return {
+      ok: false,
+      registry_count: 0,
+      manifests_refreshed: 0,
+      error: sourceResults.map(s => `${s.source_id}: ${s.error}`).join('; '),
+      sources: sourceResults,
+    };
+  }
 
   let refreshed = 0;
   if (opts.refreshInstalled) {
-    // Two cases we want to pull manifests for:
-    //   1. local manifest exists but version is behind the registry
-    //   2. config row exists but no manifest on disk yet (post-migration
-    //      from pre-v0.9 plugin-configs.json — we have the user's PAT
-    //      but the manifest must come from the remote)
-    const configOnly = new Set(listConfigOnlyIds());
-    for (const e of entries) {
-      const local = getConnector(e.id);
-      const isConfigOnly = configOnly.has(e.id);
-      if (!local && !isConfigOnly) continue;
-      // Normally skip when local matches registry — saves bandwidth.
-      // `force` overrides so the user can pull a fresh manifest even
-      // when the version string hasn't changed (e.g. param-description
-      // tweaks shipped under the same patch version).
-      if (!opts.force && local && local.version === e.version) continue;
-      try {
-        const yamlText = await fetchManifest(e.id, e.manifest);
-        installConnector(e.id, yamlText);
-        refreshed += 1;
-      } catch (err) {
-        console.warn(`[connectors] refresh failed for ${e.id}:`, err);
-      }
-    }
+    refreshed = await refreshInstalledManifests(sources, opts.force === true);
   }
 
   return {
     ok: true,
-    registry_count: entries.length,
+    registry_count: uniqueIds.size,
     manifests_refreshed: refreshed,
-    fetched_at: cache.fetched_at,
+    fetched_at,
+    sources: sourceResults,
   };
 }
 
-/** Fetch one connector's manifest from the configured repo. */
+/** For each locally installed id, find the highest-priority source that
+ *  lists it and re-pull its manifest. Skips ids whose version matches
+ *  unless `force` is set. */
+async function refreshInstalledManifests(sources: SourceMeta[], force: boolean): Promise<number> {
+  const configOnly = new Set(listConfigOnlyIds());
+  const installedIds = new Set<string>(listInstalledConnectors().map(i => i.definition.id));
+  for (const id of configOnly) installedIds.add(id);
+
+  let refreshed = 0;
+  for (const id of installedIds) {
+    const match = findEntryAcrossSources(sources, id);
+    if (!match) continue;
+    const local = getConnector(id);
+    if (!force && local && local.version === match.entry.version) continue;
+    try {
+      const yamlText = await fetchSourceFile(
+        match.source,
+        match.entry.manifest || `${id}/manifest.yaml`,
+      );
+      installConnector(id, yamlText, { source_id: match.source.id });
+      refreshed += 1;
+    } catch (err) {
+      console.warn(`[connectors] refresh failed for ${id} via ${match.source.id}:`, err);
+    }
+  }
+  return refreshed;
+}
+
+interface SourceEntryMatch {
+  source: SourceMeta;
+  entry: NonNullable<RegistryFile['connectors']>[number];
+}
+
+/** First source (highest priority) whose cache contains a registry entry for `id`. */
+function findEntryAcrossSources(sources: SourceMeta[], id: string): SourceEntryMatch | null {
+  for (const source of sources) {
+    const cache = readSourceCache(source.id);
+    const entry = cache?.data.connectors?.find(c => c.id === id);
+    if (entry) return { source, entry };
+  }
+  return null;
+}
+
+/** Fetch one connector's manifest. Picks the highest-priority source
+ *  that lists the id; falls back to public's `<id>/manifest.yaml` URL. */
 export async function fetchManifest(id: string, manifestPath?: string): Promise<string> {
-  const base = baseUrl();
-  const path = manifestPath || `${id}/manifest.yaml`;
-  return fetchText(`${base}/${path}${cacheBust()}`);
+  const sources = listSources();
+  const match = findEntryAcrossSources(sources, id);
+  if (match) {
+    return fetchSourceFile(match.source, manifestPath || match.entry.manifest || `${id}/manifest.yaml`);
+  }
+  // No source lists this id — try public's conventional path as a last
+  // resort (preserves the old behaviour for ids not yet in the registry).
+  const publicSrc = sources.find(s => s.id === 'public');
+  if (!publicSrc) throw new Error(`no source available for ${id}`);
+  return fetchSourceFile(publicSrc, manifestPath || `${id}/manifest.yaml`);
 }
 
-/**
- * Install a connector by id — looks it up in the registry cache (or
- * fetches a fresh one), pulls the manifest, and hands off to the
- * registry. Returns the chosen version on success.
- */
-export async function installFromRegistry(id: string): Promise<{ ok: boolean; version?: string; error?: string }> {
-  let cache = readCache();
-  if (!cache) {
+/** Install a connector by id — looks it up across all source caches in
+ *  priority order, pulls the manifest from the winning source, and hands
+ *  off to the registry. Returns the chosen version on success. */
+export async function installFromRegistry(id: string): Promise<{ ok: boolean; version?: string; error?: string; source_id?: string }> {
+  let sources = listSources();
+  let match = findEntryAcrossSources(sources, id);
+
+  // Cache cold — try one sync round to populate it.
+  if (!match) {
     const r = await syncRegistry();
     if (!r.ok) return { ok: false, error: r.error || 'registry unavailable' };
-    cache = readCache();
+    sources = listSources();
+    match = findEntryAcrossSources(sources, id);
   }
-  const entry = cache?.data.connectors?.find((c) => c.id === id);
-  if (!entry) return { ok: false, error: `${id} not in registry` };
+
+  if (!match) return { ok: false, error: `${id} not in registry` };
+
   try {
-    const yamlText = await fetchManifest(id, entry.manifest);
-    const ok = installConnector(id, yamlText);
-    return ok ? { ok: true, version: entry.version } : { ok: false, error: 'manifest invalid' };
+    const yamlText = await fetchSourceFile(
+      match.source,
+      match.entry.manifest || `${id}/manifest.yaml`,
+    );
+    const ok = installConnector(id, yamlText, { source_id: match.source.id });
+    return ok
+      ? { ok: true, version: match.entry.version, source_id: match.source.id }
+      : { ok: false, error: 'manifest invalid' };
   } catch (err) {
     return { ok: false, error: err instanceof Error ? err.message : String(err) };
   }
@@ -229,48 +586,86 @@ export async function installFromRegistry(id: string): Promise<{ ok: boolean; ve
 // ─── Marketplace listing ──────────────────────────────────
 
 /**
- * Merge the registry cache with installed state to produce the
- * marketplace view. Pure read; no network.
+ * Merge per-source caches with installed state into a single marketplace
+ * view. Higher-priority sources win for same id; outranked entries land
+ * in `hidden_sources`. Pure read; no network.
  *
- * If the registry cache is missing (first boot, offline, sync failed)
- * we still surface locally-installed connectors so the user sees
- * something — labelled as installed without a remote counterpart.
- * Refresh button is the path back to a real registry listing.
+ * `fetched_at` / `base_url` refer to the highest-priority source that
+ * has a cache (mostly enterprise[0] when configured, else public) — kept
+ * for back-compat with UI that reads them.
  */
 export function listMarketplace(): {
   fetched_at?: string;
   base_url?: string;
   entries: ConnectorMarketEntry[];
 } {
-  const cache = readCache();
+  const sources = listSources();
   const installed = listInstalledConnectors();
   const installedById = new Map(installed.map((i) => [i.definition.id, i] as const));
 
-  const entries: ConnectorMarketEntry[] = [];
+  // Walk sources in priority order; first occurrence wins, later ones
+  // collect into `hidden_sources` on the winning entry.
+  interface PendingEntry {
+    winning_source: SourceMeta;
+    entry: NonNullable<RegistryFile['connectors']>[number];
+    hidden: { source_id: string; display_name: string; version: string }[];
+  }
+  const pending = new Map<string, PendingEntry>();
+  let primaryCache: SourceCache | null = null;
 
-  if (cache?.data.connectors?.length) {
+  for (const source of sources) {
+    const cache = readSourceCache(source.id);
+    if (!cache?.data.connectors?.length) continue;
+    if (!primaryCache) primaryCache = cache;
     for (const c of cache.data.connectors) {
-      const local = installedById.get(c.id);
-      entries.push({
-        id: c.id,
-        name: c.name,
-        version: c.version,
-        icon: c.icon,
-        description: c.description,
-        author: c.author,
-        installed_version: local?.installed_version,
-        update_available: !!local && compareVersions(c.version, local.installed_version) > 0,
-        compatible: isVersionCompatible(c.min_forge_version),
-        source: 'registry',
-      });
-      installedById.delete(c.id);
+      const existing = pending.get(c.id);
+      if (!existing) {
+        pending.set(c.id, { winning_source: source, entry: c, hidden: [] });
+      } else {
+        existing.hidden.push({
+          source_id: source.id,
+          display_name: source.display_name,
+          version: c.version,
+        });
+      }
     }
   }
 
-  // Locally-installed connectors with no registry counterpart — either
-  // installed via /api/connectors/install-local, or the registry is
-  // currently unreachable. Either way: show them with source=local so
-  // the UI suppresses the Update path.
+  const entries: ConnectorMarketEntry[] = [];
+  for (const { winning_source, entry, hidden } of pending.values()) {
+    const local = installedById.get(entry.id);
+    // Installed source vs. registry-winning source can differ when the
+    // user installed from one source and then added a higher-priority
+    // source that hasn't refreshed yet. Surface the *installed* one
+    // so the badge reflects on-disk truth; refresh updates it.
+    const surfaced_source_id = local?.installed_source_id || winning_source.id;
+    const updateAvailable = !!local && compareVersions(entry.version, local.installed_version) > 0;
+    entries.push({
+      id: entry.id,
+      name: entry.name,
+      version: entry.version,
+      icon: entry.icon,
+      description: entry.description,
+      author: entry.author,
+      installed_version: local?.installed_version,
+      update_available: updateAvailable,
+      // Source where a fresh install / update / reinstall would pull from
+      // (highest-priority registry that lists this id). Always emitted so
+      // the UI can flag "this is installed from public but enterprise also
+      // publishes it" even when the versions match — user still wants to
+      // know they can flip the source via Reinstall.
+      update_source_id: winning_source.id,
+      compatible: isVersionCompatible(entry.min_forge_version),
+      source: 'registry',
+      source_id: surfaced_source_id,
+      hidden_sources: hidden.length ? hidden : undefined,
+    });
+    installedById.delete(entry.id);
+  }
+
+  // Locally-installed connectors that no source lists — install-local
+  // uploads, or every registry currently unreachable. Show with
+  // source=local so UI suppresses the Update path.
   for (const local of installedById.values()) {
     const def = local.definition;
     entries.push({
@@ -287,7 +682,11 @@ export function listMarketplace(): {
     });
   }
 
-  return { fetched_at: cache?.fetched_at, base_url: cache?.base_url, entries };
+  return {
+    fetched_at: primaryCache?.fetched_at,
+    base_url: primaryCache?.base_url,
+    entries,
+  };
 }
 
 // ─── Version helpers ──────────────────────────────────────
@@ -302,10 +701,9 @@ function compareVersions(a: string, b: string): number {
   return 0;
 }
 
-function isVersionCompatible(min?: string): boolean {
-  if (!min) return true;
+function isVersionCompatible(_min?: string): boolean {
   // Soft check — we don't ship a hard FORGE_VERSION constant on the
-  // server; rely on the user keeping Forge current.
-  // TODO: read version from package.json at boot if we want to enforce.
+  // server; rely on the user keeping Forge current. The unused arg is
+  // kept so the schema field stays meaningful for future enforcement.
   return true;
 }