@aion0/forge 0.8.1 → 0.8.2

package/lib/connectors/registry.ts

@@ -0,0 +1,328 @@
+/**
+ * Connector Registry — load, install, uninstall, list connectors.
+ *
+ * Manifests live in `<dataDir>/connectors/<id>/manifest.yaml`, fetched
+ * by lib/connectors/sync.ts from the remote forge-connectors repo.
+ * There are no built-in connector manifests — a fresh install has zero
+ * connectors until the first registry sync completes.
+ *
+ * Per-instance config (settings filled in by the user — host URL,
+ * PAT, etc.) lives in `<dataDir>/connector-configs.json`. Secret
+ * fields are encrypted at rest with AES-256-GCM via lib/crypto.ts.
+ */
+
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import YAML from 'yaml';
+import { getDataDir } from '../dirs';
+import { encryptSecret, decryptSecret, isEncrypted } from '../crypto';
+import type {
+  ConnectorDefinition,
+  ConnectorEntry,
+  ConnectorFieldSchema,
+} from './types';
+
+// ─── Paths ────────────────────────────────────────────────
+
+function connectorsDir(): string {
+  return join(getDataDir(), 'connectors');
+}
+
+function manifestPath(id: string): string {
+  return join(connectorsDir(), id, 'manifest.yaml');
+}
+
+function configsFile(): string {
+  return join(getDataDir(), 'connector-configs.json');
+}
+
+function ensureDir(p: string): void {
+  if (!existsSync(p)) mkdirSync(p, { recursive: true, mode: 0o700 });
+}
+
+// ─── Manifest loading ─────────────────────────────────────
+
+function parseManifest(raw: string): ConnectorDefinition | null {
+  try {
+    const def = YAML.parse(raw) as ConnectorDefinition;
+    if (!def?.id || !def?.name) return null;
+    if (!def.tools && !def.connectors?.length) return null;
+    if (!def.version) def.version = '0.0.0';
+    if (!def.icon) def.icon = '🔌';
+    return def;
+  } catch {
+    return null;
+  }
+}
+
+/** Read the manifest for a single connector id; returns null if missing. */
+export function getConnector(id: string): ConnectorDefinition | null {
+  const p = manifestPath(id);
+  if (!existsSync(p)) return null;
+  try {
+    return parseManifest(readFileSync(p, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+/** List installed connector ids (presence of `<dir>/manifest.yaml`). */
+export function listConnectorIds(): string[] {
+  const dir = connectorsDir();
+  if (!existsSync(dir)) return [];
+  const out: string[] = [];
+  for (const name of readdirSync(dir)) {
+    try {
+      const sub = join(dir, name);
+      if (!statSync(sub).isDirectory()) continue;
+      if (existsSync(join(sub, 'manifest.yaml'))) out.push(name);
+    } catch {}
+  }
+  return out;
+}
+
+/**
+ * Normalize a connector to its entry list.
+ *   1:1 shape (top-level `tools`) → synthesizes one entry with id === connector id.
+ *   1:N shape (`connectors[]`) → returned as-is.
+ */
+export function getConnectorEntries(def: ConnectorDefinition): ConnectorEntry[] {
+  if (def.connectors?.length) return def.connectors;
+  if (def.tools) {
+    return [
+      {
+        id: def.id,
+        tools: def.tools,
+        settings: def.settings,
+        host_match: def.host_match,
+        login_redirect: def.login_redirect,
+        runner: def.runner,
+      },
+    ];
+  }
+  return [];
+}
+
+// ─── Config store ─────────────────────────────────────────
+
+/**
+ * Wire shape of `<dataDir>/connector-configs.json`:
+ *   { "<id>": { config, installed_version, enabled } }
+ */
+interface ConnectorConfigRow {
+  config: Record<string, unknown>;
+  installed_version: string;
+  enabled?: boolean;
+}
+
+type ConfigStore = Record<string, ConnectorConfigRow>;
+
+function getSecretFieldNames(def: ConnectorDefinition | null): string[] {
+  if (!def) return [];
+  const fields: Array<[string, ConnectorFieldSchema]> = [];
+  if (def.settings) fields.push(...Object.entries(def.settings));
+  for (const entry of def.connectors || []) {
+    if (entry.settings) fields.push(...Object.entries(entry.settings));
+  }
+  return fields
+    .filter(([, schema]) => {
+      const t = String((schema as any)?.type || '');
+      return t === 'secret' || t === 'password';
+    })
+    .map(([name]) => name);
+}
+
+function loadStoreRaw(): ConfigStore {
+  const p = configsFile();
+  if (!existsSync(p)) return {};
+  try {
+    return JSON.parse(readFileSync(p, 'utf-8')) as ConfigStore;
+  } catch {
+    return {};
+  }
+}
+
+function loadStore(): ConfigStore {
+  const store = loadStoreRaw();
+  // Decrypt secret fields lazily — readers expect plaintext.
+  for (const [id, row] of Object.entries(store)) {
+    if (!row?.config) continue;
+    const def = getConnector(id);
+    const secrets = getSecretFieldNames(def);
+    if (!secrets.length) continue;
+    for (const key of secrets) {
+      const v = row.config[key];
+      if (typeof v === 'string' && isEncrypted(v)) {
+        try { row.config[key] = decryptSecret(v); }
+        catch (err) {
+          console.warn(`[connectors] failed to decrypt ${id}.${key}`, err);
+        }
+      }
+    }
+  }
+  return store;
+}
+
+function saveStore(store: ConfigStore): void {
+  ensureDir(getDataDir());
+  // Deep-copy so we can encrypt without mutating caller-visible values.
+  const out: ConfigStore = {};
+  for (const [id, row] of Object.entries(store)) {
+    const def = getConnector(id);
+    const secrets = new Set(getSecretFieldNames(def));
+    const encryptedConfig: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(row.config || {})) {
+      if (secrets.has(k) && typeof v === 'string' && v.length > 0 && !isEncrypted(v)) {
+        try { encryptedConfig[k] = encryptSecret(v); }
+        catch { encryptedConfig[k] = v; }
+      } else {
+        encryptedConfig[k] = v;
+      }
+    }
+    out[id] = {
+      config: encryptedConfig,
+      installed_version: row.installed_version,
+      enabled: row.enabled !== false,
+    };
+  }
+  writeFileSync(configsFile(), JSON.stringify(out, null, 2), { mode: 0o600 });
+}
+
+// ─── Public API ───────────────────────────────────────────
+
+export interface InstalledConnector {
+  definition: ConnectorDefinition;
+  config: Record<string, unknown>;
+  installed_version: string;
+  enabled: boolean;
+}
+
+export function isInstalled(id: string): boolean {
+  return existsSync(manifestPath(id)) && !!loadStoreRaw()[id];
+}
+
+/**
+ * IDs that have a config row but no manifest on disk yet. Happens
+ * during the migration window after upgrading from pre-v0.9 Forge —
+ * the row is moved across, but the manifest can only be fetched
+ * from the remote registry. Sync uses this to know what to pull.
+ */
+export function listConfigOnlyIds(): string[] {
+  const store = loadStoreRaw();
+  const out: string[] = [];
+  for (const id of Object.keys(store)) {
+    if (!existsSync(manifestPath(id))) out.push(id);
+  }
+  return out;
+}
+
+export function listInstalledConnectors(): InstalledConnector[] {
+  const store = loadStore();
+  const out: InstalledConnector[] = [];
+  for (const id of listConnectorIds()) {
+    const def = getConnector(id);
+    if (!def) continue;
+    const row = store[id];
+    if (!row) continue; // manifest on disk but no config row → not installed
+    out.push({
+      definition: def,
+      config: row.config || {},
+      installed_version: row.installed_version || def.version,
+      enabled: row.enabled !== false,
+    });
+  }
+  return out;
+}
+
+export function getInstalledConnector(id: string): InstalledConnector | null {
+  const def = getConnector(id);
+  if (!def) return null;
+  const row = loadStore()[id];
+  if (!row) return null;
+  return {
+    definition: def,
+    config: row.config || {},
+    installed_version: row.installed_version || def.version,
+    enabled: row.enabled !== false,
+  };
+}
+
+/**
+ * Install (or upgrade) a connector by writing its manifest YAML and
+ * recording the version in the config store. Existing user config is
+ * preserved across upgrades.
+ */
+export function installConnector(id: string, manifestYaml: string): boolean {
+  const def = parseManifest(manifestYaml);
+  if (!def || def.id !== id) return false;
+  const dir = join(connectorsDir(), id);
+  ensureDir(dir);
+  writeFileSync(join(dir, 'manifest.yaml'), manifestYaml, { mode: 0o600 });
+
+  const store = loadStore();
+  const prev = store[id]?.config || {};
+  store[id] = {
+    config: prev,
+    installed_version: def.version,
+    enabled: true,
+  };
+  saveStore(store);
+  return true;
+}
+
+/**
+ * Uninstall a connector.
+ *   keepConfig=true (default): drop the manifest but retain settings/PAT
+ *     so re-installing later restores them.
+ *   keepConfig=false: also delete the config row.
+ */
+export function uninstallConnector(id: string, keepConfig = true): boolean {
+  const dir = join(connectorsDir(), id);
+  if (existsSync(dir)) {
+    try { rmSync(dir, { recursive: true, force: true }); }
+    catch { return false; }
+  }
+  if (!keepConfig) {
+    const store = loadStore();
+    delete store[id];
+    saveStore(store);
+  } else {
+    // Mark disabled so the runtime doesn't try to use a now-missing manifest.
+    const store = loadStore();
+    if (store[id]) {
+      store[id].enabled = false;
+      saveStore(store);
+    }
+  }
+  return true;
+}
+
+export function setConnectorConfig(
+  id: string,
+  config: Record<string, unknown>,
+): boolean {
+  const store = loadStore();
+  const row = store[id];
+  if (!row) {
+    // Allow setting config before install if a manifest already exists
+    // — supports auto-install-then-configure flows in the UI.
+    const def = getConnector(id);
+    if (!def) return false;
+    store[id] = { config, installed_version: def.version, enabled: true };
+  } else {
+    store[id] = { ...row, config };
+  }
+  saveStore(store);
+  return true;
+}
+
+export function setConnectorEnabled(id: string, enabled: boolean): boolean {
+  const store = loadStore();
+  if (!store[id]) return false;
+  store[id].enabled = enabled;
+  saveStore(store);
+  return true;
+}
+
+// Re-export for callers (matches plugin-registry surface so it's familiar).
+export { getSecretFieldNames };

package/lib/connectors/sync.ts

@@ -0,0 +1,305 @@
+/**
+ * Connector sync — pull registry + manifests from forge-connectors.
+ *
+ * Default base URL: https://raw.githubusercontent.com/aiwatching/forge-connectors/main
+ * Override via settings.connectorsRepoUrl.
+ *
+ * The remote repo is expected to expose:
+ *   /registry.json       — listing of all available connectors (light)
+ *   /<id>/manifest.yaml  — the full YAML for one connector
+ *
+ * Local state:
+ *   <dataDir>/connectors/registry-cache.json — last successful fetch
+ *   <dataDir>/connectors/<id>/manifest.yaml  — per-connector manifest
+ *     (installed copy, refreshed on update)
+ *
+ * Sync is called non-blocking on startup and on-demand from the
+ * Settings Marketplace UI. Network failures are silent — callers
+ * keep the cached state. No built-in fallback: if the cache is empty
+ * and the network is down, the marketplace is just empty.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { loadSettings } from '../settings';
+import { getDataDir } from '../dirs';
+import { getConnector, installConnector, listConfigOnlyIds, listInstalledConnectors } from './registry';
+import type { ConnectorMarketEntry } from './types';
+
+const DEFAULT_REPO = 'https://raw.githubusercontent.com/aiwatching/forge-connectors/main';
+const FETCH_TIMEOUT_MS = 10_000;
+
+interface RegistryFile {
+  version?: number;
+  connectors?: Array<{
+    id: string;
+    name: string;
+    version: string;
+    icon?: string;
+    description?: string;
+    author?: string;
+    min_forge_version?: string;
+    /** Override manifest path; defaults to `<id>/manifest.yaml`. */
+    manifest?: string;
+  }>;
+}
+
+interface RegistryCache {
+  fetched_at: string;
+  base_url: string;
+  data: RegistryFile;
+}
+
+// ─── Paths ────────────────────────────────────────────────
+
+function connectorsDir(): string {
+  return join(getDataDir(), 'connectors');
+}
+
+function cacheFile(): string {
+  return join(connectorsDir(), 'registry-cache.json');
+}
+
+function ensureDir(p: string): void {
+  if (!existsSync(p)) mkdirSync(p, { recursive: true, mode: 0o700 });
+}
+
+function baseUrl(): string {
+  return (loadSettings().connectorsRepoUrl || '').trim() || DEFAULT_REPO;
+}
+
+// ─── Cache I/O ────────────────────────────────────────────
+
+export function readCache(): RegistryCache | null {
+  const p = cacheFile();
+  if (!existsSync(p)) return null;
+  try {
+    return JSON.parse(readFileSync(p, 'utf-8')) as RegistryCache;
+  } catch {
+    return null;
+  }
+}
+
+function writeCache(cache: RegistryCache): void {
+  ensureDir(connectorsDir());
+  writeFileSync(cacheFile(), JSON.stringify(cache, null, 2), { mode: 0o600 });
+}
+
+// ─── Fetch helpers ────────────────────────────────────────
+
+async function fetchText(url: string): Promise<string> {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT_MS);
+  try {
+    const r = await fetch(url, {
+      signal: ctrl.signal,
+      headers: {
+        // Some CDN edges (cloudflare in front of raw.githubusercontent.com)
+        // refuse default Node UA — set an explicit one.
+        'User-Agent': 'forge-connectors-sync/1.0',
+        'Cache-Control': 'no-cache',
+      },
+    });
+    if (!r.ok) throw new Error(`HTTP ${r.status} for ${url}`);
+    return await r.text();
+  } catch (err) {
+    // undici wraps the real failure under `err.cause`. Surface it so
+    // the user gets a useful message instead of a bare "fetch failed".
+    const e = err as Error & { cause?: unknown };
+    const cause = e.cause instanceof Error ? `: ${e.cause.message}` : e.cause ? `: ${String(e.cause)}` : '';
+    throw new Error(`${e.message}${cause} (${url})`);
+  } finally {
+    clearTimeout(t);
+  }
+}
+
+async function fetchJson<T>(url: string): Promise<T> {
+  const txt = await fetchText(url);
+  return JSON.parse(txt) as T;
+}
+
+function cacheBust(): string {
+  return `?_t=${Date.now()}`;
+}
+
+// ─── Sync API ─────────────────────────────────────────────
+
+export interface SyncResult {
+  ok: boolean;
+  registry_count: number;
+  manifests_refreshed: number;
+  error?: string;
+  fetched_at?: string;
+}
+
+/**
+ * Pull the registry.json and refresh on-disk manifests for any
+ * connector the user has installed (so install_version stays accurate
+ * and scripts get bug fixes). For not-yet-installed entries we only
+ * cache the registry row — the manifest is pulled on demand at install
+ * time (see installFromRegistry).
+ */
+export async function syncRegistry(opts: { refreshInstalled?: boolean } = {}): Promise<SyncResult> {
+  const base = baseUrl();
+  console.log(`[connectors] Syncing registry from ${base}`);
+  let registry: RegistryFile;
+  try {
+    registry = await fetchJson<RegistryFile>(`${base}/registry.json${cacheBust()}`);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn(`[connectors] registry fetch failed: ${msg}`);
+    return { ok: false, registry_count: 0, manifests_refreshed: 0, error: msg };
+  }
+
+  const entries = registry.connectors || [];
+  const cache: RegistryCache = {
+    fetched_at: new Date().toISOString(),
+    base_url: base,
+    data: registry,
+  };
+  writeCache(cache);
+
+  let refreshed = 0;
+  if (opts.refreshInstalled) {
+    // Two cases we want to pull manifests for:
+    //   1. local manifest exists but version is behind the registry
+    //   2. config row exists but no manifest on disk yet (post-migration
+    //      from pre-v0.9 plugin-configs.json — we have the user's PAT
+    //      but the manifest must come from the remote)
+    const configOnly = new Set(listConfigOnlyIds());
+    for (const e of entries) {
+      const local = getConnector(e.id);
+      const isConfigOnly = configOnly.has(e.id);
+      if (!local && !isConfigOnly) continue;
+      if (local && local.version === e.version) continue;
+      try {
+        const yamlText = await fetchManifest(e.id, e.manifest);
+        installConnector(e.id, yamlText);
+        refreshed += 1;
+      } catch (err) {
+        console.warn(`[connectors] refresh failed for ${e.id}:`, err);
+      }
+    }
+  }
+
+  return {
+    ok: true,
+    registry_count: entries.length,
+    manifests_refreshed: refreshed,
+    fetched_at: cache.fetched_at,
+  };
+}
+
+/** Fetch one connector's manifest from the configured repo. */
+export async function fetchManifest(id: string, manifestPath?: string): Promise<string> {
+  const base = baseUrl();
+  const path = manifestPath || `${id}/manifest.yaml`;
+  return fetchText(`${base}/${path}${cacheBust()}`);
+}
+
+/**
+ * Install a connector by id — looks it up in the registry cache (or
+ * fetches a fresh one), pulls the manifest, and hands off to the
+ * registry. Returns the chosen version on success.
+ */
+export async function installFromRegistry(id: string): Promise<{ ok: boolean; version?: string; error?: string }> {
+  let cache = readCache();
+  if (!cache) {
+    const r = await syncRegistry();
+    if (!r.ok) return { ok: false, error: r.error || 'registry unavailable' };
+    cache = readCache();
+  }
+  const entry = cache?.data.connectors?.find((c) => c.id === id);
+  if (!entry) return { ok: false, error: `${id} not in registry` };
+  try {
+    const yamlText = await fetchManifest(id, entry.manifest);
+    const ok = installConnector(id, yamlText);
+    return ok ? { ok: true, version: entry.version } : { ok: false, error: 'manifest invalid' };
+  } catch (err) {
+    return { ok: false, error: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+// ─── Marketplace listing ──────────────────────────────────
+
+/**
+ * Merge the registry cache with installed state to produce the
+ * marketplace view. Pure read; no network.
+ *
+ * If the registry cache is missing (first boot, offline, sync failed)
+ * we still surface locally-installed connectors so the user sees
+ * something — labelled as installed without a remote counterpart.
+ * Refresh button is the path back to a real registry listing.
+ */
+export function listMarketplace(): {
+  fetched_at?: string;
+  base_url?: string;
+  entries: ConnectorMarketEntry[];
+} {
+  const cache = readCache();
+  const installed = listInstalledConnectors();
+  const installedById = new Map(installed.map((i) => [i.definition.id, i] as const));
+
+  const entries: ConnectorMarketEntry[] = [];
+
+  if (cache?.data.connectors?.length) {
+    for (const c of cache.data.connectors) {
+      const local = installedById.get(c.id);
+      entries.push({
+        id: c.id,
+        name: c.name,
+        version: c.version,
+        icon: c.icon,
+        description: c.description,
+        author: c.author,
+        installed_version: local?.installed_version,
+        update_available: !!local && compareVersions(c.version, local.installed_version) > 0,
+        compatible: isVersionCompatible(c.min_forge_version),
+        source: 'registry',
+      });
+      installedById.delete(c.id);
+    }
+  }
+
+  // Locally-installed connectors with no registry counterpart — either
+  // installed via /api/connectors/install-local, or the registry is
+  // currently unreachable. Either way: show them with source=local so
+  // the UI suppresses the Update path.
+  for (const local of installedById.values()) {
+    const def = local.definition;
+    entries.push({
+      id: def.id,
+      name: def.name,
+      version: local.installed_version,
+      icon: def.icon,
+      description: def.description,
+      author: def.author,
+      installed_version: local.installed_version,
+      update_available: false,
+      compatible: true,
+      source: 'local',
+    });
+  }
+
+  return { fetched_at: cache?.fetched_at, base_url: cache?.base_url, entries };
+}
+
+// ─── Version helpers ──────────────────────────────────────
+
+function compareVersions(a: string, b: string): number {
+  const pa = (a || '0.0.0').split('.').map((n) => Number(n) || 0);
+  const pb = (b || '0.0.0').split('.').map((n) => Number(n) || 0);
+  for (let i = 0; i < 3; i++) {
+    const d = (pa[i] || 0) - (pb[i] || 0);
+    if (d !== 0) return d;
+  }
+  return 0;
+}
+
+function isVersionCompatible(min?: string): boolean {
+  if (!min) return true;
+  // Soft check — we don't ship a hard FORGE_VERSION constant on the
+  // server; rely on the user keeping Forge current.
+  // TODO: read version from package.json at boot if we want to enforce.
+  return true;
+}