@checkstack/integration-backend 0.1.30 → 0.3.0

package/src/connection-credentials.test.ts

@@ -0,0 +1,180 @@
+import { describe, it, expect } from "bun:test";
+import { z } from "zod";
+import { configString } from "@checkstack/backend-api";
+import type {
+  SecretResolverService,
+  InternalSecretsService,
+} from "@checkstack/secrets-backend";
+import {
+  inflateConnectionCredentials,
+  extractInlineCredentials,
+  internalRefMarker,
+  isInternalRefMarker,
+  connectionSecretParts,
+} from "./connection-credentials";
+
+// A Jira-like connection schema: baseUrl (non-secret) + apiToken (x-secret).
+const connectionSchema = z.object({
+  baseUrl: z.string(),
+  email: z.string(),
+  apiToken: configString({ "x-secret": true }),
+});
+
+/** In-memory internal-secrets fake (the local store). */
+function fakeInternal(): InternalSecretsService & { store: Map<string, string> } {
+  const store = new Map<string, string>();
+  const key = (parts: string[]) => parts.join("|");
+  return {
+    store,
+    set: async ({ parts, value }) => {
+      store.set(key(parts), value);
+    },
+    get: async ({ parts }) => store.get(key(parts)),
+    delete: async ({ parts }) => {
+      store.delete(key(parts));
+    },
+  };
+}
+
+/** Resolver fake backed by a name->value map (simulates the active backend). */
+function fakeResolver(values: Record<string, string>): SecretResolverService {
+  const RE = /\$\{\{\s*secrets\.([a-zA-Z0-9_-]+)\s*\}\}/g;
+  return {
+    resolveSecret: async ({ name }) => values[name] ?? "",
+    resolveBySchema: async ({ value }) => ({ resolved: value, warnings: [] }),
+    resolveForRun: async ({ secretEnv }) => {
+      const env: Record<string, string> = {};
+      for (const [k, template] of Object.entries(secretEnv)) {
+        RE.lastIndex = 0;
+        env[k] = template.replaceAll(RE, (_m, n: string) => values[n] ?? "");
+      }
+      return {
+        env,
+        masking: { size: 0, maskText: (t) => t, maskDeep: (v) => v },
+      };
+    },
+  };
+}
+
+const PROVIDER = "integration-jira.jira";
+const CONN = "conn-1";
+
+describe("extractInlineCredentials", () => {
+  it("moves an inline x-secret value into an internal secret + leaves a marker", async () => {
+    const internal = fakeInternal();
+    const { config, extracted } = await extractInlineCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: { baseUrl: "https://x", email: "a@b.c", apiToken: "tok-INLINE" },
+      schema: connectionSchema,
+      internalSecrets: internal,
+    });
+    expect(extracted).toBe(1);
+    expect(config.baseUrl).toBe("https://x"); // non-secret untouched
+    expect(isInternalRefMarker(config.apiToken as string)).toBe(true);
+    expect(config.apiToken).toBe(internalRefMarker("apiToken"));
+    // The value moved to the internal store.
+    expect(
+      internal.store.get(
+        connectionSecretParts({
+          providerId: PROVIDER,
+          connectionId: CONN,
+          fieldPath: "apiToken",
+        }).join("|"),
+      ),
+    ).toBe("tok-INLINE");
+  });
+
+  it("is idempotent: a marker or reference extracts nothing", async () => {
+    const internal = fakeInternal();
+    const { extracted } = await extractInlineCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: {
+        baseUrl: "https://x",
+        email: "a@b.c",
+        apiToken: internalRefMarker("apiToken"),
+      },
+      schema: connectionSchema,
+      internalSecrets: internal,
+    });
+    expect(extracted).toBe(0);
+
+    const ref = await extractInlineCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: { baseUrl: "https://x", email: "a@b.c", apiToken: "${{ secrets.jira }}" },
+      schema: connectionSchema,
+      internalSecrets: internal,
+    });
+    expect(ref.extracted).toBe(0);
+  });
+});
+
+describe("inflateConnectionCredentials", () => {
+  it("inflates an internal-ref marker (inline path) from the local store", async () => {
+    const internal = fakeInternal();
+    await internal.set({
+      parts: connectionSecretParts({ providerId: PROVIDER, connectionId: CONN, fieldPath: "apiToken" }),
+      value: "tok-RESOLVED",
+    });
+    const { config, values } = await inflateConnectionCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: { baseUrl: "https://x", email: "a@b.c", apiToken: internalRefMarker("apiToken") },
+      schema: connectionSchema,
+      deps: { internalSecrets: internal, secretResolver: fakeResolver({}) },
+    });
+    expect(config.apiToken).toBe("tok-RESOLVED");
+    expect(values).toContain("tok-RESOLVED");
+  });
+
+  it("inflates a ${{ secrets.NAME }} reference (reference path) via the active backend", async () => {
+    const internal = fakeInternal();
+    const { config } = await inflateConnectionCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: { baseUrl: "https://x", email: "a@b.c", apiToken: "${{ secrets.jira_token }}" },
+      schema: connectionSchema,
+      deps: {
+        internalSecrets: internal,
+        secretResolver: fakeResolver({ jira_token: "tok-FROM-VAULT" }),
+      },
+    });
+    expect(config.apiToken).toBe("tok-FROM-VAULT");
+  });
+
+  it("round-trips: extract then inflate yields the original plaintext", async () => {
+    const internal = fakeInternal();
+    const original = { baseUrl: "https://x", email: "a@b.c", apiToken: "round-trip-tok" };
+    const { config: stored } = await extractInlineCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: original,
+      schema: connectionSchema,
+      internalSecrets: internal,
+    });
+    // Stored form is reference-ized (no plaintext).
+    expect(JSON.stringify(stored)).not.toContain("round-trip-tok");
+    const { config: inflated } = await inflateConnectionCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: stored,
+      schema: connectionSchema,
+      deps: { internalSecrets: internal, secretResolver: fakeResolver({}) },
+    });
+    expect(inflated).toEqual(original);
+  });
+
+  it("leaves a bare legacy literal unchanged (pre-migration safety)", async () => {
+    const internal = fakeInternal();
+    const { config } = await inflateConnectionCredentials({
+      providerId: PROVIDER,
+      connectionId: CONN,
+      config: { baseUrl: "https://x", email: "a@b.c", apiToken: "legacy-inline" },
+      schema: connectionSchema,
+      deps: { internalSecrets: internal, secretResolver: fakeResolver({}) },
+    });
+    expect(config.apiToken).toBe("legacy-inline");
+  });
+});

package/src/connection-credentials.ts

@@ -0,0 +1,165 @@
+/**
+ * Unified credential resolution for integration connections.
+ *
+ * Connection credential fields (the provider `connectionSchema`'s
+ * `x-secret` fields) resolve through the ONE secrets channel, two entry
+ * forms:
+ *
+ *   1. Reference form: the field holds a `${{ secrets.NAME }}` template.
+ *      It resolves through the ACTIVE backend (local or Vault) via
+ *      `secretResolverRef` — so a credential that "originates from Vault"
+ *      just works.
+ *   2. Inline form: an operator-typed value, extracted into an INTERNAL
+ *      secret on the local backend (Vault is read-through / unwritable) and
+ *      represented in the stored config by an internal-reference marker.
+ *      It resolves via `internalSecretsRef`.
+ *
+ * Both forms are walked with the shared `walkSecretFields` machinery
+ * (acting only on `x-secret` fields) — no per-provider logic is
+ * re-implemented. Non-secret config is never touched.
+ */
+import { z } from "zod";
+import {
+  SECRET_TEMPLATE_REGEX,
+  type SecretEnvMapping,
+} from "@checkstack/secrets-common";
+import {
+  walkSecretFields,
+  type SecretResolverService,
+  type InternalSecretsService,
+} from "@checkstack/secrets-backend";
+
+/**
+ * Marker stored in a connection config field for an extracted INLINE
+ * credential. The actual value lives in an internal secret keyed by
+ * (providerId, connectionId, fieldPath). The marker carries the field path
+ * so the inflate can rebuild the internal-secret key.
+ */
+const INTERNAL_REF_PREFIX = "__connref__:";
+
+export function internalRefMarker(fieldPath: string): string {
+  return `${INTERNAL_REF_PREFIX}${fieldPath}`;
+}
+
+export function isInternalRefMarker(value: string): boolean {
+  return value.startsWith(INTERNAL_REF_PREFIX);
+}
+
+function fieldPathFromMarker(marker: string): string {
+  return marker.slice(INTERNAL_REF_PREFIX.length);
+}
+
+/** Internal-secret name parts for a connection credential field. */
+export function connectionSecretParts(input: {
+  providerId: string;
+  connectionId: string;
+  fieldPath: string;
+}): string[] {
+  return ["connection", input.providerId, input.connectionId, input.fieldPath];
+}
+
+/** Whether a string is a `${{ secrets.NAME }}` reference (whole-value). */
+function isSecretReference(value: string): boolean {
+  SECRET_TEMPLATE_REGEX.lastIndex = 0;
+  return SECRET_TEMPLATE_REGEX.test(value);
+}
+
+export interface ConnectionCredentialDeps {
+  secretResolver: SecretResolverService;
+  internalSecrets: InternalSecretsService;
+}
+
+/**
+ * Inflate a stored connection config to full credentials by resolving its
+ * `x-secret` fields through the unified channel. A reference form resolves
+ * through the active backend; an internal-ref marker resolves the inline
+ * value from the local internal store; a bare literal (legacy, pre-migration)
+ * is returned unchanged.
+ *
+ * Returns the inflated config plus the set of resolved credential VALUES
+ * (for registering with the run-scoped masking context).
+ */
+export async function inflateConnectionCredentials(params: {
+  providerId: string;
+  connectionId: string;
+  config: Record<string, unknown>;
+  schema: z.ZodTypeAny;
+  deps: ConnectionCredentialDeps;
+}): Promise<{ config: Record<string, unknown>; values: string[] }> {
+  const { providerId, connectionId, config, schema, deps } = params;
+  const values: string[] = [];
+
+  const inflated = await walkSecretFields({
+    value: config,
+    schema,
+    visit: async ({ value }) => {
+      let resolved = value;
+      if (isInternalRefMarker(value)) {
+        const fieldPath = fieldPathFromMarker(value);
+        const got = await deps.internalSecrets.get({
+          parts: connectionSecretParts({ providerId, connectionId, fieldPath }),
+        });
+        if (got === undefined) {
+          throw new Error(
+            `Connection ${connectionId}: internal credential for "${fieldPath}" not found.`,
+          );
+        }
+        resolved = got;
+      } else if (isSecretReference(value)) {
+        // `${{ secrets.NAME }}` — resolve via the active backend (Vault/local).
+        const mapping: SecretEnvMapping = { CRED: value };
+        const { env } = await deps.secretResolver.resolveForRun({
+          secretEnv: mapping,
+        });
+        resolved = env.CRED;
+      }
+      // else: bare literal (legacy) — leave as-is.
+      if (resolved.length > 0) values.push(resolved);
+      return resolved;
+    },
+  });
+
+  return { config: inflated as Record<string, unknown>, values };
+}
+
+/**
+ * Extract INLINE `x-secret` values out of a stored connection config into
+ * internal secrets, replacing each with an internal-ref marker. Reference
+ * (`${{ secrets.NAME }}`) values and already-extracted markers are left
+ * untouched. Returns the rewritten config + how many inline values moved.
+ *
+ * Used by the one-time migration. Idempotent: re-running over an
+ * already-migrated config (markers everywhere) extracts nothing.
+ */
+export async function extractInlineCredentials(params: {
+  providerId: string;
+  connectionId: string;
+  config: Record<string, unknown>;
+  schema: z.ZodTypeAny;
+  internalSecrets: InternalSecretsService;
+}): Promise<{ config: Record<string, unknown>; extracted: number }> {
+  const { providerId, connectionId, config, schema, internalSecrets } = params;
+  let extracted = 0;
+
+  const rewritten = await walkSecretFields({
+    value: config,
+    schema,
+    visit: async ({ path, value }) => {
+      // Already a marker or a reference — nothing to extract.
+      if (isInternalRefMarker(value) || isSecretReference(value)) {
+        return value;
+      }
+      // Empty literal — leave as-is (nothing to protect).
+      if (value.length === 0) return value;
+      // Inline literal: move it to an internal secret keyed by field path.
+      await internalSecrets.set({
+        parts: connectionSecretParts({ providerId, connectionId, fieldPath: path }),
+        value,
+      });
+      extracted++;
+      return internalRefMarker(path);
+    },
+  });
+
+  return { config: rewritten as Record<string, unknown>, extracted };
+}

package/src/connection-store.ts

@@ -17,6 +17,15 @@ import type {
   ProviderConnection,
   ProviderConnectionRedacted,
 } from "@checkstack/integration-common";
+import {
+  inflateConnectionCredentials,
+  extractInlineCredentials,
+  type ConnectionCredentialDeps,
+} from "./connection-credentials";
+import {
+  migrateConnectionCredentials,
+  type ConnectionForMigration,
+} from "./connection-credentials-migration";
 
 // Schema for connection metadata (stored separately from config)
 const ConnectionMetadataSchema = z.object({
@@ -105,12 +114,27 @@ interface ConnectionStoreDeps {
   configService: ConfigService;
   providerRegistry: IntegrationProviderRegistry;
   logger: Logger;
+  /**
+   * Unified secret-credential resolution. When provided,
+   * `getConnectionWithCredentials` inflates `x-secret` fields through the
+   * ONE secrets channel: `${{ secrets.NAME }}` references resolve via the
+   * active backend (local or Vault), inline values resolve from the local
+   * internal store. Optional so tests / older boots without the secrets
+   * platform degrade to the legacy inline-config behavior.
+   */
+  credentials?: ConnectionCredentialDeps;
 }
 
+/** The public store plus the internal one-time credential migration. */
+export type ConnectionStoreWithMigration = ConnectionStore & {
+  /** Consolidate legacy inline credentials onto the secrets platform. */
+  runCredentialMigration(): Promise<void>;
+};
+
 export function createConnectionStore(
   deps: ConnectionStoreDeps
-): ConnectionStore {
-  const { configService, providerRegistry, logger } = deps;
+): ConnectionStoreWithMigration {
+  const { configService, providerRegistry, logger, credentials } = deps;
 
   // Cache of connectionId -> providerId for efficient lookups
   const connectionProviderCache = new Map<string, string>();
@@ -242,6 +266,33 @@ export function createConnectionStore(
     );
   }
 
+  /**
+   * Persist a connection config, extracting any INLINE `x-secret` value into
+   * an internal secret first (so the stored config holds references /
+   * markers, never raw inline credentials). `${{ secrets.NAME }}`
+   * references are left as-is (they resolve via the active backend at use
+   * time). Falls back to plain storage when the credentials deps are absent.
+   */
+  async function persistConnectionConfig(
+    providerId: string,
+    connectionId: string,
+    config: Record<string, unknown>
+  ): Promise<void> {
+    const provider = providerRegistry.getProvider(providerId);
+    if (credentials && provider?.connectionSchema) {
+      const { config: rewritten } = await extractInlineCredentials({
+        providerId,
+        connectionId,
+        config,
+        schema: provider.connectionSchema.schema,
+        internalSecrets: credentials.internalSecrets,
+      });
+      await setConnectionConfig(providerId, connectionId, rewritten);
+      return;
+    }
+    await setConnectionConfig(providerId, connectionId, config);
+  }
+
   /**
    * Delete connection config and metadata.
    */
@@ -335,11 +386,29 @@ export function createConnectionStore(
 
       if (!metadata || !config) return;
 
+      // Inflate x-secret fields through the unified secrets channel:
+      // `${{ secrets.NAME }}` references resolve via the active backend
+      // (local or Vault); inline values resolve from the local internal
+      // store. When the credentials deps are absent (legacy / tests), the
+      // raw config is used as-is.
+      const provider = providerRegistry.getProvider(providerId);
+      let resolvedConfig = config;
+      if (credentials && provider?.connectionSchema) {
+        const inflated = await inflateConnectionCredentials({
+          providerId,
+          connectionId,
+          config,
+          schema: provider.connectionSchema.schema,
+          deps: credentials,
+        });
+        resolvedConfig = inflated.config;
+      }
+
       return {
         id: metadata.id,
         providerId: metadata.providerId,
         name: metadata.name,
-        config,
+        config: resolvedConfig,
         createdAt: metadata.createdAt,
         updatedAt: metadata.updatedAt,
       };
@@ -357,9 +426,10 @@ export function createConnectionStore(
         updatedAt: now,
       };
 
-      // Save metadata and config separately
+      // Save metadata and config separately. Inline credentials are
+      // extracted to internal secrets so the stored config holds references.
       await setConnectionMetadata(providerId, id, metadata);
-      await setConnectionConfig(providerId, id, config);
+      await persistConnectionConfig(providerId, id, config);
 
       // Add to index
       const connectionIds = await getConnectionIndex(providerId);
@@ -371,6 +441,9 @@ export function createConnectionStore(
         `Created connection "${name}" (${id}) for provider ${providerId}`
       );
 
+      // Return the caller's config as submitted (so the create call's return
+      // is consistent with what was requested). The stored form is
+      // reference-ized; the router returns the redacted preview separately.
       return { ...metadata, config };
     },
 
@@ -404,7 +477,7 @@ export function createConnectionStore(
         : existingConfig;
 
       await setConnectionMetadata(providerId, connectionId, updatedMetadata);
-      await setConnectionConfig(providerId, connectionId, updatedConfig);
+      await persistConnectionConfig(providerId, connectionId, updatedConfig);
 
       logger.info(
         `Updated connection "${updatedMetadata.name}" (${connectionId})`
@@ -459,5 +532,45 @@ export function createConnectionStore(
 
       return;
     },
+
+    // Not part of the public ConnectionStore interface (so callers are
+    // unchanged) — invoked once from afterPluginsReady to consolidate any
+    // legacy inline credentials onto the secrets platform.
+    async runCredentialMigration() {
+      if (!credentials) return;
+      await migrateConnectionCredentials({
+        configService,
+        internalSecrets: credentials.internalSecrets,
+        secretResolver: credentials.secretResolver,
+        logger,
+        loadConnections: async () => {
+          const out: ConnectionForMigration[] = [];
+          for (const provider of providerRegistry.getProviders()) {
+            if (!provider.connectionSchema) continue;
+            const ids = await getConnectionIndex(provider.qualifiedId);
+            for (const connectionId of ids) {
+              const config = await getConnectionConfigRaw(
+                provider.qualifiedId,
+                connectionId,
+              );
+              if (!config) continue;
+              out.push({
+                providerId: provider.qualifiedId,
+                connectionId,
+                schema: provider.connectionSchema.schema,
+                schemaVersion: provider.connectionSchema.version,
+                config,
+              });
+            }
+          }
+          return out;
+        },
+        persistConfig: async ({ providerId, connectionId, config }) => {
+          // The config is already reference-ized by the migration; store it
+          // verbatim (do not re-extract).
+          await setConnectionConfig(providerId, connectionId, config);
+        },
+      });
+    },
   };
 }