@jskit-ai/connectors-core 0.1.1

package/src/server/credentialProtection.js

@@ -0,0 +1,34 @@
+import { CompactEncrypt, compactDecrypt } from "jose";
+import { ConnectorError } from "./errors.js";
+
+function createCredentialProtection({ keys, activeKeyId }) {
+  const keyring = new Map(Object.entries(keys || {}));
+  if (!keyring.has(activeKeyId) || [...keyring.values()].some((key) => !(key instanceof Uint8Array) || key.length !== 32)) {
+    throw new TypeError("Credential protection requires named 32-byte keys and an active key ID.");
+  }
+  const encoder = new TextEncoder();
+  const decoder = new TextDecoder();
+  return Object.freeze({
+    async seal(value, binding) {
+      return new CompactEncrypt(encoder.encode(JSON.stringify({ binding, value })))
+        .setProtectedHeader({ alg: "dir", enc: "A256GCM", kid: activeKeyId })
+        .encrypt(keyring.get(activeKeyId));
+    },
+    async open(ciphertext, binding) {
+      try {
+        const { plaintext } = await compactDecrypt(ciphertext, (header) => {
+          const key = keyring.get(header.kid);
+          if (!key) throw new Error("Unknown key.");
+          return key;
+        }, { keyManagementAlgorithms: ["dir"], contentEncryptionAlgorithms: ["A256GCM"] });
+        const envelope = JSON.parse(decoder.decode(plaintext));
+        if (envelope.binding !== binding) throw new Error("Incorrect record binding.");
+        return envelope.value;
+      } catch {
+        throw new ConnectorError("connector_credentials_unavailable", "Stored credentials could not be opened.", { statusCode: 500 });
+      }
+    }
+  });
+}
+
+export { createCredentialProtection };

package/src/server/environmentReferences.js

@@ -0,0 +1,13 @@
+import { ConnectorError } from "./errors.js";
+
+function createEnvironmentReferenceResolver(env = process.env) {
+  return async (reference) => {
+    const match = /^env:([A-Z_][A-Z0-9_]*)$/u.exec(reference);
+    if (!match || typeof env[match[1]] !== "string" || (!env[match[1]].trim() || env[match[1]].trim() === "MISSING")) {
+      throw new ConnectorError("connector_binding_missing", "A required environment binding is missing.");
+    }
+    return env[match[1]];
+  };
+}
+
+export { createEnvironmentReferenceResolver };

package/src/server/errors.js

@@ -0,0 +1,27 @@
+class ConnectorError extends Error {
+  constructor(code, message, { statusCode = 400, retryAfterSeconds } = {}) {
+    super(message);
+    this.name = "ConnectorError";
+    this.code = code;
+    this.statusCode = statusCode;
+    if (retryAfterSeconds !== undefined) this.retryAfterSeconds = retryAfterSeconds;
+  }
+}
+
+function providerError(error) {
+  if (error instanceof ConnectorError) return error;
+  const status = Number(error?.status || error?.statusCode || 0);
+  if (error?.name === "AbortError") return new ConnectorError("connector_cancelled", "The operation was cancelled.");
+  if (error?.name === "TimeoutError") return new ConnectorError("connector_provider_timeout", "The provider request timed out. Try again.", { statusCode: 504 });
+  if (["invalid_grant", "invalid_client"].includes(error?.error) || status === 401) {
+    return new ConnectorError("connector_reconnect_required", "Connect this account again.", { statusCode: 401 });
+  }
+  if (["access_denied", "user_cancelled_login", "user_cancelled_authorize"].includes(error?.error)) {
+    return new ConnectorError("connector_consent_denied", "Account access was declined.");
+  }
+  if (status === 403) return new ConnectorError("connector_permission_denied", "The provider denied this operation.", { statusCode: 403 });
+  if (status === 429) return new ConnectorError("connector_rate_limited", "The provider's request limit was reached.", { statusCode: 429 });
+  return new ConnectorError("connector_provider_failed", "The provider request failed. Try again.", { statusCode: 502 });
+}
+
+export { ConnectorError, providerError };

package/src/server/fileConnectionStore.js

@@ -0,0 +1,86 @@
+import { createHash, randomUUID } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, mkdir, open, realpath, rename, rm } from "node:fs/promises";
+import path from "node:path";
+import lockfile from "proper-lockfile";
+import { ConnectorError } from "./errors.js";
+
+function createFileConnectionStore({ directory, protection, directoryMode = 0o700, fileMode = 0o600, now = Date.now }) {
+  if (typeof directory !== "string" || !path.isAbsolute(directory)) throw new TypeError("Use an absolute private runtime directory.");
+  if (typeof protection?.seal !== "function" || typeof protection?.open !== "function") throw new TypeError("Credential protection is required.");
+  const root = path.resolve(directory);
+  const storageError = () => new ConnectorError("connector_storage_invalid", "Connection state could not be read safely.", { statusCode: 500 });
+
+  async function withConnection({ owner, integrationId }, work) {
+    await mkdir(root, { recursive: true, mode: directoryMode });
+    if (!(await lstat(root)).isDirectory() || await realpath(root) !== root) throw storageError();
+    const identity = [owner.applicationId, owner.subjectId, integrationId];
+    if (identity.some((value) => typeof value !== "string" || !value)) throw new TypeError("Connection storage requires a complete owner and slot.");
+    const key = createHash("sha256").update(JSON.stringify(identity)).digest("hex");
+    const file = path.join(root, `${key}.json`);
+    const binding = `file-connection:${key}`;
+    let compromised = false;
+    const release = await lockfile.lock(file, {
+      realpath: false, stale: 60_000, update: 10_000,
+      retries: { retries: 80, minTimeout: 100, maxTimeout: 500 },
+      onCompromised() { compromised = true; }
+    });
+    try {
+      let record = { connection: null, attempts: {} };
+      let handle;
+      try {
+        handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW);
+        const info = await handle.stat();
+        if (!info.isFile() || info.size > 4 * 1024 * 1024) throw storageError();
+        const envelope = JSON.parse(await handle.readFile("utf8"));
+        if (envelope.schemaVersion !== 1 || typeof envelope.payload !== "string") throw storageError();
+        record = await protection.open(envelope.payload, binding);
+        if (!record || !Object.hasOwn(record, "connection") || !record.attempts || typeof record.attempts !== "object" || Array.isArray(record.attempts)) throw storageError();
+      } catch (error) {
+        if (error.code !== "ENOENT") throw storageError();
+      } finally { await handle?.close(); }
+      let changed = false;
+      for (const [state, attempt] of Object.entries(record.attempts)) {
+        if (attempt.expiresAt <= now()) { delete record.attempts[state]; changed = true; }
+      }
+      const attemptKey = (state) => createHash("sha256").update(state).digest("hex");
+      const result = await work({
+        connection: structuredClone(record.connection),
+        save: async (connection) => { record.connection = structuredClone(connection); changed = true; },
+        remove: async () => { record = { connection: null, attempts: {} }; changed = true; },
+        putAttempt: async (attempt) => { record.attempts[attemptKey(attempt.state)] = structuredClone(attempt); changed = true; },
+        latestAttempt: async ({ after }) => structuredClone(Object.values(record.attempts)
+          .filter((attempt) => attempt.expiresAt > after)
+          .sort((a, b) => b.expiresAt - a.expiresAt)[0] || null),
+        consumeAttempt: async (state) => {
+          const id = attemptKey(state);
+          const attempt = record.attempts[id] || null;
+          if (attempt) { delete record.attempts[id]; changed = true; }
+          return attempt;
+        }
+      });
+      if (compromised) throw storageError();
+      if (changed) {
+        const payload = await protection.seal(record, binding);
+        const temporary = path.join(root, `.${key}.${randomUUID()}.tmp`);
+        let output;
+        try {
+          output = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, fileMode);
+          await output.writeFile(`${JSON.stringify({ schemaVersion: 1, payload })}\n`, "utf8");
+          await output.sync();
+          await output.close();
+          output = null;
+          if (compromised) throw storageError();
+          await rename(temporary, file);
+        } finally {
+          await output?.close();
+          await rm(temporary, { force: true });
+        }
+      }
+      return result;
+    } finally { if (!compromised) await release(); }
+  }
+  return Object.freeze({ withConnection });
+}
+
+export { createFileConnectionStore };

package/src/server/fileStorage.js

@@ -0,0 +1,2 @@
+export { createFileConnectionStore } from "./fileConnectionStore.js";
+export { createCredentialProtection } from "./credentialProtection.js";

package/src/server/index.js

@@ -0,0 +1,4 @@
+export { createConnectionService } from "./connectionService.js";
+export { createEnvironmentReferenceResolver } from "./environmentReferences.js";
+export { ConnectorError } from "./errors.js";
+export { createConnectorsFeature } from "./ConnectorsFeature.js";

package/src/server/knexConnectionStore.js

@@ -0,0 +1,65 @@
+import { createHash } from "node:crypto";
+import { createWithTransaction } from "@jskit-ai/database-runtime/shared";
+
+const hash = (value) => createHash("sha256").update(value).digest("hex");
+
+function createKnexConnectionStore({ knex, protection }) {
+  if (typeof knex !== "function" || typeof knex.transaction !== "function") {
+    throw new TypeError("Connector storage requires the application's transactional Knex client.");
+  }
+  if (typeof protection?.seal !== "function" || typeof protection?.open !== "function") {
+    throw new TypeError("Connector storage requires credential protection.");
+  }
+  const transaction = createWithTransaction(knex);
+
+  async function withConnection({ owner, integrationId }, work) {
+    const connectionKey = hash(JSON.stringify([owner.applicationId, owner.subjectId, integrationId]));
+    const connectionBinding = `connection:${connectionKey}`;
+    return transaction(async (trx) => {
+      // Retain this row on disconnect so other processes continue to lock the same identity.
+      await trx("connector_connections").insert({ connection_key: connectionKey })
+        .onConflict("connection_key").merge({ connection_key: connectionKey });
+      const row = await trx("connector_connections").where({ connection_key: connectionKey }).forUpdate().first();
+      return work({
+        connection: row.payload ? await protection.open(row.payload, connectionBinding) : null,
+        save: async (connection) => {
+          const payload = await protection.seal(connection, connectionBinding);
+          await trx("connector_connections").where({ connection_key: connectionKey }).update({ payload });
+        },
+        remove: async () => {
+          await trx("connector_connections").where({ connection_key: connectionKey }).update({ payload: null });
+          await trx("connector_authorization_attempts").where({ connection_key: connectionKey }).delete();
+        },
+        putAttempt: async (attempt) => {
+          const attemptKey = hash(attempt.state);
+          await trx("connector_authorization_attempts").insert({
+            attempt_key: attemptKey, connection_key: connectionKey, expires_at: attempt.expiresAt,
+            payload: await protection.seal(attempt, `attempt:${connectionKey}:${attemptKey}`)
+          });
+        },
+        latestAttempt: async ({ after }) => {
+          const attempt = await trx("connector_authorization_attempts")
+            .where({ connection_key: connectionKey }).where("expires_at", ">", after)
+            .orderBy("expires_at", "desc").first();
+          return attempt ? protection.open(attempt.payload, `attempt:${connectionKey}:${attempt.attempt_key}`) : null;
+        },
+        consumeAttempt: async (state) => {
+          const attemptKey = hash(state);
+          const where = { attempt_key: attemptKey, connection_key: connectionKey };
+          const attempt = await trx("connector_authorization_attempts").where(where).first();
+          if (!attempt) return null;
+          await trx("connector_authorization_attempts").where(where).delete();
+          return protection.open(attempt.payload, `attempt:${connectionKey}:${attemptKey}`);
+        }
+      });
+    });
+  }
+
+  async function pruneExpiredAttempts({ before = Date.now() } = {}) {
+    return knex("connector_authorization_attempts").where("expires_at", "<=", before).delete();
+  }
+
+  return Object.freeze({ withConnection, pruneExpiredAttempts });
+}
+
+export { createKnexConnectionStore };

package/src/server/storage.js

@@ -0,0 +1,2 @@
+export { createCredentialProtection } from "./credentialProtection.js";
+export { createKnexConnectionStore } from "./knexConnectionStore.js";

package/src/shared/configuration.js

@@ -0,0 +1,225 @@
+import { createSchema } from "json-rest-schema";
+import { validateSchemaPayload } from "@jskit-ai/kernel/shared/validators";
+
+const name = { type: "string", required: true, minLength: 1, maxLength: 200 };
+const secretReference = {
+  ...name,
+  validator: (value) => (/^[a-z][a-z0-9-]*:[^\s]+$/u.test(value) && !/^https?:/u.test(value)) || "Use a reference such as env:VARIABLE_NAME."
+};
+const authenticationSchema = createSchema({
+  method: { ...name, enum: ["oauth2", "api-key", "service-account", "none"] },
+  registrationRef: { ...name, required: false },
+  secretRef: { ...secretReference, required: false }
+});
+const assistantPermission = { type: "string", enum: ["ask", "always", "never"] };
+const assistantPolicySchema = createSchema({
+  enabled: { type: "boolean", strictBoolean: true, defaultTo: true },
+  defaultPermission: { ...assistantPermission, defaultTo: "ask" },
+  actions: { type: "object", values: assistantPermission, defaultTo: {} }
+});
+const integrationSchema = createSchema({
+  provider: name,
+  displayName: { ...name, required: false },
+  accountMode: { ...name, enum: ["shared", "per-user", "assistant"] },
+  scopes: { type: "array", items: { ...name, maxLength: 2048 }, required: true },
+  authentication: { type: "object", schema: authenticationSchema, required: true },
+  settings: { type: "object", additionalProperties: true, required: false },
+  assistantPolicy: { type: "object", schema: assistantPolicySchema, required: false },
+  extensions: { type: "object", additionalProperties: true, required: false }
+});
+const registrationSchema = createSchema({
+  source: { ...name, enum: ["own"] },
+  grantType: { ...name, enum: ["authorization_code", "client_credentials"], required: false },
+  clientId: { ...name, maxLength: 2048, required: false },
+  clientSecretRef: { ...secretReference, required: false },
+  tokenEndpointAuthMethod: { ...name, enum: ["client_secret_post", "client_secret_basic", "none"], required: false },
+  callbackUrlRef: { ...secretReference, required: false }
+});
+
+const integrationsSchema = createSchema({
+  schemaVersion: { type: "integer", enum: [1], required: true },
+  integrations: { type: "object", values: integrationSchema, required: true },
+  registrations: { type: "object", values: registrationSchema, required: true },
+  extensions: { type: "object", additionalProperties: true, required: false }
+});
+
+class IntegrationConfigurationError extends Error {
+  constructor(fieldErrors) {
+    super("Integration configuration is invalid.");
+    this.name = "IntegrationConfigurationError";
+    this.code = "integration_configuration_invalid";
+    this.statusCode = 422;
+    this.fieldErrors = fieldErrors;
+  }
+}
+
+function getProviderSettingsSchema(provider, settings = {}) {
+  return typeof provider.settingsSchema === "function" ? provider.settingsSchema(settings) : provider.settingsSchema;
+}
+
+function getProviderAuthenticationMethods(provider, settings = {}) {
+  return provider.authenticationMethodsForSettings ? provider.authenticationMethodsForSettings(settings) : provider.authenticationMethods;
+}
+
+function getProviderAccountModes(provider, settings = {}) {
+  return provider.accountModesForSettings ? provider.accountModesForSettings(settings) : provider.accountModes;
+}
+
+function getProviderScopes(provider, settings = {}, grantType = "authorization_code", authenticationMethod) {
+  const scopes = provider.scopesForGrantType ? provider.scopesForGrantType(grantType, settings)
+    : provider.scopesForSettings ? provider.scopesForSettings(settings) : provider.scopes;
+  return authenticationMethod ? scopes.filter(scope => !scope.authenticationMethods || scope.authenticationMethods.includes(authenticationMethod)) : scopes;
+}
+
+function getProviderClientAuthenticationMethods(provider, grantType = "authorization_code", settings = {}) {
+  const methods = provider.oauthClientAuthenticationMethods;
+  return typeof methods === "function" ? methods(grantType, settings) : methods || ["client_secret_post"];
+}
+
+function validateIntegrationConfiguration(input, { providers = [], allowUnknownProviders = false } = {}) {
+  if (input?.schemaVersion !== 1) {
+    throw new IntegrationConfigurationError({ schemaVersion: "Use numeric schemaVersion 1; other versions require an explicit migration." });
+  }
+  let config;
+  try {
+    config = validateSchemaPayload({ schema: integrationsSchema, mode: "replace" }, input);
+  } catch (error) {
+    throw new IntegrationConfigurationError(error.fieldErrors || { configuration: "Invalid configuration." });
+  }
+  const fieldErrors = {};
+  const definitions = new Map(providers.map((provider) => [provider.id, provider]));
+  const requireValue = (record, key, path) => {
+    if (!record[key]) fieldErrors[`${path}.${key}`] = "This value is required.";
+  };
+  for (const field of ["integrations", "registrations"]) {
+    for (const id of Object.keys(config[field])) {
+      if (!/^[a-z][a-z0-9-]*$/u.test(id) || ["constructor", "prototype"].includes(id)) {
+        fieldErrors[`${field}.${id}`] = "Use a stable lowercase name containing letters, digits and hyphens.";
+      }
+    }
+  }
+  for (const [id, registration] of Object.entries(config.registrations)) {
+    const path = `registrations.${id}`;
+    const clientCredentials = registration.grantType === "client_credentials";
+    const required = ["clientId", ...(clientCredentials ? [] : ["callbackUrlRef"]), ...(registration.tokenEndpointAuthMethod === "none" ? [] : ["clientSecretRef"])];
+    for (const key of required) requireValue(registration, key, path);
+    const forbidden = [...(clientCredentials ? ["callbackUrlRef"] : []), ...(registration.tokenEndpointAuthMethod === "none" ? ["clientSecretRef"] : [])];
+    for (const key of forbidden) {
+      if (Object.hasOwn(registration, key)) fieldErrors[`${path}.${key}`] = "This field belongs to another credential mode.";
+    }
+    if (clientCredentials && registration.tokenEndpointAuthMethod === "none") {
+      fieldErrors[`${path}.tokenEndpointAuthMethod`] = "Client credentials require a confidential client.";
+    }
+  }
+  for (const [id, integration] of Object.entries(config.integrations)) {
+    const path = `integrations.${id}`;
+    const auth = integration.authentication;
+    const provider = definitions.get(integration.provider);
+    const registration = config.registrations[auth.registrationRef];
+    const grantType = registration?.grantType || "authorization_code";
+    if (auth.method === "oauth2") {
+      if (!auth.registrationRef || !Object.hasOwn(config.registrations, auth.registrationRef)) {
+        fieldErrors[`${path}.authentication.registrationRef`] = "Select a registration defined in this file.";
+      }
+      if (auth.secretRef) fieldErrors[`${path}.authentication.secretRef`] = "Use the registration's secret reference.";
+      if (grantType === "client_credentials" && integration.accountMode === "per-user") {
+        fieldErrors[`${path}.accountMode`] = "A service account cannot connect as each app user.";
+      }
+    } else if (auth.method === "api-key") {
+      if (!provider?.apiKeySecretOptional) requireValue(auth, "secretRef", `${path}.authentication`);
+      if (auth.registrationRef) fieldErrors[`${path}.authentication.registrationRef`] = "API keys do not use an OAuth registration.";
+    } else if (auth.method === "service-account") {
+      requireValue(auth, "secretRef", `${path}.authentication`);
+      if (auth.registrationRef) fieldErrors[`${path}.authentication.registrationRef`] = "Service-account credentials do not use a browser OAuth registration.";
+      if (integration.accountMode === "per-user") fieldErrors[`${path}.accountMode`] = "A service account cannot connect as each app user.";
+    } else {
+      for (const field of ["secretRef", "registrationRef"]) {
+        if (Object.hasOwn(auth, field)) fieldErrors[`${path}.authentication.${field}`] = "This mode does not use credentials.";
+      }
+    }
+    if (new Set(integration.scopes).size !== integration.scopes.length) {
+      fieldErrors[`${path}.scopes`] = "Select each permission only once.";
+    }
+    if (providers.length) {
+      if (!provider) {
+        if (!allowUnknownProviders) fieldErrors[`${path}.provider`] = "Install and register this provider before using it.";
+        continue;
+      }
+      for (const action of Object.keys(integration.assistantPolicy?.actions || {})) {
+        if (!provider.assistantActions?.some((item) => item.value === action)) {
+          fieldErrors[`${path}.assistantPolicy.actions.${action}`] = "This provider does not declare this assistant action.";
+        }
+      }
+      if (provider.settingsSchema) {
+        try {
+          integration.settings = validateSchemaPayload({ schema: getProviderSettingsSchema(provider, integration.settings), mode: "replace" }, integration.settings || {});
+        } catch (error) {
+          for (const [field, message] of Object.entries(error.fieldErrors || { settings: "Check this provider's settings." })) {
+            fieldErrors[`${path}.settings.${field}`] = message;
+          }
+        }
+      }
+      for (const field of provider.settingsFields || []) {
+        if (field.authenticationMethods && !field.authenticationMethods.includes(auth.method) && Object.hasOwn(integration.settings || {}, field.name)) {
+          fieldErrors[`${path}.settings.${field.name}`] = "This setting belongs to another credential mode.";
+        }
+      }
+      if (!getProviderAccountModes(provider, integration.settings || {}).includes(integration.accountMode)) {
+        fieldErrors[`${path}.accountMode`] = "This provider does not support this account mode.";
+      }
+      if (!getProviderAuthenticationMethods(provider, integration.settings).includes(auth.method)) {
+        fieldErrors[`${path}.authentication.method`] = "This provider configuration does not support this credential mode.";
+      }
+      if (auth.method === "oauth2" && registration?.source === "own") {
+        const registrationPath = `registrations.${auth.registrationRef}`;
+        if (!(provider.oauthGrantTypes || ["authorization_code"]).includes(grantType)) {
+          fieldErrors[`${registrationPath}.grantType`] = "This provider does not support this OAuth flow.";
+        }
+        const methods = getProviderClientAuthenticationMethods(provider, grantType, integration.settings || {});
+        if (!methods.includes(registration.tokenEndpointAuthMethod || "client_secret_post")) {
+          fieldErrors[`${registrationPath}.tokenEndpointAuthMethod`] = "This provider does not support this client authentication method.";
+        }
+        if (registration.clientId && provider.validateClientId) {
+          const result = provider.validateClientId(registration.clientId);
+          if (result !== true) fieldErrors[`${registrationPath}.clientId`] = result || "Check this provider's client identifier.";
+        }
+      }
+      const scopes = getProviderScopes(provider, integration.settings || {}, grantType, auth.method);
+      const allowedScopes = new Set(scopes.map((scope) => scope.value));
+      if (scopes.some((scope) => scope.required && !integration.scopes.includes(scope.value))) {
+        fieldErrors[`${path}.scopes`] = "Include this provider's required permissions.";
+      }
+      if (integration.scopes.some((scope) => !allowedScopes.has(scope))) {
+        fieldErrors[`${path}.scopes`] = "A selected permission is not supported by this provider configuration.";
+      }
+      if (["oauth2", "service-account"].includes(auth.method) && scopes.length && !integration.scopes.length) {
+        fieldErrors[`${path}.scopes`] = "Select at least one permission.";
+      }
+    }
+  }
+  if (Object.keys(fieldErrors).length) throw new IntegrationConfigurationError(fieldErrors);
+  return config;
+}
+
+function parseIntegrationConfiguration(text, options) {
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    throw new IntegrationConfigurationError({ configuration: "Enter valid JSON." });
+  }
+  return validateIntegrationConfiguration(value, options);
+}
+
+export {
+  getProviderAuthenticationMethods,
+  getProviderClientAuthenticationMethods,
+  getProviderAccountModes,
+  getProviderScopes,
+  getProviderSettingsSchema,
+  secretReference,
+  integrationsSchema,
+  IntegrationConfigurationError,
+  parseIntegrationConfiguration,
+  validateIntegrationConfiguration
+};