@shipfox/api-agent 2.0.0

package/src/core/model-provider-config-service.ts

@@ -0,0 +1,207 @@
+import {
+  DEFAULT_AGENT_THINKING,
+  getModelProviderEntry,
+  type ModelProviderRef,
+  modelProviderCredentialKeysMatch,
+  type SupportedModelProviderId,
+} from '@shipfox/api-agent-dto';
+import {deleteSecrets, getSecretsByNamespace, setSecrets} from '@shipfox/api-secrets';
+import {
+  deleteModelProviderConfig as deleteModelProviderConfigRow,
+  getModelProviderConfig,
+  updateModelProviderDefaultModel,
+  upsertModelProviderConfig,
+} from '#db/index.js';
+import {agentSystemNamespace, credentialsToStoreValues} from './credential-fingerprints.js';
+import type {ModelProviderConfig} from './entities/model-provider-config.js';
+import {
+  InvalidAgentModelError,
+  InvalidCredentialFieldsError,
+  ModelProviderConfigNotFoundError,
+  UnsupportedModelProviderError,
+} from './errors.js';
+import {buildModelProviderCatalog} from './model-provider-catalog.js';
+import {probeModelProviderCredentials, runProviderProbe} from './model-provider-validation.js';
+
+export interface TestAndSaveModelProviderConfigParams {
+  workspaceId: string;
+  providerId: SupportedModelProviderId;
+  defaultModel?: string | null | undefined;
+  credentials: Record<string, string>;
+  editedBy?: string | null | undefined;
+  setAsDefault?: boolean | undefined;
+  signal?: AbortSignal | undefined;
+}
+
+export interface TestAndSaveModelProviderConfigOptions {
+  probe?: typeof probeModelProviderCredentials;
+  pruneStaleSecrets?:
+    | ((params: {workspaceId: string; namespace: string; expectedKeys: string[]}) => Promise<void>)
+    | undefined;
+}
+
+export interface UpdateModelProviderConfigDefaultModelParams {
+  workspaceId: string;
+  providerId: ModelProviderRef;
+  defaultModel: string | null;
+}
+
+export async function testAndSaveModelProviderConfig(
+  params: TestAndSaveModelProviderConfigParams,
+  options: TestAndSaveModelProviderConfigOptions = {},
+): Promise<ModelProviderConfig> {
+  const probe = options.probe ?? probeModelProviderCredentials;
+  const entry = getModelProviderEntry(params.providerId);
+  if (entry === undefined || entry.support_status !== 'supported') {
+    throw new UnsupportedModelProviderError(params.providerId);
+  }
+  if (entry.default_model === null) throw new UnsupportedModelProviderError(params.providerId);
+
+  if (!modelProviderCredentialKeysMatch(params.providerId, params.credentials)) {
+    throw new InvalidCredentialFieldsError(params.providerId);
+  }
+
+  const existingConfig = await getModelProviderConfig({
+    workspaceId: params.workspaceId,
+    providerId: params.providerId,
+  });
+  const modelSelection = resolveDefaultModel(
+    params.providerId,
+    params.defaultModel !== undefined ? params.defaultModel : existingConfig?.defaultModel,
+  );
+
+  await runProviderProbe({
+    probe,
+    args: {
+      providerId: params.providerId,
+      model: modelSelection.probeModel,
+      credentials: params.credentials,
+      ...(params.signal ? {signal: params.signal} : {}),
+    },
+    metricLabel: params.providerId,
+    providerId: params.providerId,
+    secrets: Object.values(params.credentials),
+    signal: params.signal,
+  });
+  const pruneStaleSecrets = options.pruneStaleSecrets ?? pruneStaleProviderCredentialSecrets;
+  const namespace = agentSystemNamespace(params.providerId);
+  const values = credentialsToStoreValues(params.providerId, params.credentials);
+  await setSecrets({
+    workspaceId: params.workspaceId,
+    namespace,
+    values,
+    editedBy: params.editedBy,
+  });
+
+  const config = await upsertModelProviderConfig({
+    workspaceId: params.workspaceId,
+    providerId: params.providerId,
+    defaultModel: modelSelection.storedModel,
+    defaultThinking: DEFAULT_AGENT_THINKING,
+    setAsDefault: params.setAsDefault,
+  });
+
+  await pruneStaleSecrets({
+    workspaceId: params.workspaceId,
+    namespace,
+    expectedKeys: Object.keys(values),
+  }).catch(() => undefined);
+
+  return config;
+}
+
+export async function updateModelProviderConfigDefaultModel(
+  params: UpdateModelProviderConfigDefaultModelParams,
+): Promise<ModelProviderConfig> {
+  const existingConfig = await getModelProviderConfig({
+    workspaceId: params.workspaceId,
+    providerId: params.providerId,
+  });
+  if (!existingConfig) {
+    throw new ModelProviderConfigNotFoundError(params.workspaceId, params.providerId);
+  }
+  if (existingConfig.kind === 'custom') {
+    if (
+      params.defaultModel !== null &&
+      !existingConfig.models?.some((model) => model.id === params.defaultModel)
+    ) {
+      throw new InvalidAgentModelError('pi', params.providerId, params.defaultModel);
+    }
+
+    const config = await updateModelProviderDefaultModel({
+      workspaceId: params.workspaceId,
+      providerId: params.providerId,
+      defaultModel: params.defaultModel,
+    });
+    if (!config) throw new ModelProviderConfigNotFoundError(params.workspaceId, params.providerId);
+    return config;
+  }
+
+  const providerId = params.providerId as SupportedModelProviderId;
+  const entry = getModelProviderEntry(providerId);
+  if (entry === undefined || entry.support_status !== 'supported') {
+    throw new UnsupportedModelProviderError(params.providerId);
+  }
+  if (entry.default_model === null) throw new UnsupportedModelProviderError(params.providerId);
+
+  const modelSelection = resolveDefaultModel(providerId, params.defaultModel);
+  const config = await updateModelProviderDefaultModel({
+    workspaceId: params.workspaceId,
+    providerId: params.providerId,
+    defaultModel: modelSelection.storedModel,
+  });
+  if (!config) throw new ModelProviderConfigNotFoundError(params.workspaceId, params.providerId);
+  return config;
+}
+
+export async function deleteModelProviderConfig(params: {
+  workspaceId: string;
+  providerId: ModelProviderRef;
+}): Promise<boolean> {
+  const deleted = await deleteModelProviderConfigRow(params);
+  await deleteSecrets({
+    workspaceId: params.workspaceId,
+    namespace: agentSystemNamespace(params.providerId),
+  });
+  return deleted;
+}
+
+function resolveDefaultModel(
+  providerId: SupportedModelProviderId,
+  requestedModel: string | null | undefined,
+): {probeModel: string; storedModel: string | null} {
+  const catalogEntry = buildModelProviderCatalog().find((entry) => entry.id === providerId);
+  if (
+    catalogEntry === undefined ||
+    catalogEntry.support_status !== 'supported' ||
+    catalogEntry.default_model === null
+  ) {
+    throw new UnsupportedModelProviderError(providerId);
+  }
+
+  const model = requestedModel ?? catalogEntry.default_model;
+  if (!catalogEntry.models.some((candidate) => candidate.id === model)) {
+    throw new InvalidAgentModelError('pi', providerId, model);
+  }
+  return {probeModel: model, storedModel: requestedModel ?? null};
+}
+
+async function pruneStaleProviderCredentialSecrets(params: {
+  workspaceId: string;
+  namespace: string;
+  expectedKeys: string[];
+}): Promise<void> {
+  const stored = await getSecretsByNamespace({
+    workspaceId: params.workspaceId,
+    namespace: params.namespace,
+  });
+  const expected = new Set(params.expectedKeys);
+  const staleKeys = Object.keys(stored).filter((key) => !expected.has(key));
+  if (staleKeys.length === 0) return;
+
+  await deleteSecrets({
+    workspaceId: params.workspaceId,
+    namespace: params.namespace,
+    keys: staleKeys,
+  });
+}

package/src/core/model-provider-validation.test.ts

@@ -0,0 +1,354 @@
+import type {AssistantMessage, Context, ProviderStreamOptions} from '@earendil-works/pi-ai';
+import {InvalidAgentModelError} from './errors.js';
+import {
+  probeCustomModelProviderCredentials,
+  probeModelProviderCredentials,
+  sanitizeModelProviderError,
+} from './model-provider-validation.js';
+
+const piAi = vi.hoisted(() => ({
+  complete: vi.fn(),
+  getModels: vi.fn(),
+}));
+
+vi.mock('@earendil-works/pi-ai', () => piAi);
+
+describe('probeModelProviderCredentials', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    piAi.getModels.mockReturnValue([
+      {
+        id: 'claude-opus-4-8',
+        provider: 'anthropic',
+        api: 'anthropic-messages',
+        name: 'Claude Opus',
+        baseUrl: 'https://api.anthropic.com',
+        reasoning: true,
+        input: ['text'],
+        cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+        contextWindow: 200000,
+        maxTokens: 32000,
+      },
+    ]);
+    piAi.complete.mockResolvedValue(createAssistantMessage());
+  });
+
+  it('resolves the catalog model and sends the catalog secret field as apiKey', async () => {
+    const credentials = {api_key: 'sk-ant-secret'};
+
+    await probeModelProviderCredentials({
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials,
+    });
+
+    expect(piAi.getModels).toHaveBeenCalledWith('anthropic');
+    const [model, context, options] = piAi.complete.mock.calls[0] as [
+      {id: string},
+      Context,
+      ProviderStreamOptions,
+    ];
+    expect(model.id).toBe('claude-opus-4-8');
+    expect(context.messages).toHaveLength(1);
+    expect(options).toMatchObject({
+      apiKey: 'sk-ant-secret',
+      maxRetries: 0,
+      maxTokens: 64,
+      timeoutMs: 10000,
+    });
+  });
+
+  it('throws InvalidAgentModelError when the catalog model is missing from Pi', async () => {
+    piAi.getModels.mockReturnValue([]);
+
+    const probe = probeModelProviderCredentials({
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials: {api_key: 'sk-ant-secret'},
+    });
+
+    await expect(probe).rejects.toThrow(InvalidAgentModelError);
+    expect(piAi.complete).not.toHaveBeenCalled();
+  });
+
+  it('rejects provider error results from Pi complete', async () => {
+    piAi.complete.mockResolvedValue(
+      createAssistantMessage({stopReason: 'error', errorMessage: '401 Unauthorized'}),
+    );
+
+    const probe = probeModelProviderCredentials({
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials: {api_key: 'sk-ant-secret'},
+    });
+
+    await expect(probe).rejects.toThrow('401 Unauthorized');
+  });
+
+  it('passes Azure endpoint credentials as Pi Azure base URL options', async () => {
+    piAi.getModels.mockReturnValue([
+      {
+        id: 'gpt-5.5-pro',
+        provider: 'azure-openai-responses',
+        api: 'azure-openai-responses',
+        name: 'GPT',
+        baseUrl: '',
+        reasoning: true,
+        input: ['text'],
+        cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+        contextWindow: 200000,
+        maxTokens: 32000,
+      },
+    ]);
+
+    await probeModelProviderCredentials({
+      providerId: 'azure-openai-responses',
+      model: 'gpt-5.5-pro',
+      credentials: {
+        endpoint: 'https://azure.example.test/openai/v1',
+        api_key: 'sk-azure-secret',
+      },
+    });
+
+    const [, , options] = piAi.complete.mock.calls[0] as [
+      {id: string},
+      Context,
+      ProviderStreamOptions,
+    ];
+    expect(options).toMatchObject({
+      apiKey: 'sk-azure-secret',
+      azureBaseUrl: 'https://azure.example.test/openai/v1',
+    });
+  });
+
+  it('passes Cloudflare AI Gateway credentials through provider env options', async () => {
+    piAi.getModels.mockReturnValue([
+      {
+        id: 'claude-opus-4-8',
+        provider: 'cloudflare-ai-gateway',
+        api: 'anthropic-messages',
+        name: 'Claude',
+        baseUrl:
+          'https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic',
+        reasoning: true,
+        input: ['text'],
+        cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+        contextWindow: 200000,
+        maxTokens: 32000,
+      },
+    ]);
+
+    await probeModelProviderCredentials({
+      providerId: 'cloudflare-ai-gateway',
+      model: 'claude-opus-4-8',
+      credentials: {
+        api_key: 'cf-secret',
+        account_id: 'account-123',
+        gateway_id: 'gateway-456',
+      },
+    });
+
+    const [, , options] = piAi.complete.mock.calls[0] as [
+      {id: string},
+      Context,
+      ProviderStreamOptions,
+    ];
+    expect(options).toMatchObject({
+      apiKey: 'cf-secret',
+      env: {
+        CLOUDFLARE_ACCOUNT_ID: 'account-123',
+        CLOUDFLARE_GATEWAY_ID: 'gateway-456',
+      },
+    });
+  });
+
+  it('passes Cloudflare Workers AI credentials through provider env options', async () => {
+    piAi.getModels.mockReturnValue([
+      {
+        id: '@cf/moonshotai/kimi-k2.7-code',
+        provider: 'cloudflare-workers-ai',
+        api: 'openai-completions',
+        name: 'Kimi',
+        baseUrl: 'https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1',
+        reasoning: true,
+        input: ['text'],
+        cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+        contextWindow: 200000,
+        maxTokens: 32000,
+      },
+    ]);
+
+    await probeModelProviderCredentials({
+      providerId: 'cloudflare-workers-ai',
+      model: '@cf/moonshotai/kimi-k2.7-code',
+      credentials: {
+        api_key: 'cf-secret',
+        account_id: 'account-123',
+      },
+    });
+
+    const [, , options] = piAi.complete.mock.calls[0] as [
+      {id: string},
+      Context,
+      ProviderStreamOptions,
+    ];
+    expect(options).toMatchObject({
+      apiKey: 'cf-secret',
+      env: {
+        CLOUDFLARE_ACCOUNT_ID: 'account-123',
+      },
+    });
+  });
+});
+
+describe('probeCustomModelProviderCredentials', () => {
+  it('probes OpenAI-compatible custom providers with redirects disabled', async () => {
+    const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response('{}'));
+
+    await probeCustomModelProviderCredentials({
+      providerId: 'local-vllm',
+      api: 'openai-responses',
+      baseUrl: 'https://llm.example.test/v1',
+      model: {id: 'llama-3.1', label: 'Llama 3.1'},
+      apiKey: 'sk-local',
+      headers: {'x-region': 'local'},
+    });
+
+    const [, init] = fetchSpy.mock.calls[0] as [string, RequestInit];
+    expect(fetchSpy).toHaveBeenCalledWith(
+      'https://llm.example.test/v1/responses',
+      expect.objectContaining({method: 'POST', redirect: 'error'}),
+    );
+    expect((init.headers as Headers).get('authorization')).toBe('Bearer sk-local');
+    expect((init.headers as Headers).get('x-region')).toBe('local');
+    expect(JSON.parse(init.body as string)).toMatchObject({
+      model: 'llama-3.1',
+      input: 'Reply with OK.',
+      stream: false,
+    });
+  });
+
+  it('uses provider-specific auth headers for Google custom probes', async () => {
+    const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response('{}'));
+
+    await probeCustomModelProviderCredentials({
+      providerId: 'local-gemini',
+      api: 'google-generative-ai',
+      baseUrl: 'https://generativelanguage.googleapis.com/v1beta',
+      model: {id: 'gemini-3-pro', label: 'Gemini 3 Pro'},
+      apiKey: 'google-key',
+    });
+
+    const [, init] = fetchSpy.mock.calls[0] as [string, RequestInit];
+    expect(fetchSpy).toHaveBeenCalledWith(
+      'https://generativelanguage.googleapis.com/v1beta/models/gemini-3-pro:generateContent',
+      expect.objectContaining({redirect: 'error'}),
+    );
+    expect((init.headers as Headers).get('x-goog-api-key')).toBe('google-key');
+  });
+
+  it('encodes Google custom probe model ids as one URL path segment', async () => {
+    const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response('{}'));
+
+    await probeCustomModelProviderCredentials({
+      providerId: 'local-gemini',
+      api: 'google-generative-ai',
+      baseUrl: 'https://generativelanguage.googleapis.com/v1beta',
+      model: {id: 'models/gemini-3/pro', label: 'Gemini 3 Pro'},
+      apiKey: 'google-key',
+    });
+
+    expect(fetchSpy).toHaveBeenCalledWith(
+      'https://generativelanguage.googleapis.com/v1beta/models/models%2Fgemini-3%2Fpro:generateContent',
+      expect.any(Object),
+    );
+  });
+
+  it('cancels custom probe response bodies after checking the status', async () => {
+    const response = new Response('{}');
+    const body = response.body;
+    if (body === null) throw new Error('Expected response body');
+    const cancelSpy = vi.spyOn(body, 'cancel');
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(response);
+
+    await probeCustomModelProviderCredentials({
+      providerId: 'local-vllm',
+      api: 'openai-responses',
+      baseUrl: 'https://llm.example.test/v1',
+      model: {id: 'llama-3.1', label: 'Llama 3.1'},
+    });
+
+    expect(cancelSpy).toHaveBeenCalledOnce();
+  });
+
+  it('rejects non-2xx custom probe responses', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response('{}', {status: 401}));
+
+    const probe = probeCustomModelProviderCredentials({
+      providerId: 'local-vllm',
+      api: 'openai-responses',
+      baseUrl: 'https://llm.example.test/v1',
+      model: {id: 'llama-3.1', label: 'Llama 3.1'},
+    });
+
+    await expect(probe).rejects.toThrow('Provider returned HTTP 401.');
+  });
+});
+
+function createAssistantMessage(overrides: Partial<AssistantMessage> = {}): AssistantMessage {
+  return {
+    role: 'assistant',
+    content: [{type: 'text', text: 'OK'}],
+    api: 'anthropic-messages',
+    provider: 'anthropic',
+    model: 'claude-opus-4-8',
+    usage: {
+      input: 0,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      totalTokens: 0,
+      cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0},
+    },
+    stopReason: 'stop',
+    timestamp: Date.now(),
+    ...overrides,
+  };
+}
+
+describe('sanitizeModelProviderError', () => {
+  it('redacts literal and derived secret wire forms from provider messages', () => {
+    const secret = 'sk-ant-api-key-secret-abcd';
+    const base64Secret = Buffer.from(secret, 'utf8').toString('base64');
+    const error = new Error(`Authentication failed for ${secret} and ${base64Secret}`);
+
+    const sanitized = sanitizeModelProviderError(error, [secret]);
+
+    expect(sanitized).not.toContain(secret);
+    expect(sanitized).not.toContain(base64Secret);
+    expect(sanitized).toContain('***');
+  });
+
+  it('caps long provider messages', () => {
+    const error = new Error('x'.repeat(1000));
+
+    const sanitized = sanitizeModelProviderError(error, ['sk-ant-secret']);
+
+    expect(sanitized).toHaveLength(500);
+  });
+
+  it('omits the stack from Error objects', () => {
+    const error = new Error('Provider said no');
+
+    const sanitized = sanitizeModelProviderError(error, []);
+
+    expect(sanitized).toBe('Provider said no');
+    expect(sanitized).not.toContain('at ');
+  });
+
+  it('handles non-Error thrown values', () => {
+    const sanitized = sanitizeModelProviderError('raw thrown value', ['secret']);
+
+    expect(sanitized).toBe('Provider validation failed.');
+  });
+});