@shipfox/api-agent 2.0.0

package/src/core/entities/agent-workspace-settings.ts

@@ -0,0 +1,9 @@
+import type {Harness, ModelProviderRef} from '@shipfox/api-agent-dto';
+
+export interface AgentWorkspaceSettings {
+  workspaceId: string;
+  defaultProviderId: ModelProviderRef | null;
+  defaultHarnessId: Harness | null;
+  createdAt: Date;
+  updatedAt: Date;
+}

package/src/core/entities/model-provider-config.ts

@@ -0,0 +1,25 @@
+import type {
+  AgentThinking,
+  CustomAgentModelDto,
+  CustomModelProviderHeaderDto,
+  ModelProviderApi,
+  ModelProviderRef,
+} from '@shipfox/api-agent-dto';
+
+export interface ModelProviderConfig {
+  id: string;
+  workspaceId: string;
+  providerId: ModelProviderRef;
+  kind: 'builtin' | 'custom';
+  displayName: string | null;
+  api: ModelProviderApi | null;
+  baseUrl: string | null;
+  headers: CustomModelProviderHeaderDto[] | null;
+  secretHeaderNames: string[] | null;
+  models: CustomAgentModelDto[] | null;
+  requiresApiKey: boolean;
+  defaultModel: string | null;
+  defaultThinking: AgentThinking;
+  createdAt: Date;
+  updatedAt: Date;
+}

package/src/core/errors.ts

@@ -0,0 +1,139 @@
+import type {
+  AgentThinking,
+  Harness,
+  ModelProviderRef,
+  SupportedModelProviderId,
+} from '@shipfox/api-agent-dto';
+
+export class ModelProviderValidationError extends Error {
+  constructor(
+    public readonly providerId: ModelProviderRef,
+    public readonly sanitizedMessage: string,
+  ) {
+    super(`Model provider validation failed for ${providerId}: ${sanitizedMessage}`);
+    this.name = 'ModelProviderValidationError';
+  }
+}
+
+export class ModelProviderConfigNotFoundError extends Error {
+  constructor(
+    public readonly workspaceId: string,
+    public readonly providerId: string,
+  ) {
+    super(`Model provider config not found: ${workspaceId}/${providerId}`);
+    this.name = 'ModelProviderConfigNotFoundError';
+  }
+}
+
+export class UnsupportedModelProviderError extends Error {
+  constructor(public readonly providerId: string) {
+    super(`Unsupported model provider: ${providerId}`);
+    this.name = 'UnsupportedModelProviderError';
+  }
+}
+
+export class UnsupportedHarnessProviderError extends Error {
+  constructor(
+    public readonly harness: Harness,
+    public readonly providerId: string,
+    public readonly supportedProviderIds: readonly string[],
+  ) {
+    super(
+      `Harness ${harness} does not support model provider ${providerId}. Supported providers: ${supportedProviderIds.join(', ')}`,
+    );
+    this.name = 'UnsupportedHarnessProviderError';
+  }
+}
+
+export class UnsupportedHarnessThinkingError extends Error {
+  constructor(
+    public readonly harness: Harness,
+    public readonly thinking: AgentThinking,
+    public readonly supportedLevels: readonly AgentThinking[],
+  ) {
+    super(
+      `Harness ${harness} does not support thinking ${thinking}. Supported levels: ${supportedLevels.join(', ')}`,
+    );
+    this.name = 'UnsupportedHarnessThinkingError';
+  }
+}
+
+export class InvalidCredentialFieldsError extends Error {
+  constructor(public readonly providerId: SupportedModelProviderId) {
+    super(`Invalid credential fields for model provider: ${providerId}`);
+    this.name = 'InvalidCredentialFieldsError';
+  }
+}
+
+export class InvalidAgentModelError extends Error {
+  public readonly harness: Harness;
+  public readonly providerId: ModelProviderRef;
+  public readonly model: string;
+
+  constructor(providerId: ModelProviderRef, model: string);
+  constructor(harness: Harness, providerId: ModelProviderRef, model: string);
+  constructor(
+    harnessOrProviderId: Harness | ModelProviderRef,
+    providerIdOrModel: ModelProviderRef | string,
+    model?: string,
+  ) {
+    const harness = model === undefined ? 'pi' : (harnessOrProviderId as Harness);
+    const providerId =
+      model === undefined
+        ? (harnessOrProviderId as ModelProviderRef)
+        : (providerIdOrModel as ModelProviderRef);
+    const resolvedModel = model ?? (providerIdOrModel as string);
+
+    super(`Model is not available for harness ${harness}: ${providerId}/${resolvedModel}`);
+    this.name = 'InvalidAgentModelError';
+    this.harness = harness;
+    this.providerId = providerId;
+    this.model = resolvedModel;
+  }
+}
+
+export class ModelProviderValidationUnavailableError extends Error {
+  constructor(public readonly providerId: SupportedModelProviderId) {
+    super(`Model provider validation is not available for model provider: ${providerId}`);
+    this.name = 'ModelProviderValidationUnavailableError';
+  }
+}
+
+export class CustomModelProviderSlugCollisionError extends Error {
+  constructor(
+    public readonly workspaceId: string,
+    public readonly providerId: ModelProviderRef,
+  ) {
+    super(`Custom model provider slug already exists: ${workspaceId}/${providerId}`);
+    this.name = 'CustomModelProviderSlugCollisionError';
+  }
+}
+
+export class CustomModelProviderConfigNotFoundError extends Error {
+  constructor(
+    public readonly workspaceId: string,
+    public readonly providerId: ModelProviderRef,
+  ) {
+    super(`Custom model provider config not found: ${workspaceId}/${providerId}`);
+    this.name = 'CustomModelProviderConfigNotFoundError';
+  }
+}
+
+export class InvalidCustomModelProviderHeaderKeepError extends Error {
+  constructor(
+    public readonly providerId: ModelProviderRef,
+    public readonly headerName: string,
+  ) {
+    super(`Custom model provider secret header cannot be kept: ${providerId}/${headerName}`);
+    this.name = 'InvalidCustomModelProviderHeaderKeepError';
+  }
+}
+
+export class CustomModelProviderStoredSecretBaseUrlChangeError extends Error {
+  constructor(public readonly providerId: ModelProviderRef) {
+    super(
+      `Stored custom model provider secrets cannot be reused with a changed base URL: ${providerId}`,
+    );
+    this.name = 'CustomModelProviderStoredSecretBaseUrlChangeError';
+  }
+}

package/src/core/harness/claude.ts

@@ -0,0 +1,75 @@
+import {type AgentModelOptionDto, CLAUDE_HARNESS} from '@shipfox/api-agent-dto';
+import {config} from '#config.js';
+import type {HarnessProviderCatalog} from './registry.js';
+
+const ANTHROPIC_MESSAGES_URL = 'https://api.anthropic.com/v1/messages';
+const ANTHROPIC_VERSION = '2023-06-01';
+const PROBE_MAX_TOKENS = 64;
+const PROBE_PROMPT = 'Reply with OK.';
+
+export const CLAUDE_MODEL_LINE: AgentModelOptionDto[] = [
+  {id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5 (latest)'},
+  {id: 'claude-haiku-4-5-20251001', label: 'Claude Haiku 4.5'},
+  {id: 'claude-opus-4-1', label: 'Claude Opus 4.1 (latest)'},
+  {id: 'claude-opus-4-1-20250805', label: 'Claude Opus 4.1'},
+  {id: 'claude-opus-4-5', label: 'Claude Opus 4.5 (latest)'},
+  {id: 'claude-opus-4-5-20251101', label: 'Claude Opus 4.5'},
+  {id: 'claude-opus-4-6', label: 'Claude Opus 4.6'},
+  {id: 'claude-opus-4-7', label: 'Claude Opus 4.7'},
+  {id: 'claude-opus-4-8', label: 'Claude Opus 4.8'},
+  {id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5 (latest)'},
+  {id: 'claude-sonnet-4-5-20250929', label: 'Claude Sonnet 4.5'},
+  {id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6'},
+];
+
+export {CLAUDE_HARNESS};
+
+export const claudeHarnessCatalog: HarnessProviderCatalog = {
+  listModels: () => CLAUDE_MODEL_LINE,
+  validateCredentials: probeClaudeCredentials,
+};
+
+async function probeClaudeCredentials(params: {
+  model: string;
+  credentials: Record<string, string>;
+  signal?: AbortSignal | undefined;
+}): Promise<void> {
+  const apiKey = params.credentials.api_key;
+  if (apiKey === undefined) throw new Error('Missing Anthropic API key credential.');
+
+  const response = await fetch(ANTHROPIC_MESSAGES_URL, {
+    method: 'POST',
+    headers: claudeProbeHeaders(apiKey),
+    body: JSON.stringify(claudeProbeBody(params.model)),
+    redirect: 'error',
+    signal: timeoutSignal(params.signal, config.AGENT_PROVIDER_VALIDATION_TIMEOUT_MS),
+  });
+
+  try {
+    if (!response.ok) throw new Error(`Provider returned HTTP ${response.status}.`);
+  } finally {
+    await response.body?.cancel();
+  }
+}
+
+function claudeProbeHeaders(apiKey: string): Headers {
+  const headers = new Headers();
+  headers.set('content-type', 'application/json');
+  headers.set('x-api-key', apiKey);
+  headers.set('anthropic-version', ANTHROPIC_VERSION);
+  return headers;
+}
+
+function claudeProbeBody(model: string): unknown {
+  return {
+    model,
+    max_tokens: PROBE_MAX_TOKENS,
+    messages: [{role: 'user', content: PROBE_PROMPT}],
+  };
+}
+
+function timeoutSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
+  return signal === undefined
+    ? AbortSignal.timeout(timeoutMs)
+    : AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
+}

package/src/core/harness/index.ts

@@ -0,0 +1,19 @@
+export {CLAUDE_HARNESS, CLAUDE_MODEL_LINE, claudeHarnessCatalog} from './claude.js';
+export {listPiProviderModels, PI_HARNESS, piHarnessCatalog} from './pi.js';
+export {
+  getHarnessDescriptor,
+  getHarnessToolDescriptor,
+  type HarnessDescriptor,
+  type HarnessProviderCatalog,
+  type HarnessToolDeploymentConfig,
+  type HarnessToolDescriptor,
+  type HarnessToolPackageName,
+  harnessSupportsProvider,
+  harnessSupportsTool,
+  listEnabledHarnessTools,
+  listHarnessDescriptors,
+  listHarnessProviderModels,
+  listHarnessTools,
+  type ProbeHarnessProviderCredentialsParams,
+  probeHarnessProviderCredentials,
+} from './registry.js';

package/src/core/harness/pi.ts

@@ -0,0 +1,18 @@
+import {getModels, type KnownProvider} from '@earendil-works/pi-ai';
+import {type AgentModelOptionDto, PI_HARNESS} from '@shipfox/api-agent-dto';
+import {probeModelProviderCredentials} from '../model-provider-validation.js';
+import type {HarnessProviderCatalog} from './registry.js';
+
+export {PI_HARNESS};
+
+export function listPiProviderModels(providerId: string): AgentModelOptionDto[] {
+  return getModels(providerId as KnownProvider).map((model) => ({
+    id: model.id,
+    label: model.name,
+  }));
+}
+
+export const piHarnessCatalog: HarnessProviderCatalog = {
+  listModels: listPiProviderModels,
+  validateCredentials: probeModelProviderCredentials,
+};

package/src/core/harness/registry.test.ts

@@ -0,0 +1,281 @@
+import type {AssistantMessage, Context, ProviderStreamOptions} from '@earendil-works/pi-ai';
+import {getModels} from '@earendil-works/pi-ai';
+import {
+  claudeAgentThinkingSchema,
+  harnessSchema,
+  piAgentThinkingSchema,
+} from '@shipfox/api-agent-dto';
+import {
+  getHarnessDescriptor as getCoreHarnessDescriptor,
+  listHarnessProviderModels as listCoreHarnessProviderModels,
+  ModelProviderValidationError,
+  UnsupportedHarnessProviderError,
+} from '#core/index.js';
+import {CLAUDE_HARNESS, CLAUDE_MODEL_LINE} from './claude.js';
+import {PI_HARNESS} from './pi.js';
+import {
+  getHarnessDescriptor,
+  harnessSupportsProvider,
+  listHarnessDescriptors,
+  listHarnessProviderModels,
+  probeHarnessProviderCredentials,
+} from './registry.js';
+
+const piAi = vi.hoisted(() => ({
+  complete: vi.fn(),
+  getModels: vi.fn(),
+}));
+
+const metrics = vi.hoisted(() => ({
+  modelProviderValidationCount: {
+    add: vi.fn(),
+  },
+}));
+
+vi.mock('@earendil-works/pi-ai', () => piAi);
+vi.mock('#metrics/index.js', () => metrics);
+
+describe('harness registry', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    piAi.getModels.mockImplementation((providerId: string) =>
+      providerId === 'anthropic' ? anthropicModels() : [],
+    );
+    piAi.complete.mockResolvedValue(createAssistantMessage());
+  });
+
+  it('returns the pi and Claude descriptors', () => {
+    expect(getHarnessDescriptor('pi')).toEqual({
+      id: 'pi',
+      label: 'pi',
+      description: 'Works with 30+ model providers',
+      supportedProviderIds: expect.arrayContaining(['anthropic', 'openai']),
+      thinkingLevels: piAgentThinkingSchema.options,
+      defaultThinking: 'xhigh',
+      defaultProviderId: 'anthropic',
+      tools: expect.arrayContaining([
+        expect.objectContaining({name: 'read'}),
+        expect.objectContaining({name: 'web_search'}),
+      ]),
+    });
+    expect(getHarnessDescriptor('claude')).toEqual({
+      id: 'claude',
+      label: 'Claude',
+      description: 'Runs on your Anthropic API key',
+      supportedProviderIds: ['anthropic'],
+      thinkingLevels: claudeAgentThinkingSchema.options,
+      defaultThinking: 'xhigh',
+      defaultProviderId: 'anthropic',
+      tools: expect.arrayContaining([
+        expect.objectContaining({name: 'Read'}),
+        expect.objectContaining({name: 'WebSearch'}),
+      ]),
+    });
+    expect(listHarnessDescriptors().map((descriptor) => descriptor.id)).toEqual(['pi', 'claude']);
+  });
+
+  it('registers every harness declared by the shared schema', () => {
+    const registeredHarnessIds = listHarnessDescriptors().map((descriptor) => descriptor.id);
+
+    expect(registeredHarnessIds).toEqual(harnessSchema.options);
+  });
+
+  it('keeps harness descriptor invariants synced with their source registries', () => {
+    const anthropicModelIds = new Set(getModels('anthropic').map((model) => model.id));
+    const unknownClaudeModelIds = CLAUDE_MODEL_LINE.filter(
+      (model) => !anthropicModelIds.has(model.id),
+    ).map((model) => model.id);
+
+    expect(PI_HARNESS.thinkingLevels).toBe(piAgentThinkingSchema.options);
+    expect(CLAUDE_HARNESS.defaultThinking).toBe('xhigh');
+    expect(unknownClaudeModelIds).toEqual([]);
+  });
+
+  it('lists pi provider models from pi-ai', () => {
+    const models = listHarnessProviderModels('pi', 'anthropic');
+
+    expect(piAi.getModels).toHaveBeenCalledWith('anthropic');
+    expect(models).toContainEqual({id: 'claude-opus-4-8', label: 'Claude Opus 4.8'});
+    expect(models).toContainEqual({id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5 (latest)'});
+  });
+
+  it('lists the Claude model line for Anthropic', () => {
+    const models = listHarnessProviderModels('claude', 'anthropic');
+
+    expect(models).toContainEqual({id: 'claude-opus-4-8', label: 'Claude Opus 4.8'});
+    expect(models).toEqual(CLAUDE_MODEL_LINE);
+  });
+
+  it('rejects unsupported harness/provider pairs before entering the catalog', () => {
+    expect(harnessSupportsProvider('claude', 'openai')).toBe(false);
+    expect(() => listHarnessProviderModels('claude', 'openai')).toThrow(
+      UnsupportedHarnessProviderError,
+    );
+  });
+
+  it('exports the public seam through the core barrel', () => {
+    expect(getCoreHarnessDescriptor('claude')).toBe(getHarnessDescriptor('claude'));
+    expect(listCoreHarnessProviderModels('claude', 'anthropic')).toEqual(CLAUDE_MODEL_LINE);
+  });
+
+  it('probes pi credentials through pi-ai complete', async () => {
+    await probeHarnessProviderCredentials({
+      harness: 'pi',
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials: {api_key: 'sk-ant-secret'},
+    });
+
+    const [model, context, options] = piAi.complete.mock.calls[0] as [
+      {id: string},
+      Context,
+      ProviderStreamOptions,
+    ];
+    expect(model.id).toBe('claude-opus-4-8');
+    expect(context.messages).toHaveLength(1);
+    expect(options).toMatchObject({apiKey: 'sk-ant-secret', maxRetries: 0});
+    expect(metrics.modelProviderValidationCount.add).toHaveBeenCalledWith(1, {
+      model_provider: 'anthropic',
+      outcome: 'succeeded',
+    });
+  });
+
+  it('wraps pi probe failures in sanitized validation errors', async () => {
+    piAi.complete.mockResolvedValue(
+      createAssistantMessage({
+        stopReason: 'error',
+        errorMessage: 'Authentication failed for sk-ant-secret',
+      }),
+    );
+
+    const probe = probeHarnessProviderCredentials({
+      harness: 'pi',
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials: {api_key: 'sk-ant-secret'},
+    });
+
+    await expect(probe).rejects.toMatchObject({
+      providerId: 'anthropic',
+      sanitizedMessage: expect.not.stringContaining('sk-ant-secret'),
+    });
+    await expect(probe).rejects.toThrow(ModelProviderValidationError);
+  });
+
+  it('probes Claude credentials against Anthropic messages', async () => {
+    const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response('{}'));
+
+    await probeHarnessProviderCredentials({
+      harness: 'claude',
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials: {api_key: 'sk-ant-secret'},
+    });
+
+    const [, init] = fetchSpy.mock.calls[0] as [string, RequestInit];
+    expect(fetchSpy).toHaveBeenCalledWith(
+      'https://api.anthropic.com/v1/messages',
+      expect.objectContaining({method: 'POST', redirect: 'error'}),
+    );
+    expect((init.headers as Headers).get('x-api-key')).toBe('sk-ant-secret');
+    expect((init.headers as Headers).get('anthropic-version')).toBe('2023-06-01');
+    expect(JSON.parse(init.body as string)).toEqual({
+      model: 'claude-opus-4-8',
+      max_tokens: 64,
+      messages: [{role: 'user', content: 'Reply with OK.'}],
+    });
+  });
+
+  it('wraps Claude non-2xx responses as sanitized validation errors', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response('{}', {status: 401}));
+
+    const probe = probeHarnessProviderCredentials({
+      harness: 'claude',
+      providerId: 'anthropic',
+      model: 'claude-opus-4-8',
+      credentials: {api_key: 'sk-ant-secret'},
+    });
+
+    await expect(probe).rejects.toMatchObject({
+      providerId: 'anthropic',
+      sanitizedMessage: 'Provider returned HTTP 401.',
+    });
+    await expect(probe).rejects.toThrow(ModelProviderValidationError);
+  });
+
+  it('rejects unsupported probes before recording validation metrics', async () => {
+    const probe = probeHarnessProviderCredentials({
+      harness: 'claude',
+      providerId: 'openai',
+      model: 'gpt-5.5-pro',
+      credentials: {api_key: 'sk-openai-secret'},
+    });
+
+    await expect(probe).rejects.toThrow(UnsupportedHarnessProviderError);
+    expect(metrics.modelProviderValidationCount.add).not.toHaveBeenCalled();
+  });
+});
+
+function anthropicModels() {
+  return [
+    {
+      id: 'claude-opus-4-8',
+      provider: 'anthropic',
+      api: 'anthropic-messages',
+      name: 'Claude Opus 4.8',
+      baseUrl: 'https://api.anthropic.com',
+      reasoning: true,
+      input: ['text'],
+      cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+      contextWindow: 200000,
+      maxTokens: 32000,
+    },
+    {
+      id: 'claude-haiku-4-5',
+      provider: 'anthropic',
+      api: 'anthropic-messages',
+      name: 'Claude Haiku 4.5 (latest)',
+      baseUrl: 'https://api.anthropic.com',
+      reasoning: true,
+      input: ['text'],
+      cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+      contextWindow: 200000,
+      maxTokens: 32000,
+    },
+    ...CLAUDE_MODEL_LINE.filter(
+      (model) => model.id !== 'claude-opus-4-8' && model.id !== 'claude-haiku-4-5',
+    ).map((model) => ({
+      id: model.id,
+      provider: 'anthropic',
+      api: 'anthropic-messages',
+      name: model.label,
+      baseUrl: 'https://api.anthropic.com',
+      reasoning: true,
+      input: ['text'],
+      cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0},
+      contextWindow: 200000,
+      maxTokens: 32000,
+    })),
+  ];
+}
+
+function createAssistantMessage(overrides: Partial<AssistantMessage> = {}): AssistantMessage {
+  return {
+    role: 'assistant',
+    content: [{type: 'text', text: 'OK'}],
+    api: 'anthropic-messages',
+    provider: 'anthropic',
+    model: 'claude-opus-4-8',
+    usage: {
+      input: 0,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      totalTokens: 0,
+      cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0},
+    },
+    stopReason: 'stop',
+    timestamp: Date.now(),
+    ...overrides,
+  };
+}

package/src/core/harness/registry.ts

@@ -0,0 +1,108 @@
+import type {
+  AgentModelOptionDto,
+  Harness,
+  HarnessDescriptor,
+  HarnessToolDeploymentConfig,
+  HarnessToolDescriptor,
+  HarnessToolPackageName,
+  SupportedModelProviderId,
+} from '@shipfox/api-agent-dto';
+import {
+  getHarnessDescriptor,
+  getHarnessToolDescriptor,
+  harnessSupportsProvider,
+  harnessSupportsTool,
+  listEnabledHarnessTools,
+  listHarnessDescriptors,
+  listHarnessTools,
+  SUPPORTED_MODEL_PROVIDER_IDS,
+} from '@shipfox/api-agent-dto';
+import {UnsupportedHarnessProviderError} from '../errors.js';
+import {runProviderProbe} from '../model-provider-validation.js';
+import {claudeHarnessCatalog} from './claude.js';
+import {piHarnessCatalog} from './pi.js';
+
+export {
+  getHarnessDescriptor,
+  getHarnessToolDescriptor,
+  type HarnessDescriptor,
+  type HarnessToolDeploymentConfig,
+  type HarnessToolDescriptor,
+  type HarnessToolPackageName,
+  harnessSupportsProvider,
+  harnessSupportsTool,
+  listEnabledHarnessTools,
+  listHarnessDescriptors,
+  listHarnessTools,
+};
+
+export interface HarnessProviderCatalog {
+  listModels(providerId: string): AgentModelOptionDto[];
+  validateCredentials(params: {
+    providerId: string;
+    model: string;
+    credentials: Record<string, string>;
+    signal?: AbortSignal | undefined;
+  }): Promise<void>;
+}
+
+export interface ProbeHarnessProviderCredentialsParams {
+  harness: Harness;
+  providerId: string;
+  model: string;
+  credentials: Record<string, string>;
+  signal?: AbortSignal | undefined;
+}
+
+const CATALOGS: Record<Harness, HarnessProviderCatalog> = {
+  pi: piHarnessCatalog,
+  claude: claudeHarnessCatalog,
+};
+
+const supportedModelProviderIds = new Set<string>(SUPPORTED_MODEL_PROVIDER_IDS);
+
+export function listHarnessProviderModels(
+  harness: Harness,
+  providerId: string,
+): AgentModelOptionDto[] {
+  const descriptor = getHarnessDescriptor(harness);
+  assertHarnessSupportsProvider(descriptor, providerId);
+  return CATALOGS[harness].listModels(providerId);
+}
+
+export async function probeHarnessProviderCredentials(
+  params: ProbeHarnessProviderCredentialsParams,
+): Promise<void> {
+  const descriptor = getHarnessDescriptor(params.harness);
+  assertHarnessSupportsProvider(descriptor, params.providerId);
+
+  await runProviderProbe({
+    probe: CATALOGS[params.harness].validateCredentials,
+    args: {
+      providerId: params.providerId,
+      model: params.model,
+      credentials: params.credentials,
+      ...(params.signal ? {signal: params.signal} : {}),
+    },
+    metricLabel: metricLabel(params.providerId),
+    providerId: params.providerId,
+    secrets: Object.values(params.credentials),
+    ...(params.signal ? {signal: params.signal} : {}),
+  });
+}
+
+function assertHarnessSupportsProvider(descriptor: HarnessDescriptor, providerId: string): void {
+  if (descriptor.supportedProviderIds.includes(providerId)) return;
+
+  throw new UnsupportedHarnessProviderError(
+    descriptor.id,
+    providerId,
+    descriptor.supportedProviderIds,
+  );
+}
+
+function metricLabel(providerId: string): SupportedModelProviderId | 'custom' {
+  return supportedModelProviderIds.has(providerId)
+    ? (providerId as SupportedModelProviderId)
+    : 'custom';
+}