@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/src/loader/__tests__/registry.test.ts

@@ -0,0 +1,134 @@
+import { z } from 'zod'
+import { defineAdapterModule, type AdapterModule } from '../../contract/adapter'
+import { CONTRACT_VERSION } from '../../contract/version'
+import { createFakeAdapter } from '../../testing/fakeAdapter'
+import { createTestContext } from '../../testing/contract'
+import { readAdapterManifest } from '../manifest'
+import { instantiateAdapter, resolveAdapterModules } from '../registry'
+
+const workingModule = defineAdapterModule({
+  id: 'working',
+  kind: 'api',
+  contractVersion: CONTRACT_VERSION,
+  optionsSchema: z.object({ apiKey: z.string().min(1) }),
+  createAdapter: (options) => createFakeAdapter({ id: 'working', results: [{ url: `https://x/${options.apiKey}` }] }),
+})
+
+describe('readAdapterManifest', () => {
+  it('reads a well-formed manifest', () => {
+    const result = readAdapterManifest({
+      name: '@acme/web-research-brave',
+      openMercato: { webResearchAdapter: { id: 'brave' } },
+    })
+    expect(result).toEqual({ packageName: '@acme/web-research-brave', manifest: { id: 'brave' } })
+  })
+
+  it('ignores a package that is not an adapter', () => {
+    expect(readAdapterManifest({ name: 'lodash' })).toBeNull()
+    expect(readAdapterManifest({ name: 'x', openMercato: {} })).toBeNull()
+    expect(readAdapterManifest(null)).toBeNull()
+  })
+
+  it('reports a malformed manifest instead of ignoring it', () => {
+    const result = readAdapterManifest({
+      name: '@acme/broken',
+      openMercato: { webResearchAdapter: { id: '' } },
+    })
+    expect(result).toMatchObject({ error: expect.stringContaining('@acme/broken') })
+  })
+})
+
+describe('resolveAdapterModules', () => {
+  it('accepts a valid descriptor and unwraps a default export', () => {
+    const registry = resolveAdapterModules([
+      { packageName: 'a', module: workingModule },
+      { packageName: 'b', module: { default: { ...workingModule, id: 'other' } } },
+    ])
+    expect(registry.rejected).toEqual([])
+    expect(registry.loaded.map((entry) => entry.module.id)).toEqual(['working', 'other'])
+  })
+
+  it('rejects a contract-version mismatch without throwing', () => {
+    const registry = resolveAdapterModules([
+      { packageName: 'stale', module: { ...workingModule, contractVersion: CONTRACT_VERSION + 1 } },
+    ])
+    expect(registry.loaded).toEqual([])
+    expect(registry.rejected[0].reason).toContain('contract version')
+  })
+
+  it.each([
+    ['not an object', 42],
+    ['missing id', { ...workingModule, id: '' }],
+    ['unknown kind', { ...workingModule, kind: 'telepathy' }],
+    ['missing createAdapter', { ...workingModule, createAdapter: undefined }],
+    ['missing optionsSchema', { ...workingModule, optionsSchema: undefined }],
+  ])('rejects a descriptor that is %s', (_label, module) => {
+    const registry = resolveAdapterModules([{ packageName: 'bad', module }])
+    expect(registry.loaded).toEqual([])
+    expect(registry.rejected).toHaveLength(1)
+  })
+
+  it('keeps the first package to claim an adapter id and rejects the duplicate', () => {
+    const registry = resolveAdapterModules([
+      { packageName: 'first', module: workingModule },
+      { packageName: 'second', module: { ...workingModule } },
+    ])
+    expect(registry.loaded.map((entry) => entry.packageName)).toEqual(['first'])
+    expect(registry.rejected[0].reason).toContain('already provided by first')
+  })
+
+  // The MCP server loads every module tool through one ESM graph, where a single
+  // failed import drops all of them. One broken adapter must never do that.
+  it('isolates a bad descriptor from the good ones in the same batch', () => {
+    const registry = resolveAdapterModules([
+      { packageName: 'broken', module: null },
+      { packageName: 'fine', module: workingModule },
+    ])
+    expect(registry.loaded.map((entry) => entry.module.id)).toEqual(['working'])
+    expect(registry.rejected.map((entry) => entry.packageName)).toEqual(['broken'])
+  })
+})
+
+describe('instantiateAdapter', () => {
+  const loaded = { packageName: 'pkg', module: workingModule as AdapterModule<unknown> }
+
+  it('builds an adapter from valid options', () => {
+    const result = instantiateAdapter(loaded, { apiKey: 'secret' })
+    expect(result.error).toBeUndefined()
+    expect(result.adapter.readiness().ready).toBe(true)
+  })
+
+  it('stands in an unavailable adapter when options do not validate', async () => {
+    const result = instantiateAdapter(loaded, { apiKey: '' })
+    expect(result.error).toContain('invalid options')
+    const readiness = result.adapter.readiness()
+    expect(readiness.ready).toBe(false)
+    await expect(result.adapter.search({ query: 'x', limit: 5 }, createTestContext())).resolves.toMatchObject({
+      status: 'unavailable',
+    })
+  })
+
+  it('stands in an unavailable adapter when the constructor throws', () => {
+    const exploding = {
+      packageName: 'pkg',
+      module: {
+        ...workingModule,
+        createAdapter: () => {
+          throw new Error('boom')
+        },
+      } as AdapterModule<unknown>,
+    }
+    const result = instantiateAdapter(exploding, { apiKey: 'k' })
+    expect(result.error).toContain('threw while being constructed')
+    expect(result.adapter.readiness().ready).toBe(false)
+  })
+
+  it('rejects an object that does not implement the adapter interface', () => {
+    const bogus = {
+      packageName: 'pkg',
+      module: { ...workingModule, createAdapter: () => ({ id: 'x' }) } as unknown as AdapterModule<unknown>,
+    }
+    const result = instantiateAdapter(bogus, { apiKey: 'k' })
+    expect(result.error).toContain('does not implement SearchAdapter')
+  })
+})

package/src/loader/describe.ts

@@ -0,0 +1,145 @@
+import type { AdapterModule } from '../contract/adapter'
+
+export type OptionFieldKind = 'string' | 'number' | 'boolean' | 'enum' | 'stringList'
+
+export type OptionField = {
+  readonly name: string
+  readonly kind: OptionFieldKind
+  readonly required: boolean
+  /** True when the value must never be echoed back to a browser. */
+  readonly secret: boolean
+  readonly choices?: readonly string[]
+  readonly format?: string
+  readonly min?: number
+  readonly max?: number
+}
+
+const SECRET_WORD = /(^|_)(api_?key|key|token|secret|password|passwd|credential)s?(_|$)/
+
+/**
+ * Whether a field name looks like a credential. The name is split at camelCase
+ * boundaries first, so `accessToken` and `access_token` are treated alike — a
+ * plain word-boundary match misses the camelCase spelling entirely.
+ */
+function isSecretName(name: string): boolean {
+  return SECRET_WORD.test(name.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase())
+}
+
+type ZodInternals = {
+  type?: string
+  innerType?: { def?: ZodInternals }
+  entries?: Record<string, unknown>
+  format?: string
+  element?: { def?: ZodInternals }
+  checks?: Array<{ _zod?: { def?: { check?: string; value?: number } } }>
+}
+
+type ZodLike = { def?: ZodInternals; shape?: Record<string, ZodLike> }
+
+function unwrap(schema: ZodLike): { inner: ZodInternals; required: boolean } {
+  let def = schema.def
+  let required = true
+  // `.optional()` / `.nullable()` / `.default()` wrap the real type.
+  while (def && (def.type === 'optional' || def.type === 'nullable' || def.type === 'default')) {
+    if (def.type !== 'nullable') required = false
+    def = def.innerType?.def
+  }
+  return { inner: def ?? {}, required }
+}
+
+function boundsOf(inner: ZodInternals): { min?: number; max?: number } {
+  const bounds: { min?: number; max?: number } = {}
+  for (const check of inner.checks ?? []) {
+    const def = check?._zod?.def
+    if (!def || typeof def.value !== 'number') continue
+    if (def.check === 'greater_than' || def.check === 'min_length') bounds.min = def.value
+    if (def.check === 'less_than' || def.check === 'max_length') bounds.max = def.value
+  }
+  return bounds
+}
+
+function kindOf(inner: ZodInternals): OptionFieldKind | null {
+  switch (inner.type) {
+    case 'string':
+      return 'string'
+    case 'number':
+    case 'int':
+      return 'number'
+    case 'boolean':
+      return 'boolean'
+    case 'enum':
+      return 'enum'
+    case 'array':
+      // Only string arrays are renderable as a plain comma-separated input.
+      return inner.element?.def?.type === 'string' ? 'stringList' : null
+    default:
+      // `custom` is how an adapter declares a host-supplied capability
+      // (a model resolver, a sidecar handle). Not user-configurable, so it must
+      // never reach the admin form.
+      return null
+  }
+}
+
+/**
+ * Describes an adapter's options as renderable fields, so the settings UI can
+ * build a form for an adapter it has never heard of — including one published by
+ * a third party — without the adapter shipping any UI of its own.
+ */
+export function describeOptionsSchema(module: AdapterModule<unknown>): OptionField[] {
+  const schema = module.optionsSchema as unknown as ZodLike
+  const shape = schema.shape ?? (schema.def as { shape?: Record<string, ZodLike> } | undefined)?.shape
+  if (!shape) return []
+
+  const fields: OptionField[] = []
+  for (const [name, entry] of Object.entries(shape)) {
+    const { inner, required } = unwrap(entry)
+    const kind = kindOf(inner)
+    if (kind === null) continue
+    const bounds = boundsOf(inner)
+    fields.push({
+      name,
+      kind,
+      required,
+      secret: isSecretName(name),
+      ...(kind === 'enum' && inner.entries ? { choices: Object.keys(inner.entries) } : {}),
+      ...(inner.format ? { format: inner.format } : {}),
+      ...(bounds.min === undefined ? {} : { min: bounds.min }),
+      ...(bounds.max === undefined ? {} : { max: bounds.max }),
+    })
+  }
+  return fields
+}
+
+/** Placeholder returned instead of a stored secret, and accepted back unchanged. */
+export const SECRET_PLACEHOLDER = '__om_secret_unchanged__'
+
+export function maskSecrets(
+  values: Record<string, unknown>,
+  fields: readonly OptionField[],
+): Record<string, unknown> {
+  const masked: Record<string, unknown> = { ...values }
+  for (const field of fields) {
+    if (!field.secret) continue
+    const value = masked[field.name]
+    if (typeof value === 'string' && value.length > 0) masked[field.name] = SECRET_PLACEHOLDER
+  }
+  return masked
+}
+
+/** Restores stored secrets the client echoed back as the placeholder. */
+export function unmaskSecrets(
+  incoming: Record<string, unknown>,
+  stored: Record<string, unknown>,
+  fields: readonly OptionField[],
+): Record<string, unknown> {
+  const merged: Record<string, unknown> = { ...incoming }
+  for (const field of fields) {
+    if (!field.secret) continue
+    if (merged[field.name] === SECRET_PLACEHOLDER) {
+      const previous = stored[field.name]
+      if (previous === undefined) delete merged[field.name]
+      else merged[field.name] = previous
+    }
+  }
+  return merged
+}

package/src/loader/index.ts

@@ -0,0 +1,25 @@
+export {
+  SECRET_PLACEHOLDER,
+  describeOptionsSchema,
+  maskSecrets,
+  unmaskSecrets,
+  type OptionField,
+  type OptionFieldKind,
+} from './describe'
+export {
+  ADAPTER_MANIFEST_KEY,
+  MANIFEST_NAMESPACE,
+  adapterManifestSchema,
+  readAdapterManifest,
+  type AdapterManifest,
+} from './manifest'
+export {
+  createUnavailableAdapter,
+  instantiateAdapter,
+  resolveAdapterModules,
+  type AdapterModuleRegistry,
+  type AdapterRegistryEntry,
+  type InstantiatedAdapter,
+  type LoadedAdapterModule,
+  type RejectedAdapterModule,
+} from './registry'

package/src/loader/manifest.ts

@@ -0,0 +1,44 @@
+import { z } from 'zod'
+
+/** `package.json` → `openMercato.webResearchAdapter`. */
+export const MANIFEST_NAMESPACE = 'openMercato'
+export const ADAPTER_MANIFEST_KEY = 'webResearchAdapter'
+
+export const adapterManifestSchema = z.object({
+  id: z.string().min(1).max(64),
+  /** Import specifier relative to the package root; defaults to the package main. */
+  entry: z.string().min(1).optional(),
+})
+
+export type AdapterManifest = z.infer<typeof adapterManifestSchema>
+
+type PackageJsonLike = {
+  readonly name?: unknown
+  readonly [MANIFEST_NAMESPACE]?: unknown
+}
+
+/**
+ * Reads the adapter manifest out of a parsed `package.json`. Returns null for any
+ * package that simply is not an adapter, and only reports a problem when a
+ * package claims to be one but declares it wrongly.
+ */
+export function readAdapterManifest(
+  packageJson: unknown,
+): { manifest: AdapterManifest; packageName: string } | { error: string } | null {
+  if (typeof packageJson !== 'object' || packageJson === null) return null
+  const parsed = packageJson as PackageJsonLike
+  const namespace = parsed[MANIFEST_NAMESPACE]
+  if (typeof namespace !== 'object' || namespace === null) return null
+  const raw = (namespace as Record<string, unknown>)[ADAPTER_MANIFEST_KEY]
+  if (raw === undefined) return null
+
+  const packageName = typeof parsed.name === 'string' ? parsed.name : ''
+  if (packageName.length === 0) {
+    return { error: 'package declares a web-research adapter but has no name' }
+  }
+  const result = adapterManifestSchema.safeParse(raw)
+  if (!result.success) {
+    return { error: `${packageName} has an invalid ${MANIFEST_NAMESPACE}.${ADAPTER_MANIFEST_KEY}: ${result.error.message}` }
+  }
+  return { manifest: result.data, packageName }
+}

package/src/loader/registry.ts

@@ -0,0 +1,156 @@
+import type { AdapterModule, SearchAdapter } from '../contract/adapter'
+import { notReady } from '../contract/adapter'
+import { errorMessage } from '../contract/errors'
+import { unavailable } from '../contract/outcomes'
+import { CONTRACT_VERSION, MIN_SUPPORTED_CONTRACT_VERSION, isSupportedContractVersion } from '../contract/version'
+
+export type AdapterRegistryEntry = {
+  readonly packageName: string
+  /** The statically imported module namespace or its default export. */
+  readonly module: unknown
+}
+
+export type LoadedAdapterModule = {
+  readonly packageName: string
+  readonly module: AdapterModule<unknown>
+}
+
+export type RejectedAdapterModule = {
+  readonly packageName: string
+  readonly id: string | null
+  readonly reason: string
+}
+
+export type AdapterModuleRegistry = {
+  readonly loaded: readonly LoadedAdapterModule[]
+  readonly rejected: readonly RejectedAdapterModule[]
+}
+
+const ADAPTER_KINDS = new Set(['serp', 'api', 'model', 'browser'])
+
+/**
+ * A stand-in for an adapter that could not be built. Returning one of these
+ * instead of throwing is what keeps a broken third-party package from taking the
+ * whole tool set down: the scheduler simply reports it as skipped, with the
+ * reason, alongside every adapter that did work.
+ */
+export function createUnavailableAdapter(id: string, reason: string): SearchAdapter {
+  return {
+    id,
+    kind: 'api',
+    capabilities: {
+      search: false,
+      fetch: false,
+      snippets: false,
+      content: false,
+      freshness: false,
+      siteFilter: false,
+      cost: 'free',
+    },
+    readiness: () => notReady(reason),
+    search: async () => unavailable(reason),
+    healthCheck: async () => ({ ok: false, detail: reason }),
+  }
+}
+
+function describeModule(candidate: unknown): AdapterModule<unknown> | string {
+  const unwrapped =
+    typeof candidate === 'object' &&
+    candidate !== null &&
+    'default' in candidate &&
+    !('createAdapter' in candidate)
+      ? (candidate as { default: unknown }).default
+      : candidate
+
+  if (typeof unwrapped !== 'object' || unwrapped === null) {
+    return 'module does not export an adapter descriptor'
+  }
+  const module = unwrapped as Partial<AdapterModule<unknown>>
+  if (typeof module.id !== 'string' || module.id.length === 0) return 'adapter descriptor has no id'
+  if (typeof module.kind !== 'string' || !ADAPTER_KINDS.has(module.kind)) {
+    return `adapter "${module.id}" declares an unknown kind`
+  }
+  if (typeof module.createAdapter !== 'function') {
+    return `adapter "${module.id}" does not expose createAdapter()`
+  }
+  if (
+    typeof module.optionsSchema !== 'object' ||
+    module.optionsSchema === null ||
+    typeof (module.optionsSchema as { safeParse?: unknown }).safeParse !== 'function'
+  ) {
+    return `adapter "${module.id}" does not expose a zod optionsSchema`
+  }
+  if (!isSupportedContractVersion(module.contractVersion)) {
+    return `adapter "${module.id}" targets contract version ${String(module.contractVersion)}, this engine speaks ${MIN_SUPPORTED_CONTRACT_VERSION}..${CONTRACT_VERSION}`
+  }
+  return module as AdapterModule<unknown>
+}
+
+/**
+ * Validates statically imported adapter modules. Discovery itself is done by the
+ * generator so imports stay static — a bundler cannot follow an `import()` with a
+ * runtime-computed specifier.
+ */
+export function resolveAdapterModules(entries: readonly AdapterRegistryEntry[]): AdapterModuleRegistry {
+  const loaded: LoadedAdapterModule[] = []
+  const rejected: RejectedAdapterModule[] = []
+  const seen = new Map<string, string>()
+
+  for (const entry of entries) {
+    const described = describeModule(entry.module)
+    if (typeof described === 'string') {
+      rejected.push({ packageName: entry.packageName, id: null, reason: described })
+      continue
+    }
+    const owner = seen.get(described.id)
+    if (owner !== undefined) {
+      rejected.push({
+        packageName: entry.packageName,
+        id: described.id,
+        reason: `adapter id "${described.id}" is already provided by ${owner}`,
+      })
+      continue
+    }
+    seen.set(described.id, entry.packageName)
+    loaded.push({ packageName: entry.packageName, module: described })
+  }
+
+  return { loaded, rejected }
+}
+
+export type InstantiatedAdapter = {
+  readonly id: string
+  readonly packageName: string
+  readonly adapter: SearchAdapter
+  /** Present when the adapter could not be built and a placeholder is standing in. */
+  readonly error?: string
+}
+
+/** Builds one adapter, converting any options or constructor fault into a placeholder. */
+export function instantiateAdapter(
+  entry: LoadedAdapterModule,
+  rawOptions: unknown,
+): InstantiatedAdapter {
+  const { module, packageName } = entry
+  const parsed = module.optionsSchema.safeParse(rawOptions ?? {})
+  if (!parsed.success) {
+    const reason = `invalid options for "${module.id}": ${parsed.error.message}`
+    return { id: module.id, packageName, adapter: createUnavailableAdapter(module.id, reason), error: reason }
+  }
+  try {
+    const adapter = module.createAdapter(parsed.data)
+    if (typeof adapter?.search !== 'function' || typeof adapter?.readiness !== 'function') {
+      const reason = `adapter "${module.id}" returned an object that does not implement SearchAdapter`
+      return {
+        id: module.id,
+        packageName,
+        adapter: createUnavailableAdapter(module.id, reason),
+        error: reason,
+      }
+    }
+    return { id: module.id, packageName, adapter }
+  } catch (error) {
+    const reason = `adapter "${module.id}" threw while being constructed: ${errorMessage(error)}`
+    return { id: module.id, packageName, adapter: createUnavailableAdapter(module.id, reason), error: reason }
+  }
+}