@agent-native/core 0.113.0 → 0.114.0

package/src/provider-api/corpus-jobs.ts

@@ -0,0 +1,1817 @@
+import { randomUUID } from "node:crypto";
+
+import { z } from "zod";
+
+import { defineAction } from "../action.js";
+import { getCredentialContext } from "../server/request-context.js";
+import {
+  appendProviderCorpusJobHits,
+  createProviderCorpusJob,
+  deleteProviderCorpusJob,
+  getProviderCorpusJob,
+  getProviderCorpusJobHits,
+  listProviderCorpusJobs,
+  updateProviderCorpusJob,
+  type ProviderCorpusJobRecord,
+  type ProviderCorpusJobStatus,
+} from "./corpus-jobs-store.js";
+import type { ProviderApiMethod, ProviderApiRequestArgs } from "./index.js";
+import { getStagedDatasetRows } from "./staged-datasets-store.js";
+
+type ProviderCorpusRuntime = {
+  executeRequest(args: ProviderApiRequestArgs): Promise<unknown>;
+};
+
+export interface CreateProviderCorpusJobActionOptions {
+  appId: string;
+  getRuntime: () => ProviderCorpusRuntime;
+}
+
+const MethodSchema = z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"]);
+
+const ProviderRequestSchema = z.object({
+  provider: z.string().min(1).describe("Configured provider API id."),
+  method: MethodSchema.default("GET").describe("HTTP method to use."),
+  path: z.string().min(1).describe("Provider API path or allowed full URL."),
+  query: z.unknown().optional().describe("Optional query parameters."),
+  headers: z.record(z.string(), z.unknown()).optional(),
+  body: z.unknown().optional(),
+  auth: z.enum(["default", "none"]).default("default"),
+  connectionId: z.string().trim().min(1).optional(),
+  accountId: z.string().optional(),
+  timeoutMs: z.coerce.number().int().min(1_000).max(120_000).optional(),
+  maxBytes: z.coerce
+    .number()
+    .int()
+    .min(1_000)
+    .max(4 * 1024 * 1024)
+    .optional(),
+});
+
+const PaginationSchema = z.object({
+  itemsPath: z
+    .string()
+    .optional()
+    .describe("Dot-path to records in each page, e.g. records.calls."),
+  nextCursorPath: z
+    .string()
+    .optional()
+    .describe("Dot-path to the next cursor in each response."),
+  cursorPath: z.string().optional().describe("Alias for nextCursorPath."),
+  cursorParam: z
+    .string()
+    .optional()
+    .describe("Query parameter that receives the next cursor."),
+  cursorBodyPath: z
+    .string()
+    .optional()
+    .describe("Request body path that receives the next cursor."),
+  pageParam: z
+    .string()
+    .optional()
+    .describe("Query parameter for page-number pagination."),
+  startPage: z.coerce.number().int().min(0).optional(),
+  offsetParam: z
+    .string()
+    .optional()
+    .describe("Query parameter for offset pagination."),
+  startOffset: z.coerce.number().int().min(0).optional(),
+  pageSize: z.coerce.number().int().min(1).optional(),
+  maxPages: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(2_000)
+    .optional()
+    .describe("Overall page cap for the job. Default 500."),
+});
+
+const BatchSchema = z.object({
+  items: z
+    .array(z.unknown())
+    .max(10_000)
+    .optional()
+    .describe("Input records or ids to process in batches."),
+  inputDatasetId: z
+    .string()
+    .optional()
+    .describe("Optional staged dataset id to read input records from."),
+  inputValuePath: z
+    .string()
+    .optional()
+    .describe("Dot-path to extract the provider id/value from each input row."),
+  batchSize: z.coerce.number().int().min(1).max(100).optional(),
+  itemBodyPath: z
+    .string()
+    .optional()
+    .describe("Dot-path in request body where the current batch array goes."),
+  itemQueryParam: z
+    .string()
+    .optional()
+    .describe("Query parameter where the current batch goes."),
+  responseItemsPath: z
+    .string()
+    .optional()
+    .describe("Dot-path to records in each batch response."),
+});
+
+const SearchSchema = z.object({
+  query: z.string().optional().describe("Phrase to search for."),
+  queries: z.array(z.string()).max(50).optional(),
+  terms: z
+    .array(z.string())
+    .max(50)
+    .optional()
+    .describe("Terms for allTerms/anyTerm search."),
+  regex: z.string().optional(),
+  regexFlags: z.string().optional(),
+  matchMode: z.enum(["query", "allTerms", "anyTerm"]).optional(),
+  caseSensitive: z.boolean().optional(),
+  textPaths: z
+    .array(z.string())
+    .max(100)
+    .optional()
+    .describe("Only search these dot-paths. Omit to search text recursively."),
+  idPaths: z.array(z.string()).max(100).optional(),
+  metadataPaths: z.array(z.string()).max(100).optional(),
+  contextChars: z.coerce.number().int().min(20).max(1_000).optional(),
+  maxHits: z.coerce.number().int().min(1).max(10_000).optional(),
+  maxHitsPerItem: z.coerce.number().int().min(1).max(100).optional(),
+  maxFieldsPerItem: z.coerce.number().int().min(1).max(50_000).optional(),
+});
+
+const LimitsSchema = z.object({
+  pageBudget: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(200)
+    .optional()
+    .describe("Pages to process in this action call. Default 25."),
+  batchBudget: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(500)
+    .optional()
+    .describe("Batches to process in this action call. Default 25."),
+  runtimeMs: z.coerce
+    .number()
+    .int()
+    .min(1_000)
+    .max(110_000)
+    .optional()
+    .describe("Wall-clock budget for this action call. Default 90000."),
+  itemBudget: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(200_000)
+    .optional()
+    .describe("Response records to search in this action call."),
+  maxHits: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(10_000)
+    .optional()
+    .describe("Stored hit cap for the whole job. Default comes from search."),
+});
+
+const ProviderCorpusJobSchema = z.object({
+  operation: z
+    .enum(["start", "continue", "status", "results", "list", "delete"])
+    .default("start"),
+  jobId: z
+    .string()
+    .optional()
+    .describe("Existing job id for continue/status/results/delete."),
+  name: z.string().optional().describe("Human-readable job name."),
+  mode: z.enum(["paginated-search", "batch-search"]).optional(),
+  request: ProviderRequestSchema.optional(),
+  pagination: PaginationSchema.optional(),
+  batch: BatchSchema.optional(),
+  search: SearchSchema.optional(),
+  limits: LimitsSchema.optional(),
+  offset: z.coerce
+    .number()
+    .int()
+    .min(0)
+    .optional()
+    .describe("Results/list offset."),
+  limit: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(1_000)
+    .optional()
+    .describe("Results/list limit."),
+});
+
+const ProviderCorpusJobReadSchema = z.object({
+  operation: z.enum(["list", "status", "results"]).default("list"),
+  jobId: z.string().optional().describe("Existing job id for status/results."),
+  offset: z.coerce.number().int().min(0).optional().describe("Results offset."),
+  limit: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(1_000)
+    .optional()
+    .describe("Results/list limit."),
+});
+
+type ProviderCorpusJobArgs = z.infer<typeof ProviderCorpusJobSchema>;
+type ProviderCorpusJobReadArgs = z.infer<typeof ProviderCorpusJobReadSchema>;
+type ProviderRequest = z.infer<typeof ProviderRequestSchema>;
+type PaginationConfig = z.infer<typeof PaginationSchema>;
+type BatchConfig = z.infer<typeof BatchSchema>;
+type SearchConfig = z.infer<typeof SearchSchema>;
+type LimitsConfig = z.infer<typeof LimitsSchema>;
+
+interface SearchHit {
+  id: string | null;
+  idPath: string | null;
+  itemIndex: number;
+  path: string;
+  kind: string;
+  query: string;
+  match: string;
+  snippet: string;
+  matchedTerms?: string[];
+  metadata?: Record<string, unknown>;
+  pageIndex?: number;
+  pageItemIndex?: number;
+  batchIndex?: number;
+  batchItemIndex?: number;
+}
+
+interface SearchPageResult {
+  searchedItems: number;
+  matchedItems: number;
+  totalHits: number;
+  storedHits: SearchHit[];
+}
+
+interface ProviderBodyResult {
+  body: unknown;
+  quota?: { retryAt: string; retryAfterMs: number; reason: string };
+}
+
+interface MatchRecord {
+  kind: string;
+  query: string;
+  index: number;
+  endIndex?: number;
+  match: string;
+  matchedTerms?: string[];
+  snippet?: string;
+}
+
+const DEFAULT_PAGE_BUDGET = 25;
+const DEFAULT_BATCH_BUDGET = 25;
+const DEFAULT_RUNTIME_MS = 90_000;
+const DEFAULT_BATCH_SIZE = 25;
+const DEFAULT_OVERALL_MAX_PAGES = 500;
+const DEFAULT_MAX_HITS = 500;
+
+export function createProviderCorpusJobAction(
+  options: CreateProviderCorpusJobActionOptions,
+) {
+  return defineAction({
+    description:
+      "Create, continue, inspect, list, read results from, or delete a durable provider corpus search job. " +
+      "Use this for broad provider searches, cross-source joins, transcript/message/ticket/issue/document scans, batch endpoint walks, and absence-sensitive questions that may exceed one tool call. " +
+      "The job is generic: paginated-search walks any paginated provider endpoint; batch-search walks a supplied id/record list or staged dataset through any provider endpoint by injecting each batch into a query param or request body path. " +
+      "Every provider request still goes through provider-api-request credentials, host allow-listing, SSRF blocking, secret redaction, and provider quota cooldown. " +
+      "When status is paused or quota_wait, call operation=continue with the jobId after the indicated time/budget. Report coverage counts, pagination status, and any remaining gaps.",
+    schema: ProviderCorpusJobSchema,
+    http: false,
+    run: async (args) => runProviderCorpusJobAction(args, options),
+  });
+}
+
+export function createProviderCorpusJobReadAction(
+  options: Pick<CreateProviderCorpusJobActionOptions, "appId">,
+) {
+  return defineAction({
+    description:
+      "Read provider corpus job status and stored results for the current user. " +
+      "This read-only companion is for UI polling and status surfaces; it cannot start or continue provider requests.",
+    schema: ProviderCorpusJobReadSchema,
+    http: { method: "GET" },
+    readOnly: true,
+    agentTool: false,
+    run: async (args) => runProviderCorpusJobReadAction(args, options),
+  });
+}
+
+async function runProviderCorpusJobAction(
+  args: ProviderCorpusJobArgs,
+  options: CreateProviderCorpusJobActionOptions,
+) {
+  const ctx = getCredentialContext();
+  if (!ctx)
+    throw new Error("No authenticated context for provider-corpus-job.");
+
+  if (args.operation === "list") {
+    const jobs = await listProviderCorpusJobs({
+      appId: options.appId,
+      ownerEmail: ctx.userEmail,
+      limit: args.limit,
+    });
+    return { jobs: jobs.map(jobSummary), total: jobs.length };
+  }
+
+  if (args.operation === "start") {
+    const job = await startJob(args, options.appId, ctx.userEmail);
+    return continueJob(job, args.limits, options.getRuntime());
+  }
+
+  const jobId = args.jobId?.trim();
+  if (!jobId) throw new Error(`${args.operation} requires jobId.`);
+
+  if (args.operation === "delete") {
+    const deleted = await deleteProviderCorpusJob({
+      id: jobId,
+      appId: options.appId,
+      ownerEmail: ctx.userEmail,
+    });
+    if (!deleted) {
+      throw new Error(
+        `Provider corpus job ${jobId} not found or belongs to another owner/app.`,
+      );
+    }
+    return { deleted: true, jobId };
+  }
+
+  const job = await getProviderCorpusJob({
+    id: jobId,
+    appId: options.appId,
+    ownerEmail: ctx.userEmail,
+  });
+  if (!job) {
+    throw new Error(
+      `Provider corpus job ${jobId} not found or belongs to another owner/app.`,
+    );
+  }
+
+  if (args.operation === "status") return jobStatus(job);
+  if (args.operation === "results") {
+    const hits = await getProviderCorpusJobHits({
+      jobId,
+      appId: options.appId,
+      ownerEmail: ctx.userEmail,
+      offset: args.offset,
+      limit: args.limit,
+    });
+    return {
+      ...jobStatus(job),
+      hits,
+      offset: args.offset ?? 0,
+      limit: args.limit ?? 100,
+    };
+  }
+  if (args.operation === "continue") {
+    return continueJob(job, args.limits, options.getRuntime());
+  }
+
+  throw new Error(`Unsupported provider corpus operation ${args.operation}.`);
+}
+
+async function runProviderCorpusJobReadAction(
+  args: ProviderCorpusJobReadArgs,
+  options: Pick<CreateProviderCorpusJobActionOptions, "appId">,
+) {
+  const ctx = getCredentialContext();
+  if (!ctx)
+    throw new Error("No authenticated context for provider corpus jobs.");
+
+  if (args.operation === "list") {
+    const jobs = await listProviderCorpusJobs({
+      appId: options.appId,
+      ownerEmail: ctx.userEmail,
+      limit: args.limit,
+    });
+    return { jobs: jobs.map(jobStatus), total: jobs.length };
+  }
+
+  const jobId = args.jobId?.trim();
+  if (!jobId) throw new Error(`${args.operation} requires jobId.`);
+
+  const job = await getProviderCorpusJob({
+    id: jobId,
+    appId: options.appId,
+    ownerEmail: ctx.userEmail,
+  });
+  if (!job) {
+    throw new Error(
+      `Provider corpus job ${jobId} not found or belongs to another owner/app.`,
+    );
+  }
+
+  if (args.operation === "status") return jobStatus(job);
+  const hits = await getProviderCorpusJobHits({
+    jobId,
+    appId: options.appId,
+    ownerEmail: ctx.userEmail,
+    offset: args.offset,
+    limit: args.limit,
+  });
+  return {
+    ...jobStatus(job),
+    hits,
+    offset: args.offset ?? 0,
+    limit: args.limit ?? 100,
+  };
+}
+
+async function startJob(
+  args: ProviderCorpusJobArgs,
+  appId: string,
+  ownerEmail: string,
+): Promise<ProviderCorpusJobRecord> {
+  if (!args.mode) throw new Error("start requires mode.");
+  if (!args.request) throw new Error("start requires request.");
+  const search = args.search ?? {};
+  if (!hasSearchNeedle(search)) {
+    throw new Error(
+      "start requires search.query, search.queries, search.terms, or search.regex.",
+    );
+  }
+  if (args.mode === "paginated-search" && !args.pagination) {
+    throw new Error("paginated-search requires pagination.");
+  }
+  if (args.mode === "batch-search") {
+    if (!args.batch) throw new Error("batch-search requires batch config.");
+    if (!args.batch.itemBodyPath && !args.batch.itemQueryParam) {
+      throw new Error(
+        "batch-search requires batch.itemBodyPath or batch.itemQueryParam.",
+      );
+    }
+    if (!args.batch.items?.length && !args.batch.inputDatasetId) {
+      throw new Error("batch-search requires batch.items or inputDatasetId.");
+    }
+  }
+
+  const id = args.jobId?.trim() || `pcj_${randomUUID().replace(/-/g, "")}`;
+  const request = cleanRequest(args.request);
+  const checkpoint =
+    args.mode === "batch-search"
+      ? { nextIndex: 0, sourceItemCount: null }
+      : {
+          pageIndex: 0,
+          cursor: null,
+          pageNumber: args.pagination?.startPage ?? 1,
+          offset: args.pagination?.startOffset ?? 0,
+          lastCursor: null,
+          stoppedReason: null,
+        };
+
+  return createProviderCorpusJob({
+    id,
+    appId,
+    ownerEmail,
+    name: args.name?.trim() || id,
+    mode: args.mode,
+    status: "paused",
+    provider: request.provider,
+    request: request as unknown as Record<string, unknown>,
+    pagination:
+      (args.pagination as Record<string, unknown> | undefined) ?? null,
+    batch: (args.batch as Record<string, unknown> | undefined) ?? null,
+    search: search as Record<string, unknown>,
+    limits: (args.limits as Record<string, unknown> | undefined) ?? {},
+    checkpoint,
+  });
+}
+
+async function continueJob(
+  job: ProviderCorpusJobRecord,
+  overrideLimits: LimitsConfig | undefined,
+  runtime: ProviderCorpusRuntime,
+) {
+  if (job.status === "completed") return jobStatus(job);
+
+  const limits = normalizeLimits({
+    ...(job.limits as LimitsConfig),
+    ...(overrideLimits ?? {}),
+  });
+  await updateProviderCorpusJob({
+    id: job.id,
+    appId: job.appId,
+    ownerEmail: job.ownerEmail,
+    status: "running",
+    error: null,
+    nextResumeAt: null,
+  });
+
+  try {
+    const refreshed = await getProviderCorpusJob({
+      id: job.id,
+      appId: job.appId,
+      ownerEmail: job.ownerEmail,
+    });
+    if (!refreshed)
+      throw new Error(`Provider corpus job ${job.id} disappeared.`);
+    const next =
+      refreshed.mode === "batch-search"
+        ? await runBatchSearch(refreshed, limits, runtime)
+        : await runPaginatedSearch(refreshed, limits, runtime);
+    const hits = await getProviderCorpusJobHits({
+      jobId: next.id,
+      appId: next.appId,
+      ownerEmail: next.ownerEmail,
+      limit: 25,
+    });
+    return {
+      ...jobStatus(next),
+      sampleHits: hits,
+      nextAction: nextAction(next),
+    };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const failed =
+      (await updateProviderCorpusJob({
+        id: job.id,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+        status: "failed",
+        error: message,
+      })) ?? job;
+    return { ...jobStatus(failed), nextAction: nextAction(failed) };
+  }
+}
+
+async function runPaginatedSearch(
+  job: ProviderCorpusJobRecord,
+  limits: Required<LimitsConfig>,
+  runtime: ProviderCorpusRuntime,
+): Promise<ProviderCorpusJobRecord> {
+  const startedAt = Date.now();
+  const pagination = normalizePagination(job.pagination);
+  const request = job.request as ProviderRequest;
+  const search = job.search as SearchConfig;
+  const checkpoint = { ...job.checkpoint };
+  const maxPages = pagination.maxPages ?? DEFAULT_OVERALL_MAX_PAGES;
+  let pageIndex = numberValue(checkpoint.pageIndex, 0);
+  let cursor = valueOrNull(checkpoint.cursor);
+  let pageNumber = numberValue(
+    checkpoint.pageNumber,
+    pagination.startPage ?? 1,
+  );
+  let offset = numberValue(checkpoint.offset, pagination.startOffset ?? 0);
+  let pagesThisCall = 0;
+  let itemsThisCall = 0;
+  let current = job;
+
+  while (
+    pagesThisCall < limits.pageBudget &&
+    pageIndex < maxPages &&
+    itemsThisCall < limits.itemBudget &&
+    Date.now() - startedAt < limits.runtimeMs
+  ) {
+    const pageRequest = buildPaginatedRequest(request, pagination, {
+      pageIndex,
+      cursor,
+      pageNumber,
+      offset,
+    });
+    const provider = await callProvider(runtime, pageRequest);
+    if (provider.quota) {
+      return pauseForQuota(
+        current,
+        {
+          ...checkpoint,
+          pageIndex,
+          cursor,
+          pageNumber,
+          offset,
+        },
+        provider.quota,
+      );
+    }
+
+    const items = extractItemsArray(provider.body, pagination.itemsPath, false);
+    const nextCursorPath = pagination.nextCursorPath || pagination.cursorPath;
+    const nextCursor = nextCursorPath
+      ? valueOrNull(getAtPath(provider.body, nextCursorPath))
+      : null;
+    const searchResult = searchItems(items, search, {
+      baseItemIndex: current.itemsProcessed,
+      maxStoredHits: Math.max(0, limits.maxHits - current.storedHits),
+      pageIndex,
+    });
+    const storedHits = searchResult.storedHits.map((hit) => hitToRecord(hit));
+    await appendProviderCorpusJobHits({
+      jobId: job.id,
+      startIndex: current.storedHits,
+      hits: storedHits,
+    });
+
+    pagesThisCall++;
+    itemsThisCall += searchResult.searchedItems;
+    const nextCheckpoint: Record<string, unknown> = {
+      ...checkpoint,
+      pageIndex: pageIndex + 1,
+      cursor: nextCursor,
+      pageNumber,
+      offset,
+      lastCursor: nextCursor,
+      stoppedReason: null,
+    };
+
+    let status: ProviderCorpusJobStatus = "paused";
+    let stoppedReason: string | null = null;
+    if (items.length === 0) {
+      status = "completed";
+      stoppedReason = "empty-page";
+    } else if (nextCursorPath) {
+      if (!nextCursor) {
+        status = "completed";
+        stoppedReason = "no-next-cursor";
+      } else if (cursor !== null && String(nextCursor) === String(cursor)) {
+        status = "completed";
+        stoppedReason = "repeated-cursor";
+      } else {
+        cursor = nextCursor;
+      }
+    } else if (pagination.pageParam) {
+      pageNumber += 1;
+      nextCheckpoint.pageNumber = pageNumber;
+    } else if (pagination.offsetParam) {
+      const step = pagination.pageSize || items.length;
+      offset += step;
+      nextCheckpoint.offset = offset;
+      if (pagination.pageSize && items.length < pagination.pageSize) {
+        status = "completed";
+        stoppedReason = "short-page";
+      }
+    } else {
+      status = "completed";
+      stoppedReason = "single-page";
+    }
+
+    if (pageIndex + 1 >= maxPages && status !== "completed") {
+      status = "completed";
+      stoppedReason = "max-pages";
+    }
+
+    nextCheckpoint.stoppedReason = stoppedReason;
+    current =
+      (await updateProviderCorpusJob({
+        id: job.id,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+        status,
+        checkpoint: nextCheckpoint,
+        pagesProcessed: current.pagesProcessed + 1,
+        itemsProcessed: current.itemsProcessed + searchResult.searchedItems,
+        matchedItems: current.matchedItems + searchResult.matchedItems,
+        totalHits: current.totalHits + searchResult.totalHits,
+        storedHits: current.storedHits + storedHits.length,
+        error: null,
+        nextResumeAt: null,
+      })) ?? current;
+
+    pageIndex += 1;
+    if (status === "completed") return current;
+  }
+
+  return pauseForBudget(current, {
+    ...checkpoint,
+    pageIndex,
+    cursor,
+    pageNumber,
+    offset,
+    stoppedReason: "budget",
+  });
+}
+
+async function runBatchSearch(
+  job: ProviderCorpusJobRecord,
+  limits: Required<LimitsConfig>,
+  runtime: ProviderCorpusRuntime,
+): Promise<ProviderCorpusJobRecord> {
+  const startedAt = Date.now();
+  const batch = normalizeBatch(job.batch);
+  const request = job.request as ProviderRequest;
+  const search = job.search as SearchConfig;
+  const sourceItems = await loadBatchSourceItems(job, batch);
+  const totalSourceItems = sourceItems.length;
+  let nextIndex = numberValue(job.checkpoint.nextIndex, 0);
+  let batchesThisCall = 0;
+  let itemsThisCall = 0;
+  let current = job;
+
+  while (
+    nextIndex < totalSourceItems &&
+    batchesThisCall < limits.batchBudget &&
+    itemsThisCall < limits.itemBudget &&
+    Date.now() - startedAt < limits.runtimeMs
+  ) {
+    const slice = sourceItems.slice(nextIndex, nextIndex + batch.batchSize);
+    const values = slice
+      .map((item) =>
+        batch.inputValuePath ? getAtPath(item, batch.inputValuePath) : item,
+      )
+      .filter((value) => value !== undefined && value !== null);
+    if (values.length === 0) {
+      nextIndex += slice.length || batch.batchSize;
+      continue;
+    }
+
+    const batchRequest = buildBatchRequest(request, batch, values);
+    const provider = await callProvider(runtime, batchRequest);
+    if (provider.quota) {
+      return pauseForQuota(
+        current,
+        {
+          ...job.checkpoint,
+          nextIndex,
+          sourceItemCount: totalSourceItems,
+        },
+        provider.quota,
+      );
+    }
+
+    const items = extractItemsArray(
+      provider.body,
+      batch.responseItemsPath,
+      true,
+    );
+    const searchResult = searchItems(items, search, {
+      baseItemIndex: current.itemsProcessed,
+      maxStoredHits: Math.max(0, limits.maxHits - current.storedHits),
+      batchIndex: current.batchesProcessed,
+    });
+    const storedHits = searchResult.storedHits.map((hit) => hitToRecord(hit));
+    await appendProviderCorpusJobHits({
+      jobId: job.id,
+      startIndex: current.storedHits,
+      hits: storedHits,
+    });
+
+    batchesThisCall++;
+    itemsThisCall += searchResult.searchedItems;
+    nextIndex += slice.length;
+    const status: ProviderCorpusJobStatus =
+      nextIndex >= totalSourceItems ? "completed" : "paused";
+
+    current =
+      (await updateProviderCorpusJob({
+        id: job.id,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+        status,
+        checkpoint: {
+          ...job.checkpoint,
+          nextIndex,
+          sourceItemCount: totalSourceItems,
+          stoppedReason: status === "completed" ? "completed" : "budget",
+        },
+        batchesProcessed: current.batchesProcessed + 1,
+        itemsProcessed: current.itemsProcessed + searchResult.searchedItems,
+        matchedItems: current.matchedItems + searchResult.matchedItems,
+        totalHits: current.totalHits + searchResult.totalHits,
+        storedHits: current.storedHits + storedHits.length,
+        error: null,
+        nextResumeAt: null,
+      })) ?? current;
+
+    if (status === "completed") return current;
+  }
+
+  return pauseForBudget(current, {
+    ...job.checkpoint,
+    nextIndex,
+    sourceItemCount: totalSourceItems,
+    stoppedReason: "budget",
+  });
+}
+
+async function pauseForQuota(
+  job: ProviderCorpusJobRecord,
+  checkpoint: Record<string, unknown>,
+  quota: { retryAt: string; retryAfterMs: number; reason: string },
+): Promise<ProviderCorpusJobRecord> {
+  return (
+    (await updateProviderCorpusJob({
+      id: job.id,
+      appId: job.appId,
+      ownerEmail: job.ownerEmail,
+      status: "quota_wait",
+      checkpoint,
+      error: `Provider quota exhausted. Retry after ${quota.retryAt}.`,
+      nextResumeAt: parseRetryAt(quota.retryAt, quota.retryAfterMs),
+    })) ?? job
+  );
+}
+
+async function pauseForBudget(
+  job: ProviderCorpusJobRecord,
+  checkpoint: Record<string, unknown>,
+): Promise<ProviderCorpusJobRecord> {
+  return (
+    (await updateProviderCorpusJob({
+      id: job.id,
+      appId: job.appId,
+      ownerEmail: job.ownerEmail,
+      status: "paused",
+      checkpoint,
+      error: null,
+      nextResumeAt: null,
+    })) ?? job
+  );
+}
+
+async function callProvider(
+  runtime: ProviderCorpusRuntime,
+  request: ProviderRequest,
+): Promise<ProviderBodyResult> {
+  const raw = (await runtime.executeRequest(cleanRequest(request))) as Record<
+    string,
+    unknown
+  >;
+  const response = raw.response as Record<string, unknown> | undefined;
+  if (!response) {
+    return { body: raw };
+  }
+  const quota = readQuota(response);
+  if (quota) return { body: null, quota };
+  if (response.ok !== true) {
+    const status = response.status ?? "unknown";
+    const detail = response.text ?? response.json ?? response.statusText ?? "";
+    throw new Error(
+      `Provider request failed (${status}): ${stringifyBrief(detail, 500)}`,
+    );
+  }
+  return {
+    body:
+      response.json ??
+      (typeof response.text === "string" ? tryParseJson(response.text) : null),
+  };
+}
+
+async function loadBatchSourceItems(
+  job: ProviderCorpusJobRecord,
+  batch: Required<BatchConfig>,
+): Promise<unknown[]> {
+  if (batch.items.length > 0) return batch.items;
+  if (!batch.inputDatasetId) return [];
+  return getStagedDatasetRows({
+    id: batch.inputDatasetId,
+    appId: job.appId,
+    ownerEmail: job.ownerEmail,
+  });
+}
+
+function cleanRequest(request: ProviderRequest): ProviderApiRequestArgs {
+  return {
+    provider: request.provider,
+    method: request.method as ProviderApiMethod,
+    path: request.path,
+    query: request.query,
+    headers: request.headers,
+    body: request.body,
+    auth: request.auth,
+    connectionId: request.connectionId,
+    accountId: request.accountId,
+    timeoutMs: request.timeoutMs,
+    maxBytes: request.maxBytes,
+  };
+}
+
+function buildPaginatedRequest(
+  request: ProviderRequest,
+  pagination: Required<PaginationConfig>,
+  state: {
+    pageIndex: number;
+    cursor: unknown;
+    pageNumber: number;
+    offset: number;
+  },
+): ProviderRequest {
+  let query = cloneRecord(request.query);
+  let body = request.body;
+  if (pagination.pageParam) {
+    query = { ...query, [pagination.pageParam]: state.pageNumber };
+  }
+  if (pagination.offsetParam) {
+    query = { ...query, [pagination.offsetParam]: state.offset };
+  }
+  if (state.pageIndex > 0 && state.cursor != null) {
+    if (pagination.cursorParam) {
+      query = { ...query, [pagination.cursorParam]: state.cursor };
+    }
+    if (pagination.cursorBodyPath) {
+      body = setAtPath(body, pagination.cursorBodyPath, state.cursor);
+    }
+  }
+  return { ...request, query, body };
+}
+
+function buildBatchRequest(
+  request: ProviderRequest,
+  batch: Required<BatchConfig>,
+  values: unknown[],
+): ProviderRequest {
+  let query = cloneRecord(request.query);
+  let body = request.body;
+  const payload = values.length === 1 ? values[0] : values;
+  if (batch.itemQueryParam) {
+    query = { ...query, [batch.itemQueryParam]: payload };
+  }
+  if (batch.itemBodyPath) {
+    body = setAtPath(body, batch.itemBodyPath, values);
+  }
+  return { ...request, query, body };
+}
+
+function searchItems(
+  items: unknown[],
+  search: SearchConfig,
+  options: {
+    baseItemIndex: number;
+    maxStoredHits: number;
+    pageIndex?: number;
+    batchIndex?: number;
+  },
+): SearchPageResult {
+  const maxStoredHits = Math.max(0, options.maxStoredHits);
+  const maxHitsPerItem = boundedNumber(search.maxHitsPerItem, 3, 1, 100);
+  const maxFieldsPerItem = boundedNumber(
+    search.maxFieldsPerItem,
+    5_000,
+    1,
+    50_000,
+  );
+  let matchedItems = 0;
+  let totalHits = 0;
+  const storedHits: SearchHit[] = [];
+
+  for (let itemOffset = 0; itemOffset < items.length; itemOffset++) {
+    const item = items[itemOffset];
+    const fields = collectSearchStrings(
+      item,
+      search.textPaths,
+      maxFieldsPerItem,
+    );
+    const identity = extractItemIdentity(item, search.idPaths);
+    const metadata = extractMetadata(item, search.metadataPaths);
+    let itemMatched = false;
+    let storedForItem = 0;
+    const itemWideTermMatch = findItemWideTermMatch(fields, search);
+
+    const addHit = (field: TextField, match: MatchRecord) => {
+      totalHits++;
+      if (!itemMatched) {
+        matchedItems++;
+        itemMatched = true;
+      }
+      if (storedHits.length < maxStoredHits && storedForItem < maxHitsPerItem) {
+        storedForItem++;
+        storedHits.push({
+          id: identity.id,
+          idPath: identity.idPath,
+          itemIndex: options.baseItemIndex + itemOffset,
+          pageIndex: options.pageIndex,
+          pageItemIndex:
+            options.pageIndex === undefined ? undefined : itemOffset,
+          batchIndex: options.batchIndex,
+          batchItemIndex:
+            options.batchIndex === undefined ? undefined : itemOffset,
+          path: field.path,
+          kind: match.kind,
+          query: match.query,
+          match: match.match,
+          snippet:
+            match.snippet ??
+            makeSnippet(
+              field.text,
+              match.index,
+              search.contextChars,
+              match.endIndex,
+            ),
+          ...(match.matchedTerms?.length
+            ? { matchedTerms: match.matchedTerms }
+            : {}),
+          ...(Object.keys(metadata).length ? { metadata } : {}),
+        });
+      }
+    };
+
+    if (itemWideTermMatch) {
+      addHit(itemWideTermMatch.field, itemWideTermMatch.match);
+    }
+
+    for (const field of fields) {
+      const matches = findSearchMatches(field.text, search, !itemWideTermMatch);
+      for (const match of matches) addHit(field, match);
+    }
+  }
+
+  return {
+    searchedItems: items.length,
+    matchedItems,
+    totalHits,
+    storedHits,
+  };
+}
+
+function normalizeLimits(input: LimitsConfig): Required<LimitsConfig> {
+  const maxHits = input.maxHits ?? DEFAULT_MAX_HITS;
+  return {
+    pageBudget: input.pageBudget ?? DEFAULT_PAGE_BUDGET,
+    batchBudget: input.batchBudget ?? DEFAULT_BATCH_BUDGET,
+    runtimeMs: input.runtimeMs ?? DEFAULT_RUNTIME_MS,
+    itemBudget: input.itemBudget ?? 200_000,
+    maxHits,
+  };
+}
+
+function normalizePagination(
+  input: Record<string, unknown> | null,
+): Required<PaginationConfig> {
+  const parsed = PaginationSchema.parse(input ?? {});
+  return {
+    itemsPath: parsed.itemsPath ?? "",
+    nextCursorPath: parsed.nextCursorPath ?? "",
+    cursorPath: parsed.cursorPath ?? "",
+    cursorParam: parsed.cursorParam ?? "",
+    cursorBodyPath: parsed.cursorBodyPath ?? "",
+    pageParam: parsed.pageParam ?? "",
+    startPage: parsed.startPage ?? 1,
+    offsetParam: parsed.offsetParam ?? "",
+    startOffset: parsed.startOffset ?? 0,
+    pageSize: parsed.pageSize ?? 0,
+    maxPages: parsed.maxPages ?? DEFAULT_OVERALL_MAX_PAGES,
+  };
+}
+
+function normalizeBatch(
+  input: Record<string, unknown> | null,
+): Required<BatchConfig> {
+  const parsed = BatchSchema.parse(input ?? {});
+  return {
+    items: parsed.items ?? [],
+    inputDatasetId: parsed.inputDatasetId ?? "",
+    inputValuePath: parsed.inputValuePath ?? "",
+    batchSize: parsed.batchSize ?? DEFAULT_BATCH_SIZE,
+    itemBodyPath: parsed.itemBodyPath ?? "",
+    itemQueryParam: parsed.itemQueryParam ?? "",
+    responseItemsPath: parsed.responseItemsPath ?? "",
+  };
+}
+
+function jobStatus(job: ProviderCorpusJobRecord) {
+  const nextResumeAtIso = job.nextResumeAt
+    ? new Date(job.nextResumeAt).toISOString()
+    : null;
+  return {
+    job: jobSummary(job),
+    source: jobSourceSummary(job),
+    coverage: {
+      pagesProcessed: job.pagesProcessed,
+      batchesProcessed: job.batchesProcessed,
+      itemsProcessed: job.itemsProcessed,
+      matchedItems: job.matchedItems,
+      totalHits: job.totalHits,
+      storedHits: job.storedHits,
+      truncatedHits: job.totalHits > job.storedHits,
+    },
+    checkpoint: job.checkpoint,
+    error: job.error,
+    nextResumeAt: nextResumeAtIso,
+  };
+}
+
+function jobSourceSummary(job: ProviderCorpusJobRecord) {
+  const request = job.request as Partial<ProviderRequest>;
+  const pagination = (job.pagination ?? {}) as Partial<PaginationConfig>;
+  const batch = (job.batch ?? {}) as Partial<BatchConfig>;
+  const search = (job.search ?? {}) as Partial<SearchConfig>;
+  return {
+    provider: job.provider,
+    mode: job.mode,
+    request: {
+      method: request.method ?? "GET",
+      path: request.path ?? "",
+    },
+    pagination: {
+      itemsPath: pagination.itemsPath ?? null,
+      nextCursorPath:
+        pagination.nextCursorPath ?? pagination.cursorPath ?? null,
+      cursorParam: pagination.cursorParam ?? null,
+      cursorBodyPath: pagination.cursorBodyPath ?? null,
+      pageParam: pagination.pageParam ?? null,
+      offsetParam: pagination.offsetParam ?? null,
+    },
+    batch: {
+      inputDatasetId: batch.inputDatasetId ?? null,
+      inputValuePath: batch.inputValuePath ?? null,
+      batchSize: batch.batchSize ?? null,
+      itemBodyPath: batch.itemBodyPath ?? null,
+      itemQueryParam: batch.itemQueryParam ?? null,
+      responseItemsPath: batch.responseItemsPath ?? null,
+      sourceItemCount:
+        typeof job.checkpoint.sourceItemCount === "number"
+          ? job.checkpoint.sourceItemCount
+          : null,
+    },
+    search: {
+      textPaths: search.textPaths ?? [],
+      idPaths: search.idPaths ?? [],
+      metadataPaths: search.metadataPaths ?? [],
+      matchMode: search.matchMode ?? null,
+      queryCount:
+        (search.query ? 1 : 0) +
+        (search.regex ? 1 : 0) +
+        (search.queries?.length ?? 0) +
+        (search.terms?.length ?? 0),
+    },
+  };
+}
+
+function jobSummary(job: ProviderCorpusJobRecord) {
+  return {
+    id: job.id,
+    name: job.name,
+    mode: job.mode,
+    status: job.status,
+    provider: job.provider,
+    createdAt: new Date(job.createdAt).toISOString(),
+    updatedAt: new Date(job.updatedAt).toISOString(),
+  };
+}
+
+function nextAction(job: ProviderCorpusJobRecord): string {
+  if (job.status === "completed") {
+    return `Read all stored hits with operation="results", jobId="${job.id}".`;
+  }
+  if (job.status === "quota_wait") {
+    const at = job.nextResumeAt
+      ? new Date(job.nextResumeAt).toISOString()
+      : "later";
+    return `Provider quota is cooling down. Continue this job after ${at} with operation="continue", jobId="${job.id}".`;
+  }
+  if (job.status === "paused") {
+    return `Continue this job with operation="continue", jobId="${job.id}" until completed or quota_wait.`;
+  }
+  if (job.status === "failed") {
+    return 'Fix the request/configuration or start a new job; progress and stored hits are still inspectable with operation="results".';
+  }
+  return `Check status with operation="status", jobId="${job.id}".`;
+}
+
+function extractItemsArray(
+  body: unknown,
+  itemsPath: string | undefined,
+  wrapObjectFallback: boolean,
+): unknown[] {
+  if (Array.isArray(body)) return body;
+  if (!body || typeof body !== "object") return [];
+  if (itemsPath) {
+    const found = getAtPath(body, itemsPath);
+    if (Array.isArray(found)) return found;
+    if (found !== undefined && wrapObjectFallback) return [found];
+    return [];
+  }
+  const obj = body as Record<string, unknown>;
+  for (const key of [
+    "data",
+    "results",
+    "items",
+    "records",
+    "rows",
+    "calls",
+    "callTranscripts",
+    "transcripts",
+    "messages",
+    "tickets",
+    "issues",
+    "deals",
+    "events",
+    "notes",
+    "documents",
+    "entries",
+    "objects",
+  ]) {
+    if (Array.isArray(obj[key])) return obj[key] as unknown[];
+  }
+  const arrayFields = Object.values(obj).filter(Array.isArray);
+  if (arrayFields.length === 1) return arrayFields[0] as unknown[];
+  return wrapObjectFallback ? [body] : [];
+}
+
+function hasSearchNeedle(search: SearchConfig): boolean {
+  return Boolean(
+    search.query ||
+    search.regex ||
+    search.queries?.length ||
+    search.terms?.length,
+  );
+}
+
+function readQuota(response: Record<string, unknown>) {
+  const quota = response.quota as Record<string, unknown> | undefined;
+  if (quota?.exhausted === true && typeof quota.retryAt === "string") {
+    return {
+      retryAt: quota.retryAt,
+      retryAfterMs: numberValue(quota.retryAfterMs, 0),
+      reason: String(quota.reason ?? "quota"),
+    };
+  }
+  const json = response.json as Record<string, unknown> | undefined;
+  if (
+    json?.error === "provider_quota_exhausted" &&
+    typeof json.retryAt === "string"
+  ) {
+    return {
+      retryAt: json.retryAt,
+      retryAfterMs: numberValue(json.retryAfterMs, 0),
+      reason: String(json.reason ?? "quota"),
+    };
+  }
+  return null;
+}
+
+function hitToRecord(hit: SearchHit): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(hit).filter(([, value]) => value !== undefined),
+  );
+}
+
+function tryParseJson(text: string): unknown {
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    return text;
+  }
+}
+
+function stringifyBrief(value: unknown, maxChars: number): string {
+  const text = typeof value === "string" ? value : JSON.stringify(value);
+  return text.slice(0, maxChars);
+}
+
+function cloneRecord(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return JSON.parse(JSON.stringify(value)) as Record<string, unknown>;
+}
+
+function valueOrNull(value: unknown): unknown | null {
+  if (
+    value === undefined ||
+    value === null ||
+    value === "" ||
+    value === false
+  ) {
+    return null;
+  }
+  return value;
+}
+
+function numberValue(value: unknown, fallback: number): number {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function parseRetryAt(retryAt: string, retryAfterMs: number): number {
+  const parsed = Date.parse(retryAt);
+  if (Number.isFinite(parsed)) return parsed;
+  return Date.now() + Math.max(0, retryAfterMs);
+}
+
+function boundedNumber(
+  value: unknown,
+  fallback: number,
+  min: number,
+  max: number,
+): number {
+  const parsed = Number(value);
+  const finite = Number.isFinite(parsed) ? parsed : fallback;
+  return Math.max(min, Math.min(max, finite));
+}
+
+function pathParts(path: string | undefined): string[] {
+  if (!path) return [];
+  return path
+    .replace(/\[(\d+)\]/g, ".$1")
+    .split(".")
+    .map((part) => part.trim())
+    .filter(Boolean);
+}
+
+function getAtPath(value: unknown, path: string | undefined): unknown {
+  let current = value;
+  for (const part of pathParts(path)) {
+    if (current === undefined || current === null) return undefined;
+    if (typeof current !== "object") return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}
+
+function setAtPath(base: unknown, path: string, value: unknown): unknown {
+  const parts = pathParts(path);
+  if (!parts.length) return value;
+  const root =
+    base && typeof base === "object" && !Array.isArray(base)
+      ? (JSON.parse(JSON.stringify(base)) as Record<string, unknown>)
+      : {};
+  let current = root;
+  for (const part of parts.slice(0, -1)) {
+    const existing = current[part];
+    const next =
+      existing && typeof existing === "object" && !Array.isArray(existing)
+        ? { ...(existing as Record<string, unknown>) }
+        : {};
+    current[part] = next;
+    current = next;
+  }
+  current[parts[parts.length - 1]!] = value;
+  return root;
+}
+
+interface TextField {
+  path: string;
+  text: string;
+}
+
+function collectSearchStrings(
+  item: unknown,
+  textPaths: string[] | undefined,
+  maxFields: number,
+): TextField[] {
+  const paths = (textPaths ?? []).filter((path) => path.trim());
+  if (!paths.length) return collectStrings(item, "", [], maxFields);
+  const fields: TextField[] = [];
+  for (const path of paths) {
+    const value = getAtPath(item, path);
+    if (value !== undefined) collectStrings(value, path, fields, maxFields);
+    if (fields.length >= maxFields) break;
+  }
+  return fields;
+}
+
+function collectStrings(
+  value: unknown,
+  basePath: string,
+  out: TextField[],
+  limit: number,
+): TextField[] {
+  if (out.length >= limit || value === undefined || value === null) return out;
+  if (
+    typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "bigint"
+  ) {
+    out.push({ path: basePath || "$", text: String(value) });
+    return out;
+  }
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length && out.length < limit; i++) {
+      collectStrings(
+        value[i],
+        basePath ? `${basePath}[${i}]` : `[${i}]`,
+        out,
+        limit,
+      );
+    }
+    return out;
+  }
+  if (typeof value === "object") {
+    for (const key of Object.keys(value as Record<string, unknown>)) {
+      if (out.length >= limit) break;
+      collectStrings(
+        (value as Record<string, unknown>)[key],
+        basePath ? `${basePath}.${key}` : key,
+        out,
+        limit,
+      );
+    }
+  }
+  return out;
+}
+
+const DEFAULT_ID_PATHS = [
+  "id",
+  "callId",
+  "callID",
+  "call_id",
+  "call.id",
+  "call.metaData.id",
+  "metaData.id",
+  "metadata.id",
+  "recordId",
+  "record_id",
+  "objectId",
+  "object_id",
+  "ticketId",
+  "ticket_id",
+  "issueId",
+  "issue_id",
+  "messageId",
+  "message_id",
+  "conversationId",
+  "conversation_id",
+  "eventId",
+  "event_id",
+  "documentId",
+  "document_id",
+  "url",
+  "webUrl",
+  "permalink",
+];
+
+function extractItemIdentity(
+  item: unknown,
+  idPaths: string[] | undefined,
+): { id: string | null; idPath: string | null } {
+  for (const path of [...(idPaths ?? []), ...DEFAULT_ID_PATHS]) {
+    const value = getAtPath(item, path);
+    if (value !== undefined && value !== null && String(value) !== "") {
+      return { id: stringifySearchValue(value), idPath: path };
+    }
+  }
+  return { id: null, idPath: null };
+}
+
+function extractMetadata(
+  item: unknown,
+  metadataPaths: string[] | undefined,
+): Record<string, unknown> {
+  const metadata: Record<string, unknown> = {};
+  for (const path of metadataPaths ?? []) {
+    const value = getAtPath(item, path);
+    if (value !== undefined) metadata[path] = value;
+  }
+  return metadata;
+}
+
+function stringifySearchValue(value: unknown): string {
+  if (typeof value === "string") return value;
+  if (value === undefined || value === null) return "";
+  if (
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "bigint"
+  ) {
+    return String(value);
+  }
+  return stringifyBrief(value, 500);
+}
+
+function normalizedTerms(search: SearchConfig): string[] {
+  const explicit = (search.terms ?? [])
+    .map((term) => term.trim())
+    .filter(Boolean);
+  if (explicit.length) return explicit;
+  if (search.matchMode === "allTerms" && search.query) {
+    return search.query
+      .split(/\s+/)
+      .map((term) => term.trim())
+      .filter(Boolean);
+  }
+  return [];
+}
+
+function findItemWideTermMatch(fields: TextField[], search: SearchConfig) {
+  const terms = normalizedTerms(search);
+  if (!terms.length || search.matchMode === "anyTerm") return null;
+  const caseSensitive = Boolean(search.caseSensitive);
+  const fieldWindows = fields
+    .map((field) => {
+      const hits = bestTermClusterInText(field.text, terms, caseSensitive).map(
+        (hit) => ({ ...hit, field }),
+      );
+      if (hits.length !== terms.length) return null;
+      const start = Math.min(...hits.map((hit) => hit.index));
+      const end = Math.max(...hits.map((hit) => hit.end));
+      return {
+        field,
+        hits,
+        start,
+        end,
+        length: end - start,
+      };
+    })
+    .filter(
+      (
+        result,
+      ): result is {
+        field: TextField;
+        hits: TermHit[];
+        start: number;
+        end: number;
+        length: number;
+      } => Boolean(result),
+    )
+    .sort((a, b) => a.length - b.length || a.start - b.start);
+
+  const bestFieldWindow = fieldWindows[0];
+  if (bestFieldWindow) {
+    const context = boundedNumber(search.contextChars, 180, 20, 1_000);
+    const snippet =
+      bestFieldWindow.length > context * 4
+        ? makeCombinedTermSnippet(bestFieldWindow.hits, search.contextChars)
+        : undefined;
+    return {
+      field: bestFieldWindow.field,
+      match: {
+        kind: "allTerms",
+        query: terms.join(" "),
+        index: bestFieldWindow.start,
+        endIndex: bestFieldWindow.end,
+        match: bestFieldWindow.field.text.slice(
+          bestFieldWindow.start,
+          bestFieldWindow.end,
+        ),
+        matchedTerms: terms,
+        snippet,
+      },
+    };
+  }
+
+  const termHits = terms.map((term) => {
+    for (const field of fields) {
+      const hit = firstTermHitInField(field, term, caseSensitive);
+      if (hit) return hit;
+    }
+    return null;
+  });
+  if (termHits.some((hit) => !hit)) return null;
+  const hits = termHits.filter(isTermHit);
+  const first = hits.sort((a, b) => a.index - b.index)[0];
+  if (!first) return null;
+  return {
+    field: first.field,
+    match: {
+      kind: "allTerms",
+      query: terms.join(" "),
+      index: first.index,
+      endIndex: first.end,
+      match: first.match,
+      matchedTerms: terms,
+      snippet: makeCombinedTermSnippet(hits, search.contextChars),
+    },
+  };
+}
+
+function findSearchMatches(
+  text: string,
+  search: SearchConfig,
+  includeTerms: boolean,
+): MatchRecord[] {
+  const source = String(text);
+  const caseSensitive = Boolean(search.caseSensitive);
+  const haystack = caseSensitive ? source : source.toLowerCase();
+  const maxMatchesPerField = 1_000;
+  const matches: MatchRecord[] = [];
+
+  const addSubstring = (needle: unknown, label: string, kind: string) => {
+    if (needle === undefined || needle === null) return;
+    const rawNeedle = String(needle);
+    if (!rawNeedle) return;
+    const searchNeedle = caseSensitive ? rawNeedle : rawNeedle.toLowerCase();
+    let from = 0;
+    while (from <= haystack.length) {
+      const index = haystack.indexOf(searchNeedle, from);
+      if (index < 0) break;
+      matches.push({
+        kind,
+        query: label,
+        index,
+        match: source.slice(index, index + rawNeedle.length),
+      });
+      from = index + Math.max(1, searchNeedle.length);
+      if (matches.length >= maxMatchesPerField) break;
+    }
+  };
+
+  if (search.regex) {
+    const regex = new RegExp(
+      search.regex,
+      normalizeRegexFlags(search.regexFlags, caseSensitive),
+    );
+    let match: RegExpExecArray | null;
+    while ((match = regex.exec(source)) && typeof match.index === "number") {
+      matches.push({
+        kind: "regex",
+        query: search.regex,
+        index: match.index,
+        match: match[0],
+      });
+      if (matches.length >= maxMatchesPerField) break;
+      if (match[0] === "") regex.lastIndex += 1;
+    }
+  }
+
+  if (search.query) addSubstring(search.query, search.query, "query");
+  for (const query of search.queries ?? []) addSubstring(query, query, "query");
+
+  const terms = includeTerms ? normalizedTerms(search) : [];
+  if (terms.length) {
+    const mode = search.matchMode === "anyTerm" ? "anyTerm" : "allTerms";
+    const hits =
+      mode === "allTerms"
+        ? bestTermClusterInText(source, terms, caseSensitive)
+        : terms
+            .map((term) =>
+              firstTermHitInText(
+                source,
+                haystack,
+                caseSensitive ? term : term.toLowerCase(),
+                term,
+              ),
+            )
+            .filter(isInlineTermHit)
+            .sort((a, b) => a.index - b.index);
+    if (
+      (mode === "allTerms" && hits.length === terms.length) ||
+      (mode === "anyTerm" && hits.length > 0)
+    ) {
+      const first = hits[0];
+      if (first) {
+        const endIndex =
+          mode === "allTerms"
+            ? Math.max(...hits.map((hit) => hit.end))
+            : first.end;
+        matches.push({
+          kind: mode,
+          query: terms.join(" "),
+          index: first.index,
+          endIndex,
+          match:
+            mode === "allTerms"
+              ? source.slice(first.index, endIndex)
+              : first.match,
+          matchedTerms:
+            mode === "allTerms" ? terms : hits.map((hit) => hit.term),
+        });
+      }
+    }
+  }
+
+  return matches.sort((a, b) => a.index - b.index);
+}
+
+function normalizeRegexFlags(
+  flags: string | undefined,
+  caseSensitive: boolean,
+): string {
+  const allowed = (flags ?? "")
+    .replace(/[^dgimsuvy]/g, "")
+    .replace(/[gy]/g, "");
+  const withCase = caseSensitive || /i/.test(allowed) ? allowed : `${allowed}i`;
+  return `${withCase}g`;
+}
+
+function makeSnippet(
+  text: string,
+  index: number,
+  contextChars: number | undefined,
+  endIndex = index,
+): string {
+  const context = boundedNumber(contextChars, 180, 20, 1_000);
+  const start = Math.max(0, index - context);
+  const end = Math.min(text.length, Math.max(index, endIndex) + context);
+  const prefix = start > 0 ? "..." : "";
+  const suffix = end < text.length ? "..." : "";
+  return `${prefix}${text.slice(start, end)}${suffix}`
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+interface InlineTermHit {
+  term: string;
+  index: number;
+  end: number;
+  match: string;
+}
+
+interface TermHit extends InlineTermHit {
+  field: TextField;
+}
+
+function firstTermHitInText(
+  source: string,
+  haystack: string,
+  needle: string,
+  term: string,
+): InlineTermHit | null {
+  const index = haystack.indexOf(needle);
+  return index >= 0
+    ? {
+        term,
+        index,
+        end: index + term.length,
+        match: source.slice(index, index + term.length),
+      }
+    : null;
+}
+
+function firstTermHitInField(
+  field: TextField,
+  term: string,
+  caseSensitive: boolean,
+): TermHit | null {
+  const source = field.text;
+  const haystack = caseSensitive ? source : source.toLowerCase();
+  const needle = caseSensitive ? term : term.toLowerCase();
+  const hit = firstTermHitInText(source, haystack, needle, term);
+  return hit ? { ...hit, term, field } : null;
+}
+
+function isInlineTermHit(
+  hit: InlineTermHit | null | undefined,
+): hit is InlineTermHit {
+  return Boolean(hit);
+}
+
+function isTermHit(hit: TermHit | null): hit is TermHit {
+  return Boolean(hit);
+}
+
+function bestTermClusterInText(
+  source: string,
+  terms: string[],
+  caseSensitive: boolean,
+): InlineTermHit[] {
+  const haystack = caseSensitive ? source : source.toLowerCase();
+  const allHits = terms.map((term) => {
+    const needle = caseSensitive ? term : term.toLowerCase();
+    const hits: InlineTermHit[] = [];
+    let from = 0;
+    while (from <= haystack.length) {
+      const index = haystack.indexOf(needle, from);
+      if (index < 0) break;
+      hits.push({
+        term,
+        index,
+        end: index + term.length,
+        match: source.slice(index, index + term.length),
+      });
+      from = index + Math.max(1, needle.length);
+      if (hits.length >= 100) break;
+    }
+    return hits;
+  });
+  if (allHits.some((hits) => hits.length === 0)) return [];
+
+  const flattened = allHits
+    .flat()
+    .sort((a, b) => a.index - b.index || a.end - b.end);
+  let best: InlineTermHit[] | null = null;
+  let bestLength = Infinity;
+
+  for (let start = 0; start < flattened.length; start++) {
+    const selected = new Map<string, InlineTermHit>();
+    for (let end = start; end < flattened.length; end++) {
+      const hit = flattened[end]!;
+      selected.set(hit.term, hit);
+      if (selected.size !== terms.length) continue;
+      const cluster = terms
+        .map((term) => selected.get(term))
+        .filter(isInlineTermHit)
+        .sort((a, b) => a.index - b.index);
+      const length =
+        Math.max(...cluster.map((item) => item.end)) -
+        Math.min(...cluster.map((item) => item.index));
+      if (length < bestLength) {
+        best = cluster;
+        bestLength = length;
+      }
+      break;
+    }
+  }
+
+  return best ?? [];
+}
+
+function makeCombinedTermSnippet(
+  hits: TermHit[],
+  contextChars: number | undefined,
+): string {
+  const context = boundedNumber(contextChars, 180, 20, 1_000);
+  const shownHits = hits.slice(0, 5);
+  const perHitContext = Math.max(
+    20,
+    Math.min(120, Math.floor(context / Math.max(1, Math.min(3, hits.length)))),
+  );
+  const parts = shownHits.map((hit) => {
+    const snippet = makeSnippet(
+      hit.field.text,
+      hit.index,
+      perHitContext,
+      hit.end,
+    );
+    return `${hit.field.path}: ${snippet}`;
+  });
+  if (hits.length > shownHits.length) {
+    parts.push(`... +${hits.length - shownHits.length} more terms`);
+  }
+  return parts.join(" | ");
+}