@agent-native/core 0.51.5 → 0.51.7

package/dist/provider-api/corpus-jobs.js

@@ -0,0 +1,1192 @@
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import { defineAction } from "../action.js";
+import { getCredentialContext } from "../server/request-context.js";
+import { getStagedDatasetRows } from "./staged-datasets-store.js";
+import { appendProviderCorpusJobHits, createProviderCorpusJob, deleteProviderCorpusJob, getProviderCorpusJob, getProviderCorpusJobHits, listProviderCorpusJobs, updateProviderCorpusJob, } from "./corpus-jobs-store.js";
+const MethodSchema = z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"]);
+const ProviderRequestSchema = z.object({
+    provider: z.string().min(1).describe("Configured provider API id."),
+    method: MethodSchema.default("GET").describe("HTTP method to use."),
+    path: z.string().min(1).describe("Provider API path or allowed full URL."),
+    query: z.unknown().optional().describe("Optional query parameters."),
+    headers: z.record(z.string(), z.unknown()).optional(),
+    body: z.unknown().optional(),
+    auth: z.enum(["default", "none"]).default("default"),
+    connectionId: z.string().trim().min(1).optional(),
+    accountId: z.string().optional(),
+    timeoutMs: z.coerce.number().int().min(1_000).max(120_000).optional(),
+    maxBytes: z.coerce
+        .number()
+        .int()
+        .min(1_000)
+        .max(4 * 1024 * 1024)
+        .optional(),
+});
+const PaginationSchema = z.object({
+    itemsPath: z
+        .string()
+        .optional()
+        .describe("Dot-path to records in each page, e.g. records.calls."),
+    nextCursorPath: z
+        .string()
+        .optional()
+        .describe("Dot-path to the next cursor in each response."),
+    cursorPath: z.string().optional().describe("Alias for nextCursorPath."),
+    cursorParam: z
+        .string()
+        .optional()
+        .describe("Query parameter that receives the next cursor."),
+    cursorBodyPath: z
+        .string()
+        .optional()
+        .describe("Request body path that receives the next cursor."),
+    pageParam: z
+        .string()
+        .optional()
+        .describe("Query parameter for page-number pagination."),
+    startPage: z.coerce.number().int().min(0).optional(),
+    offsetParam: z
+        .string()
+        .optional()
+        .describe("Query parameter for offset pagination."),
+    startOffset: z.coerce.number().int().min(0).optional(),
+    pageSize: z.coerce.number().int().min(1).optional(),
+    maxPages: z.coerce
+        .number()
+        .int()
+        .min(1)
+        .max(2_000)
+        .optional()
+        .describe("Overall page cap for the job. Default 500."),
+});
+const BatchSchema = z.object({
+    items: z
+        .array(z.unknown())
+        .max(10_000)
+        .optional()
+        .describe("Input records or ids to process in batches."),
+    inputDatasetId: z
+        .string()
+        .optional()
+        .describe("Optional staged dataset id to read input records from."),
+    inputValuePath: z
+        .string()
+        .optional()
+        .describe("Dot-path to extract the provider id/value from each input row."),
+    batchSize: z.coerce.number().int().min(1).max(100).optional(),
+    itemBodyPath: z
+        .string()
+        .optional()
+        .describe("Dot-path in request body where the current batch array goes."),
+    itemQueryParam: z
+        .string()
+        .optional()
+        .describe("Query parameter where the current batch goes."),
+    responseItemsPath: z
+        .string()
+        .optional()
+        .describe("Dot-path to records in each batch response."),
+});
+const SearchSchema = z.object({
+    query: z.string().optional().describe("Phrase to search for."),
+    queries: z.array(z.string()).max(50).optional(),
+    terms: z
+        .array(z.string())
+        .max(50)
+        .optional()
+        .describe("Terms for allTerms/anyTerm search."),
+    regex: z.string().optional(),
+    regexFlags: z.string().optional(),
+    matchMode: z.enum(["query", "allTerms", "anyTerm"]).optional(),
+    caseSensitive: z.boolean().optional(),
+    textPaths: z
+        .array(z.string())
+        .max(100)
+        .optional()
+        .describe("Only search these dot-paths. Omit to search text recursively."),
+    idPaths: z.array(z.string()).max(100).optional(),
+    metadataPaths: z.array(z.string()).max(100).optional(),
+    contextChars: z.coerce.number().int().min(20).max(1_000).optional(),
+    maxHits: z.coerce.number().int().min(1).max(10_000).optional(),
+    maxHitsPerItem: z.coerce.number().int().min(1).max(100).optional(),
+    maxFieldsPerItem: z.coerce.number().int().min(1).max(50_000).optional(),
+});
+const LimitsSchema = z.object({
+    pageBudget: z.coerce
+        .number()
+        .int()
+        .min(1)
+        .max(200)
+        .optional()
+        .describe("Pages to process in this action call. Default 25."),
+    batchBudget: z.coerce
+        .number()
+        .int()
+        .min(1)
+        .max(500)
+        .optional()
+        .describe("Batches to process in this action call. Default 25."),
+    runtimeMs: z.coerce
+        .number()
+        .int()
+        .min(1_000)
+        .max(110_000)
+        .optional()
+        .describe("Wall-clock budget for this action call. Default 90000."),
+    itemBudget: z.coerce
+        .number()
+        .int()
+        .min(1)
+        .max(200_000)
+        .optional()
+        .describe("Response records to search in this action call."),
+    maxHits: z.coerce
+        .number()
+        .int()
+        .min(1)
+        .max(10_000)
+        .optional()
+        .describe("Stored hit cap for the whole job. Default comes from search."),
+});
+const ProviderCorpusJobSchema = z.object({
+    operation: z
+        .enum(["start", "continue", "status", "results", "list", "delete"])
+        .default("start"),
+    jobId: z
+        .string()
+        .optional()
+        .describe("Existing job id for continue/status/results/delete."),
+    name: z.string().optional().describe("Human-readable job name."),
+    mode: z.enum(["paginated-search", "batch-search"]).optional(),
+    request: ProviderRequestSchema.optional(),
+    pagination: PaginationSchema.optional(),
+    batch: BatchSchema.optional(),
+    search: SearchSchema.optional(),
+    limits: LimitsSchema.optional(),
+    offset: z.coerce
+        .number()
+        .int()
+        .min(0)
+        .optional()
+        .describe("Results/list offset."),
+    limit: z.coerce
+        .number()
+        .int()
+        .min(1)
+        .max(1_000)
+        .optional()
+        .describe("Results/list limit."),
+});
+const DEFAULT_PAGE_BUDGET = 25;
+const DEFAULT_BATCH_BUDGET = 25;
+const DEFAULT_RUNTIME_MS = 90_000;
+const DEFAULT_BATCH_SIZE = 25;
+const DEFAULT_OVERALL_MAX_PAGES = 500;
+const DEFAULT_MAX_HITS = 500;
+export function createProviderCorpusJobAction(options) {
+    return defineAction({
+        description: "Create, continue, inspect, list, read results from, or delete a durable provider corpus search job. " +
+            "Use this for broad provider searches, cross-source joins, transcript/message/ticket/issue/document scans, batch endpoint walks, and absence-sensitive questions that may exceed one tool call. " +
+            "The job is generic: paginated-search walks any paginated provider endpoint; batch-search walks a supplied id/record list or staged dataset through any provider endpoint by injecting each batch into a query param or request body path. " +
+            "Every provider request still goes through provider-api-request credentials, host allow-listing, SSRF blocking, secret redaction, and provider quota cooldown. " +
+            "When status is paused or quota_wait, call operation=continue with the jobId after the indicated time/budget. Report coverage counts, pagination status, and any remaining gaps.",
+        schema: ProviderCorpusJobSchema,
+        http: false,
+        run: async (args) => runProviderCorpusJobAction(args, options),
+    });
+}
+async function runProviderCorpusJobAction(args, options) {
+    const ctx = getCredentialContext();
+    if (!ctx)
+        throw new Error("No authenticated context for provider-corpus-job.");
+    if (args.operation === "list") {
+        const jobs = await listProviderCorpusJobs({
+            appId: options.appId,
+            ownerEmail: ctx.userEmail,
+            limit: args.limit,
+        });
+        return { jobs: jobs.map(jobSummary), total: jobs.length };
+    }
+    if (args.operation === "start") {
+        const job = await startJob(args, options.appId, ctx.userEmail);
+        return continueJob(job, args.limits, options.getRuntime());
+    }
+    const jobId = args.jobId?.trim();
+    if (!jobId)
+        throw new Error(`${args.operation} requires jobId.`);
+    if (args.operation === "delete") {
+        const deleted = await deleteProviderCorpusJob({
+            id: jobId,
+            appId: options.appId,
+            ownerEmail: ctx.userEmail,
+        });
+        if (!deleted) {
+            throw new Error(`Provider corpus job ${jobId} not found or belongs to another owner/app.`);
+        }
+        return { deleted: true, jobId };
+    }
+    const job = await getProviderCorpusJob({
+        id: jobId,
+        appId: options.appId,
+        ownerEmail: ctx.userEmail,
+    });
+    if (!job) {
+        throw new Error(`Provider corpus job ${jobId} not found or belongs to another owner/app.`);
+    }
+    if (args.operation === "status")
+        return jobStatus(job);
+    if (args.operation === "results") {
+        const hits = await getProviderCorpusJobHits({
+            jobId,
+            appId: options.appId,
+            ownerEmail: ctx.userEmail,
+            offset: args.offset,
+            limit: args.limit,
+        });
+        return {
+            ...jobStatus(job),
+            hits,
+            offset: args.offset ?? 0,
+            limit: args.limit ?? 100,
+        };
+    }
+    if (args.operation === "continue") {
+        return continueJob(job, args.limits, options.getRuntime());
+    }
+    throw new Error(`Unsupported provider corpus operation ${args.operation}.`);
+}
+async function startJob(args, appId, ownerEmail) {
+    if (!args.mode)
+        throw new Error("start requires mode.");
+    if (!args.request)
+        throw new Error("start requires request.");
+    const search = args.search ?? {};
+    if (!hasSearchNeedle(search)) {
+        throw new Error("start requires search.query, search.queries, search.terms, or search.regex.");
+    }
+    if (args.mode === "paginated-search" && !args.pagination) {
+        throw new Error("paginated-search requires pagination.");
+    }
+    if (args.mode === "batch-search") {
+        if (!args.batch)
+            throw new Error("batch-search requires batch config.");
+        if (!args.batch.itemBodyPath && !args.batch.itemQueryParam) {
+            throw new Error("batch-search requires batch.itemBodyPath or batch.itemQueryParam.");
+        }
+        if (!args.batch.items?.length && !args.batch.inputDatasetId) {
+            throw new Error("batch-search requires batch.items or inputDatasetId.");
+        }
+    }
+    const id = args.jobId?.trim() || `pcj_${randomUUID().replace(/-/g, "")}`;
+    const request = cleanRequest(args.request);
+    const checkpoint = args.mode === "batch-search"
+        ? { nextIndex: 0, sourceItemCount: null }
+        : {
+            pageIndex: 0,
+            cursor: null,
+            pageNumber: args.pagination?.startPage ?? 1,
+            offset: args.pagination?.startOffset ?? 0,
+            lastCursor: null,
+            stoppedReason: null,
+        };
+    return createProviderCorpusJob({
+        id,
+        appId,
+        ownerEmail,
+        name: args.name?.trim() || id,
+        mode: args.mode,
+        status: "paused",
+        provider: request.provider,
+        request: request,
+        pagination: args.pagination ?? null,
+        batch: args.batch ?? null,
+        search: search,
+        limits: args.limits ?? {},
+        checkpoint,
+    });
+}
+async function continueJob(job, overrideLimits, runtime) {
+    if (job.status === "completed")
+        return jobStatus(job);
+    const limits = normalizeLimits({
+        ...job.limits,
+        ...(overrideLimits ?? {}),
+    });
+    await updateProviderCorpusJob({
+        id: job.id,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+        status: "running",
+        error: null,
+        nextResumeAt: null,
+    });
+    try {
+        const refreshed = await getProviderCorpusJob({
+            id: job.id,
+            appId: job.appId,
+            ownerEmail: job.ownerEmail,
+        });
+        if (!refreshed)
+            throw new Error(`Provider corpus job ${job.id} disappeared.`);
+        const next = refreshed.mode === "batch-search"
+            ? await runBatchSearch(refreshed, limits, runtime)
+            : await runPaginatedSearch(refreshed, limits, runtime);
+        const hits = await getProviderCorpusJobHits({
+            jobId: next.id,
+            appId: next.appId,
+            ownerEmail: next.ownerEmail,
+            limit: 25,
+        });
+        return {
+            ...jobStatus(next),
+            sampleHits: hits,
+            nextAction: nextAction(next),
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        const failed = (await updateProviderCorpusJob({
+            id: job.id,
+            appId: job.appId,
+            ownerEmail: job.ownerEmail,
+            status: "failed",
+            error: message,
+        })) ?? job;
+        return { ...jobStatus(failed), nextAction: nextAction(failed) };
+    }
+}
+async function runPaginatedSearch(job, limits, runtime) {
+    const startedAt = Date.now();
+    const pagination = normalizePagination(job.pagination);
+    const request = job.request;
+    const search = job.search;
+    const checkpoint = { ...job.checkpoint };
+    const maxPages = pagination.maxPages ?? DEFAULT_OVERALL_MAX_PAGES;
+    let pageIndex = numberValue(checkpoint.pageIndex, 0);
+    let cursor = valueOrNull(checkpoint.cursor);
+    let pageNumber = numberValue(checkpoint.pageNumber, pagination.startPage ?? 1);
+    let offset = numberValue(checkpoint.offset, pagination.startOffset ?? 0);
+    let pagesThisCall = 0;
+    let itemsThisCall = 0;
+    let current = job;
+    while (pagesThisCall < limits.pageBudget &&
+        pageIndex < maxPages &&
+        itemsThisCall < limits.itemBudget &&
+        Date.now() - startedAt < limits.runtimeMs) {
+        const pageRequest = buildPaginatedRequest(request, pagination, {
+            pageIndex,
+            cursor,
+            pageNumber,
+            offset,
+        });
+        const provider = await callProvider(runtime, pageRequest);
+        if (provider.quota) {
+            return pauseForQuota(current, {
+                ...checkpoint,
+                pageIndex,
+                cursor,
+                pageNumber,
+                offset,
+            }, provider.quota);
+        }
+        const items = extractItemsArray(provider.body, pagination.itemsPath, false);
+        const nextCursorPath = pagination.nextCursorPath || pagination.cursorPath;
+        const nextCursor = nextCursorPath
+            ? valueOrNull(getAtPath(provider.body, nextCursorPath))
+            : null;
+        const searchResult = searchItems(items, search, {
+            baseItemIndex: current.itemsProcessed,
+            maxStoredHits: Math.max(0, limits.maxHits - current.storedHits),
+            pageIndex,
+        });
+        const storedHits = searchResult.storedHits.map((hit) => hitToRecord(hit));
+        await appendProviderCorpusJobHits({
+            jobId: job.id,
+            startIndex: current.storedHits,
+            hits: storedHits,
+        });
+        pagesThisCall++;
+        itemsThisCall += searchResult.searchedItems;
+        const nextCheckpoint = {
+            ...checkpoint,
+            pageIndex: pageIndex + 1,
+            cursor: nextCursor,
+            pageNumber,
+            offset,
+            lastCursor: nextCursor,
+            stoppedReason: null,
+        };
+        let status = "paused";
+        let stoppedReason = null;
+        if (items.length === 0) {
+            status = "completed";
+            stoppedReason = "empty-page";
+        }
+        else if (nextCursorPath) {
+            if (!nextCursor) {
+                status = "completed";
+                stoppedReason = "no-next-cursor";
+            }
+            else if (cursor !== null && String(nextCursor) === String(cursor)) {
+                status = "completed";
+                stoppedReason = "repeated-cursor";
+            }
+            else {
+                cursor = nextCursor;
+            }
+        }
+        else if (pagination.pageParam) {
+            pageNumber += 1;
+            nextCheckpoint.pageNumber = pageNumber;
+        }
+        else if (pagination.offsetParam) {
+            const step = pagination.pageSize || items.length;
+            offset += step;
+            nextCheckpoint.offset = offset;
+            if (pagination.pageSize && items.length < pagination.pageSize) {
+                status = "completed";
+                stoppedReason = "short-page";
+            }
+        }
+        else {
+            status = "completed";
+            stoppedReason = "single-page";
+        }
+        if (pageIndex + 1 >= maxPages && status !== "completed") {
+            status = "completed";
+            stoppedReason = "max-pages";
+        }
+        nextCheckpoint.stoppedReason = stoppedReason;
+        current =
+            (await updateProviderCorpusJob({
+                id: job.id,
+                appId: job.appId,
+                ownerEmail: job.ownerEmail,
+                status,
+                checkpoint: nextCheckpoint,
+                pagesProcessed: current.pagesProcessed + 1,
+                itemsProcessed: current.itemsProcessed + searchResult.searchedItems,
+                matchedItems: current.matchedItems + searchResult.matchedItems,
+                totalHits: current.totalHits + searchResult.totalHits,
+                storedHits: current.storedHits + storedHits.length,
+                error: null,
+                nextResumeAt: null,
+            })) ?? current;
+        pageIndex += 1;
+        if (status === "completed")
+            return current;
+    }
+    return pauseForBudget(current, {
+        ...checkpoint,
+        pageIndex,
+        cursor,
+        pageNumber,
+        offset,
+        stoppedReason: "budget",
+    });
+}
+async function runBatchSearch(job, limits, runtime) {
+    const startedAt = Date.now();
+    const batch = normalizeBatch(job.batch);
+    const request = job.request;
+    const search = job.search;
+    const sourceItems = await loadBatchSourceItems(job, batch);
+    const totalSourceItems = sourceItems.length;
+    let nextIndex = numberValue(job.checkpoint.nextIndex, 0);
+    let batchesThisCall = 0;
+    let itemsThisCall = 0;
+    let current = job;
+    while (nextIndex < totalSourceItems &&
+        batchesThisCall < limits.batchBudget &&
+        itemsThisCall < limits.itemBudget &&
+        Date.now() - startedAt < limits.runtimeMs) {
+        const slice = sourceItems.slice(nextIndex, nextIndex + batch.batchSize);
+        const values = slice
+            .map((item) => batch.inputValuePath ? getAtPath(item, batch.inputValuePath) : item)
+            .filter((value) => value !== undefined && value !== null);
+        if (values.length === 0) {
+            nextIndex += slice.length || batch.batchSize;
+            continue;
+        }
+        const batchRequest = buildBatchRequest(request, batch, values);
+        const provider = await callProvider(runtime, batchRequest);
+        if (provider.quota) {
+            return pauseForQuota(current, {
+                ...job.checkpoint,
+                nextIndex,
+                sourceItemCount: totalSourceItems,
+            }, provider.quota);
+        }
+        const items = extractItemsArray(provider.body, batch.responseItemsPath, true);
+        const searchResult = searchItems(items, search, {
+            baseItemIndex: current.itemsProcessed,
+            maxStoredHits: Math.max(0, limits.maxHits - current.storedHits),
+            batchIndex: current.batchesProcessed,
+        });
+        const storedHits = searchResult.storedHits.map((hit) => hitToRecord(hit));
+        await appendProviderCorpusJobHits({
+            jobId: job.id,
+            startIndex: current.storedHits,
+            hits: storedHits,
+        });
+        batchesThisCall++;
+        itemsThisCall += searchResult.searchedItems;
+        nextIndex += slice.length;
+        const status = nextIndex >= totalSourceItems ? "completed" : "paused";
+        current =
+            (await updateProviderCorpusJob({
+                id: job.id,
+                appId: job.appId,
+                ownerEmail: job.ownerEmail,
+                status,
+                checkpoint: {
+                    ...job.checkpoint,
+                    nextIndex,
+                    sourceItemCount: totalSourceItems,
+                    stoppedReason: status === "completed" ? "completed" : "budget",
+                },
+                batchesProcessed: current.batchesProcessed + 1,
+                itemsProcessed: current.itemsProcessed + searchResult.searchedItems,
+                matchedItems: current.matchedItems + searchResult.matchedItems,
+                totalHits: current.totalHits + searchResult.totalHits,
+                storedHits: current.storedHits + storedHits.length,
+                error: null,
+                nextResumeAt: null,
+            })) ?? current;
+        if (status === "completed")
+            return current;
+    }
+    return pauseForBudget(current, {
+        ...job.checkpoint,
+        nextIndex,
+        sourceItemCount: totalSourceItems,
+        stoppedReason: "budget",
+    });
+}
+async function pauseForQuota(job, checkpoint, quota) {
+    return ((await updateProviderCorpusJob({
+        id: job.id,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+        status: "quota_wait",
+        checkpoint,
+        error: `Provider quota exhausted. Retry after ${quota.retryAt}.`,
+        nextResumeAt: parseRetryAt(quota.retryAt, quota.retryAfterMs),
+    })) ?? job);
+}
+async function pauseForBudget(job, checkpoint) {
+    return ((await updateProviderCorpusJob({
+        id: job.id,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+        status: "paused",
+        checkpoint,
+        error: null,
+        nextResumeAt: null,
+    })) ?? job);
+}
+async function callProvider(runtime, request) {
+    const raw = (await runtime.executeRequest(cleanRequest(request)));
+    const response = raw.response;
+    if (!response) {
+        return { body: raw };
+    }
+    const quota = readQuota(response);
+    if (quota)
+        return { body: null, quota };
+    if (response.ok !== true) {
+        const status = response.status ?? "unknown";
+        const detail = response.text ?? response.json ?? response.statusText ?? "";
+        throw new Error(`Provider request failed (${status}): ${stringifyBrief(detail, 500)}`);
+    }
+    return {
+        body: response.json ??
+            (typeof response.text === "string" ? tryParseJson(response.text) : null),
+    };
+}
+async function loadBatchSourceItems(job, batch) {
+    if (batch.items.length > 0)
+        return batch.items;
+    if (!batch.inputDatasetId)
+        return [];
+    return getStagedDatasetRows({
+        id: batch.inputDatasetId,
+        appId: job.appId,
+        ownerEmail: job.ownerEmail,
+    });
+}
+function cleanRequest(request) {
+    return {
+        provider: request.provider,
+        method: request.method,
+        path: request.path,
+        query: request.query,
+        headers: request.headers,
+        body: request.body,
+        auth: request.auth,
+        connectionId: request.connectionId,
+        accountId: request.accountId,
+        timeoutMs: request.timeoutMs,
+        maxBytes: request.maxBytes,
+    };
+}
+function buildPaginatedRequest(request, pagination, state) {
+    let query = cloneRecord(request.query);
+    let body = request.body;
+    if (pagination.pageParam) {
+        query = { ...query, [pagination.pageParam]: state.pageNumber };
+    }
+    if (pagination.offsetParam) {
+        query = { ...query, [pagination.offsetParam]: state.offset };
+    }
+    if (state.pageIndex > 0 && state.cursor != null) {
+        if (pagination.cursorParam) {
+            query = { ...query, [pagination.cursorParam]: state.cursor };
+        }
+        if (pagination.cursorBodyPath) {
+            body = setAtPath(body, pagination.cursorBodyPath, state.cursor);
+        }
+    }
+    return { ...request, query, body };
+}
+function buildBatchRequest(request, batch, values) {
+    let query = cloneRecord(request.query);
+    let body = request.body;
+    const payload = values.length === 1 ? values[0] : values;
+    if (batch.itemQueryParam) {
+        query = { ...query, [batch.itemQueryParam]: payload };
+    }
+    if (batch.itemBodyPath) {
+        body = setAtPath(body, batch.itemBodyPath, values);
+    }
+    return { ...request, query, body };
+}
+function searchItems(items, search, options) {
+    const maxStoredHits = Math.max(0, options.maxStoredHits);
+    const maxHitsPerItem = boundedNumber(search.maxHitsPerItem, 3, 1, 100);
+    const maxFieldsPerItem = boundedNumber(search.maxFieldsPerItem, 5_000, 1, 50_000);
+    let matchedItems = 0;
+    let totalHits = 0;
+    const storedHits = [];
+    for (let itemOffset = 0; itemOffset < items.length; itemOffset++) {
+        const item = items[itemOffset];
+        const fields = collectSearchStrings(item, search.textPaths, maxFieldsPerItem);
+        const identity = extractItemIdentity(item, search.idPaths);
+        const metadata = extractMetadata(item, search.metadataPaths);
+        let itemMatched = false;
+        let storedForItem = 0;
+        const itemWideTermMatch = findItemWideTermMatch(fields, search);
+        const addHit = (field, match) => {
+            totalHits++;
+            if (!itemMatched) {
+                matchedItems++;
+                itemMatched = true;
+            }
+            if (storedHits.length < maxStoredHits && storedForItem < maxHitsPerItem) {
+                storedForItem++;
+                storedHits.push({
+                    id: identity.id,
+                    idPath: identity.idPath,
+                    itemIndex: options.baseItemIndex + itemOffset,
+                    pageIndex: options.pageIndex,
+                    pageItemIndex: options.pageIndex === undefined ? undefined : itemOffset,
+                    batchIndex: options.batchIndex,
+                    batchItemIndex: options.batchIndex === undefined ? undefined : itemOffset,
+                    path: field.path,
+                    kind: match.kind,
+                    query: match.query,
+                    match: match.match,
+                    snippet: makeSnippet(field.text, match.index, search.contextChars),
+                    ...(Object.keys(metadata).length ? { metadata } : {}),
+                });
+            }
+        };
+        if (itemWideTermMatch) {
+            addHit(itemWideTermMatch.field, itemWideTermMatch.match);
+        }
+        for (const field of fields) {
+            const matches = findSearchMatches(field.text, search, !itemWideTermMatch);
+            for (const match of matches)
+                addHit(field, match);
+        }
+    }
+    return {
+        searchedItems: items.length,
+        matchedItems,
+        totalHits,
+        storedHits,
+    };
+}
+function normalizeLimits(input) {
+    const maxHits = input.maxHits ?? DEFAULT_MAX_HITS;
+    return {
+        pageBudget: input.pageBudget ?? DEFAULT_PAGE_BUDGET,
+        batchBudget: input.batchBudget ?? DEFAULT_BATCH_BUDGET,
+        runtimeMs: input.runtimeMs ?? DEFAULT_RUNTIME_MS,
+        itemBudget: input.itemBudget ?? 200_000,
+        maxHits,
+    };
+}
+function normalizePagination(input) {
+    const parsed = PaginationSchema.parse(input ?? {});
+    return {
+        itemsPath: parsed.itemsPath ?? "",
+        nextCursorPath: parsed.nextCursorPath ?? "",
+        cursorPath: parsed.cursorPath ?? "",
+        cursorParam: parsed.cursorParam ?? "",
+        cursorBodyPath: parsed.cursorBodyPath ?? "",
+        pageParam: parsed.pageParam ?? "",
+        startPage: parsed.startPage ?? 1,
+        offsetParam: parsed.offsetParam ?? "",
+        startOffset: parsed.startOffset ?? 0,
+        pageSize: parsed.pageSize ?? 0,
+        maxPages: parsed.maxPages ?? DEFAULT_OVERALL_MAX_PAGES,
+    };
+}
+function normalizeBatch(input) {
+    const parsed = BatchSchema.parse(input ?? {});
+    return {
+        items: parsed.items ?? [],
+        inputDatasetId: parsed.inputDatasetId ?? "",
+        inputValuePath: parsed.inputValuePath ?? "",
+        batchSize: parsed.batchSize ?? DEFAULT_BATCH_SIZE,
+        itemBodyPath: parsed.itemBodyPath ?? "",
+        itemQueryParam: parsed.itemQueryParam ?? "",
+        responseItemsPath: parsed.responseItemsPath ?? "",
+    };
+}
+function jobStatus(job) {
+    const nextResumeAtIso = job.nextResumeAt
+        ? new Date(job.nextResumeAt).toISOString()
+        : null;
+    return {
+        job: jobSummary(job),
+        coverage: {
+            pagesProcessed: job.pagesProcessed,
+            batchesProcessed: job.batchesProcessed,
+            itemsProcessed: job.itemsProcessed,
+            matchedItems: job.matchedItems,
+            totalHits: job.totalHits,
+            storedHits: job.storedHits,
+            truncatedHits: job.totalHits > job.storedHits,
+        },
+        checkpoint: job.checkpoint,
+        error: job.error,
+        nextResumeAt: nextResumeAtIso,
+    };
+}
+function jobSummary(job) {
+    return {
+        id: job.id,
+        name: job.name,
+        mode: job.mode,
+        status: job.status,
+        provider: job.provider,
+        createdAt: new Date(job.createdAt).toISOString(),
+        updatedAt: new Date(job.updatedAt).toISOString(),
+    };
+}
+function nextAction(job) {
+    if (job.status === "completed") {
+        return `Read all stored hits with operation="results", jobId="${job.id}".`;
+    }
+    if (job.status === "quota_wait") {
+        const at = job.nextResumeAt
+            ? new Date(job.nextResumeAt).toISOString()
+            : "later";
+        return `Provider quota is cooling down. Continue this job after ${at} with operation="continue", jobId="${job.id}".`;
+    }
+    if (job.status === "paused") {
+        return `Continue this job with operation="continue", jobId="${job.id}" until completed or quota_wait.`;
+    }
+    if (job.status === "failed") {
+        return 'Fix the request/configuration or start a new job; progress and stored hits are still inspectable with operation="results".';
+    }
+    return `Check status with operation="status", jobId="${job.id}".`;
+}
+function extractItemsArray(body, itemsPath, wrapObjectFallback) {
+    if (Array.isArray(body))
+        return body;
+    if (!body || typeof body !== "object")
+        return [];
+    if (itemsPath) {
+        const found = getAtPath(body, itemsPath);
+        if (Array.isArray(found))
+            return found;
+        if (found !== undefined && wrapObjectFallback)
+            return [found];
+        return [];
+    }
+    const obj = body;
+    for (const key of [
+        "data",
+        "results",
+        "items",
+        "records",
+        "rows",
+        "calls",
+        "callTranscripts",
+        "transcripts",
+        "messages",
+        "tickets",
+        "issues",
+        "deals",
+        "events",
+        "notes",
+        "documents",
+        "entries",
+        "objects",
+    ]) {
+        if (Array.isArray(obj[key]))
+            return obj[key];
+    }
+    const arrayFields = Object.values(obj).filter(Array.isArray);
+    if (arrayFields.length === 1)
+        return arrayFields[0];
+    return wrapObjectFallback ? [body] : [];
+}
+function hasSearchNeedle(search) {
+    return Boolean(search.query ||
+        search.regex ||
+        search.queries?.length ||
+        search.terms?.length);
+}
+function readQuota(response) {
+    const quota = response.quota;
+    if (quota?.exhausted === true && typeof quota.retryAt === "string") {
+        return {
+            retryAt: quota.retryAt,
+            retryAfterMs: numberValue(quota.retryAfterMs, 0),
+            reason: String(quota.reason ?? "quota"),
+        };
+    }
+    const json = response.json;
+    if (json?.error === "provider_quota_exhausted" &&
+        typeof json.retryAt === "string") {
+        return {
+            retryAt: json.retryAt,
+            retryAfterMs: numberValue(json.retryAfterMs, 0),
+            reason: String(json.reason ?? "quota"),
+        };
+    }
+    return null;
+}
+function hitToRecord(hit) {
+    return Object.fromEntries(Object.entries(hit).filter(([, value]) => value !== undefined));
+}
+function tryParseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return text;
+    }
+}
+function stringifyBrief(value, maxChars) {
+    const text = typeof value === "string" ? value : JSON.stringify(value);
+    return text.slice(0, maxChars);
+}
+function cloneRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return {};
+    return JSON.parse(JSON.stringify(value));
+}
+function valueOrNull(value) {
+    if (value === undefined ||
+        value === null ||
+        value === "" ||
+        value === false) {
+        return null;
+    }
+    return value;
+}
+function numberValue(value, fallback) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : fallback;
+}
+function parseRetryAt(retryAt, retryAfterMs) {
+    const parsed = Date.parse(retryAt);
+    if (Number.isFinite(parsed))
+        return parsed;
+    return Date.now() + Math.max(0, retryAfterMs);
+}
+function boundedNumber(value, fallback, min, max) {
+    const parsed = Number(value);
+    const finite = Number.isFinite(parsed) ? parsed : fallback;
+    return Math.max(min, Math.min(max, finite));
+}
+function pathParts(path) {
+    if (!path)
+        return [];
+    return path
+        .replace(/\[(\d+)\]/g, ".$1")
+        .split(".")
+        .map((part) => part.trim())
+        .filter(Boolean);
+}
+function getAtPath(value, path) {
+    let current = value;
+    for (const part of pathParts(path)) {
+        if (current === undefined || current === null)
+            return undefined;
+        if (typeof current !== "object")
+            return undefined;
+        current = current[part];
+    }
+    return current;
+}
+function setAtPath(base, path, value) {
+    const parts = pathParts(path);
+    if (!parts.length)
+        return value;
+    const root = base && typeof base === "object" && !Array.isArray(base)
+        ? JSON.parse(JSON.stringify(base))
+        : {};
+    let current = root;
+    for (const part of parts.slice(0, -1)) {
+        const existing = current[part];
+        const next = existing && typeof existing === "object" && !Array.isArray(existing)
+            ? { ...existing }
+            : {};
+        current[part] = next;
+        current = next;
+    }
+    current[parts[parts.length - 1]] = value;
+    return root;
+}
+function collectSearchStrings(item, textPaths, maxFields) {
+    const paths = (textPaths ?? []).filter((path) => path.trim());
+    if (!paths.length)
+        return collectStrings(item, "", [], maxFields);
+    const fields = [];
+    for (const path of paths) {
+        const value = getAtPath(item, path);
+        if (value !== undefined)
+            collectStrings(value, path, fields, maxFields);
+        if (fields.length >= maxFields)
+            break;
+    }
+    return fields;
+}
+function collectStrings(value, basePath, out, limit) {
+    if (out.length >= limit || value === undefined || value === null)
+        return out;
+    if (typeof value === "string" ||
+        typeof value === "number" ||
+        typeof value === "boolean" ||
+        typeof value === "bigint") {
+        out.push({ path: basePath || "$", text: String(value) });
+        return out;
+    }
+    if (Array.isArray(value)) {
+        for (let i = 0; i < value.length && out.length < limit; i++) {
+            collectStrings(value[i], basePath ? `${basePath}[${i}]` : `[${i}]`, out, limit);
+        }
+        return out;
+    }
+    if (typeof value === "object") {
+        for (const key of Object.keys(value)) {
+            if (out.length >= limit)
+                break;
+            collectStrings(value[key], basePath ? `${basePath}.${key}` : key, out, limit);
+        }
+    }
+    return out;
+}
+const DEFAULT_ID_PATHS = [
+    "id",
+    "callId",
+    "callID",
+    "call_id",
+    "call.id",
+    "call.metaData.id",
+    "metaData.id",
+    "metadata.id",
+    "recordId",
+    "record_id",
+    "objectId",
+    "object_id",
+    "ticketId",
+    "ticket_id",
+    "issueId",
+    "issue_id",
+    "messageId",
+    "message_id",
+    "conversationId",
+    "conversation_id",
+    "eventId",
+    "event_id",
+    "documentId",
+    "document_id",
+    "url",
+    "webUrl",
+    "permalink",
+];
+function extractItemIdentity(item, idPaths) {
+    for (const path of [...(idPaths ?? []), ...DEFAULT_ID_PATHS]) {
+        const value = getAtPath(item, path);
+        if (value !== undefined && value !== null && String(value) !== "") {
+            return { id: stringifySearchValue(value), idPath: path };
+        }
+    }
+    return { id: null, idPath: null };
+}
+function extractMetadata(item, metadataPaths) {
+    const metadata = {};
+    for (const path of metadataPaths ?? []) {
+        const value = getAtPath(item, path);
+        if (value !== undefined)
+            metadata[path] = value;
+    }
+    return metadata;
+}
+function stringifySearchValue(value) {
+    if (typeof value === "string")
+        return value;
+    if (value === undefined || value === null)
+        return "";
+    if (typeof value === "number" ||
+        typeof value === "boolean" ||
+        typeof value === "bigint") {
+        return String(value);
+    }
+    return stringifyBrief(value, 500);
+}
+function normalizedTerms(search) {
+    const explicit = (search.terms ?? [])
+        .map((term) => term.trim())
+        .filter(Boolean);
+    if (explicit.length)
+        return explicit;
+    if (search.matchMode === "allTerms" && search.query) {
+        return search.query
+            .split(/\s+/)
+            .map((term) => term.trim())
+            .filter(Boolean);
+    }
+    return [];
+}
+function findItemWideTermMatch(fields, search) {
+    const terms = normalizedTerms(search);
+    if (!terms.length || search.matchMode === "anyTerm")
+        return null;
+    const caseSensitive = Boolean(search.caseSensitive);
+    const termHits = terms.map((term) => {
+        const needle = caseSensitive ? term : term.toLowerCase();
+        for (const field of fields) {
+            const haystack = caseSensitive ? field.text : field.text.toLowerCase();
+            const index = haystack.indexOf(needle);
+            if (index >= 0)
+                return { term, field, index };
+        }
+        return { term, field: null, index: -1 };
+    });
+    if (termHits.some((hit) => !hit.field || hit.index < 0))
+        return null;
+    const first = termHits
+        .filter((hit) => Boolean(hit.field))
+        .sort((a, b) => a.index - b.index)[0];
+    if (!first)
+        return null;
+    return {
+        field: first.field,
+        match: {
+            kind: "allTerms",
+            query: terms.join(" "),
+            index: first.index,
+            match: first.term,
+        },
+    };
+}
+function findSearchMatches(text, search, includeTerms) {
+    const source = String(text);
+    const caseSensitive = Boolean(search.caseSensitive);
+    const haystack = caseSensitive ? source : source.toLowerCase();
+    const maxMatchesPerField = 1_000;
+    const matches = [];
+    const addSubstring = (needle, label, kind) => {
+        if (needle === undefined || needle === null)
+            return;
+        const rawNeedle = String(needle);
+        if (!rawNeedle)
+            return;
+        const searchNeedle = caseSensitive ? rawNeedle : rawNeedle.toLowerCase();
+        let from = 0;
+        while (from <= haystack.length) {
+            const index = haystack.indexOf(searchNeedle, from);
+            if (index < 0)
+                break;
+            matches.push({
+                kind,
+                query: label,
+                index,
+                match: source.slice(index, index + rawNeedle.length),
+            });
+            from = index + Math.max(1, searchNeedle.length);
+            if (matches.length >= maxMatchesPerField)
+                break;
+        }
+    };
+    if (search.regex) {
+        const regex = new RegExp(search.regex, normalizeRegexFlags(search.regexFlags, caseSensitive));
+        let match;
+        while ((match = regex.exec(source)) && typeof match.index === "number") {
+            matches.push({
+                kind: "regex",
+                query: search.regex,
+                index: match.index,
+                match: match[0],
+            });
+            if (matches.length >= maxMatchesPerField)
+                break;
+            if (match[0] === "")
+                regex.lastIndex += 1;
+        }
+    }
+    if (search.query)
+        addSubstring(search.query, search.query, "query");
+    for (const query of search.queries ?? [])
+        addSubstring(query, query, "query");
+    const terms = includeTerms ? normalizedTerms(search) : [];
+    if (terms.length) {
+        const hits = terms
+            .map((term) => {
+            const needle = caseSensitive ? term : term.toLowerCase();
+            return { term, index: haystack.indexOf(needle) };
+        })
+            .filter((hit) => hit.index >= 0);
+        const mode = search.matchMode === "anyTerm" ? "anyTerm" : "allTerms";
+        if ((mode === "allTerms" && hits.length === terms.length) ||
+            (mode === "anyTerm" && hits.length > 0)) {
+            const first = hits.sort((a, b) => a.index - b.index)[0];
+            if (first) {
+                matches.push({
+                    kind: mode,
+                    query: terms.join(" "),
+                    index: first.index,
+                    match: first.term,
+                });
+            }
+        }
+    }
+    return matches.sort((a, b) => a.index - b.index);
+}
+function normalizeRegexFlags(flags, caseSensitive) {
+    const allowed = (flags ?? "")
+        .replace(/[^dgimsuvy]/g, "")
+        .replace(/[gy]/g, "");
+    const withCase = caseSensitive || /i/.test(allowed) ? allowed : `${allowed}i`;
+    return `${withCase}g`;
+}
+function makeSnippet(text, index, contextChars) {
+    const context = boundedNumber(contextChars, 180, 20, 1_000);
+    const start = Math.max(0, index - context);
+    const end = Math.min(text.length, index + context);
+    const prefix = start > 0 ? "..." : "";
+    const suffix = end < text.length ? "..." : "";
+    return `${prefix}${text.slice(start, end)}${suffix}`
+        .replace(/\s+/g, " ")
+        .trim();
+}
+//# sourceMappingURL=corpus-jobs.js.map