@agent-native/core 0.51.5 → 0.51.7

package/dist/provider-api/corpus-jobs-store.js

@@ -0,0 +1,394 @@
+/**
+ * Durable state for provider corpus jobs.
+ *
+ * Jobs are scoped by (app_id, owner_email), matching staged datasets. The
+ * runner stores only request configuration, checkpoints, and compact search
+ * hits so large provider corpora never enter chat context or filesystem
+ * scratch files.
+ */
+import { getDbExec, intType, isPostgres } from "../db/client.js";
+let initPromise;
+async function ensureTables() {
+    if (!initPromise) {
+        initPromise = (async () => {
+            const db = getDbExec();
+            const integerType = intType();
+            await db.execute(`
+        CREATE TABLE IF NOT EXISTS provider_corpus_jobs (
+          id TEXT NOT NULL,
+          app_id TEXT NOT NULL,
+          owner_email TEXT NOT NULL,
+          name TEXT NOT NULL,
+          mode TEXT NOT NULL,
+          status TEXT NOT NULL,
+          provider TEXT NOT NULL,
+          request_json TEXT NOT NULL,
+          pagination_json TEXT,
+          batch_json TEXT,
+          search_json TEXT NOT NULL,
+          limits_json TEXT NOT NULL,
+          checkpoint_json TEXT NOT NULL,
+          pages_processed ${integerType} NOT NULL DEFAULT 0,
+          batches_processed ${integerType} NOT NULL DEFAULT 0,
+          items_processed ${integerType} NOT NULL DEFAULT 0,
+          matched_items ${integerType} NOT NULL DEFAULT 0,
+          total_hits ${integerType} NOT NULL DEFAULT 0,
+          stored_hits ${integerType} NOT NULL DEFAULT 0,
+          error TEXT,
+          next_resume_at ${integerType},
+          created_at ${integerType} NOT NULL,
+          updated_at ${integerType} NOT NULL,
+          PRIMARY KEY (id)
+        )
+      `);
+            await db.execute(`
+        CREATE TABLE IF NOT EXISTS provider_corpus_job_hits (
+          job_id TEXT NOT NULL,
+          hit_index ${integerType} NOT NULL,
+          hit_data TEXT NOT NULL,
+          PRIMARY KEY (job_id, hit_index)
+        )
+      `);
+            if (isPostgres()) {
+                await widenPostgresIntegerColumns(db);
+            }
+            for (const ddl of [
+                `CREATE INDEX IF NOT EXISTS provider_corpus_jobs_scope_idx ON provider_corpus_jobs (app_id, owner_email, updated_at)`,
+                `CREATE INDEX IF NOT EXISTS provider_corpus_jobs_status_idx ON provider_corpus_jobs (app_id, owner_email, status)`,
+                `CREATE INDEX IF NOT EXISTS provider_corpus_job_hits_job_idx ON provider_corpus_job_hits (job_id)`,
+            ]) {
+                try {
+                    await db.execute(ddl);
+                }
+                catch {
+                    // Index already exists or the backend rejected best-effort indexing.
+                }
+            }
+        })().catch((err) => {
+            initPromise = undefined;
+            throw err;
+        });
+    }
+    return initPromise;
+}
+async function widenPostgresIntegerColumns(db) {
+    const statements = [
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN pages_processed TYPE BIGINT USING pages_processed::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN batches_processed TYPE BIGINT USING batches_processed::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN items_processed TYPE BIGINT USING items_processed::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN matched_items TYPE BIGINT USING matched_items::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN total_hits TYPE BIGINT USING total_hits::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN stored_hits TYPE BIGINT USING stored_hits::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN next_resume_at TYPE BIGINT USING next_resume_at::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN created_at TYPE BIGINT USING created_at::bigint`,
+        `ALTER TABLE provider_corpus_jobs ALTER COLUMN updated_at TYPE BIGINT USING updated_at::bigint`,
+        `ALTER TABLE provider_corpus_job_hits ALTER COLUMN hit_index TYPE BIGINT USING hit_index::bigint`,
+    ];
+    for (const sql of statements) {
+        try {
+            await db.execute(sql);
+        }
+        catch {
+            // Best-effort compatibility for older deployments.
+        }
+    }
+}
+export async function createProviderCorpusJob(options) {
+    await ensureTables();
+    const db = getDbExec();
+    const now = Date.now();
+    await db.execute({
+        sql: isPostgres()
+            ? `
+        INSERT INTO provider_corpus_jobs
+          (id, app_id, owner_email, name, mode, status, provider, request_json, pagination_json, batch_json, search_json, limits_json, checkpoint_json, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT (id) DO UPDATE SET
+          app_id = EXCLUDED.app_id,
+          owner_email = EXCLUDED.owner_email,
+          name = EXCLUDED.name,
+          mode = EXCLUDED.mode,
+          status = EXCLUDED.status,
+          provider = EXCLUDED.provider,
+          request_json = EXCLUDED.request_json,
+          pagination_json = EXCLUDED.pagination_json,
+          batch_json = EXCLUDED.batch_json,
+          search_json = EXCLUDED.search_json,
+          limits_json = EXCLUDED.limits_json,
+          checkpoint_json = EXCLUDED.checkpoint_json,
+          pages_processed = 0,
+          batches_processed = 0,
+          items_processed = 0,
+          matched_items = 0,
+          total_hits = 0,
+          stored_hits = 0,
+          error = NULL,
+          next_resume_at = NULL,
+          updated_at = EXCLUDED.updated_at
+        WHERE provider_corpus_jobs.app_id = EXCLUDED.app_id
+          AND provider_corpus_jobs.owner_email = EXCLUDED.owner_email
+      `
+            : `
+        INSERT INTO provider_corpus_jobs
+          (id, app_id, owner_email, name, mode, status, provider, request_json, pagination_json, batch_json, search_json, limits_json, checkpoint_json, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT (id) DO UPDATE SET
+          app_id = excluded.app_id,
+          owner_email = excluded.owner_email,
+          name = excluded.name,
+          mode = excluded.mode,
+          status = excluded.status,
+          provider = excluded.provider,
+          request_json = excluded.request_json,
+          pagination_json = excluded.pagination_json,
+          batch_json = excluded.batch_json,
+          search_json = excluded.search_json,
+          limits_json = excluded.limits_json,
+          checkpoint_json = excluded.checkpoint_json,
+          pages_processed = 0,
+          batches_processed = 0,
+          items_processed = 0,
+          matched_items = 0,
+          total_hits = 0,
+          stored_hits = 0,
+          error = NULL,
+          next_resume_at = NULL,
+          updated_at = excluded.updated_at
+        WHERE provider_corpus_jobs.app_id = excluded.app_id
+          AND provider_corpus_jobs.owner_email = excluded.owner_email
+      `,
+        args: [
+            options.id,
+            options.appId,
+            options.ownerEmail,
+            options.name,
+            options.mode,
+            options.status,
+            options.provider,
+            JSON.stringify(options.request),
+            jsonOrNull(options.pagination),
+            jsonOrNull(options.batch),
+            JSON.stringify(options.search),
+            JSON.stringify(options.limits),
+            JSON.stringify(options.checkpoint),
+            now,
+            now,
+        ],
+    });
+    await db.execute({
+        sql: `DELETE FROM provider_corpus_job_hits WHERE job_id = ?`,
+        args: [options.id],
+    });
+    const job = await getProviderCorpusJob({
+        id: options.id,
+        appId: options.appId,
+        ownerEmail: options.ownerEmail,
+    });
+    if (!job) {
+        // The scoped upsert above only updates a row that already belongs to this
+        // (app_id, owner_email). A null read here means a job with this id exists
+        // under a different owner, so the conflicting insert was skipped rather
+        // than clobbering the other tenant's job.
+        throw new Error(`Failed to create provider corpus job ${options.id}: a job with this id ` +
+            `already exists for a different owner. Use a different jobId.`);
+    }
+    return job;
+}
+export async function getProviderCorpusJob(options) {
+    await ensureTables();
+    const db = getDbExec();
+    const { rows } = await db.execute({
+        sql: `SELECT * FROM provider_corpus_jobs WHERE id = ? AND app_id = ? AND owner_email = ?`,
+        args: [options.id, options.appId, options.ownerEmail],
+    });
+    return rows[0] ? rowToJob(rows[0]) : null;
+}
+export async function listProviderCorpusJobs(options) {
+    await ensureTables();
+    const db = getDbExec();
+    const limit = Math.max(1, Math.min(options.limit ?? 20, 100));
+    const { rows } = await db.execute({
+        sql: `SELECT * FROM provider_corpus_jobs WHERE app_id = ? AND owner_email = ? ORDER BY updated_at DESC LIMIT ?`,
+        args: [options.appId, options.ownerEmail, limit],
+    });
+    return rows.map(rowToJob);
+}
+export async function updateProviderCorpusJob(options) {
+    await ensureTables();
+    const existing = await getProviderCorpusJob({
+        id: options.id,
+        appId: options.appId,
+        ownerEmail: options.ownerEmail,
+    });
+    if (!existing)
+        return null;
+    const next = {
+        status: options.status ?? existing.status,
+        checkpoint: options.checkpoint ?? existing.checkpoint,
+        pagesProcessed: options.pagesProcessed ?? existing.pagesProcessed,
+        batchesProcessed: options.batchesProcessed ?? existing.batchesProcessed,
+        itemsProcessed: options.itemsProcessed ?? existing.itemsProcessed,
+        matchedItems: options.matchedItems ?? existing.matchedItems,
+        totalHits: options.totalHits ?? existing.totalHits,
+        storedHits: options.storedHits ?? existing.storedHits,
+        error: options.error === undefined ? existing.error : options.error,
+        nextResumeAt: options.nextResumeAt === undefined
+            ? existing.nextResumeAt
+            : options.nextResumeAt,
+    };
+    const db = getDbExec();
+    await db.execute({
+        sql: `
+      UPDATE provider_corpus_jobs SET
+        status = ?,
+        checkpoint_json = ?,
+        pages_processed = ?,
+        batches_processed = ?,
+        items_processed = ?,
+        matched_items = ?,
+        total_hits = ?,
+        stored_hits = ?,
+        error = ?,
+        next_resume_at = ?,
+        updated_at = ?
+      WHERE id = ? AND app_id = ? AND owner_email = ?
+    `,
+        args: [
+            next.status,
+            JSON.stringify(next.checkpoint),
+            next.pagesProcessed,
+            next.batchesProcessed,
+            next.itemsProcessed,
+            next.matchedItems,
+            next.totalHits,
+            next.storedHits,
+            next.error,
+            next.nextResumeAt,
+            Date.now(),
+            options.id,
+            options.appId,
+            options.ownerEmail,
+        ],
+    });
+    return getProviderCorpusJob({
+        id: options.id,
+        appId: options.appId,
+        ownerEmail: options.ownerEmail,
+    });
+}
+export async function appendProviderCorpusJobHits(options) {
+    if (options.hits.length === 0)
+        return;
+    await ensureTables();
+    const db = getDbExec();
+    // Insert in chunked multi-row statements rather than one round-trip per hit,
+    // and ignore conflicts on (job_id, hit_index). The runner writes hits before
+    // it advances the stored-hits counter and checkpoint, so a crash between the
+    // two means resume re-fetches the same page and re-appends the same indices;
+    // DO NOTHING makes that retry idempotent instead of a primary-key violation.
+    const CHUNK = 100;
+    for (let start = 0; start < options.hits.length; start += CHUNK) {
+        const chunk = options.hits.slice(start, start + CHUNK);
+        const placeholders = chunk.map(() => "(?, ?, ?)").join(", ");
+        const args = [];
+        for (let j = 0; j < chunk.length; j++) {
+            args.push(options.jobId, options.startIndex + start + j, JSON.stringify(chunk[j]));
+        }
+        await db.execute({
+            sql: `INSERT INTO provider_corpus_job_hits (job_id, hit_index, hit_data) VALUES ${placeholders} ON CONFLICT (job_id, hit_index) DO NOTHING`,
+            args,
+        });
+    }
+}
+export async function getProviderCorpusJobHits(options) {
+    await ensureTables();
+    const job = await getProviderCorpusJob({
+        id: options.jobId,
+        appId: options.appId,
+        ownerEmail: options.ownerEmail,
+    });
+    if (!job)
+        return [];
+    const offset = Math.max(0, options.offset ?? 0);
+    const limit = Math.max(1, Math.min(options.limit ?? 100, 1000));
+    const db = getDbExec();
+    const { rows } = await db.execute({
+        sql: `SELECT hit_data FROM provider_corpus_job_hits WHERE job_id = ? ORDER BY hit_index ASC LIMIT ? OFFSET ?`,
+        args: [options.jobId, limit, offset],
+    });
+    return rows.map((row) => parseJsonRecord(row.hit_data));
+}
+export async function deleteProviderCorpusJob(options) {
+    await ensureTables();
+    const existing = await getProviderCorpusJob(options);
+    if (!existing)
+        return false;
+    const db = getDbExec();
+    await db.execute({
+        sql: `DELETE FROM provider_corpus_job_hits WHERE job_id = ?`,
+        args: [options.id],
+    });
+    const result = await db.execute({
+        sql: `DELETE FROM provider_corpus_jobs WHERE id = ? AND app_id = ? AND owner_email = ?`,
+        args: [options.id, options.appId, options.ownerEmail],
+    });
+    return result.rowsAffected > 0;
+}
+function jsonOrNull(value) {
+    return value ? JSON.stringify(value) : null;
+}
+function parseJsonRecord(value) {
+    if (typeof value !== "string")
+        return {};
+    try {
+        const parsed = JSON.parse(value);
+        return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+            ? parsed
+            : {};
+    }
+    catch {
+        return {};
+    }
+}
+function parseNullableJsonRecord(value) {
+    if (typeof value !== "string" || !value)
+        return null;
+    return parseJsonRecord(value);
+}
+function nullableNumber(value) {
+    if (value === null || value === undefined)
+        return null;
+    const number = Number(value);
+    return Number.isFinite(number) ? number : null;
+}
+function rowToJob(row) {
+    return {
+        id: String(row.id),
+        appId: String(row.app_id),
+        ownerEmail: String(row.owner_email),
+        name: String(row.name),
+        mode: String(row.mode),
+        status: String(row.status),
+        provider: String(row.provider),
+        request: parseJsonRecord(row.request_json),
+        pagination: parseNullableJsonRecord(row.pagination_json),
+        batch: parseNullableJsonRecord(row.batch_json),
+        search: parseJsonRecord(row.search_json),
+        limits: parseJsonRecord(row.limits_json),
+        checkpoint: parseJsonRecord(row.checkpoint_json),
+        pagesProcessed: Number(row.pages_processed ?? 0),
+        batchesProcessed: Number(row.batches_processed ?? 0),
+        itemsProcessed: Number(row.items_processed ?? 0),
+        matchedItems: Number(row.matched_items ?? 0),
+        totalHits: Number(row.total_hits ?? 0),
+        storedHits: Number(row.stored_hits ?? 0),
+        error: row.error == null ? null : String(row.error),
+        nextResumeAt: nullableNumber(row.next_resume_at),
+        createdAt: Number(row.created_at ?? 0),
+        updatedAt: Number(row.updated_at ?? 0),
+    };
+}
+export function _resetProviderCorpusJobsStoreForTests() {
+    initPromise = undefined;
+}
+//# sourceMappingURL=corpus-jobs-store.js.map

package/dist/provider-api/corpus-jobs-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-jobs-store.js","sourceRoot":"","sources":["../../src/provider-api/corpus-jobs-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAe,MAAM,iBAAiB,CAAC;AAmE9E,IAAI,WAAsC,CAAC;AAE3C,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACxB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,EAAE,CAAC;YAC9B,MAAM,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;4BAeK,WAAW;8BACT,WAAW;4BACb,WAAW;0BACb,WAAW;uBACd,WAAW;wBACV,WAAW;;2BAER,WAAW;uBACf,WAAW;uBACX,WAAW;;;OAG3B,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,OAAO,CAAC;;;sBAGD,WAAW;;;;OAI1B,CAAC,CAAC;YACH,IAAI,UAAU,EAAE,EAAE,CAAC;gBACjB,MAAM,2BAA2B,CAAC,EAAE,CAAC,CAAC;YACxC,CAAC;YACD,KAAK,MAAM,GAAG,IAAI;gBAChB,qHAAqH;gBACrH,kHAAkH;gBAClH,kGAAkG;aACnG,EAAE,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBACP,qEAAqE;gBACvE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;YACxB,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,EAAU;IACnD,MAAM,UAAU,GAAG;QACjB,yGAAyG;QACzG,6GAA6G;QAC7G,yGAAyG;QACzG,qGAAqG;QACrG,+FAA+F;QAC/F,iGAAiG;QACjG,uGAAuG;QACvG,+FAA+F;QAC/F,+FAA+F;QAC/F,iGAAiG;KAClG,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;QACrD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAuC;IAEvC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,EAAE,CAAC,OAAO,CAAC;QACf,GAAG,EAAE,UAAU,EAAE;YACf,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BD;YACD,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BD;QACH,IAAI,EAAE;YACJ,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,KAAK;YACb,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,QAAQ;YAChB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;YAC/B,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC;YAC9B,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;YACzB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC;YAClC,GAAG;YACH,GAAG;SACJ;KACF,CAAC,CAAC;IACH,MAAM,EAAE,CAAC,OAAO,CAAC;QACf,GAAG,EAAE,uDAAuD;QAC5D,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,MAAM,oBAAoB,CAAC;QACrC,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,0EAA0E;QAC1E,0EAA0E;QAC1E,wEAAwE;QACxE,0CAA0C;QAC1C,MAAM,IAAI,KAAK,CACb,wCAAwC,OAAO,CAAC,EAAE,uBAAuB;YACvE,8DAA8D,CACjE,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAI1C;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,oFAAoF;QACzF,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;KACtD,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAI5C;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,0GAA0G;QAC/G,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;KACjD,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAuC;IAEvC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC;QAC1C,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,IAAI,GAAG;QACX,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM;QACzC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU;QACrD,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC,cAAc;QACjE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB;QACvE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC,cAAc;QACjE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY;QAC3D,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAClD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU;QACrD,KAAK,EAAE,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK;QACnE,YAAY,EACV,OAAO,CAAC,YAAY,KAAK,SAAS;YAChC,CAAC,CAAC,QAAQ,CAAC,YAAY;YACvB,CAAC,CAAC,OAAO,CAAC,YAAY;KAC3B,CAAC;IACF,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,CAAC,OAAO,CAAC;QACf,GAAG,EAAE;;;;;;;;;;;;;;KAcJ;QACD,IAAI,EAAE;YACJ,IAAI,CAAC,MAAM;YACX,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC;YAC/B,IAAI,CAAC,cAAc;YACnB,IAAI,CAAC,gBAAgB;YACrB,IAAI,CAAC,cAAc;YACnB,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,SAAS;YACd,IAAI,CAAC,UAAU;YACf,IAAI,CAAC,KAAK;YACV,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,KAAK;YACb,OAAO,CAAC,UAAU;SACnB;KACF,CAAC,CAAC;IACH,OAAO,oBAAoB,CAAC;QAC1B,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,OAIjD;IACC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACtC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,6EAA6E;IAC7E,6EAA6E;IAC7E,6EAA6E;IAC7E,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,KAAK,GAAG,GAAG,CAAC;IAClB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,KAAK,EAAE,CAAC;QAChE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAc,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,UAAU,GAAG,KAAK,GAAG,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACzB,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,CAAC,OAAO,CAAC;YACf,GAAG,EAAE,6EAA6E,YAAY,6CAA6C;YAC3I,IAAI;SACL,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAM9C;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,GAAG,GAAG,MAAM,oBAAoB,CAAC;QACrC,EAAE,EAAE,OAAO,CAAC,KAAK;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IAChE,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,wGAAwG;QAC7G,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;KACrC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAI7C;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,CAAC,OAAO,CAAC;QACf,GAAG,EAAE,uDAAuD;QAC5D,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAC9B,GAAG,EAAE,kFAAkF;QACvF,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;KACtD,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CACjB,KAAiD;IAEjD,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAY,CAAC;QAC5C,OAAO,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACnE,CAAC,CAAE,MAAkC;YACrC,CAAC,CAAC,EAAE,CAAC;IACT,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC9B,KAAc;IAEd,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACrD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,SAAS,QAAQ,CAAC,GAA4B;IAC5C,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAA4B;QACrD,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC;QAC1C,UAAU,EAAE,uBAAuB,CAAC,GAAG,CAAC,eAAe,CAAC;QACxD,KAAK,EAAE,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC;QAC9C,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC;QACxC,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC;QACxC,UAAU,EAAE,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC;QAChD,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC;QAChD,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,CAAC;QACpD,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC;QAChD,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC;QAC5C,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;QACnD,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,cAAc,CAAC;QAChD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qCAAqC;IACnD,WAAW,GAAG,SAAS,CAAC;AAC1B,CAAC","sourcesContent":["/**\n * Durable state for provider corpus jobs.\n *\n * Jobs are scoped by (app_id, owner_email), matching staged datasets. The\n * runner stores only request configuration, checkpoints, and compact search\n * hits so large provider corpora never enter chat context or filesystem\n * scratch files.\n */\n\nimport { getDbExec, intType, isPostgres, type DbExec } from \"../db/client.js\";\n\nexport type ProviderCorpusJobStatus =\n  | \"running\"\n  | \"paused\"\n  | \"quota_wait\"\n  | \"completed\"\n  | \"failed\";\n\nexport interface ProviderCorpusJobRecord {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n  name: string;\n  mode: string;\n  status: ProviderCorpusJobStatus;\n  provider: string;\n  request: Record<string, unknown>;\n  pagination: Record<string, unknown> | null;\n  batch: Record<string, unknown> | null;\n  search: Record<string, unknown>;\n  limits: Record<string, unknown>;\n  checkpoint: Record<string, unknown>;\n  pagesProcessed: number;\n  batchesProcessed: number;\n  itemsProcessed: number;\n  matchedItems: number;\n  totalHits: number;\n  storedHits: number;\n  error: string | null;\n  nextResumeAt: number | null;\n  createdAt: number;\n  updatedAt: number;\n}\n\nexport interface CreateProviderCorpusJobOptions {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n  name: string;\n  mode: string;\n  status: ProviderCorpusJobStatus;\n  provider: string;\n  request: Record<string, unknown>;\n  pagination?: Record<string, unknown> | null;\n  batch?: Record<string, unknown> | null;\n  search: Record<string, unknown>;\n  limits: Record<string, unknown>;\n  checkpoint: Record<string, unknown>;\n}\n\nexport interface UpdateProviderCorpusJobOptions {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n  status?: ProviderCorpusJobStatus;\n  checkpoint?: Record<string, unknown>;\n  pagesProcessed?: number;\n  batchesProcessed?: number;\n  itemsProcessed?: number;\n  matchedItems?: number;\n  totalHits?: number;\n  storedHits?: number;\n  error?: string | null;\n  nextResumeAt?: number | null;\n}\n\nlet initPromise: Promise<void> | undefined;\n\nasync function ensureTables(): Promise<void> {\n  if (!initPromise) {\n    initPromise = (async () => {\n      const db = getDbExec();\n      const integerType = intType();\n      await db.execute(`\n        CREATE TABLE IF NOT EXISTS provider_corpus_jobs (\n          id TEXT NOT NULL,\n          app_id TEXT NOT NULL,\n          owner_email TEXT NOT NULL,\n          name TEXT NOT NULL,\n          mode TEXT NOT NULL,\n          status TEXT NOT NULL,\n          provider TEXT NOT NULL,\n          request_json TEXT NOT NULL,\n          pagination_json TEXT,\n          batch_json TEXT,\n          search_json TEXT NOT NULL,\n          limits_json TEXT NOT NULL,\n          checkpoint_json TEXT NOT NULL,\n          pages_processed ${integerType} NOT NULL DEFAULT 0,\n          batches_processed ${integerType} NOT NULL DEFAULT 0,\n          items_processed ${integerType} NOT NULL DEFAULT 0,\n          matched_items ${integerType} NOT NULL DEFAULT 0,\n          total_hits ${integerType} NOT NULL DEFAULT 0,\n          stored_hits ${integerType} NOT NULL DEFAULT 0,\n          error TEXT,\n          next_resume_at ${integerType},\n          created_at ${integerType} NOT NULL,\n          updated_at ${integerType} NOT NULL,\n          PRIMARY KEY (id)\n        )\n      `);\n      await db.execute(`\n        CREATE TABLE IF NOT EXISTS provider_corpus_job_hits (\n          job_id TEXT NOT NULL,\n          hit_index ${integerType} NOT NULL,\n          hit_data TEXT NOT NULL,\n          PRIMARY KEY (job_id, hit_index)\n        )\n      `);\n      if (isPostgres()) {\n        await widenPostgresIntegerColumns(db);\n      }\n      for (const ddl of [\n        `CREATE INDEX IF NOT EXISTS provider_corpus_jobs_scope_idx ON provider_corpus_jobs (app_id, owner_email, updated_at)`,\n        `CREATE INDEX IF NOT EXISTS provider_corpus_jobs_status_idx ON provider_corpus_jobs (app_id, owner_email, status)`,\n        `CREATE INDEX IF NOT EXISTS provider_corpus_job_hits_job_idx ON provider_corpus_job_hits (job_id)`,\n      ]) {\n        try {\n          await db.execute(ddl);\n        } catch {\n          // Index already exists or the backend rejected best-effort indexing.\n        }\n      }\n    })().catch((err) => {\n      initPromise = undefined;\n      throw err;\n    });\n  }\n  return initPromise;\n}\n\nasync function widenPostgresIntegerColumns(db: DbExec): Promise<void> {\n  const statements = [\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN pages_processed TYPE BIGINT USING pages_processed::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN batches_processed TYPE BIGINT USING batches_processed::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN items_processed TYPE BIGINT USING items_processed::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN matched_items TYPE BIGINT USING matched_items::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN total_hits TYPE BIGINT USING total_hits::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN stored_hits TYPE BIGINT USING stored_hits::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN next_resume_at TYPE BIGINT USING next_resume_at::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN created_at TYPE BIGINT USING created_at::bigint`,\n    `ALTER TABLE provider_corpus_jobs ALTER COLUMN updated_at TYPE BIGINT USING updated_at::bigint`,\n    `ALTER TABLE provider_corpus_job_hits ALTER COLUMN hit_index TYPE BIGINT USING hit_index::bigint`,\n  ];\n  for (const sql of statements) {\n    try {\n      await db.execute(sql);\n    } catch {\n      // Best-effort compatibility for older deployments.\n    }\n  }\n}\n\nexport async function createProviderCorpusJob(\n  options: CreateProviderCorpusJobOptions,\n): Promise<ProviderCorpusJobRecord> {\n  await ensureTables();\n  const db = getDbExec();\n  const now = Date.now();\n  await db.execute({\n    sql: isPostgres()\n      ? `\n        INSERT INTO provider_corpus_jobs\n          (id, app_id, owner_email, name, mode, status, provider, request_json, pagination_json, batch_json, search_json, limits_json, checkpoint_json, created_at, updated_at)\n        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n        ON CONFLICT (id) DO UPDATE SET\n          app_id = EXCLUDED.app_id,\n          owner_email = EXCLUDED.owner_email,\n          name = EXCLUDED.name,\n          mode = EXCLUDED.mode,\n          status = EXCLUDED.status,\n          provider = EXCLUDED.provider,\n          request_json = EXCLUDED.request_json,\n          pagination_json = EXCLUDED.pagination_json,\n          batch_json = EXCLUDED.batch_json,\n          search_json = EXCLUDED.search_json,\n          limits_json = EXCLUDED.limits_json,\n          checkpoint_json = EXCLUDED.checkpoint_json,\n          pages_processed = 0,\n          batches_processed = 0,\n          items_processed = 0,\n          matched_items = 0,\n          total_hits = 0,\n          stored_hits = 0,\n          error = NULL,\n          next_resume_at = NULL,\n          updated_at = EXCLUDED.updated_at\n        WHERE provider_corpus_jobs.app_id = EXCLUDED.app_id\n          AND provider_corpus_jobs.owner_email = EXCLUDED.owner_email\n      `\n      : `\n        INSERT INTO provider_corpus_jobs\n          (id, app_id, owner_email, name, mode, status, provider, request_json, pagination_json, batch_json, search_json, limits_json, checkpoint_json, created_at, updated_at)\n        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n        ON CONFLICT (id) DO UPDATE SET\n          app_id = excluded.app_id,\n          owner_email = excluded.owner_email,\n          name = excluded.name,\n          mode = excluded.mode,\n          status = excluded.status,\n          provider = excluded.provider,\n          request_json = excluded.request_json,\n          pagination_json = excluded.pagination_json,\n          batch_json = excluded.batch_json,\n          search_json = excluded.search_json,\n          limits_json = excluded.limits_json,\n          checkpoint_json = excluded.checkpoint_json,\n          pages_processed = 0,\n          batches_processed = 0,\n          items_processed = 0,\n          matched_items = 0,\n          total_hits = 0,\n          stored_hits = 0,\n          error = NULL,\n          next_resume_at = NULL,\n          updated_at = excluded.updated_at\n        WHERE provider_corpus_jobs.app_id = excluded.app_id\n          AND provider_corpus_jobs.owner_email = excluded.owner_email\n      `,\n    args: [\n      options.id,\n      options.appId,\n      options.ownerEmail,\n      options.name,\n      options.mode,\n      options.status,\n      options.provider,\n      JSON.stringify(options.request),\n      jsonOrNull(options.pagination),\n      jsonOrNull(options.batch),\n      JSON.stringify(options.search),\n      JSON.stringify(options.limits),\n      JSON.stringify(options.checkpoint),\n      now,\n      now,\n    ],\n  });\n  await db.execute({\n    sql: `DELETE FROM provider_corpus_job_hits WHERE job_id = ?`,\n    args: [options.id],\n  });\n  const job = await getProviderCorpusJob({\n    id: options.id,\n    appId: options.appId,\n    ownerEmail: options.ownerEmail,\n  });\n  if (!job) {\n    // The scoped upsert above only updates a row that already belongs to this\n    // (app_id, owner_email). A null read here means a job with this id exists\n    // under a different owner, so the conflicting insert was skipped rather\n    // than clobbering the other tenant's job.\n    throw new Error(\n      `Failed to create provider corpus job ${options.id}: a job with this id ` +\n        `already exists for a different owner. Use a different jobId.`,\n    );\n  }\n  return job;\n}\n\nexport async function getProviderCorpusJob(options: {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n}): Promise<ProviderCorpusJobRecord | null> {\n  await ensureTables();\n  const db = getDbExec();\n  const { rows } = await db.execute({\n    sql: `SELECT * FROM provider_corpus_jobs WHERE id = ? AND app_id = ? AND owner_email = ?`,\n    args: [options.id, options.appId, options.ownerEmail],\n  });\n  return rows[0] ? rowToJob(rows[0]) : null;\n}\n\nexport async function listProviderCorpusJobs(options: {\n  appId: string;\n  ownerEmail: string;\n  limit?: number;\n}): Promise<ProviderCorpusJobRecord[]> {\n  await ensureTables();\n  const db = getDbExec();\n  const limit = Math.max(1, Math.min(options.limit ?? 20, 100));\n  const { rows } = await db.execute({\n    sql: `SELECT * FROM provider_corpus_jobs WHERE app_id = ? AND owner_email = ? ORDER BY updated_at DESC LIMIT ?`,\n    args: [options.appId, options.ownerEmail, limit],\n  });\n  return rows.map(rowToJob);\n}\n\nexport async function updateProviderCorpusJob(\n  options: UpdateProviderCorpusJobOptions,\n): Promise<ProviderCorpusJobRecord | null> {\n  await ensureTables();\n  const existing = await getProviderCorpusJob({\n    id: options.id,\n    appId: options.appId,\n    ownerEmail: options.ownerEmail,\n  });\n  if (!existing) return null;\n  const next = {\n    status: options.status ?? existing.status,\n    checkpoint: options.checkpoint ?? existing.checkpoint,\n    pagesProcessed: options.pagesProcessed ?? existing.pagesProcessed,\n    batchesProcessed: options.batchesProcessed ?? existing.batchesProcessed,\n    itemsProcessed: options.itemsProcessed ?? existing.itemsProcessed,\n    matchedItems: options.matchedItems ?? existing.matchedItems,\n    totalHits: options.totalHits ?? existing.totalHits,\n    storedHits: options.storedHits ?? existing.storedHits,\n    error: options.error === undefined ? existing.error : options.error,\n    nextResumeAt:\n      options.nextResumeAt === undefined\n        ? existing.nextResumeAt\n        : options.nextResumeAt,\n  };\n  const db = getDbExec();\n  await db.execute({\n    sql: `\n      UPDATE provider_corpus_jobs SET\n        status = ?,\n        checkpoint_json = ?,\n        pages_processed = ?,\n        batches_processed = ?,\n        items_processed = ?,\n        matched_items = ?,\n        total_hits = ?,\n        stored_hits = ?,\n        error = ?,\n        next_resume_at = ?,\n        updated_at = ?\n      WHERE id = ? AND app_id = ? AND owner_email = ?\n    `,\n    args: [\n      next.status,\n      JSON.stringify(next.checkpoint),\n      next.pagesProcessed,\n      next.batchesProcessed,\n      next.itemsProcessed,\n      next.matchedItems,\n      next.totalHits,\n      next.storedHits,\n      next.error,\n      next.nextResumeAt,\n      Date.now(),\n      options.id,\n      options.appId,\n      options.ownerEmail,\n    ],\n  });\n  return getProviderCorpusJob({\n    id: options.id,\n    appId: options.appId,\n    ownerEmail: options.ownerEmail,\n  });\n}\n\nexport async function appendProviderCorpusJobHits(options: {\n  jobId: string;\n  startIndex: number;\n  hits: Record<string, unknown>[];\n}): Promise<void> {\n  if (options.hits.length === 0) return;\n  await ensureTables();\n  const db = getDbExec();\n  // Insert in chunked multi-row statements rather than one round-trip per hit,\n  // and ignore conflicts on (job_id, hit_index). The runner writes hits before\n  // it advances the stored-hits counter and checkpoint, so a crash between the\n  // two means resume re-fetches the same page and re-appends the same indices;\n  // DO NOTHING makes that retry idempotent instead of a primary-key violation.\n  const CHUNK = 100;\n  for (let start = 0; start < options.hits.length; start += CHUNK) {\n    const chunk = options.hits.slice(start, start + CHUNK);\n    const placeholders = chunk.map(() => \"(?, ?, ?)\").join(\", \");\n    const args: unknown[] = [];\n    for (let j = 0; j < chunk.length; j++) {\n      args.push(\n        options.jobId,\n        options.startIndex + start + j,\n        JSON.stringify(chunk[j]),\n      );\n    }\n    await db.execute({\n      sql: `INSERT INTO provider_corpus_job_hits (job_id, hit_index, hit_data) VALUES ${placeholders} ON CONFLICT (job_id, hit_index) DO NOTHING`,\n      args,\n    });\n  }\n}\n\nexport async function getProviderCorpusJobHits(options: {\n  jobId: string;\n  appId: string;\n  ownerEmail: string;\n  offset?: number;\n  limit?: number;\n}): Promise<Record<string, unknown>[]> {\n  await ensureTables();\n  const job = await getProviderCorpusJob({\n    id: options.jobId,\n    appId: options.appId,\n    ownerEmail: options.ownerEmail,\n  });\n  if (!job) return [];\n  const offset = Math.max(0, options.offset ?? 0);\n  const limit = Math.max(1, Math.min(options.limit ?? 100, 1000));\n  const db = getDbExec();\n  const { rows } = await db.execute({\n    sql: `SELECT hit_data FROM provider_corpus_job_hits WHERE job_id = ? ORDER BY hit_index ASC LIMIT ? OFFSET ?`,\n    args: [options.jobId, limit, offset],\n  });\n  return rows.map((row) => parseJsonRecord(row.hit_data));\n}\n\nexport async function deleteProviderCorpusJob(options: {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n}): Promise<boolean> {\n  await ensureTables();\n  const existing = await getProviderCorpusJob(options);\n  if (!existing) return false;\n  const db = getDbExec();\n  await db.execute({\n    sql: `DELETE FROM provider_corpus_job_hits WHERE job_id = ?`,\n    args: [options.id],\n  });\n  const result = await db.execute({\n    sql: `DELETE FROM provider_corpus_jobs WHERE id = ? AND app_id = ? AND owner_email = ?`,\n    args: [options.id, options.appId, options.ownerEmail],\n  });\n  return result.rowsAffected > 0;\n}\n\nfunction jsonOrNull(\n  value: Record<string, unknown> | null | undefined,\n): string | null {\n  return value ? JSON.stringify(value) : null;\n}\n\nfunction parseJsonRecord(value: unknown): Record<string, unknown> {\n  if (typeof value !== \"string\") return {};\n  try {\n    const parsed = JSON.parse(value) as unknown;\n    return parsed && typeof parsed === \"object\" && !Array.isArray(parsed)\n      ? (parsed as Record<string, unknown>)\n      : {};\n  } catch {\n    return {};\n  }\n}\n\nfunction parseNullableJsonRecord(\n  value: unknown,\n): Record<string, unknown> | null {\n  if (typeof value !== \"string\" || !value) return null;\n  return parseJsonRecord(value);\n}\n\nfunction nullableNumber(value: unknown): number | null {\n  if (value === null || value === undefined) return null;\n  const number = Number(value);\n  return Number.isFinite(number) ? number : null;\n}\n\nfunction rowToJob(row: Record<string, unknown>): ProviderCorpusJobRecord {\n  return {\n    id: String(row.id),\n    appId: String(row.app_id),\n    ownerEmail: String(row.owner_email),\n    name: String(row.name),\n    mode: String(row.mode),\n    status: String(row.status) as ProviderCorpusJobStatus,\n    provider: String(row.provider),\n    request: parseJsonRecord(row.request_json),\n    pagination: parseNullableJsonRecord(row.pagination_json),\n    batch: parseNullableJsonRecord(row.batch_json),\n    search: parseJsonRecord(row.search_json),\n    limits: parseJsonRecord(row.limits_json),\n    checkpoint: parseJsonRecord(row.checkpoint_json),\n    pagesProcessed: Number(row.pages_processed ?? 0),\n    batchesProcessed: Number(row.batches_processed ?? 0),\n    itemsProcessed: Number(row.items_processed ?? 0),\n    matchedItems: Number(row.matched_items ?? 0),\n    totalHits: Number(row.total_hits ?? 0),\n    storedHits: Number(row.stored_hits ?? 0),\n    error: row.error == null ? null : String(row.error),\n    nextResumeAt: nullableNumber(row.next_resume_at),\n    createdAt: Number(row.created_at ?? 0),\n    updatedAt: Number(row.updated_at ?? 0),\n  };\n}\n\nexport function _resetProviderCorpusJobsStoreForTests(): void {\n  initPromise = undefined;\n}\n"]}

package/dist/provider-api/corpus-jobs.d.ts

@@ -0,0 +1,146 @@
+import type { ProviderApiRequestArgs } from "./index.js";
+import { type ProviderCorpusJobStatus } from "./corpus-jobs-store.js";
+type ProviderCorpusRuntime = {
+    executeRequest(args: ProviderApiRequestArgs): Promise<unknown>;
+};
+export interface CreateProviderCorpusJobActionOptions {
+    appId: string;
+    getRuntime: () => ProviderCorpusRuntime;
+}
+export declare function createProviderCorpusJobAction(options: CreateProviderCorpusJobActionOptions): import("../action.js").ActionDefinition<{
+    operation?: "start" | "status" | "continue" | "list" | "results" | "delete" | undefined;
+    jobId?: string | undefined;
+    name?: string | undefined;
+    mode?: "paginated-search" | "batch-search" | undefined;
+    request?: {
+        provider: string;
+        path: string;
+        method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "PATCH" | undefined;
+        query?: unknown;
+        headers?: Record<string, unknown> | undefined;
+        body?: unknown;
+        auth?: "none" | "default" | undefined;
+        connectionId?: string | undefined;
+        accountId?: string | undefined;
+        timeoutMs?: unknown;
+        maxBytes?: unknown;
+    } | undefined;
+    pagination?: {
+        itemsPath?: string | undefined;
+        nextCursorPath?: string | undefined;
+        cursorPath?: string | undefined;
+        cursorParam?: string | undefined;
+        cursorBodyPath?: string | undefined;
+        pageParam?: string | undefined;
+        startPage?: unknown;
+        offsetParam?: string | undefined;
+        startOffset?: unknown;
+        pageSize?: unknown;
+        maxPages?: unknown;
+    } | undefined;
+    batch?: {
+        items?: unknown[] | undefined;
+        inputDatasetId?: string | undefined;
+        inputValuePath?: string | undefined;
+        batchSize?: unknown;
+        itemBodyPath?: string | undefined;
+        itemQueryParam?: string | undefined;
+        responseItemsPath?: string | undefined;
+    } | undefined;
+    search?: {
+        query?: string | undefined;
+        queries?: string[] | undefined;
+        terms?: string[] | undefined;
+        regex?: string | undefined;
+        regexFlags?: string | undefined;
+        matchMode?: "query" | "allTerms" | "anyTerm" | undefined;
+        caseSensitive?: boolean | undefined;
+        textPaths?: string[] | undefined;
+        idPaths?: string[] | undefined;
+        metadataPaths?: string[] | undefined;
+        contextChars?: unknown;
+        maxHits?: unknown;
+        maxHitsPerItem?: unknown;
+        maxFieldsPerItem?: unknown;
+    } | undefined;
+    limits?: {
+        pageBudget?: unknown;
+        batchBudget?: unknown;
+        runtimeMs?: unknown;
+        itemBudget?: unknown;
+        maxHits?: unknown;
+    } | undefined;
+    offset?: unknown;
+    limit?: unknown;
+}, {
+    job: {
+        id: string;
+        name: string;
+        mode: string;
+        status: ProviderCorpusJobStatus;
+        provider: string;
+        createdAt: string;
+        updatedAt: string;
+    };
+    coverage: {
+        pagesProcessed: number;
+        batchesProcessed: number;
+        itemsProcessed: number;
+        matchedItems: number;
+        totalHits: number;
+        storedHits: number;
+        truncatedHits: boolean;
+    };
+    checkpoint: Record<string, unknown>;
+    error: string | null;
+    nextResumeAt: string | null;
+} | {
+    jobs: {
+        id: string;
+        name: string;
+        mode: string;
+        status: ProviderCorpusJobStatus;
+        provider: string;
+        createdAt: string;
+        updatedAt: string;
+    }[];
+    total: number;
+    deleted?: undefined;
+    jobId?: undefined;
+} | {
+    deleted: boolean;
+    jobId: string;
+    jobs?: undefined;
+    total?: undefined;
+} | {
+    hits: Record<string, unknown>[];
+    offset: number;
+    limit: number;
+    job: {
+        id: string;
+        name: string;
+        mode: string;
+        status: ProviderCorpusJobStatus;
+        provider: string;
+        createdAt: string;
+        updatedAt: string;
+    };
+    coverage: {
+        pagesProcessed: number;
+        batchesProcessed: number;
+        itemsProcessed: number;
+        matchedItems: number;
+        totalHits: number;
+        storedHits: number;
+        truncatedHits: boolean;
+    };
+    checkpoint: Record<string, unknown>;
+    error: string | null;
+    nextResumeAt: string | null;
+    jobs?: undefined;
+    total?: undefined;
+    deleted?: undefined;
+    jobId?: undefined;
+}>;
+export {};
+//# sourceMappingURL=corpus-jobs.d.ts.map

package/dist/provider-api/corpus-jobs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-jobs.d.ts","sourceRoot":"","sources":["../../src/provider-api/corpus-jobs.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAqB,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,EASL,KAAK,uBAAuB,EAC7B,MAAM,wBAAwB,CAAC;AAEhC,KAAK,qBAAqB,GAAG;IAC3B,cAAc,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChE,CAAC;AAEF,MAAM,WAAW,oCAAoC;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,qBAAqB,CAAC;CACzC;AAiOD,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAa9C"}