@agent-native/core 0.47.1 → 0.48.2

package/dist/provider-api/staged-datasets-store.js

@@ -0,0 +1,319 @@
+/**
+ * Scratch storage for staged provider-API datasets.
+ *
+ * Rows are stored as JSON text — no dialect-specific JSON-column types needed.
+ * Aggregation is done in TypeScript server-side (portable across Postgres and
+ * SQLite) rather than via JSON SQL fragments. See `staged-datasets-aggregate.ts`.
+ *
+ * Storage caps:
+ *   - MAX_ROWS_PER_APP: 200 000 rows
+ *   - MAX_BYTES_PER_APP: 50 MB (approximate, measured as raw JSON text length)
+ *
+ * Scoping: datasets are owned by (app_id, owner_email) so different apps or
+ * different users never share or cross-read each other's scratch data.
+ */
+import { getDbExec, isPostgres } from "../db/client.js";
+// ---------------------------------------------------------------------------
+// Caps
+// ---------------------------------------------------------------------------
+export const MAX_ROWS_PER_APP = 200_000;
+export const MAX_BYTES_PER_APP = 50 * 1024 * 1024; // 50 MB
+// ---------------------------------------------------------------------------
+// Table initialisation
+// ---------------------------------------------------------------------------
+let _initPromise;
+async function ensureTables() {
+    if (!_initPromise) {
+        _initPromise = (async () => {
+            const db = getDbExec();
+            await db.execute(`
+        CREATE TABLE IF NOT EXISTS staged_datasets (
+          id TEXT NOT NULL,
+          app_id TEXT NOT NULL,
+          owner_email TEXT NOT NULL,
+          name TEXT NOT NULL,
+          columns TEXT NOT NULL,
+          row_count INTEGER NOT NULL DEFAULT 0,
+          byte_size INTEGER NOT NULL DEFAULT 0,
+          created_at INTEGER NOT NULL,
+          updated_at INTEGER NOT NULL,
+          PRIMARY KEY (id)
+        )
+      `);
+            await db.execute(`
+        CREATE TABLE IF NOT EXISTS staged_dataset_rows (
+          dataset_id TEXT NOT NULL,
+          row_index INTEGER NOT NULL,
+          row_data TEXT NOT NULL,
+          PRIMARY KEY (dataset_id, row_index)
+        )
+      `);
+            for (const ddl of [
+                `CREATE INDEX IF NOT EXISTS staged_datasets_scope_idx ON staged_datasets (app_id, owner_email)`,
+                `CREATE INDEX IF NOT EXISTS staged_dataset_rows_dataset_idx ON staged_dataset_rows (dataset_id)`,
+            ]) {
+                try {
+                    await db.execute(ddl);
+                }
+                catch {
+                    // Index already exists — harmless.
+                }
+            }
+        })().catch((err) => {
+            _initPromise = undefined;
+            throw err;
+        });
+    }
+    return _initPromise;
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function generateId() {
+    return `ds_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+/** Derive column names from a set of rows (first 20 rows surveyed). */
+export function deriveColumns(rows) {
+    const seen = new Set();
+    for (const row of rows.slice(0, 20)) {
+        for (const key of Object.keys(row))
+            seen.add(key);
+    }
+    return Array.from(seen);
+}
+// ---------------------------------------------------------------------------
+// Scope check
+// ---------------------------------------------------------------------------
+async function getAppRowCount(appId) {
+    const db = getDbExec();
+    const { rows } = await db.execute({
+        sql: `SELECT COALESCE(SUM(row_count), 0) as total FROM staged_datasets WHERE app_id = ?`,
+        args: [appId],
+    });
+    return Number(rows[0]?.total ?? 0);
+}
+async function getAppByteSize(appId) {
+    const db = getDbExec();
+    const { rows } = await db.execute({
+        sql: `SELECT COALESCE(SUM(byte_size), 0) as total FROM staged_datasets WHERE app_id = ?`,
+        args: [appId],
+    });
+    return Number(rows[0]?.total ?? 0);
+}
+// ---------------------------------------------------------------------------
+// Write
+// ---------------------------------------------------------------------------
+/**
+ * Create or replace a staged dataset.  Returns metadata including the
+ * resolved dataset ID (generated when not supplied in options).
+ */
+export async function upsertStagedDataset(options) {
+    await ensureTables();
+    const db = getDbExec();
+    const now = Date.now();
+    const id = options.id || generateId();
+    // If appending, load existing rows first.
+    let existingRows = [];
+    if (options.append) {
+        existingRows = await getStagedDatasetRows({
+            id,
+            appId: options.appId,
+            ownerEmail: options.ownerEmail,
+        });
+    }
+    const allRows = options.append
+        ? [...existingRows, ...options.rows]
+        : options.rows;
+    // Check caps (net new count after this write).
+    const currentAppRows = await getAppRowCount(options.appId);
+    const currentDatasetRows = options.append
+        ? existingRows.length
+        : await (async () => {
+            const { rows } = await db.execute({
+                sql: `SELECT row_count FROM staged_datasets WHERE id = ?`,
+                args: [id],
+            });
+            return Number(rows[0]?.row_count ?? 0);
+        })();
+    const netNewRows = allRows.length - currentDatasetRows;
+    if (currentAppRows + netNewRows > MAX_ROWS_PER_APP) {
+        throw new Error(`Staged dataset cap exceeded: this app already has ${currentAppRows} rows stored ` +
+            `(limit ${MAX_ROWS_PER_APP}). Delete older datasets before staging more data.`);
+    }
+    // Serialize rows and measure byte size.
+    const serialized = allRows.map((row) => JSON.stringify(row));
+    const byteSize = serialized.reduce((sum, s) => sum + s.length, 0);
+    const currentAppBytes = await getAppByteSize(options.appId);
+    const existingDatasetBytes = options.append
+        ? await (async () => {
+            const { rows } = await db.execute({
+                sql: `SELECT byte_size FROM staged_datasets WHERE id = ?`,
+                args: [id],
+            });
+            return Number(rows[0]?.byte_size ?? 0);
+        })()
+        : 0;
+    const netNewBytes = byteSize - existingDatasetBytes;
+    if (currentAppBytes + netNewBytes > MAX_BYTES_PER_APP) {
+        throw new Error(`Staged dataset byte cap exceeded: this app already stores ` +
+            `${(currentAppBytes / 1024 / 1024).toFixed(1)} MB (limit 50 MB). ` +
+            `Delete older datasets before staging more data.`);
+    }
+    const columns = options.columns.length
+        ? options.columns
+        : deriveColumns(allRows);
+    const columnsJson = JSON.stringify(columns);
+    await withDbTransaction(db, async (tx) => {
+        // Delete old rows and insert the replacement row set atomically with the
+        // metadata update so a failed batch cannot leave an empty or partial dataset.
+        await tx.execute({
+            sql: `DELETE FROM staged_dataset_rows WHERE dataset_id = ?`,
+            args: [id],
+        });
+        const BATCH = 500;
+        for (let i = 0; i < serialized.length; i += BATCH) {
+            const slice = serialized.slice(i, i + BATCH);
+            for (let j = 0; j < slice.length; j++) {
+                await tx.execute({
+                    sql: `INSERT INTO staged_dataset_rows (dataset_id, row_index, row_data) VALUES (?, ?, ?)`,
+                    args: [id, i + j, slice[j]],
+                });
+            }
+        }
+        await tx.execute({
+            sql: isPostgres()
+                ? `INSERT INTO staged_datasets (id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET name=EXCLUDED.name, columns=EXCLUDED.columns, row_count=EXCLUDED.row_count, byte_size=EXCLUDED.byte_size, updated_at=EXCLUDED.updated_at`
+                : `INSERT OR REPLACE INTO staged_datasets (id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+            args: [
+                id,
+                options.appId,
+                options.ownerEmail,
+                options.name,
+                columnsJson,
+                allRows.length,
+                byteSize,
+                now,
+                now,
+            ],
+        });
+    });
+    return {
+        id,
+        appId: options.appId,
+        ownerEmail: options.ownerEmail,
+        name: options.name,
+        columns,
+        rowCount: allRows.length,
+        byteSize,
+        createdAt: now,
+        updatedAt: now,
+    };
+}
+// ---------------------------------------------------------------------------
+// Read
+// ---------------------------------------------------------------------------
+/**
+ * Fetch all rows for a dataset, asserting scope (appId + ownerEmail).
+ */
+export async function getStagedDatasetRows(options) {
+    await ensureTables();
+    const db = getDbExec();
+    // Scope check
+    const { rows: meta } = await db.execute({
+        sql: `SELECT id FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,
+        args: [options.id, options.appId, options.ownerEmail],
+    });
+    if (meta.length === 0)
+        return [];
+    const { rows } = await db.execute({
+        sql: `SELECT row_data FROM staged_dataset_rows WHERE dataset_id = ? ORDER BY row_index ASC`,
+        args: [options.id],
+    });
+    return rows.map((r) => JSON.parse(r.row_data));
+}
+/**
+ * Get dataset metadata (no rows), asserting scope.
+ */
+export async function getStagedDatasetMeta(options) {
+    await ensureTables();
+    const db = getDbExec();
+    const { rows } = await db.execute({
+        sql: `SELECT id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,
+        args: [options.id, options.appId, options.ownerEmail],
+    });
+    if (rows.length === 0)
+        return null;
+    return rowToMeta(rows[0]);
+}
+/**
+ * List all datasets scoped to (appId, ownerEmail), newest first.
+ */
+export async function listStagedDatasets(options) {
+    await ensureTables();
+    const db = getDbExec();
+    const { rows } = await db.execute({
+        sql: `SELECT id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at FROM staged_datasets WHERE app_id = ? AND owner_email = ? ORDER BY updated_at DESC`,
+        args: [options.appId, options.ownerEmail],
+    });
+    return rows.map(rowToMeta);
+}
+// ---------------------------------------------------------------------------
+// Delete
+// ---------------------------------------------------------------------------
+export async function deleteStagedDataset(options) {
+    await ensureTables();
+    const db = getDbExec();
+    // Scope check
+    const { rows: meta } = await db.execute({
+        sql: `SELECT id FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,
+        args: [options.id, options.appId, options.ownerEmail],
+    });
+    if (meta.length === 0)
+        return false;
+    const result = await withDbTransaction(db, async (tx) => {
+        await tx.execute({
+            sql: `DELETE FROM staged_dataset_rows WHERE dataset_id = ?`,
+            args: [options.id],
+        });
+        return tx.execute({
+            sql: `DELETE FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,
+            args: [options.id, options.appId, options.ownerEmail],
+        });
+    });
+    return result.rowsAffected > 0;
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function rowToMeta(row) {
+    return {
+        id: row.id,
+        appId: row.app_id,
+        ownerEmail: row.owner_email,
+        name: row.name,
+        columns: JSON.parse(row.columns),
+        rowCount: Number(row.row_count),
+        byteSize: Number(row.byte_size),
+        createdAt: Number(row.created_at),
+        updatedAt: Number(row.updated_at),
+    };
+}
+async function withDbTransaction(db, fn) {
+    if (db.transaction)
+        return db.transaction(fn);
+    await db.execute("BEGIN");
+    try {
+        const result = await fn(db);
+        await db.execute("COMMIT");
+        return result;
+    }
+    catch (err) {
+        await db.execute("ROLLBACK").catch(() => { });
+        throw err;
+    }
+}
+/** Reset the init promise (only used in tests). */
+export function _resetInitPromiseForTests() {
+    _initPromise = undefined;
+}
+//# sourceMappingURL=staged-datasets-store.js.map

package/dist/provider-api/staged-datasets-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"staged-datasets-store.js","sourceRoot":"","sources":["../../src/provider-api/staged-datasets-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAe,MAAM,iBAAiB,CAAC;AAErE,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAC9E,MAAM,CAAC,MAAM,gBAAgB,GAAG,OAAO,CAAC;AACxC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAE3D,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,IAAI,YAAuC,CAAC;AAE5C,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;YACvB,MAAM,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;OAahB,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,OAAO,CAAC;;;;;;;OAOhB,CAAC,CAAC;YACH,KAAK,MAAM,GAAG,IAAI;gBAChB,+FAA+F;gBAC/F,gGAAgG;aACjG,EAAE,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBACP,mCAAmC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,YAAY,GAAG,SAAS,CAAC;YACzB,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AA8BD,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,UAAU;IACjB,OAAO,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACnF,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,aAAa,CAAC,IAA+B;IAC3D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACpC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,KAAK,UAAU,cAAc,CAAC,KAAa;IACzC,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,mFAAmF;QACxF,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,KAAa;IACzC,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,mFAAmF;QACxF,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAA6B;IAE7B,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IAEtC,0CAA0C;IAC1C,IAAI,YAAY,GAA8B,EAAE,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,YAAY,GAAG,MAAM,oBAAoB,CAAC;YACxC,EAAE;YACF,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM;QAC5B,CAAC,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QACpC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAEjB,+CAA+C;IAC/C,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC3D,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM;QACvC,CAAC,CAAC,YAAY,CAAC,MAAM;QACrB,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE;YAChB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAChC,GAAG,EAAE,oDAAoD;gBACzD,IAAI,EAAE,CAAC,EAAE,CAAC;aACX,CAAC,CAAC;YACH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;QACzC,CAAC,CAAC,EAAE,CAAC;IACT,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC;IACvD,IAAI,cAAc,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,qDAAqD,cAAc,eAAe;YAChF,UAAU,gBAAgB,oDAAoD,CACjF,CAAC;IACJ,CAAC;IAED,wCAAwC;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAElE,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5D,MAAM,oBAAoB,GAAG,OAAO,CAAC,MAAM;QACzC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE;YAChB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAChC,GAAG,EAAE,oDAAoD;gBACzD,IAAI,EAAE,CAAC,EAAE,CAAC;aACX,CAAC,CAAC;YACH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;QACzC,CAAC,CAAC,EAAE;QACN,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,WAAW,GAAG,QAAQ,GAAG,oBAAoB,CAAC;IACpD,IAAI,eAAe,GAAG,WAAW,GAAG,iBAAiB,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,4DAA4D;YAC1D,GAAG,CAAC,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB;YAClE,iDAAiD,CACpD,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM;QACpC,CAAC,CAAC,OAAO,CAAC,OAAO;QACjB,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAE5C,MAAM,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;QACvC,yEAAyE;QACzE,8EAA8E;QAC9E,MAAM,EAAE,CAAC,OAAO,CAAC;YACf,GAAG,EAAE,sDAAsD;YAC3D,IAAI,EAAE,CAAC,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,GAAG,CAAC;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,EAAE,CAAC,OAAO,CAAC;oBACf,GAAG,EAAE,oFAAoF;oBACzF,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;iBAC5B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,EAAE,CAAC,OAAO,CAAC;YACf,GAAG,EAAE,UAAU,EAAE;gBACf,CAAC,CAAC,+TAA+T;gBACjU,CAAC,CAAC,kKAAkK;YACtK,IAAI,EAAE;gBACJ,EAAE;gBACF,OAAO,CAAC,KAAK;gBACb,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,IAAI;gBACZ,WAAW;gBACX,OAAO,CAAC,MAAM;gBACd,QAAQ;gBACR,GAAG;gBACH,GAAG;aACJ;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,EAAE;QACF,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,OAAO;QACP,QAAQ,EAAE,OAAO,CAAC,MAAM;QACxB,QAAQ;QACR,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAI1C;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,cAAc;IACd,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QACtC,GAAG,EAAE,gFAAgF;QACrF,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,sFAAsF;QAC3F,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAkB,CAA4B,CACnE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAI1C;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,kKAAkK;QACvK,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAGxC;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,gLAAgL;QACrL,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;KAC1C,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAIzC;IACC,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IACvB,cAAc;IACd,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;QACtC,GAAG,EAAE,gFAAgF;QACrF,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;QACtD,MAAM,EAAE,CAAC,OAAO,CAAC;YACf,GAAG,EAAE,sDAAsD;YAC3D,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;SACnB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC,OAAO,CAAC;YAChB,GAAG,EAAE,6EAA6E;YAClF,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC;SACtD,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,SAAS,CAAC,GAA4B;IAC7C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAY;QACpB,KAAK,EAAE,GAAG,CAAC,MAAgB;QAC3B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,IAAI,EAAE,GAAG,CAAC,IAAc;QACxB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAiB,CAAa;QACtD,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;QAC/B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;QACjC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;KAClC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,EAAU,EACV,EAA8B;IAE9B,IAAI,EAAE,CAAC,WAAW;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAE9C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC7C,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,yBAAyB;IACvC,YAAY,GAAG,SAAS,CAAC;AAC3B,CAAC","sourcesContent":["/**\n * Scratch storage for staged provider-API datasets.\n *\n * Rows are stored as JSON text — no dialect-specific JSON-column types needed.\n * Aggregation is done in TypeScript server-side (portable across Postgres and\n * SQLite) rather than via JSON SQL fragments. See `staged-datasets-aggregate.ts`.\n *\n * Storage caps:\n *   - MAX_ROWS_PER_APP: 200 000 rows\n *   - MAX_BYTES_PER_APP: 50 MB (approximate, measured as raw JSON text length)\n *\n * Scoping: datasets are owned by (app_id, owner_email) so different apps or\n * different users never share or cross-read each other's scratch data.\n */\n\nimport { getDbExec, isPostgres, type DbExec } from \"../db/client.js\";\n\n// ---------------------------------------------------------------------------\n// Caps\n// ---------------------------------------------------------------------------\nexport const MAX_ROWS_PER_APP = 200_000;\nexport const MAX_BYTES_PER_APP = 50 * 1024 * 1024; // 50 MB\n\n// ---------------------------------------------------------------------------\n// Table initialisation\n// ---------------------------------------------------------------------------\n\nlet _initPromise: Promise<void> | undefined;\n\nasync function ensureTables(): Promise<void> {\n  if (!_initPromise) {\n    _initPromise = (async () => {\n      const db = getDbExec();\n      await db.execute(`\n        CREATE TABLE IF NOT EXISTS staged_datasets (\n          id TEXT NOT NULL,\n          app_id TEXT NOT NULL,\n          owner_email TEXT NOT NULL,\n          name TEXT NOT NULL,\n          columns TEXT NOT NULL,\n          row_count INTEGER NOT NULL DEFAULT 0,\n          byte_size INTEGER NOT NULL DEFAULT 0,\n          created_at INTEGER NOT NULL,\n          updated_at INTEGER NOT NULL,\n          PRIMARY KEY (id)\n        )\n      `);\n      await db.execute(`\n        CREATE TABLE IF NOT EXISTS staged_dataset_rows (\n          dataset_id TEXT NOT NULL,\n          row_index INTEGER NOT NULL,\n          row_data TEXT NOT NULL,\n          PRIMARY KEY (dataset_id, row_index)\n        )\n      `);\n      for (const ddl of [\n        `CREATE INDEX IF NOT EXISTS staged_datasets_scope_idx ON staged_datasets (app_id, owner_email)`,\n        `CREATE INDEX IF NOT EXISTS staged_dataset_rows_dataset_idx ON staged_dataset_rows (dataset_id)`,\n      ]) {\n        try {\n          await db.execute(ddl);\n        } catch {\n          // Index already exists — harmless.\n        }\n      }\n    })().catch((err) => {\n      _initPromise = undefined;\n      throw err;\n    });\n  }\n  return _initPromise;\n}\n\n// ---------------------------------------------------------------------------\n// Types\n// ---------------------------------------------------------------------------\n\nexport interface StagedDatasetMeta {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n  name: string;\n  columns: string[];\n  rowCount: number;\n  byteSize: number;\n  createdAt: number;\n  updatedAt: number;\n}\n\nexport interface UpsertDatasetOptions {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n  name: string;\n  /** All rows to store; replaces any existing rows. */\n  rows: Record<string, unknown>[];\n  columns: string[];\n  /** When true, append rows instead of replacing. */\n  append?: boolean;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\nfunction generateId(): string {\n  return `ds_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;\n}\n\n/** Derive column names from a set of rows (first 20 rows surveyed). */\nexport function deriveColumns(rows: Record<string, unknown>[]): string[] {\n  const seen = new Set<string>();\n  for (const row of rows.slice(0, 20)) {\n    for (const key of Object.keys(row)) seen.add(key);\n  }\n  return Array.from(seen);\n}\n\n// ---------------------------------------------------------------------------\n// Scope check\n// ---------------------------------------------------------------------------\n\nasync function getAppRowCount(appId: string): Promise<number> {\n  const db = getDbExec();\n  const { rows } = await db.execute({\n    sql: `SELECT COALESCE(SUM(row_count), 0) as total FROM staged_datasets WHERE app_id = ?`,\n    args: [appId],\n  });\n  return Number(rows[0]?.total ?? 0);\n}\n\nasync function getAppByteSize(appId: string): Promise<number> {\n  const db = getDbExec();\n  const { rows } = await db.execute({\n    sql: `SELECT COALESCE(SUM(byte_size), 0) as total FROM staged_datasets WHERE app_id = ?`,\n    args: [appId],\n  });\n  return Number(rows[0]?.total ?? 0);\n}\n\n// ---------------------------------------------------------------------------\n// Write\n// ---------------------------------------------------------------------------\n\n/**\n * Create or replace a staged dataset.  Returns metadata including the\n * resolved dataset ID (generated when not supplied in options).\n */\nexport async function upsertStagedDataset(\n  options: UpsertDatasetOptions,\n): Promise<StagedDatasetMeta> {\n  await ensureTables();\n  const db = getDbExec();\n  const now = Date.now();\n  const id = options.id || generateId();\n\n  // If appending, load existing rows first.\n  let existingRows: Record<string, unknown>[] = [];\n  if (options.append) {\n    existingRows = await getStagedDatasetRows({\n      id,\n      appId: options.appId,\n      ownerEmail: options.ownerEmail,\n    });\n  }\n\n  const allRows = options.append\n    ? [...existingRows, ...options.rows]\n    : options.rows;\n\n  // Check caps (net new count after this write).\n  const currentAppRows = await getAppRowCount(options.appId);\n  const currentDatasetRows = options.append\n    ? existingRows.length\n    : await (async () => {\n        const { rows } = await db.execute({\n          sql: `SELECT row_count FROM staged_datasets WHERE id = ?`,\n          args: [id],\n        });\n        return Number(rows[0]?.row_count ?? 0);\n      })();\n  const netNewRows = allRows.length - currentDatasetRows;\n  if (currentAppRows + netNewRows > MAX_ROWS_PER_APP) {\n    throw new Error(\n      `Staged dataset cap exceeded: this app already has ${currentAppRows} rows stored ` +\n        `(limit ${MAX_ROWS_PER_APP}). Delete older datasets before staging more data.`,\n    );\n  }\n\n  // Serialize rows and measure byte size.\n  const serialized = allRows.map((row) => JSON.stringify(row));\n  const byteSize = serialized.reduce((sum, s) => sum + s.length, 0);\n\n  const currentAppBytes = await getAppByteSize(options.appId);\n  const existingDatasetBytes = options.append\n    ? await (async () => {\n        const { rows } = await db.execute({\n          sql: `SELECT byte_size FROM staged_datasets WHERE id = ?`,\n          args: [id],\n        });\n        return Number(rows[0]?.byte_size ?? 0);\n      })()\n    : 0;\n  const netNewBytes = byteSize - existingDatasetBytes;\n  if (currentAppBytes + netNewBytes > MAX_BYTES_PER_APP) {\n    throw new Error(\n      `Staged dataset byte cap exceeded: this app already stores ` +\n        `${(currentAppBytes / 1024 / 1024).toFixed(1)} MB (limit 50 MB). ` +\n        `Delete older datasets before staging more data.`,\n    );\n  }\n\n  const columns = options.columns.length\n    ? options.columns\n    : deriveColumns(allRows);\n  const columnsJson = JSON.stringify(columns);\n\n  await withDbTransaction(db, async (tx) => {\n    // Delete old rows and insert the replacement row set atomically with the\n    // metadata update so a failed batch cannot leave an empty or partial dataset.\n    await tx.execute({\n      sql: `DELETE FROM staged_dataset_rows WHERE dataset_id = ?`,\n      args: [id],\n    });\n\n    const BATCH = 500;\n    for (let i = 0; i < serialized.length; i += BATCH) {\n      const slice = serialized.slice(i, i + BATCH);\n      for (let j = 0; j < slice.length; j++) {\n        await tx.execute({\n          sql: `INSERT INTO staged_dataset_rows (dataset_id, row_index, row_data) VALUES (?, ?, ?)`,\n          args: [id, i + j, slice[j]],\n        });\n      }\n    }\n\n    await tx.execute({\n      sql: isPostgres()\n        ? `INSERT INTO staged_datasets (id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET name=EXCLUDED.name, columns=EXCLUDED.columns, row_count=EXCLUDED.row_count, byte_size=EXCLUDED.byte_size, updated_at=EXCLUDED.updated_at`\n        : `INSERT OR REPLACE INTO staged_datasets (id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n      args: [\n        id,\n        options.appId,\n        options.ownerEmail,\n        options.name,\n        columnsJson,\n        allRows.length,\n        byteSize,\n        now,\n        now,\n      ],\n    });\n  });\n\n  return {\n    id,\n    appId: options.appId,\n    ownerEmail: options.ownerEmail,\n    name: options.name,\n    columns,\n    rowCount: allRows.length,\n    byteSize,\n    createdAt: now,\n    updatedAt: now,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Read\n// ---------------------------------------------------------------------------\n\n/**\n * Fetch all rows for a dataset, asserting scope (appId + ownerEmail).\n */\nexport async function getStagedDatasetRows(options: {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n}): Promise<Record<string, unknown>[]> {\n  await ensureTables();\n  const db = getDbExec();\n  // Scope check\n  const { rows: meta } = await db.execute({\n    sql: `SELECT id FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,\n    args: [options.id, options.appId, options.ownerEmail],\n  });\n  if (meta.length === 0) return [];\n\n  const { rows } = await db.execute({\n    sql: `SELECT row_data FROM staged_dataset_rows WHERE dataset_id = ? ORDER BY row_index ASC`,\n    args: [options.id],\n  });\n  return rows.map(\n    (r) => JSON.parse(r.row_data as string) as Record<string, unknown>,\n  );\n}\n\n/**\n * Get dataset metadata (no rows), asserting scope.\n */\nexport async function getStagedDatasetMeta(options: {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n}): Promise<StagedDatasetMeta | null> {\n  await ensureTables();\n  const db = getDbExec();\n  const { rows } = await db.execute({\n    sql: `SELECT id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,\n    args: [options.id, options.appId, options.ownerEmail],\n  });\n  if (rows.length === 0) return null;\n  return rowToMeta(rows[0]);\n}\n\n/**\n * List all datasets scoped to (appId, ownerEmail), newest first.\n */\nexport async function listStagedDatasets(options: {\n  appId: string;\n  ownerEmail: string;\n}): Promise<StagedDatasetMeta[]> {\n  await ensureTables();\n  const db = getDbExec();\n  const { rows } = await db.execute({\n    sql: `SELECT id, app_id, owner_email, name, columns, row_count, byte_size, created_at, updated_at FROM staged_datasets WHERE app_id = ? AND owner_email = ? ORDER BY updated_at DESC`,\n    args: [options.appId, options.ownerEmail],\n  });\n  return rows.map(rowToMeta);\n}\n\n// ---------------------------------------------------------------------------\n// Delete\n// ---------------------------------------------------------------------------\n\nexport async function deleteStagedDataset(options: {\n  id: string;\n  appId: string;\n  ownerEmail: string;\n}): Promise<boolean> {\n  await ensureTables();\n  const db = getDbExec();\n  // Scope check\n  const { rows: meta } = await db.execute({\n    sql: `SELECT id FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,\n    args: [options.id, options.appId, options.ownerEmail],\n  });\n  if (meta.length === 0) return false;\n\n  const result = await withDbTransaction(db, async (tx) => {\n    await tx.execute({\n      sql: `DELETE FROM staged_dataset_rows WHERE dataset_id = ?`,\n      args: [options.id],\n    });\n    return tx.execute({\n      sql: `DELETE FROM staged_datasets WHERE id = ? AND app_id = ? AND owner_email = ?`,\n      args: [options.id, options.appId, options.ownerEmail],\n    });\n  });\n  return result.rowsAffected > 0;\n}\n\n// ---------------------------------------------------------------------------\n// Internal helpers\n// ---------------------------------------------------------------------------\n\nfunction rowToMeta(row: Record<string, unknown>): StagedDatasetMeta {\n  return {\n    id: row.id as string,\n    appId: row.app_id as string,\n    ownerEmail: row.owner_email as string,\n    name: row.name as string,\n    columns: JSON.parse(row.columns as string) as string[],\n    rowCount: Number(row.row_count),\n    byteSize: Number(row.byte_size),\n    createdAt: Number(row.created_at),\n    updatedAt: Number(row.updated_at),\n  };\n}\n\nasync function withDbTransaction<T>(\n  db: DbExec,\n  fn: (tx: DbExec) => Promise<T>,\n): Promise<T> {\n  if (db.transaction) return db.transaction(fn);\n\n  await db.execute(\"BEGIN\");\n  try {\n    const result = await fn(db);\n    await db.execute(\"COMMIT\");\n    return result;\n  } catch (err) {\n    await db.execute(\"ROLLBACK\").catch(() => {});\n    throw err;\n  }\n}\n\n/** Reset the init promise (only used in tests). */\nexport function _resetInitPromiseForTests(): void {\n  _initPromise = undefined;\n}\n"]}

package/dist/provider-api/staging.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Server-side staging layer for provider-api responses.
+ *
+ * ## P0 — stageAs
+ * Instead of returning a raw response body to the model (which hits the 50K-char
+ * context-window truncation and biases aggregates), the caller can pass
+ * `stageAs: "dataset_name"` to `stagingExecuteRequest`. The runtime will:
+ *   1. Execute the provider HTTP request.
+ *   2. Auto-detect the items array in common response shapes.
+ *   3. Write rows into `staged_datasets` + `staged_dataset_rows`.
+ *   4. Return only { dataset, rowCount, columns, sampleRows: first 5 }.
+ *
+ * ## P1 — fetchAll
+ * When `pagination` config is supplied alongside `stageAs`, the runtime
+ * iterates pages server-side:
+ *   - Supports nextCursor (embedded in response), offset, and page modes.
+ *   - Handles 429 / Retry-After with exponential back-off.
+ *   - Caps at maxPages (default 50, up to 200).
+ *   - Returns { dataset, pages, rows, truncated, lastCursor } so the agent
+ *     can resume with another call when capped.
+ *
+ * Security: re-uses executeProviderApiRequest from index.ts — SSRF-safety and
+ * secret redaction are fully preserved. We read the raw (pre-redacted) JSON
+ * here; redaction is applied to the response returned to the model.
+ */
+import type { ProviderApiRequestArgs } from "./index.js";
+/** Auto-detect or explicit path to the items array in a JSON response. */
+export type ItemsPath = "auto" | "data" | "results" | "items" | "records" | "rows" | string;
+export interface PaginationConfig {
+    /**
+     * Dot-path in the response JSON for the next cursor/token value.
+     * e.g. "next_cursor", "meta.next", "pagination.cursor"
+     */
+    nextCursorPath?: string;
+    /**
+     * Query parameter name to use for the cursor in the next request.
+     * Required when nextCursorPath is set.
+     */
+    cursorParam?: string;
+    /**
+     * Use page-number mode: send `pageParam=N` for each subsequent page.
+     */
+    pageParam?: string;
+    /** Starting page number (default 1). */
+    startPage?: number;
+    /**
+     * Use offset mode: send `offsetParam=N` for each subsequent request.
+     */
+    offsetParam?: string;
+    /**
+     * Hint at expected page size for offset increments. Defaults to the
+     * actual item count of the first page.
+     */
+    pageSize?: number;
+    /** Maximum pages to fetch (default 50, max 200). */
+    maxPages?: number;
+}
+export interface StageAsOptions {
+    stageAs: string;
+    itemsPath?: ItemsPath;
+    pagination?: PaginationConfig;
+}
+export interface StagingRequestArgs extends ProviderApiRequestArgs {
+    stageAs?: string;
+    itemsPath?: ItemsPath;
+    pagination?: PaginationConfig;
+}
+export interface StagingResult {
+    dataset: {
+        id: string;
+        name: string;
+        rowCount: number;
+        columns: string[];
+        sampleRows: Record<string, unknown>[];
+    };
+    pages?: number;
+    rows?: number;
+    truncated?: boolean;
+    lastCursor?: string | number | null;
+    provider: unknown;
+    request: unknown;
+    guidance: string;
+}
+/** Minimal callable interface accepted by stagingExecuteRequest. */
+export type ProviderApiExecutor = (args: ProviderApiRequestArgs) => Promise<unknown>;
+export interface StagingRuntimeContext {
+    appId: string;
+    ownerEmail: string;
+}
+export declare function extractItemsArray(body: unknown, itemsPath?: ItemsPath): unknown[];
+/**
+ * Execute a provider API request and stage the result into scratch storage.
+ * Returns a compact summary instead of the raw body.
+ *
+ * @param args  — request args including stageAs / itemsPath / pagination
+ * @param execute — callable provider executor (e.g. `runtime.executeRequest.bind(runtime)`)
+ * @param ctx   — (appId, ownerEmail) for dataset ownership scoping
+ */
+export declare function stagingExecuteRequest(args: StagingRequestArgs, execute: ProviderApiExecutor, ctx: StagingRuntimeContext): Promise<StagingResult>;
+//# sourceMappingURL=staging.d.ts.map

package/dist/provider-api/staging.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"staging.d.ts","sourceRoot":"","sources":["../../src/provider-api/staging.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAWzD,0EAA0E;AAC1E,MAAM,MAAM,SAAS,GACjB,MAAM,GACN,MAAM,GACN,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,GACN,MAAM,CAAC;AAEX,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAmB,SAAQ,sBAAsB;IAChE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE;QACP,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;KACvC,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAMD,oEAAoE;AACpE,MAAM,MAAM,mBAAmB,GAAG,CAChC,IAAI,EAAE,sBAAsB,KACzB,OAAO,CAAC,OAAO,CAAC,CAAC;AAEtB,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAsBD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,OAAO,EACb,SAAS,GAAE,SAAkB,GAC5B,OAAO,EAAE,CAwBX;AAqCD;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,kBAAkB,EACxB,OAAO,EAAE,mBAAmB,EAC5B,GAAG,EAAE,qBAAqB,GACzB,OAAO,CAAC,aAAa,CAAC,CA2LxB"}