@agent-native/core 0.46.0 → 0.48.1

package/dist/provider-api/staged-datasets-aggregate.d.ts

@@ -0,0 +1,46 @@
+/**
+ * In-memory aggregation over staged dataset rows.
+ *
+ * Deliberately avoids dialect-specific JSON SQL so the same code runs on both
+ * Postgres and SQLite without any JSON_EXTRACT / -> operators.
+ *
+ * Supported aggregations (all portable, no SQL required):
+ *   - groupBy + sum / avg / count / min / max per group
+ *   - where filters: equals / not_equals / contains / gt / gte / lt / lte / exists
+ *   - orderBy + limit
+ *   - select (column projection)
+ *
+ * All numeric operations coerce values to numbers; non-numeric strings produce
+ * NaN which surfaces as null in the output.
+ */
+export type AggregateOp = "sum" | "avg" | "count" | "min" | "max" | "count_distinct";
+export interface AggregateField {
+    column: string;
+    op: AggregateOp;
+    /** Output column name. Defaults to `${op}_${column}`. */
+    as?: string;
+}
+export type FilterOp = "equals" | "not_equals" | "contains" | "not_contains" | "gt" | "gte" | "lt" | "lte" | "exists" | "not_exists";
+export interface WhereClause {
+    column: string;
+    op: FilterOp;
+    value?: unknown;
+}
+export interface AggregateQuery {
+    /** Row-level filters applied before aggregation. */
+    where?: WhereClause[];
+    /** Column(s) to group by. Omit for a single aggregate over all rows. */
+    groupBy?: string[];
+    /** Aggregation operations to compute. */
+    aggregate?: AggregateField[];
+    /** Column(s) to select (projection). Applied when aggregate is empty. */
+    select?: string[];
+    /** Sort output by this column. */
+    orderBy?: string;
+    /** Ascending (default) or descending sort. */
+    orderDir?: "asc" | "desc";
+    /** Limit number of output rows. */
+    limit?: number;
+}
+export declare function runAggregateQuery(rows: Record<string, unknown>[], query: AggregateQuery): Record<string, unknown>[];
+//# sourceMappingURL=staged-datasets-aggregate.d.ts.map

package/dist/provider-api/staged-datasets-aggregate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"staged-datasets-aggregate.d.ts","sourceRoot":"","sources":["../../src/provider-api/staged-datasets-aggregate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAMH,MAAM,MAAM,WAAW,GACnB,KAAK,GACL,KAAK,GACL,OAAO,GACP,KAAK,GACL,KAAK,GACL,gBAAgB,CAAC;AAErB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,WAAW,CAAC;IAChB,yDAAyD;IACzD,EAAE,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,QAAQ,GAChB,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,cAAc,GACd,IAAI,GACJ,KAAK,GACL,IAAI,GACJ,KAAK,GACL,QAAQ,GACR,YAAY,CAAC;AAEjB,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,QAAQ,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,oDAAoD;IACpD,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;IACtB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,yCAAyC;IACzC,SAAS,CAAC,EAAE,cAAc,EAAE,CAAC;IAC7B,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAC1B,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA6MD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,KAAK,EAAE,cAAc,GACpB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CA2B3B"}

package/dist/provider-api/staged-datasets-aggregate.js

@@ -0,0 +1,209 @@
+/**
+ * In-memory aggregation over staged dataset rows.
+ *
+ * Deliberately avoids dialect-specific JSON SQL so the same code runs on both
+ * Postgres and SQLite without any JSON_EXTRACT / -> operators.
+ *
+ * Supported aggregations (all portable, no SQL required):
+ *   - groupBy + sum / avg / count / min / max per group
+ *   - where filters: equals / not_equals / contains / gt / gte / lt / lte / exists
+ *   - orderBy + limit
+ *   - select (column projection)
+ *
+ * All numeric operations coerce values to numbers; non-numeric strings produce
+ * NaN which surfaces as null in the output.
+ */
+// ---------------------------------------------------------------------------
+// Filter
+// ---------------------------------------------------------------------------
+function getField(row, column) {
+    return row[column];
+}
+function coerceNum(v) {
+    if (v === null || v === undefined)
+        return NaN;
+    return Number(v);
+}
+function matchesFilter(row, clause) {
+    const val = getField(row, clause.column);
+    switch (clause.op) {
+        case "equals":
+            // eslint-disable-next-line eqeqeq
+            return val == clause.value;
+        case "not_equals":
+            // eslint-disable-next-line eqeqeq
+            return val != clause.value;
+        case "contains":
+            return typeof val === "string" && typeof clause.value === "string"
+                ? val.toLowerCase().includes(clause.value.toLowerCase())
+                : false;
+        case "not_contains":
+            return typeof val === "string" && typeof clause.value === "string"
+                ? !val.toLowerCase().includes(clause.value.toLowerCase())
+                : true;
+        case "gt":
+            return coerceNum(val) > coerceNum(clause.value);
+        case "gte":
+            return coerceNum(val) >= coerceNum(clause.value);
+        case "lt":
+            return coerceNum(val) < coerceNum(clause.value);
+        case "lte":
+            return coerceNum(val) <= coerceNum(clause.value);
+        case "exists":
+            return val !== null && val !== undefined && val !== "";
+        case "not_exists":
+            return val === null || val === undefined || val === "";
+        default:
+            return true;
+    }
+}
+function applyWhere(rows, where) {
+    if (!where.length)
+        return rows;
+    return rows.filter((row) => where.every((clause) => matchesFilter(row, clause)));
+}
+function newAccumulator() {
+    return { count: 0, sums: {}, mins: {}, maxs: {}, distinctSets: {}, nums: {} };
+}
+function accumulate(acc, row, fields) {
+    acc.count++;
+    for (const f of fields) {
+        const val = getField(row, f.column);
+        const num = coerceNum(val);
+        if (f.op === "sum" || f.op === "avg") {
+            acc.sums[f.column] = (acc.sums[f.column] ?? 0) + (isNaN(num) ? 0 : num);
+            if (!acc.nums[f.column])
+                acc.nums[f.column] = [];
+            if (!isNaN(num))
+                acc.nums[f.column].push(num);
+        }
+        else if (f.op === "min") {
+            if (!isNaN(num)) {
+                acc.mins[f.column] =
+                    acc.mins[f.column] === undefined
+                        ? num
+                        : Math.min(acc.mins[f.column], num);
+            }
+        }
+        else if (f.op === "max") {
+            if (!isNaN(num)) {
+                acc.maxs[f.column] =
+                    acc.maxs[f.column] === undefined
+                        ? num
+                        : Math.max(acc.maxs[f.column], num);
+            }
+        }
+        else if (f.op === "count_distinct") {
+            if (!acc.distinctSets[f.column])
+                acc.distinctSets[f.column] = new Set();
+            acc.distinctSets[f.column].add(val);
+        }
+    }
+}
+function finalizeAccumulator(acc, fields) {
+    const out = {};
+    for (const f of fields) {
+        const outKey = f.as ?? `${f.op}_${f.column}`;
+        if (f.op === "count") {
+            out[outKey] = acc.count;
+        }
+        else if (f.op === "sum") {
+            out[outKey] = acc.sums[f.column] ?? 0;
+        }
+        else if (f.op === "avg") {
+            const nums = acc.nums[f.column] ?? [];
+            out[outKey] = nums.length
+                ? nums.reduce((a, b) => a + b, 0) / nums.length
+                : null;
+        }
+        else if (f.op === "min") {
+            out[outKey] = acc.mins[f.column] ?? null;
+        }
+        else if (f.op === "max") {
+            out[outKey] = acc.maxs[f.column] ?? null;
+        }
+        else if (f.op === "count_distinct") {
+            out[outKey] = acc.distinctSets[f.column]?.size ?? 0;
+        }
+    }
+    return out;
+}
+function applyAggregate(rows, groupBy, fields) {
+    const groups = new Map();
+    for (const row of rows) {
+        const keyObj = {};
+        for (const col of groupBy)
+            keyObj[col] = row[col] ?? null;
+        const keyStr = JSON.stringify(keyObj);
+        if (!groups.has(keyStr)) {
+            groups.set(keyStr, { keys: keyObj, acc: newAccumulator() });
+        }
+        accumulate(groups.get(keyStr).acc, row, fields);
+    }
+    if (groups.size === 0 && rows.length > 0) {
+        // No group keys — single aggregate over all rows
+        const acc = newAccumulator();
+        for (const row of rows)
+            accumulate(acc, row, fields);
+        return [finalizeAccumulator(acc, fields)];
+    }
+    if (groups.size === 0)
+        return [];
+    return Array.from(groups.values()).map(({ keys, acc }) => ({
+        ...keys,
+        ...finalizeAccumulator(acc, fields),
+    }));
+}
+// ---------------------------------------------------------------------------
+// Sort + limit
+// ---------------------------------------------------------------------------
+function applySort(rows, orderBy, dir) {
+    return [...rows].sort((a, b) => {
+        const av = a[orderBy];
+        const bv = b[orderBy];
+        const an = coerceNum(av);
+        const bn = coerceNum(bv);
+        // Numeric sort when both parse as numbers.
+        if (!isNaN(an) && !isNaN(bn)) {
+            return dir === "asc" ? an - bn : bn - an;
+        }
+        // String sort fallback.
+        const as = String(av ?? "");
+        const bs = String(bv ?? "");
+        if (as < bs)
+            return dir === "asc" ? -1 : 1;
+        if (as > bs)
+            return dir === "asc" ? 1 : -1;
+        return 0;
+    });
+}
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+export function runAggregateQuery(rows, query) {
+    // 1. Filter
+    let result = applyWhere(rows, query.where ?? []);
+    // 2. Aggregate or project
+    if (query.aggregate && query.aggregate.length > 0) {
+        result = applyAggregate(result, query.groupBy ?? [], query.aggregate);
+    }
+    else if (query.select && query.select.length > 0) {
+        const cols = query.select;
+        result = result.map((row) => {
+            const out = {};
+            for (const col of cols)
+                out[col] = row[col] ?? null;
+            return out;
+        });
+    }
+    // 3. Sort
+    if (query.orderBy) {
+        result = applySort(result, query.orderBy, query.orderDir ?? "asc");
+    }
+    // 4. Limit
+    if (query.limit && query.limit > 0) {
+        result = result.slice(0, query.limit);
+    }
+    return result;
+}
+//# sourceMappingURL=staged-datasets-aggregate.js.map

package/dist/provider-api/staged-datasets-aggregate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"staged-datasets-aggregate.js","sourceRoot":"","sources":["../../src/provider-api/staged-datasets-aggregate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAwDH,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,SAAS,QAAQ,CAAC,GAA4B,EAAE,MAAc;IAC5D,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IAC3B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IAC9C,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CACpB,GAA4B,EAC5B,MAAmB;IAEnB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACzC,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;QAClB,KAAK,QAAQ;YACX,kCAAkC;YAClC,OAAO,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;QAC7B,KAAK,YAAY;YACf,kCAAkC;YAClC,OAAO,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;QAC7B,KAAK,UAAU;YACb,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;gBAChE,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBACxD,CAAC,CAAC,KAAK,CAAC;QACZ,KAAK,cAAc;YACjB,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;gBAChE,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBACzD,CAAC,CAAC,IAAI,CAAC;QACX,KAAK,IAAI;YACP,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,KAAK,KAAK;YACR,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnD,KAAK,IAAI;YACP,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,KAAK,KAAK;YACR,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnD,KAAK,QAAQ;YACX,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,CAAC;QACzD,KAAK,YAAY;YACf,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,CAAC;QACzD;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,IAA+B,EAC/B,KAAoB;IAEpB,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACzB,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CACpD,CAAC;AACJ,CAAC;AAeD,SAAS,cAAc;IACrB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,UAAU,CACjB,GAAqB,EACrB,GAA4B,EAC5B,MAAwB;IAExB,GAAG,CAAC,KAAK,EAAE,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACxE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;oBAChB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,SAAS;wBAC9B,CAAC,CAAC,GAAG;wBACL,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;oBAChB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,SAAS;wBAC9B,CAAC,CAAC,GAAG;wBACL,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,gBAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;gBAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC;YACxE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,GAAqB,EACrB,MAAwB;IAExB,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7C,IAAI,CAAC,CAAC,EAAE,KAAK,OAAO,EAAE,CAAC;YACrB,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;QAC1B,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC1B,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACtC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM;gBACvB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;gBAC/C,CAAC,CAAC,IAAI,CAAC;QACX,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC1B,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC3C,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAC1B,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC3C,CAAC;aAAM,IAAI,CAAC,CAAC,EAAE,KAAK,gBAAgB,EAAE,CAAC;YACrC,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CACrB,IAA+B,EAC/B,OAAiB,EACjB,MAAwB;IAExB,MAAM,MAAM,GAAG,IAAI,GAAG,EAGnB,CAAC;IACJ,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,OAAO;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,iDAAiD;QACjD,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,CAAC,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QACzD,GAAG,IAAI;QACP,GAAG,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC;KACpC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,SAAS,SAAS,CAChB,IAA+B,EAC/B,OAAe,EACf,GAAmB;IAEnB,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QACzB,MAAM,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QACzB,2CAA2C;QAC3C,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC;QAC3C,CAAC;QACD,wBAAwB;QACxB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE,GAAG,EAAE;YAAE,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,IAAI,EAAE,GAAG,EAAE;YAAE,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,MAAM,UAAU,iBAAiB,CAC/B,IAA+B,EAC/B,KAAqB;IAErB,YAAY;IACZ,IAAI,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEjD,0BAA0B;IAC1B,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IACxE,CAAC;SAAM,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC;QAC1B,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1B,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,GAAG,IAAI,IAAI;gBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;YACpD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;IACL,CAAC;IAED,UAAU;IACV,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC;IACrE,CAAC;IAED,WAAW;IACX,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC","sourcesContent":["/**\n * In-memory aggregation over staged dataset rows.\n *\n * Deliberately avoids dialect-specific JSON SQL so the same code runs on both\n * Postgres and SQLite without any JSON_EXTRACT / -> operators.\n *\n * Supported aggregations (all portable, no SQL required):\n *   - groupBy + sum / avg / count / min / max per group\n *   - where filters: equals / not_equals / contains / gt / gte / lt / lte / exists\n *   - orderBy + limit\n *   - select (column projection)\n *\n * All numeric operations coerce values to numbers; non-numeric strings produce\n * NaN which surfaces as null in the output.\n */\n\n// ---------------------------------------------------------------------------\n// Types\n// ---------------------------------------------------------------------------\n\nexport type AggregateOp =\n  | \"sum\"\n  | \"avg\"\n  | \"count\"\n  | \"min\"\n  | \"max\"\n  | \"count_distinct\";\n\nexport interface AggregateField {\n  column: string;\n  op: AggregateOp;\n  /** Output column name. Defaults to `${op}_${column}`. */\n  as?: string;\n}\n\nexport type FilterOp =\n  | \"equals\"\n  | \"not_equals\"\n  | \"contains\"\n  | \"not_contains\"\n  | \"gt\"\n  | \"gte\"\n  | \"lt\"\n  | \"lte\"\n  | \"exists\"\n  | \"not_exists\";\n\nexport interface WhereClause {\n  column: string;\n  op: FilterOp;\n  value?: unknown;\n}\n\nexport interface AggregateQuery {\n  /** Row-level filters applied before aggregation. */\n  where?: WhereClause[];\n  /** Column(s) to group by. Omit for a single aggregate over all rows. */\n  groupBy?: string[];\n  /** Aggregation operations to compute. */\n  aggregate?: AggregateField[];\n  /** Column(s) to select (projection). Applied when aggregate is empty. */\n  select?: string[];\n  /** Sort output by this column. */\n  orderBy?: string;\n  /** Ascending (default) or descending sort. */\n  orderDir?: \"asc\" | \"desc\";\n  /** Limit number of output rows. */\n  limit?: number;\n}\n\n// ---------------------------------------------------------------------------\n// Filter\n// ---------------------------------------------------------------------------\n\nfunction getField(row: Record<string, unknown>, column: string): unknown {\n  return row[column];\n}\n\nfunction coerceNum(v: unknown): number {\n  if (v === null || v === undefined) return NaN;\n  return Number(v);\n}\n\nfunction matchesFilter(\n  row: Record<string, unknown>,\n  clause: WhereClause,\n): boolean {\n  const val = getField(row, clause.column);\n  switch (clause.op) {\n    case \"equals\":\n      // eslint-disable-next-line eqeqeq\n      return val == clause.value;\n    case \"not_equals\":\n      // eslint-disable-next-line eqeqeq\n      return val != clause.value;\n    case \"contains\":\n      return typeof val === \"string\" && typeof clause.value === \"string\"\n        ? val.toLowerCase().includes(clause.value.toLowerCase())\n        : false;\n    case \"not_contains\":\n      return typeof val === \"string\" && typeof clause.value === \"string\"\n        ? !val.toLowerCase().includes(clause.value.toLowerCase())\n        : true;\n    case \"gt\":\n      return coerceNum(val) > coerceNum(clause.value);\n    case \"gte\":\n      return coerceNum(val) >= coerceNum(clause.value);\n    case \"lt\":\n      return coerceNum(val) < coerceNum(clause.value);\n    case \"lte\":\n      return coerceNum(val) <= coerceNum(clause.value);\n    case \"exists\":\n      return val !== null && val !== undefined && val !== \"\";\n    case \"not_exists\":\n      return val === null || val === undefined || val === \"\";\n    default:\n      return true;\n  }\n}\n\nfunction applyWhere(\n  rows: Record<string, unknown>[],\n  where: WhereClause[],\n): Record<string, unknown>[] {\n  if (!where.length) return rows;\n  return rows.filter((row) =>\n    where.every((clause) => matchesFilter(row, clause)),\n  );\n}\n\n// ---------------------------------------------------------------------------\n// Aggregate\n// ---------------------------------------------------------------------------\n\ninterface GroupAccumulator {\n  count: number;\n  sums: Record<string, number>;\n  mins: Record<string, number>;\n  maxs: Record<string, number>;\n  distinctSets: Record<string, Set<unknown>>;\n  nums: Record<string, number[]>;\n}\n\nfunction newAccumulator(): GroupAccumulator {\n  return { count: 0, sums: {}, mins: {}, maxs: {}, distinctSets: {}, nums: {} };\n}\n\nfunction accumulate(\n  acc: GroupAccumulator,\n  row: Record<string, unknown>,\n  fields: AggregateField[],\n): void {\n  acc.count++;\n  for (const f of fields) {\n    const val = getField(row, f.column);\n    const num = coerceNum(val);\n    if (f.op === \"sum\" || f.op === \"avg\") {\n      acc.sums[f.column] = (acc.sums[f.column] ?? 0) + (isNaN(num) ? 0 : num);\n      if (!acc.nums[f.column]) acc.nums[f.column] = [];\n      if (!isNaN(num)) acc.nums[f.column].push(num);\n    } else if (f.op === \"min\") {\n      if (!isNaN(num)) {\n        acc.mins[f.column] =\n          acc.mins[f.column] === undefined\n            ? num\n            : Math.min(acc.mins[f.column], num);\n      }\n    } else if (f.op === \"max\") {\n      if (!isNaN(num)) {\n        acc.maxs[f.column] =\n          acc.maxs[f.column] === undefined\n            ? num\n            : Math.max(acc.maxs[f.column], num);\n      }\n    } else if (f.op === \"count_distinct\") {\n      if (!acc.distinctSets[f.column]) acc.distinctSets[f.column] = new Set();\n      acc.distinctSets[f.column].add(val);\n    }\n  }\n}\n\nfunction finalizeAccumulator(\n  acc: GroupAccumulator,\n  fields: AggregateField[],\n): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const f of fields) {\n    const outKey = f.as ?? `${f.op}_${f.column}`;\n    if (f.op === \"count\") {\n      out[outKey] = acc.count;\n    } else if (f.op === \"sum\") {\n      out[outKey] = acc.sums[f.column] ?? 0;\n    } else if (f.op === \"avg\") {\n      const nums = acc.nums[f.column] ?? [];\n      out[outKey] = nums.length\n        ? nums.reduce((a, b) => a + b, 0) / nums.length\n        : null;\n    } else if (f.op === \"min\") {\n      out[outKey] = acc.mins[f.column] ?? null;\n    } else if (f.op === \"max\") {\n      out[outKey] = acc.maxs[f.column] ?? null;\n    } else if (f.op === \"count_distinct\") {\n      out[outKey] = acc.distinctSets[f.column]?.size ?? 0;\n    }\n  }\n  return out;\n}\n\nfunction applyAggregate(\n  rows: Record<string, unknown>[],\n  groupBy: string[],\n  fields: AggregateField[],\n): Record<string, unknown>[] {\n  const groups = new Map<\n    string,\n    { keys: Record<string, unknown>; acc: GroupAccumulator }\n  >();\n  for (const row of rows) {\n    const keyObj: Record<string, unknown> = {};\n    for (const col of groupBy) keyObj[col] = row[col] ?? null;\n    const keyStr = JSON.stringify(keyObj);\n    if (!groups.has(keyStr)) {\n      groups.set(keyStr, { keys: keyObj, acc: newAccumulator() });\n    }\n    accumulate(groups.get(keyStr)!.acc, row, fields);\n  }\n\n  if (groups.size === 0 && rows.length > 0) {\n    // No group keys — single aggregate over all rows\n    const acc = newAccumulator();\n    for (const row of rows) accumulate(acc, row, fields);\n    return [finalizeAccumulator(acc, fields)];\n  }\n\n  if (groups.size === 0) return [];\n\n  return Array.from(groups.values()).map(({ keys, acc }) => ({\n    ...keys,\n    ...finalizeAccumulator(acc, fields),\n  }));\n}\n\n// ---------------------------------------------------------------------------\n// Sort + limit\n// ---------------------------------------------------------------------------\n\nfunction applySort(\n  rows: Record<string, unknown>[],\n  orderBy: string,\n  dir: \"asc\" | \"desc\",\n): Record<string, unknown>[] {\n  return [...rows].sort((a, b) => {\n    const av = a[orderBy];\n    const bv = b[orderBy];\n    const an = coerceNum(av);\n    const bn = coerceNum(bv);\n    // Numeric sort when both parse as numbers.\n    if (!isNaN(an) && !isNaN(bn)) {\n      return dir === \"asc\" ? an - bn : bn - an;\n    }\n    // String sort fallback.\n    const as = String(av ?? \"\");\n    const bs = String(bv ?? \"\");\n    if (as < bs) return dir === \"asc\" ? -1 : 1;\n    if (as > bs) return dir === \"asc\" ? 1 : -1;\n    return 0;\n  });\n}\n\n// ---------------------------------------------------------------------------\n// Main entry point\n// ---------------------------------------------------------------------------\n\nexport function runAggregateQuery(\n  rows: Record<string, unknown>[],\n  query: AggregateQuery,\n): Record<string, unknown>[] {\n  // 1. Filter\n  let result = applyWhere(rows, query.where ?? []);\n\n  // 2. Aggregate or project\n  if (query.aggregate && query.aggregate.length > 0) {\n    result = applyAggregate(result, query.groupBy ?? [], query.aggregate);\n  } else if (query.select && query.select.length > 0) {\n    const cols = query.select;\n    result = result.map((row) => {\n      const out: Record<string, unknown> = {};\n      for (const col of cols) out[col] = row[col] ?? null;\n      return out;\n    });\n  }\n\n  // 3. Sort\n  if (query.orderBy) {\n    result = applySort(result, query.orderBy, query.orderDir ?? \"asc\");\n  }\n\n  // 4. Limit\n  if (query.limit && query.limit > 0) {\n    result = result.slice(0, query.limit);\n  }\n\n  return result;\n}\n"]}

package/dist/provider-api/staged-datasets-store.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Scratch storage for staged provider-API datasets.
+ *
+ * Rows are stored as JSON text — no dialect-specific JSON-column types needed.
+ * Aggregation is done in TypeScript server-side (portable across Postgres and
+ * SQLite) rather than via JSON SQL fragments. See `staged-datasets-aggregate.ts`.
+ *
+ * Storage caps:
+ *   - MAX_ROWS_PER_APP: 200 000 rows
+ *   - MAX_BYTES_PER_APP: 50 MB (approximate, measured as raw JSON text length)
+ *
+ * Scoping: datasets are owned by (app_id, owner_email) so different apps or
+ * different users never share or cross-read each other's scratch data.
+ */
+export declare const MAX_ROWS_PER_APP = 200000;
+export declare const MAX_BYTES_PER_APP: number;
+export interface StagedDatasetMeta {
+    id: string;
+    appId: string;
+    ownerEmail: string;
+    name: string;
+    columns: string[];
+    rowCount: number;
+    byteSize: number;
+    createdAt: number;
+    updatedAt: number;
+}
+export interface UpsertDatasetOptions {
+    id: string;
+    appId: string;
+    ownerEmail: string;
+    name: string;
+    /** All rows to store; replaces any existing rows. */
+    rows: Record<string, unknown>[];
+    columns: string[];
+    /** When true, append rows instead of replacing. */
+    append?: boolean;
+}
+/** Derive column names from a set of rows (first 20 rows surveyed). */
+export declare function deriveColumns(rows: Record<string, unknown>[]): string[];
+/**
+ * Create or replace a staged dataset.  Returns metadata including the
+ * resolved dataset ID (generated when not supplied in options).
+ */
+export declare function upsertStagedDataset(options: UpsertDatasetOptions): Promise<StagedDatasetMeta>;
+/**
+ * Fetch all rows for a dataset, asserting scope (appId + ownerEmail).
+ */
+export declare function getStagedDatasetRows(options: {
+    id: string;
+    appId: string;
+    ownerEmail: string;
+}): Promise<Record<string, unknown>[]>;
+/**
+ * Get dataset metadata (no rows), asserting scope.
+ */
+export declare function getStagedDatasetMeta(options: {
+    id: string;
+    appId: string;
+    ownerEmail: string;
+}): Promise<StagedDatasetMeta | null>;
+/**
+ * List all datasets scoped to (appId, ownerEmail), newest first.
+ */
+export declare function listStagedDatasets(options: {
+    appId: string;
+    ownerEmail: string;
+}): Promise<StagedDatasetMeta[]>;
+export declare function deleteStagedDataset(options: {
+    id: string;
+    appId: string;
+    ownerEmail: string;
+}): Promise<boolean>;
+/** Reset the init promise (only used in tests). */
+export declare function _resetInitPromiseForTests(): void;
+//# sourceMappingURL=staged-datasets-store.d.ts.map

package/dist/provider-api/staged-datasets-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"staged-datasets-store.d.ts","sourceRoot":"","sources":["../../src/provider-api/staged-datasets-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,eAAO,MAAM,gBAAgB,SAAU,CAAC;AACxC,eAAO,MAAM,iBAAiB,QAAmB,CAAC;AAwDlD,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,mDAAmD;IACnD,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAUD,uEAAuE;AACvE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,MAAM,EAAE,CAMvE;AA4BD;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CAmH5B;AAMD;;GAEG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,EAAE;IAClD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAiBrC;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,EAAE;IAClD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CASpC;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAQ/B;AAMD,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,OAAO,CAAC,CAqBnB;AAqCD,mDAAmD;AACnD,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD"}