df-script 1.4.0 → 1.5.0

package/dist/columnExpressions/mixins/ListExpr.d.ts

@@ -1,4 +1,6 @@
-import type { ExprConstructor } from "../types";
+import { ExprBase } from "../ExprBase";
+import { StepSliceListOptions } from "../../utils";
+import type { UniqueListStatsOptions, JoinListOptions, ExplodeOptions, IExpr } from "../../types";
 export declare class ListExprNamespace {
     expr: any;
     constructor(expr: any);
@@ -6,17 +8,16 @@ export declare class ListExprNamespace {
     all(): any;
     any(): any;
     contains(item: any): any;
-    contains_all(items: any[]): any;
-    contains_any(items: any[]): any;
+    contains_all(items: ArrayLike<any>): any;
+    contains_any(items: ArrayLike<any>): any;
     count_matches(item: any): any;
     drop_nulls(): any;
+    explode({ empty_as_null, keep_nulls }?: ExplodeOptions): any;
     first(null_on_oob?: boolean): any;
-    gather(indices: number | number[], null_on_oob?: boolean): any;
-    gather_every(n: number, offset?: number): any;
+    gather(indices: number | ArrayLike<number>, null_on_oob?: boolean): any;
+    gather_every(options?: StepSliceListOptions): any;
     get(index: number, null_on_oob?: boolean): any;
-    join(separator: string, { ignoreNulls }?: {
-        ignoreNulls?: boolean;
-    }): any;
+    join(separator?: string, options?: JoinListOptions): any;
     last(null_on_oob?: boolean): any;
     len(): any;
     lengths(): any;
@@ -25,39 +26,14 @@ export declare class ListExprNamespace {
     median(): any;
     min(): any;
     mode(): any;
-    n_unique(): any;
+    n_unique(options?: UniqueListStatsOptions): any;
     reverse(): any;
     slice(offset: number, length?: number): any;
     sort(descending?: boolean): any;
     sum(): any;
-    unique(): any;
+    unique(options?: UniqueListStatsOptions): any;
+    eval(expr: IExpr): any;
+}
+export declare class ListExpr extends ExprBase {
+    get list(): ListExprNamespace;
 }
-export declare const ListExpr: <TBase extends ExprConstructor>(Base: TBase) => {
-    new (...args: any[]): {
-        get list(): ListExprNamespace;
-        ops: import("../..").OpFn[];
-        colName?: string;
-        outputName?: string;
-        isLiteral?: boolean;
-        literalValue?: any;
-        aggFn?: import("../..").AggFn<any> | null;
-        groupingOpsIndex?: number;
-        partitionOpsIndex?: number;
-        partitionBy?: (string | import("../..").IExpr)[] | null;
-        windowOp?: {
-            type: string;
-            [key: string]: any;
-        } | null;
-        isWindow?: boolean;
-        alias(name: string): /*elided*/ any;
-        cast(dataType: any): /*elided*/ any;
-        _resolve(val: any, columns: import("../..").ColumnDict, height: number): import("../..").ColumnData | any;
-        evaluate(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePreGrouping(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePostGrouping(aggregatedArray: any[], columns: import("../..").ColumnDict): import("../..").ColumnData;
-        evaluatePrePartition(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePostPartition(aggregatedArray: any[], columns: import("../..").ColumnDict): import("../..").ColumnData;
-        evaluateWindow?(groupPreValues: any[], partitionIndices: number[], currentIndex: number): any;
-        debug(label?: string): /*elided*/ any;
-    };
-} & TBase;

package/dist/columnExpressions/mixins/ListExpr.js

@@ -2,69 +2,41 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ListExpr = exports.ListExprNamespace = void 0;
 const ExprBase_1 = require("../ExprBase");
-const utils_1 = require("../../utils");
+const utils_1 = require("../utils");
+const utils_2 = require("../../utils");
 const exceptions_1 = require("../../exceptions");
+const constants_1 = require("../constants");
 class ListExprNamespace {
     expr;
     constructor(expr) {
         this.expr = expr;
     }
     _deriveList(fn) {
-        return (0, ExprBase_1.derive)(this.expr, (0, ExprBase_1.kleeneUnary)((v) => {
-            return (0, utils_1.isArrayOrTypedArray)(v) ? fn(v) : null;
+        return (0, ExprBase_1.derive)(this.expr, (0, utils_1.kleeneUnary)((v) => {
+            return (0, utils_2.isArrayOrTypedArray)(v) ? fn(v) : null;
         }));
     }
     all() {
-        return this._deriveList((arr) => {
-            const list = Array.from(arr);
-            for (let i = 0; i < list.length; i++) {
-                if (!list[i])
-                    return false;
-            }
-            return true;
-        });
+        return this._deriveList((arr) => (0, utils_2.isArrayOfType)(arr, (x) => !!x, { mode: "every" }));
     }
     any() {
-        return this._deriveList((arr) => {
-            const list = Array.from(arr);
-            for (let i = 0; i < list.length; i++) {
-                if (list[i])
-                    return true;
-            }
-            return false;
-        });
+        return this._deriveList((arr) => (0, utils_2.isArrayOfType)(arr, (x) => !!x, { mode: "some" }));
     }
     contains(item) {
-        return this._deriveList((arr) => {
-            return Array.from(arr).includes(item);
-        });
+        return this._deriveList((arr) => arr.includes(item));
     }
     contains_all(items) {
-        return this._deriveList((arr) => {
-            const list = Array.from(arr);
-            for (let i = 0; i < items.length; i++) {
-                if (!list.includes(items[i]))
-                    return false;
-            }
-            return true;
-        });
+        return this._deriveList((arr) => (0, utils_2.isArrayOfType)(items, (x) => arr.includes(x), { mode: "every" }));
     }
     contains_any(items) {
-        return this._deriveList((arr) => {
-            const list = Array.from(arr);
-            for (let i = 0; i < items.length; i++) {
-                if (list.includes(items[i]))
-                    return true;
-            }
-            return false;
-        });
+        return this._deriveList((arr) => (0, utils_2.isArrayOfType)(items, (x) => arr.includes(x), { mode: "some" }));
     }
     count_matches(item) {
         return this._deriveList((arr) => {
-            const list = Array.from(arr);
             let count = 0;
-            for (const val of list) {
-                if (val === item) {
+            const len = arr.length;
+            for (let i = 0; i < len; i++) {
+                if (arr[i] === item) {
                     count++;
                 }
             }
@@ -73,24 +45,77 @@ class ListExprNamespace {
     }
     drop_nulls() {
         return this._deriveList((arr) => {
-            const list = Array.from(arr);
+            const len = arr.length;
             const result = [];
-            for (let i = 0; i < list.length; i++) {
-                if (list[i] != null)
-                    result.push(list[i]);
+            for (let i = 0; i < len; i++) {
+                if (arr[i] != null)
+                    result.push(arr[i]);
             }
             return result;
         });
     }
+    explode({ empty_as_null = true, keep_nulls = true } = {}) {
+        return (0, ExprBase_1.derive)(this.expr, (vArray) => {
+            const height = vArray.length;
+            let newHeight = 0;
+            for (let i = 0; i < height; i++) {
+                const val = vArray[i];
+                if ((0, utils_2.isArrayOrTypedArray)(val)) {
+                    newHeight += val.length || (empty_as_null ? 1 : 0);
+                    continue;
+                }
+                if (val != null) {
+                    newHeight += 1;
+                    continue;
+                }
+                if (keep_nulls) {
+                    newHeight += 1;
+                }
+            }
+            const res = new Array(newHeight);
+            const rowMap = new Int32Array(newHeight);
+            let idx = 0;
+            for (let i = 0; i < height; i++) {
+                const val = vArray[i];
+                if ((0, utils_2.isArrayOrTypedArray)(val)) {
+                    const len = val.length;
+                    if (len > 0) {
+                        for (let j = 0; j < len; j++) {
+                            rowMap[idx] = i;
+                            res[idx++] = val[j];
+                        }
+                        continue;
+                    }
+                    if (empty_as_null) {
+                        rowMap[idx] = i;
+                        res[idx++] = null;
+                    }
+                    continue;
+                }
+                if (val != null) {
+                    rowMap[idx] = i;
+                    res[idx++] = val;
+                    continue;
+                }
+                if (keep_nulls) {
+                    rowMap[idx] = i;
+                    res[idx++] = null;
+                }
+            }
+            res.rowMap = rowMap;
+            return res;
+        });
+    }
     first(null_on_oob = true) {
         return this.get(0, null_on_oob);
     }
     gather(indices, null_on_oob = true) {
-        const idxs = Array.isArray(indices) ? indices : [indices];
         return this._deriveList((arr) => {
             const len = arr.length;
-            const res = new Array(idxs.length);
-            for (let i = 0; i < idxs.length; i++) {
+            const idxs = (0, utils_2.isArrayOrTypedArray)(indices) ? indices : [indices];
+            const numIndices = idxs.length;
+            const res = new Array(numIndices);
+            for (let i = 0; i < numIndices; i++) {
                 const index = idxs[i];
                 const val = arr.at(index);
                 if (val === undefined && !null_on_oob) {
@@ -101,18 +126,8 @@ class ListExprNamespace {
             return res;
         });
     }
-    gather_every(n, offset = 0) {
-        if (n <= 0) {
-            throw new exceptions_1.ComputeError("Step size n must be positive");
-        }
-        return this._deriveList((arr) => {
-            const len = arr.length;
-            const res = [];
-            for (let i = offset; i < len; i += n) {
-                res.push(arr[i]);
-            }
-            return res;
-        });
+    gather_every(options = {}) {
+        return this._deriveList((arr) => (0, utils_2.stepSliceList)(arr, options));
     }
     get(index, null_on_oob = true) {
         return this._deriveList((arr) => {
@@ -123,21 +138,8 @@ class ListExprNamespace {
             return val ?? null;
         });
     }
-    join(separator, { ignoreNulls = false } = {}) {
-        return this._deriveList((arr) => {
-            const list = Array.from(arr);
-            const strList = [];
-            for (let i = 0; i < list.length; i++) {
-                const x = list[i];
-                if (x != null) {
-                    strList.push(String(x));
-                }
-                else if (!ignoreNulls) {
-                    strList.push("");
-                }
-            }
-            return strList.join(separator);
-        });
+    join(separator = ",", options = {}) {
+        return this._deriveList((arr) => (0, utils_2.joinList)(arr, separator, options));
     }
     last(null_on_oob = true) {
         return this.get(-1, null_on_oob);
@@ -149,100 +151,74 @@ class ListExprNamespace {
         return this._deriveList((arr) => arr.length);
     }
     max() {
-        return this._deriveList((arr) => (0, utils_1.getListStats)(arr).max);
+        return this._deriveList((arr) => (0, utils_2.getListStats)(arr).max);
     }
     mean() {
-        return this._deriveList((arr) => {
-            const { sum, count } = (0, utils_1.getListStats)(arr);
-            return sum !== null && count > 0 ? sum / count : null;
-        });
+        return this._deriveList((arr) => (0, utils_2.getListStats)(arr).mean);
     }
     median() {
-        return this._deriveList((arr) => {
-            if (!(0, utils_1.isArrayOfType)(arr, "number", { allowNulls: true }))
-                return null;
-            const len = arr.length;
-            const nums = [];
-            for (let i = 0; i < len; i++) {
-                const val = arr[i];
-                if (val != null) {
-                    nums.push(val);
-                }
-            }
-            const numsLen = nums.length;
-            if (numsLen === 0)
-                return null;
-            nums.sort((a, b) => a - b);
-            const mid = Math.floor(numsLen / 2);
-            return numsLen % 2 !== 0
-                ? nums[mid]
-                : (nums[mid - 1] + nums[mid]) / 2;
-        });
+        return this._deriveList((arr) => (0, utils_2.computeMedian)(arr));
     }
     min() {
-        return this._deriveList((arr) => (0, utils_1.getListStats)(arr).min);
+        return this._deriveList((arr) => (0, utils_2.getListStats)(arr).min);
     }
     mode() {
-        return this._deriveList((arr) => {
-            const len = arr.length;
-            const counts = new Map();
-            let maxCount = 0;
-            for (let i = 0; i < len; i++) {
-                const val = arr[i];
-                if (val == null)
-                    continue;
-                const c = (counts.get(val) ?? 0) + 1;
-                counts.set(val, c);
-                if (c > maxCount)
-                    maxCount = c;
-            }
-            if (maxCount === 0)
-                return [];
-            const modes = [];
-            for (const [val, c] of counts.entries()) {
-                if (c === maxCount) {
-                    modes.push(val);
-                }
-            }
-            return (0, utils_1.sortList)(modes);
-        });
+        return this._deriveList((arr) => (0, utils_2.computeMode)(arr));
     }
-    n_unique() {
-        return this._deriveList((arr) => {
-            return new Set(Array.from(arr)).size;
-        });
+    n_unique(options = {}) {
+        return this._deriveList((arr) => (0, utils_2.getUniqueListStats)(arr, options).count);
     }
     reverse() {
-        return this._deriveList((arr) => {
-            return Array.from(arr).reverse();
-        });
+        return this._deriveList((arr) => arr.slice().reverse());
     }
     slice(offset, length) {
         return this._deriveList((arr) => {
-            const list = Array.from(arr);
-            const start = offset < 0 ? Math.max(0, list.length + offset) : offset;
-            const end = length !== undefined ? start + length : list.length;
-            return list.slice(start, end);
+            const len = arr.length;
+            const start = offset < 0 ? Math.max(0, len + offset) : offset;
+            const end = length !== undefined ? start + length : len;
+            return arr.slice(start, end);
         });
     }
     sort(descending = false) {
-        return this._deriveList((arr) => (0, utils_1.sortList)(arr, descending));
+        return this._deriveList((arr) => (0, utils_2.sortList)(arr, descending));
     }
     sum() {
-        return this._deriveList((arr) => (0, utils_1.getListStats)(arr).sum);
-    }
-    unique() {
-        return this._deriveList((arr) => {
-            return Array.from(new Set(arr));
+        return this._deriveList((arr) => (0, utils_2.getListStats)(arr).sum);
+    }
+    unique(options = {}) {
+        return this._deriveList((arr) => (0, utils_2.getUniqueListStats)(arr, options).values);
+    }
+    eval(expr) {
+        return (0, ExprBase_1.derive)(this.expr, (vArray, columns) => {
+            const height = vArray.length;
+            const result = new Array(height);
+            for (let i = 0; i < height; i++) {
+                const val = vArray[i];
+                if (!(0, utils_2.isArrayOrTypedArray)(val)) {
+                    result[i] = null;
+                    continue;
+                }
+                const subHeight = val.length;
+                const subColumns = { ...columns, [constants_1.ELEMENT_MARKER]: val };
+                const isGlobalAgg = expr.aggFn != null && (expr.partitionBy == null || expr.partitionBy.length === 0);
+                if (isGlobalAgg) {
+                    const preOpsIdx = expr.groupingOpsIndex !== undefined ? expr.groupingOpsIndex : expr.ops.length;
+                    const preVal = expr.evaluatePre(preOpsIdx, subColumns, subHeight);
+                    const aggVal = expr.aggFn(Array.from(preVal));
+                    result[i] = expr.evaluatePost(preOpsIdx, [aggVal], subColumns);
+                }
+                else {
+                    result[i] = (0, utils_1.evaluateExpression)(expr, subColumns, subHeight);
+                }
+            }
+            return result;
         });
     }
 }
 exports.ListExprNamespace = ListExprNamespace;
-const ListExpr = (Base) => {
-    return class extends Base {
-        get list() {
-            return new ListExprNamespace(this);
-        }
-    };
-};
+class ListExpr extends ExprBase_1.ExprBase {
+    get list() {
+        return new ListExprNamespace(this);
+    }
+}
 exports.ListExpr = ListExpr;

package/dist/columnExpressions/mixins/LogicalExpr.d.ts

@@ -1,33 +1,7 @@
-import type { ExprConstructor } from "../types";
-export declare const LogicalExpr: <TBase extends ExprConstructor>(Base: TBase) => {
-    new (...args: any[]): {
-        and(other: any): /*elided*/ any;
-        not(): /*elided*/ any;
-        or(other: any): /*elided*/ any;
-        xor(other: any): /*elided*/ any;
-        ops: import("../..").OpFn[];
-        colName?: string;
-        outputName?: string;
-        isLiteral?: boolean;
-        literalValue?: any;
-        aggFn?: import("../..").AggFn<any> | null;
-        groupingOpsIndex?: number;
-        partitionOpsIndex?: number;
-        partitionBy?: (string | import("../..").IExpr)[] | null;
-        windowOp?: {
-            type: string;
-            [key: string]: any;
-        } | null;
-        isWindow?: boolean;
-        alias(name: string): /*elided*/ any;
-        cast(dataType: any): /*elided*/ any;
-        _resolve(val: any, columns: import("../..").ColumnDict, height: number): import("../..").ColumnData | any;
-        evaluate(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePreGrouping(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePostGrouping(aggregatedArray: any[], columns: import("../..").ColumnDict): import("../..").ColumnData;
-        evaluatePrePartition(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePostPartition(aggregatedArray: any[], columns: import("../..").ColumnDict): import("../..").ColumnData;
-        evaluateWindow?(groupPreValues: any[], partitionIndices: number[], currentIndex: number): any;
-        debug(label?: string): /*elided*/ any;
-    };
-} & TBase;
+import { ExprBase } from "../ExprBase";
+export declare class LogicalExpr extends ExprBase {
+    and(other: any): this;
+    not(): this;
+    or(other: any): this;
+    xor(other: any): this;
+}

package/dist/columnExpressions/mixins/LogicalExpr.js

@@ -2,53 +2,52 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LogicalExpr = void 0;
 const ExprBase_1 = require("../ExprBase");
-const utils_1 = require("../../utils");
-const LogicalExpr = (Base) => {
-    return class extends Base {
-        and(other) {
-            return (0, ExprBase_1.derive)(this, (vArray, columns) => {
-                const height = vArray.length;
-                const otherVal = this._resolve(other, columns, height);
-                const isOtherArray = (0, utils_1.isArrayOrTypedArray)(otherVal);
-                const result = new Array(height);
-                for (let i = 0; i < height; i++) {
-                    const v = vArray[i];
-                    const w = isOtherArray ? otherVal[i] : otherVal;
-                    if (v === false || w === false)
-                        result[i] = false;
-                    else if (v == null || w == null)
-                        result[i] = null;
-                    else
-                        result[i] = true;
-                }
-                return result;
-            });
-        }
-        not() {
-            return (0, ExprBase_1.derive)(this, (0, ExprBase_1.kleeneUnary)((v) => !v));
-        }
-        or(other) {
-            return (0, ExprBase_1.derive)(this, (vArray, columns) => {
-                const height = vArray.length;
-                const otherVal = this._resolve(other, columns, height);
-                const isOtherArray = (0, utils_1.isArrayOrTypedArray)(otherVal);
-                const result = new Array(height);
-                for (let i = 0; i < height; i++) {
-                    const v = vArray[i];
-                    const w = isOtherArray ? otherVal[i] : otherVal;
-                    if (v === true || w === true)
-                        result[i] = true;
-                    else if (v == null || w == null)
-                        result[i] = null;
-                    else
-                        result[i] = false;
-                }
-                return result;
-            });
-        }
-        xor(other) {
-            return (0, ExprBase_1.derive)(this, (0, ExprBase_1.kleeneBinary)(this, other, (v, w) => !!v !== !!w));
-        }
-    };
-};
+const utils_1 = require("../utils");
+const utils_2 = require("../../utils");
+class LogicalExpr extends ExprBase_1.ExprBase {
+    and(other) {
+        return (0, ExprBase_1.derive)(this, (vArray, columns) => {
+            const height = vArray.length;
+            const otherVal = this._resolve(other, columns, height);
+            const isOtherArray = (0, utils_2.isArrayOrTypedArray)(otherVal);
+            const result = new Array(height);
+            for (let i = 0; i < height; i++) {
+                const v = vArray[i];
+                const w = isOtherArray ? otherVal[i] : otherVal;
+                if (v === false || w === false)
+                    result[i] = false;
+                else if (v == null || w == null)
+                    result[i] = null;
+                else
+                    result[i] = true;
+            }
+            return result;
+        });
+    }
+    not() {
+        return (0, ExprBase_1.derive)(this, (0, utils_1.kleeneUnary)((v) => !v));
+    }
+    or(other) {
+        return (0, ExprBase_1.derive)(this, (vArray, columns) => {
+            const height = vArray.length;
+            const otherVal = this._resolve(other, columns, height);
+            const isOtherArray = (0, utils_2.isArrayOrTypedArray)(otherVal);
+            const result = new Array(height);
+            for (let i = 0; i < height; i++) {
+                const v = vArray[i];
+                const w = isOtherArray ? otherVal[i] : otherVal;
+                if (v === true || w === true)
+                    result[i] = true;
+                else if (v == null || w == null)
+                    result[i] = null;
+                else
+                    result[i] = false;
+            }
+            return result;
+        });
+    }
+    xor(other) {
+        return (0, ExprBase_1.derive)(this, (0, utils_1.kleeneBinary)(this, other, (v, w) => !!v !== !!w));
+    }
+}
 exports.LogicalExpr = LogicalExpr;

package/dist/columnExpressions/mixins/ManipulationExpr.d.ts

@@ -1,31 +1,6 @@
-import type { ExprConstructor } from "../types";
-export declare const ManipulationExpr: <TBase extends ExprConstructor>(Base: TBase) => {
-    new (...args: any[]): {
-        fill_null(value: any): /*elided*/ any;
-        reverse(): /*elided*/ any;
-        ops: import("../..").OpFn[];
-        colName?: string;
-        outputName?: string;
-        isLiteral?: boolean;
-        literalValue?: any;
-        aggFn?: import("../..").AggFn<any> | null;
-        groupingOpsIndex?: number;
-        partitionOpsIndex?: number;
-        partitionBy?: (string | import("../..").IExpr)[] | null;
-        windowOp?: {
-            type: string;
-            [key: string]: any;
-        } | null;
-        isWindow?: boolean;
-        alias(name: string): /*elided*/ any;
-        cast(dataType: any): /*elided*/ any;
-        _resolve(val: any, columns: import("../..").ColumnDict, height: number): import("../..").ColumnData | any;
-        evaluate(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePreGrouping(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePostGrouping(aggregatedArray: any[], columns: import("../..").ColumnDict): import("../..").ColumnData;
-        evaluatePrePartition(columns: import("../..").ColumnDict, height: number): import("../..").ColumnData;
-        evaluatePostPartition(aggregatedArray: any[], columns: import("../..").ColumnDict): import("../..").ColumnData;
-        evaluateWindow?(groupPreValues: any[], partitionIndices: number[], currentIndex: number): any;
-        debug(label?: string): /*elided*/ any;
-    };
-} & TBase;
+import { ExprBase } from "../ExprBase";
+import type { FillNullOptions } from "../../types";
+export declare class ManipulationExpr extends ExprBase {
+    fill_null({ value, strategy, limit }?: FillNullOptions): this;
+    reverse(): this;
+}