df-script 1.4.0 → 1.5.0

package/dist/columnExpressions/mixins/WindowExpr.js

@@ -3,279 +3,149 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.WindowExpr = void 0;
 const ExprBase_1 = require("../ExprBase");
 const utils_1 = require("../../utils");
-const WindowExpr = (Base) => {
-    return class extends Base {
-        partitionBy = this.partitionBy || null;
-        _rolling(windowSize, aggFn) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                const start = Math.max(0, currentIndex - windowSize + 1);
-                const end = currentIndex + 1;
-                const windowVals = groupPreValues.slice(start, end);
-                return aggFn(windowVals);
-            };
-            return newInst;
-        }
-        get isWindow() {
-            return this.partitionBy !== null || this.evaluateWindow !== undefined || this.aggFn !== null;
-        }
-        cum_count(reverse = false) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let count = 0;
-                const start = reverse ? currentIndex : 0;
-                const end = reverse ? groupPreValues.length - 1 : currentIndex;
-                for (let i = start; i <= end; i++) {
-                    const val = groupPreValues[i];
-                    if (val != null)
-                        count++;
+function computeRank(arr, value, options = {}) {
+    if (value == null)
+        return null;
+    let targetArr = arr;
+    if (options.ignoreNulls) {
+        targetArr = [];
+        const len = arr.length;
+        for (let i = 0; i < len; i++) {
+            if (arr[i] != null)
+                targetArr.push(arr[i]);
+        }
+    }
+    if (options.dense) {
+        targetArr = Array.from(new Set(targetArr));
+    }
+    const sorted = [...targetArr].sort((a, b) => a - b);
+    const valueToRank = new Map();
+    const len = sorted.length;
+    for (let i = 0; i < len; i++) {
+        const v = sorted[i];
+        if (!valueToRank.has(v)) {
+            valueToRank.set(v, i + 1);
+        }
+    }
+    return valueToRank.get(value) ?? null;
+}
+class WindowExpr extends ExprBase_1.ExprBase {
+    partitionBy = this.partitionBy || null;
+    _window(evaluateWindow) {
+        const newInst = (0, ExprBase_1.derive)(this);
+        newInst.partitionOpsIndex = this.ops.length;
+        newInst.groupingOpsIndex = this.ops.length;
+        newInst.evaluateWindow = evaluateWindow;
+        return newInst;
+    }
+    _rolling(windowSize, aggFn) {
+        return this._window(function (groupPreValues, _partitionIndices, currentIndex) {
+            const start = Math.max(0, currentIndex - windowSize + 1);
+            const end = currentIndex + 1;
+            const windowVals = groupPreValues.slice(start, end);
+            return aggFn(windowVals);
+        });
+    }
+    _cum(reverse, initialVal, stepFn, postFn) {
+        return this._window(function (groupPreValues, _partitionIndices, currentIndex) {
+            let acc = initialVal;
+            let hasValid = false;
+            const start = reverse ? currentIndex : 0;
+            const end = reverse ? groupPreValues.length - 1 : currentIndex;
+            for (let i = start; i <= end; i++) {
+                const val = groupPreValues[i];
+                if (val != null) {
+                    acc = stepFn(acc, val);
+                    hasValid = true;
                 }
-                return count;
-            };
-            return newInst;
-        }
-        cum_max(reverse = false) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let maxVal = null;
-                const start = reverse ? currentIndex : 0;
-                const end = reverse ? groupPreValues.length - 1 : currentIndex;
-                for (let i = start; i <= end; i++) {
-                    const val = groupPreValues[i];
-                    if (val != null) {
-                        if (maxVal === null || val > maxVal) {
-                            maxVal = val;
-                        }
-                    }
-                }
-                return maxVal;
-            };
-            return newInst;
-        }
-        cum_min(reverse = false) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let minVal = null;
-                const start = reverse ? currentIndex : 0;
-                const end = reverse ? groupPreValues.length - 1 : currentIndex;
-                for (let i = start; i <= end; i++) {
-                    const val = groupPreValues[i];
-                    if (val != null) {
-                        if (minVal === null || val < minVal) {
-                            minVal = val;
-                        }
-                    }
-                }
-                return minVal;
-            };
-            return newInst;
-        }
-        cum_prod(reverse = false) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let prod = 1;
-                let hasValid = false;
-                const start = reverse ? currentIndex : 0;
-                const end = reverse ? groupPreValues.length - 1 : currentIndex;
-                for (let i = start; i <= end; i++) {
-                    const val = groupPreValues[i];
-                    if (val != null) {
-                        prod *= val;
-                        hasValid = true;
-                    }
-                }
-                return hasValid ? prod : null;
-            };
-            return newInst;
-        }
-        cum_sum(reverse = false) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let sum = 0;
-                const start = reverse ? currentIndex : 0;
-                const end = reverse ? groupPreValues.length - 1 : currentIndex;
-                for (let i = start; i <= end; i++) {
-                    const val = groupPreValues[i];
-                    if (val != null)
-                        sum += val;
-                }
-                return sum;
-            };
-            return newInst;
-        }
-        dense_rank() {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                const sortedUnique = Array.from(new Set(groupPreValues)).sort((a, b) => a - b);
-                const valueToRank = new Map();
-                for (let idx = 0; idx < sortedUnique.length; idx++) {
-                    valueToRank.set(sortedUnique[idx], idx + 1);
-                }
-                const currentVal = groupPreValues[currentIndex];
-                return valueToRank.get(currentVal) ?? null;
-            };
-            return newInst;
-        }
-        lag(offset = 1, defaultVal = null) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let val = defaultVal;
-                if (currentIndex - offset >= 0) {
-                    val = groupPreValues[currentIndex - offset];
-                }
-                return val;
-            };
-            return newInst;
-        }
-        lead(offset = 1, defaultVal = null) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                let val = defaultVal;
-                if (currentIndex + offset < groupPreValues.length) {
-                    val = groupPreValues[currentIndex + offset];
-                }
-                return val;
-            };
-            return newInst;
-        }
-        over(columns) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            const cols = Array.isArray(columns) ? columns : [columns];
-            newInst.partitionBy = cols;
-            return newInst;
-        }
-        rank() {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                const sorted = [...groupPreValues].sort((a, b) => a - b);
-                const valueToRank = new Map();
-                for (let i = 0; i < sorted.length; i++) {
-                    const v = sorted[i];
-                    if (!valueToRank.has(v)) {
-                        valueToRank.set(v, i + 1);
-                    }
-                }
-                const currentVal = groupPreValues[currentIndex];
-                return valueToRank.get(currentVal) ?? null;
-            };
-            return newInst;
-        }
-        rolling_max(windowSize) {
-            return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).max);
-        }
-        rolling_mean(windowSize) {
-            return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).mean);
-        }
-        rolling_median(windowSize) {
-            return this._rolling(windowSize, v => {
-                if (!(0, utils_1.isArrayOfType)(v, "number", { allowNulls: true }))
-                    return null;
-                const len = v.length;
-                const nums = [];
-                for (let i = 0; i < len; i++) {
-                    const val = v[i];
-                    if (val != null) {
-                        nums.push(val);
-                    }
-                }
-                const numsLen = nums.length;
-                if (numsLen === 0)
-                    return null;
-                nums.sort((a, b) => a - b);
-                const mid = Math.floor(numsLen / 2);
-                return numsLen % 2 !== 0
-                    ? nums[mid]
-                    : (nums[mid - 1] + nums[mid]) / 2;
-            });
-        }
-        rolling_min(windowSize) {
-            return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).min);
-        }
-        rolling_quantile(quantile, windowSize) {
-            return this._rolling(windowSize, v => {
-                const f = [];
-                for (let i = 0; i < v.length; i++) {
-                    if (v[i] != null)
-                        f.push(v[i]);
-                }
-                const n = f.length;
-                if (n === 0)
-                    return null;
-                f.sort((a, b) => a - b);
-                const idx = quantile * (n - 1);
-                const low = Math.floor(idx);
-                const high = Math.ceil(idx);
-                if (low === high)
-                    return f[low];
-                return f[low] + (idx - low) * (f[high] - f[low]);
-            });
-        }
-        rolling_rank(windowSize) {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (groupPreValues, _partitionIndices, currentIndex) {
-                const start = Math.max(0, currentIndex - windowSize + 1);
-                const end = currentIndex + 1;
-                const vals = groupPreValues.slice(start, end);
-                const currentVal = groupPreValues[currentIndex];
-                if (currentVal == null)
-                    return null;
-                const nonNullVals = [];
-                for (let j = 0; j < vals.length; j++) {
-                    if (vals[j] != null)
-                        nonNullVals.push(vals[j]);
-                }
-                if (nonNullVals.length === 0)
-                    return null;
-                const sorted = [...nonNullVals].sort((a, b) => a - b);
-                const valueToRank = new Map();
-                for (let i = 0; i < sorted.length; i++) {
-                    const v = sorted[i];
-                    if (!valueToRank.has(v)) {
-                        valueToRank.set(v, i + 1);
-                    }
-                }
-                return valueToRank.get(currentVal) ?? null;
-            };
-            return newInst;
-        }
-        rolling_std(windowSize) {
-            return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).std);
-        }
-        rolling_sum(windowSize) {
-            return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).sum);
-        }
-        row_number() {
-            const newInst = (0, ExprBase_1.derive)(this);
-            newInst.partitionOpsIndex = this.ops.length;
-            newInst.groupingOpsIndex = this.ops.length;
-            newInst.evaluateWindow = function (_groupPreValues, _partitionIndices, currentIndex) {
-                return currentIndex + 1;
-            };
-            newInst.outputName = "row_number";
-            return newInst;
-        }
-    };
-};
+            }
+            return postFn ? postFn(acc, hasValid) : acc;
+        });
+    }
+    get isWindow() {
+        return this.partitionBy !== null || this.evaluateWindow !== undefined || this.aggFn !== null;
+    }
+    cum_count(reverse = false) {
+        return this._cum(reverse, 0, (acc) => acc + 1);
+    }
+    cum_max(reverse = false) {
+        return this._cum(reverse, null, (acc, val) => (acc === null || val > acc ? val : acc));
+    }
+    cum_min(reverse = false) {
+        return this._cum(reverse, null, (acc, val) => (acc === null || val < acc ? val : acc));
+    }
+    cum_prod(reverse = false) {
+        return this._cum(reverse, 1, (acc, val) => acc * val, (acc, hasValid) => (hasValid ? acc : null));
+    }
+    cum_sum(reverse = false) {
+        return this._cum(reverse, 0, (acc, val) => acc + val);
+    }
+    dense_rank() {
+        return this._window(function (groupPreValues, _partitionIndices, currentIndex) {
+            return computeRank(groupPreValues, groupPreValues[currentIndex], { dense: true });
+        });
+    }
+    lag(offset = 1, defaultVal = null) {
+        return this._window(function (groupPreValues, _partitionIndices, currentIndex) {
+            let val = defaultVal;
+            if (currentIndex - offset >= 0) {
+                val = groupPreValues[currentIndex - offset];
+            }
+            return val;
+        });
+    }
+    lead(offset = 1, defaultVal = null) {
+        return this._window(function (groupPreValues, _partitionIndices, currentIndex) {
+            let val = defaultVal;
+            if (currentIndex + offset < groupPreValues.length) {
+                val = groupPreValues[currentIndex + offset];
+            }
+            return val;
+        });
+    }
+    over(columns) {
+        const newInst = (0, ExprBase_1.derive)(this);
+        const cols = Array.isArray(columns) ? columns : [columns];
+        newInst.partitionBy = cols;
+        return newInst;
+    }
+    rank() {
+        return this._window(function (groupPreValues, _partitionIndices, currentIndex) {
+            return computeRank(groupPreValues, groupPreValues[currentIndex]);
+        });
+    }
+    rolling_max(windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).max);
+    }
+    rolling_mean(windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).mean);
+    }
+    rolling_median(windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.computeMedian)(v));
+    }
+    rolling_min(windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).min);
+    }
+    rolling_quantile(quantile, windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.computeQuantile)(v, quantile));
+    }
+    rolling_rank(windowSize) {
+        return this._rolling(windowSize, (vals) => {
+            return computeRank(vals, vals[vals.length - 1], { ignoreNulls: true });
+        });
+    }
+    rolling_std(windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).std);
+    }
+    rolling_sum(windowSize) {
+        return this._rolling(windowSize, v => (0, utils_1.getListStats)(v).sum);
+    }
+    row_number() {
+        const newInst = this._window(function (_groupPreValues, _partitionIndices, currentIndex) {
+            return currentIndex + 1;
+        });
+        newInst.outputName = "row_number";
+        return newInst;
+    }
+}
 exports.WindowExpr = WindowExpr;

package/dist/columnExpressions/utils.d.ts

@@ -0,0 +1,4 @@
+import type { IExpr, ColumnData, ColumnDict } from "../types";
+export declare const kleeneUnary: (fn: (v: any) => any) => (vArray: ColumnData) => any[];
+export declare const kleeneBinary: (expr: IExpr, other: any, fn: (v: any, r: any) => any) => (vArray: ColumnData, columns: ColumnDict) => any[];
+export declare function evaluateExpression(expr: IExpr, columns: ColumnDict, height: number): ColumnData;

package/dist/columnExpressions/utils.js

@@ -0,0 +1,45 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.kleeneBinary = exports.kleeneUnary = void 0;
+exports.evaluateExpression = evaluateExpression;
+const utils_1 = require("../utils");
+const utils_2 = require("../dataframe/utils");
+const kleeneUnary = (fn) => {
+    return (vArray) => {
+        const height = vArray.length;
+        const result = new Array(height);
+        for (let i = 0; i < height; i++) {
+            const v = vArray[i];
+            result[i] = v == null ? null : fn(v);
+        }
+        return result;
+    };
+};
+exports.kleeneUnary = kleeneUnary;
+const kleeneBinary = (expr, other, fn) => {
+    return (vArray, columns) => {
+        const height = vArray.length;
+        const rResolved = expr._resolve(other, columns, height);
+        const result = new Array(height);
+        if ((0, utils_1.isArrayOrTypedArray)(rResolved)) {
+            for (let i = 0; i < height; i++) {
+                const v = vArray[i];
+                const r = rResolved[i];
+                result[i] = (v == null || r == null) ? null : fn(v, r);
+            }
+        }
+        else {
+            for (let i = 0; i < height; i++) {
+                const v = vArray[i];
+                result[i] = (v == null || rResolved == null) ? null : fn(v, rResolved);
+            }
+        }
+        return result;
+    };
+};
+exports.kleeneBinary = kleeneBinary;
+function evaluateExpression(expr, columns, height) {
+    return expr.isWindow
+        ? (0, utils_2.resolveWindowExpr)(expr, columns, height)
+        : expr.evaluate(columns, height);
+}

package/dist/dataframe/constants.d.ts

@@ -1 +1,4 @@
 export declare const KEY_SEPARATOR = "\0";
+export declare const NEWLINE = "\n";
+export declare const CARRIAGE_RETURN = "\r";
+export declare const UTF8_BOM = "\uFEFF";

package/dist/dataframe/constants.js

@@ -1,4 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.KEY_SEPARATOR = void 0;
+exports.UTF8_BOM = exports.CARRIAGE_RETURN = exports.NEWLINE = exports.KEY_SEPARATOR = void 0;
 exports.KEY_SEPARATOR = "\x00";
+exports.NEWLINE = "\n";
+exports.CARRIAGE_RETURN = "\r";
+exports.UTF8_BOM = "\ufeff";

package/dist/dataframe/dataframe.d.ts

@@ -1,42 +1,59 @@
 import { GroupedData } from "./grouped/grouped";
-import type { IExpr, ColumnDict, DataFrameColumns, ConcatOptions, ConcatItem, HorizontalConcatOptions, RowRecord } from "../types";
-import type { LimitOptions, SortOptions, PivotOptions, JoinOptions, UnpivotOptions } from "./types";
-import { DataType } from "../datatypes";
+import type { IExpr, ColumnData, ColumnDict, DataFrameColumns, ConcatOptions, ConcatItem, HorizontalConcatOptions, RowRecord, DataFrameSchema, RegisteredDataType, ExplodeOptions, IntoExpr, FillNullOptions } from "../types";
+import type { LimitOptions, SortOptions, PivotOptions, JoinOptions, UnpivotOptions, TransposeOptions, WriteJSONOptions, WriteCSVOptions } from "./types";
 export declare class DataFrame<T extends RowRecord = any> {
     _columns: DataFrameColumns<T>;
     private _height;
     private _schema;
-    constructor(data: T[] | ColumnDict, schema?: Record<string, DataType>, height?: number);
+    static _createDirect<U extends RowRecord = any>(columns: ColumnDict, schema: DataFrameSchema, height: number): DataFrame<U>;
+    constructor(data: T[] | ColumnDict, schema?: DataFrameSchema, height?: number);
     private inferSchema;
     private applySchema;
-    getSchema(): Record<string, DataType>;
-    to_dicts(): T[];
-    to_dict(): DataFrameColumns<T>;
     get columns(): string[];
     concat<U extends RowRecord = any>(items: ConcatItem | ConcatItem[], options?: ConcatOptions): DataFrame<U>;
     drop<K extends keyof T>(...args: (K | K[])[]): DataFrame<Omit<T, K>>;
-    get dtypes(): DataType[];
+    drop_nulls(subset?: string | string[]): DataFrame<T>;
+    get dtypes(): RegisteredDataType[];
+    explode(columns: IntoExpr | IntoExpr[], options?: ExplodeOptions): DataFrame<any>;
+    fill_null(options?: FillNullOptions): DataFrame<T>;
     filter(...exprs: (IExpr | ((row: T) => any))[]): DataFrame<T>;
+    get_schema(): DataFrameSchema;
     groupby<K extends keyof T>(keys: K | K[]): GroupedData<T, K>;
     head(n?: number): DataFrame<T>;
     get height(): number;
     hstack<U extends RowRecord = any>(other: ConcatItem | ConcatItem[], options?: HorizontalConcatOptions): DataFrame<U>;
+    insert_column(index: number, name: string, expr: IntoExpr): DataFrame<any>;
+    item(row?: number, column?: number | string): any;
+    iter_columns(): Generator<ColumnData>;
+    iter_rows({ named }?: {
+        named?: boolean;
+    }): Generator<any[] | Record<string, any>>;
     join<U extends RowRecord = any, R extends RowRecord = any>(config: JoinOptions<T, U>): DataFrame<R>;
-    limit(n: number, options?: LimitOptions): DataFrame<T>;
+    limit(n: number, { offset, from }?: LimitOptions): DataFrame<T>;
     pivot<U extends RowRecord = any>(config: PivotOptions<T>): DataFrame<U>;
     rename(mapping?: Partial<Record<keyof T, string>>): DataFrame<any>;
     reverse(): DataFrame<T>;
-    get schema(): Record<string, DataType>;
+    get schema(): DataFrameSchema;
     select<U extends RowRecord = any>(...args: (string | IExpr | Record<string, any> | (string | IExpr | Record<string, any>)[])[]): DataFrame<U>;
     get shape(): [number, number];
     slice(start: number, end?: number): DataFrame<T>;
     sort(config?: SortOptions<T>): DataFrame<T>;
     tail(n?: number): DataFrame<T>;
+    to_dict(): DataFrameColumns<T>;
+    to_dicts(): T[];
     to_list<K extends keyof T>(nameOrExpr: K | IExpr): any[];
+    transpose({ include_header: includeHeader, header_name: headerName, column_names: colNamesOpt }?: TransposeOptions): DataFrame<any>;
     unique<K extends keyof T>(columns?: K | K[]): DataFrame<T>;
     unpivot<U extends RowRecord = any>(config: UnpivotOptions<T>): DataFrame<U>;
     vstack<U extends RowRecord = any>(other: ConcatItem | ConcatItem[]): DataFrame<U>;
     get width(): number;
     private _normalizeArgs;
     with_columns(...args: (string | IExpr | Record<string, any> | (string | IExpr | Record<string, any>)[])[]): DataFrame<any>;
+    with_row_index(name?: string, offset?: number): DataFrame<any>;
+    write_json(file?: string | {
+        write: (str: string) => void;
+    }, { format, replacerOptions }?: WriteJSONOptions): string;
+    write_csv(file?: string | {
+        write: (str: string) => void;
+    }, options?: WriteCSVOptions): string;
 }