df-script 1.4.0 → 1.5.0

package/dist/dataframe/dataframe.js

@@ -3,16 +3,25 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DataFrame = void 0;
 const columnExpressions_1 = require("../columnExpressions");
 const grouped_1 = require("./grouped/grouped");
+const constants_1 = require("./constants");
+const json_1 = require("../utils/json");
 const datatypes_1 = require("../datatypes");
 const utils_1 = require("../utils");
 const exceptions_1 = require("../exceptions");
 const concat_1 = require("../functions/concat");
-const constants_1 = require("./constants");
 const utils_2 = require("./utils");
 class DataFrame {
     _columns;
     _height;
     _schema = {};
+    static _createDirect(columns, schema, height) {
+        (0, exceptions_1.assertHeight)(columns, height);
+        const df = Object.create(DataFrame.prototype);
+        df._columns = columns;
+        df._schema = schema;
+        df._height = height;
+        return df;
+    }
     constructor(data, schema, height) {
         if (Array.isArray(data)) {
             const { columns, height: h } = (0, utils_2.rowsToColumns)(data);
@@ -22,18 +31,8 @@ class DataFrame {
             return;
         }
         if ((0, utils_1.isObj)(data)) {
-            let firstLength = -1;
-            for (const [key, col] of Object.entries(data)) {
-                const colLen = (0, utils_1.isArrayOrTypedArray)(col) ? col.length : 0;
-                if (firstLength === -1) {
-                    firstLength = colLen;
-                }
-                else if (colLen !== firstLength) {
-                    throw new exceptions_1.DataFrameError(`Column height mismatch: Column "${key}" has length ${colLen}, but previous columns have length ${firstLength}`);
-                }
-            }
             this._columns = data;
-            this._height = height !== undefined ? height : (firstLength === -1 ? 0 : firstLength);
+            this._height = (0, exceptions_1.assertHeight)(data, height);
             schema ? this.applySchema(schema) : this.inferSchema();
             return;
         }
@@ -44,10 +43,12 @@ class DataFrame {
     inferSchema() {
         const schema = {};
         const keys = Object.keys(this._columns);
-        for (const key of keys) {
+        const numKeys = keys.length;
+        for (let i = 0; i < numKeys; i++) {
+            const key = keys[i];
             schema[key] = (0, utils_2.inferColumnType)(this._columns[key]);
         }
-        this._schema = schema;
+        this.applySchema(schema);
     }
     applySchema(schema) {
         this._schema = schema;
@@ -56,42 +57,12 @@ class DataFrame {
         for (const key of keys) {
             const type = schema[key];
             const oldCol = this._columns[key];
-            let newCol = type.allocate ? type.allocate(this._height) : new Array(this._height).fill(null);
-            if (!oldCol) {
-                if (this._height > 0 && (0, utils_1.isTypedArray)(newCol)) {
-                    newCol = new Array(this._height).fill(null);
-                }
-                newColumns[key] = newCol;
-                continue;
-            }
-            let hasNulls = false;
-            const coercedVals = new Array(this._height);
-            for (let i = 0; i < this._height; i++) {
-                const coerced = type.coerce(oldCol[i]);
-                coercedVals[i] = coerced;
-                if (coerced == null) {
-                    hasNulls = true;
-                }
-            }
-            if (hasNulls && (0, utils_1.isTypedArray)(newCol)) {
-                newCol = new Array(this._height);
-            }
-            for (let i = 0; i < this._height; i++) {
-                newCol[i] = coercedVals[i];
-            }
-            newColumns[key] = newCol;
+            newColumns[key] = oldCol
+                ? (0, utils_2.coerceColumn)(oldCol, type, this._height)
+                : (0, utils_2.coerceColumn)(new Array(this._height).fill(null), type, this._height);
         }
         this._columns = newColumns;
     }
-    getSchema() {
-        return this._schema;
-    }
-    to_dicts() {
-        return (0, utils_2.columnsToRows)(this._columns, this._height);
-    }
-    to_dict() {
-        return { ...this._columns };
-    }
     get columns() {
         return Object.keys(this._columns);
     }
@@ -111,7 +82,12 @@ class DataFrame {
                 outSchema[key] = this._schema[key];
             }
         }
-        return new DataFrame(newColumns, outSchema, this._height);
+        return DataFrame._createDirect(newColumns, outSchema, this._height);
+    }
+    drop_nulls(subset) {
+        if (this._height === 0)
+            return this;
+        return this.filter(subset ? new columnExpressions_1.ColumnExpr(subset).is_not_null() : (0, columnExpressions_1.all)().is_not_null());
     }
     get dtypes() {
         const keys = Object.keys(this._columns);
@@ -122,22 +98,59 @@ class DataFrame {
         }
         return result;
     }
+    explode(columns, options) {
+        const expr = columnExpressions_1.ColumnExpr.toColExpr(columns);
+        const colNames = expr.colNames || [expr.colName || expr.outputName];
+        const colsToExplode = new Set();
+        const numCols = colNames.length;
+        for (let i = 0; i < numCols; i++) {
+            const name = colNames[i];
+            if (!name) {
+                throw new exceptions_1.DataFrameError("Expression passed to explode must have a column name.");
+            }
+            (0, exceptions_1.assertColumnExists)(name, this._columns, "Explode column");
+            colsToExplode.add(name);
+        }
+        const keys = Object.keys(this._columns);
+        const selectList = [];
+        const numKeys = keys.length;
+        for (let i = 0; i < numKeys; i++) {
+            const key = keys[i];
+            selectList.push(colsToExplode.has(key)
+                ? new columnExpressions_1.ColumnExpr(key).list.explode(options)
+                : new columnExpressions_1.ColumnExpr(key));
+        }
+        return this.select(...selectList);
+    }
+    fill_null(options = {}) {
+        if (this._height === 0)
+            return this;
+        return this.with_columns((0, columnExpressions_1.all)().fill_null(options));
+    }
     filter(...exprs) {
         if (this._height === 0)
-            return new DataFrame({}, this._schema, 0);
+            return DataFrame._createDirect({}, this._schema, 0);
         const height = this._height;
         const keys = Object.keys(this._columns);
         const numKeys = keys.length;
         const evaluatedExprs = [];
         const funcPredicates = [];
-        for (const expr of exprs) {
+        const exprSelectors = [];
+        const numExprs = exprs.length;
+        for (let i = 0; i < numExprs; i++) {
+            const expr = exprs[i];
             if (typeof expr === "function") {
                 funcPredicates.push(expr);
             }
             else {
-                evaluatedExprs.push(expr.evaluate(this._columns, height));
+                exprSelectors.push(expr);
             }
         }
+        const expandedExprs = (0, columnExpressions_1.resolveColumnSelectors)(exprSelectors, keys, undefined, this._schema, this._columns);
+        const numExpanded = expandedExprs.length;
+        for (let i = 0; i < numExpanded; i++) {
+            evaluatedExprs.push(expandedExprs[i].evaluate(this._columns, height));
+        }
         const matchingIndices = [];
         let currentIndex = 0;
         let rowObj = null;
@@ -182,36 +195,21 @@ class DataFrame {
         }
         const newColumns = (0, utils_2.gatherColumnsByIndices)(this._columns, matchingIndices);
         const newHeight = matchingIndices.length;
-        return new DataFrame(newColumns, this._schema, newHeight);
+        return DataFrame._createDirect(newColumns, this._schema, newHeight);
+    }
+    get_schema() {
+        return this._schema;
     }
     groupby(keys) {
         const keysArr = (0, utils_1.toValidArray)(keys);
         const groups = new Map();
         const len = this._height;
-        const keysLen = keysArr.length;
-        const keysStr = new Array(keysLen);
-        for (let i = 0; i < keysLen; i++) {
-            keysStr[i] = String(keysArr[i]);
-        }
+        const keysStr = (0, utils_1.toValidStringArray)(keys);
         for (let j = 0; j < keysStr.length; j++) {
             (0, exceptions_1.assertColumnExists)(keysStr[j], this._columns, "Grouping key");
         }
-        const singleKey = keysLen === 1 ? keysStr[0] : null;
-        const col = singleKey ? this._columns[singleKey] : null;
         for (let i = 0; i < len; i++) {
-            let hash;
-            if (singleKey) {
-                const val = col[i];
-                hash = val == null ? "" : String(val);
-            }
-            else {
-                const vals = new Array(keysLen);
-                for (let j = 0; j < keysLen; j++) {
-                    const val = this._columns[keysStr[j]][i];
-                    vals[j] = val == null ? "" : String(val);
-                }
-                hash = vals.join(constants_1.KEY_SEPARATOR);
-            }
+            const hash = (0, utils_2.computeRowHash)(this._columns, keysStr, i);
             let group = groups.get(hash);
             if (group === undefined) {
                 groups.set(hash, group = []);
@@ -230,34 +228,109 @@ class DataFrame {
     hstack(other, options = {}) {
         return this.concat(other, { how: "horizontal", horizontal: options });
     }
+    insert_column(index, name, expr) {
+        const colExpr = columnExpressions_1.ColumnExpr.toColExpr(expr).alias(name);
+        const keys = Object.keys(this._columns);
+        const keysLen = keys.length;
+        const selectList = [];
+        for (let i = 0; i < keysLen; i++) {
+            const k = keys[i];
+            if (k !== name) {
+                selectList.push(k);
+            }
+        }
+        const targetIndex = Math.max(0, Math.min(index, selectList.length));
+        selectList.splice(targetIndex, 0, colExpr);
+        return this.select(...selectList);
+    }
+    item(row, column) {
+        const height = this._height;
+        const keys = Object.keys(this._columns);
+        const width = keys.length;
+        if (row === undefined && column === undefined) {
+            if (height !== 1 || width !== 1) {
+                throw new Error("DataFrame.item() can only be called without arguments if the shape is (1, 1).");
+            }
+            return this._columns[keys[0]][0];
+        }
+        if (row === undefined || column === undefined) {
+            throw new Error("DataFrame.item() requires both row and column to be specified if not empty.");
+        }
+        if (row < 0 || row >= height) {
+            throw new Error(`Row index ${row} is out of bounds for DataFrame height ${height}.`);
+        }
+        let colName;
+        if (typeof column === "number") {
+            if (column < 0 || column >= width) {
+                throw new Error(`Column index ${column} is out of bounds for DataFrame width ${width}.`);
+            }
+            colName = keys[column];
+        }
+        else {
+            colName = column;
+            if (this._columns[colName] === undefined) {
+                throw new Error(`Column "${colName}" does not exist.`);
+            }
+        }
+        return this._columns[colName][row];
+    }
+    *iter_columns() {
+        const keys = Object.keys(this._columns);
+        const keysLen = keys.length;
+        const columns = this._columns;
+        for (let j = 0; j < keysLen; j++) {
+            yield columns[keys[j]];
+        }
+    }
+    *iter_rows({ named = false } = {}) {
+        const keys = Object.keys(this._columns);
+        const keysLen = keys.length;
+        const columns = this._columns;
+        const height = this._height;
+        const colArrays = new Array(keysLen);
+        for (let j = 0; j < keysLen; j++) {
+            colArrays[j] = columns[keys[j]];
+        }
+        if (named) {
+            for (let i = 0; i < height; i++) {
+                const row = {};
+                for (let j = 0; j < keysLen; j++) {
+                    row[keys[j]] = colArrays[j][i];
+                }
+                yield row;
+            }
+        }
+        else {
+            for (let i = 0; i < height; i++) {
+                const row = new Array(keysLen);
+                for (let j = 0; j < keysLen; j++) {
+                    row[j] = colArrays[j][i];
+                }
+                yield row;
+            }
+        }
+    }
     join(config) {
         const { other, on, how = "inner", suffixes = ["", "_right"] } = config;
-        const joinKeys = (0, utils_1.toValidArray)(on);
-        for (const key of joinKeys) {
-            const keyStr = String(key);
+        const joinKeysStr = (0, utils_1.toValidStringArray)(on);
+        for (let i = 0; i < joinKeysStr.length; i++) {
+            const keyStr = joinKeysStr[i];
             (0, exceptions_1.assertColumnExists)(keyStr, this._columns, "Join key", " in the left DataFrame.");
             (0, exceptions_1.assertColumnExists)(keyStr, other._columns, "Join key", " in the right DataFrame.");
         }
         const [leftSuffix, rightSuffix] = suffixes;
         const leftKeys = Object.keys(this._columns);
         const rightKeys = Object.keys(other._columns);
-        const joinKeySet = new Set(joinKeys);
+        const joinKeySet = new Set(joinKeysStr);
         const leftLen = leftKeys.length;
         const rightLen = rightKeys.length;
         const getColumnHashAt = (columns, idx) => {
-            const len = joinKeys.length;
-            if (len === 1) {
-                const val = columns[joinKeys[0]][idx];
-                return val == null ? null : String(val);
-            }
-            const vals = new Array(len);
+            const len = joinKeysStr.length;
             for (let i = 0; i < len; i++) {
-                const val = columns[joinKeys[i]][idx];
-                if (val == null)
+                if (columns[joinKeysStr[i]][idx] == null)
                     return null;
-                vals[i] = String(val);
             }
-            return vals.join(constants_1.KEY_SEPARATOR);
+            return (0, utils_2.computeRowHash)(columns, joinKeysStr, idx);
         };
         const rightHash = new Map();
         const rightHeight = other._height;
@@ -356,87 +429,66 @@ class DataFrame {
                 }
             }
         }
-        return new DataFrame(newColumns, outSchema, outHeight);
+        return DataFrame._createDirect(newColumns, outSchema, outHeight);
     }
-    limit(n, options = {}) {
-        const { offset = 0, from = "start" } = options;
+    limit(n, { offset = 0, from = "start" } = {}) {
         const len = this._height;
-        const safeN = isNaN(n) ? 0 : Math.max(Math.floor(n), 0);
-        const safeOffset = isNaN(offset) ? 0 : Math.max(Math.floor(offset), 0);
-        if (safeN === 0 || len === 0 || safeOffset >= len) {
-            const newColumns = {};
-            const outSchema = {};
-            for (const key of Object.keys(this._columns)) {
-                newColumns[key] = [];
-                outSchema[key] = this._schema[key];
-            }
-            return new DataFrame(newColumns, outSchema, 0);
-        }
-        let actualStart = 0;
-        let actualEnd = 0;
+        const safeN = (0, utils_1.clamp)(Math.floor(n), { min: 0, max: len });
+        const safeOffset = (0, utils_1.clamp)(Math.floor(offset), { min: 0, max: len });
+        let actualStart = safeOffset;
+        let actualEnd = Math.min(safeOffset + safeN, len);
         if (from === "end") {
-            actualEnd = Math.max(len - safeOffset, 0);
+            actualEnd = len - safeOffset;
             actualStart = Math.max(actualEnd - safeN, 0);
         }
-        else {
-            actualEnd = Math.min(safeOffset + safeN, len);
-            actualStart = safeOffset;
-        }
-        const newHeight = Math.max(actualEnd - actualStart, 0);
+        const newHeight = actualEnd - actualStart;
         const newColumns = {};
-        for (const key of Object.keys(this._columns)) {
+        const keys = Object.keys(this._columns);
+        const keysLen = keys.length;
+        for (let i = 0; i < keysLen; i++) {
+            const key = keys[i];
             newColumns[key] = this._columns[key].slice(actualStart, actualEnd);
         }
-        return new DataFrame(newColumns, this._schema, newHeight);
+        return DataFrame._createDirect(newColumns, this._schema, newHeight);
     }
     pivot(config) {
         if (this._height === 0)
-            return new DataFrame({}, {}, 0);
+            return DataFrame._createDirect({}, {}, 0);
         const { index, columns, values } = config;
-        const indexArr = (0, utils_1.toValidArray)(index);
-        const indexLen = indexArr.length;
-        const indexStr = new Array(indexLen);
-        for (let i = 0; i < indexLen; i++) {
-            indexStr[i] = String(indexArr[i]);
+        const indexStr = (0, utils_1.toValidStringArray)(index);
+        const indexLen = indexStr.length;
+        for (let j = 0; j < indexLen; j++) {
+            (0, exceptions_1.assertColumnExists)(indexStr[j], this._columns, "Pivot index key");
         }
         const colKey = String(columns);
         const valKey = String(values);
-        for (const idxKey of indexStr) {
-            (0, exceptions_1.assertColumnExists)(idxKey, this._columns, "Pivot index key", " in the DataFrame.");
-        }
         (0, exceptions_1.assertColumnExists)(colKey, this._columns, "Pivot column key");
-        (0, exceptions_1.assertColumnExists)(valKey, this._columns, "Pivot values key", " in the DataFrame.");
+        (0, exceptions_1.assertColumnExists)(valKey, this._columns, "Pivot values key");
         const groups = new Map();
+        const firstRowIdxs = [];
         const colNames = new Set();
         const height = this._height;
         const pivotCol = this._columns[colKey];
         const valCol = this._columns[valKey];
         for (let i = 0; i < height; i++) {
-            const vals = new Array(indexLen);
-            for (let j = 0; j < indexLen; j++) {
-                const val = this._columns[indexStr[j]][i];
-                vals[j] = val == null ? "" : String(val);
-            }
-            const rowKey = vals.join(constants_1.KEY_SEPARATOR);
-            const pivotColName = String(pivotCol[i]);
-            colNames.add(pivotColName);
-            let group = groups.get(rowKey);
-            if (group === undefined) {
-                group = { firstIdx: i, indices: [] };
-                groups.set(rowKey, group);
+            const rowKey = (0, utils_2.computeRowHash)(this._columns, indexStr, i);
+            colNames.add(String(pivotCol[i]));
+            if (groups.get(rowKey) === undefined) {
+                groups.set(rowKey, groups.size);
+                firstRowIdxs.push(i);
             }
-            group.indices.push(i);
         }
         const outHeight = groups.size;
-        const newColumns = {};
+        const indexColsDict = {};
         const outSchema = {};
         for (let j = 0; j < indexLen; j++) {
-            const k = indexStr[j];
-            newColumns[k] = new Array(outHeight);
-            if (this._schema[k]) {
-                outSchema[k] = this._schema[k];
+            const idxKey = indexStr[j];
+            indexColsDict[idxKey] = this._columns[idxKey];
+            if (this._schema[idxKey]) {
+                outSchema[idxKey] = this._schema[idxKey];
             }
         }
+        const newColumns = (0, utils_2.gatherColumnsByIndices)(indexColsDict, firstRowIdxs);
         const allCols = Array.from(colNames);
         const valType = this._schema[valKey] || datatypes_1.DataTypeRegistry.Utf8;
         for (let j = 0; j < allCols.length; j++) {
@@ -444,22 +496,13 @@ class DataFrame {
             newColumns[colName] = new Array(outHeight).fill(null);
             outSchema[colName] = valType;
         }
-        let groupIdx = 0;
-        for (const group of groups.values()) {
-            const firstIdx = group.firstIdx;
-            for (let j = 0; j < indexLen; j++) {
-                const k = indexStr[j];
-                newColumns[k][groupIdx] = this._columns[k][firstIdx];
-            }
-            const indices = group.indices;
-            for (let k = 0; k < indices.length; k++) {
-                const idx = indices[k];
-                const pivotColName = String(pivotCol[idx]);
-                newColumns[pivotColName][groupIdx] = valCol[idx];
-            }
-            groupIdx++;
+        for (let i = 0; i < height; i++) {
+            const rowKey = (0, utils_2.computeRowHash)(this._columns, indexStr, i);
+            const groupIdx = groups.get(rowKey);
+            const pivotColName = String(pivotCol[i]);
+            newColumns[pivotColName][groupIdx] = valCol[i];
         }
-        return new DataFrame(newColumns, outSchema, outHeight);
+        return DataFrame._createDirect(newColumns, outSchema, outHeight);
     }
     rename(mapping) {
         const renameMapping = mapping || {};
@@ -475,7 +518,7 @@ class DataFrame {
         if (finalKeys.length < originalKeys.length) {
             throw new exceptions_1.DataFrameError("Rename collision: Multiple columns mapped to the same output name.");
         }
-        return new DataFrame(newColumns, outSchema, this._height);
+        return DataFrame._createDirect(newColumns, outSchema, this._height);
     }
     reverse() {
         if (this._height === 0)
@@ -487,7 +530,7 @@ class DataFrame {
             const key = keys[i];
             newColumns[key] = this._columns[key].slice().reverse();
         }
-        return new DataFrame(newColumns, this._schema, this._height);
+        return DataFrame._createDirect(newColumns, this._schema, this._height);
     }
     get schema() {
         return this._schema;
@@ -495,24 +538,99 @@ class DataFrame {
     select(...args) {
         const exprs = this._normalizeArgs(args);
         const allKeys = Object.keys(this._columns);
-        const expandedExprs = (0, columnExpressions_1.resolveColumnSelectors)(exprs, allKeys);
+        const expandedExprs = (0, columnExpressions_1.resolveColumnSelectors)(exprs, allKeys, undefined, this._schema, this._columns);
+        const numExprs = expandedExprs.length;
+        if (numExprs === 0) {
+            return DataFrame._createDirect({}, {}, this._height);
+        }
         const newColumns = {};
         const outSchema = {};
-        for (const expr of expandedExprs) {
+        const evaluatedCols = new Array(numExprs);
+        const targetKeys = new Array(numExprs);
+        const selectedKeys = new Set();
+        let activeRowMap = null;
+        for (let i = 0; i < numExprs; i++) {
+            const expr = expandedExprs[i];
             const targetKey = expr.outputName || expr.colName || columnExpressions_1.ALL_COLUMNS_MARKER;
-            if (targetKey in newColumns) {
+            if (selectedKeys.has(targetKey)) {
                 throw new exceptions_1.DataFrameError(`Duplicate column selection: "${targetKey}" is selected multiple times.`);
             }
-            newColumns[targetKey] = expr.isWindow
-                ? (0, utils_2.resolveWindowExpr)(expr, this._columns, this._height)
-                : expr.evaluate(this._columns, this._height);
+            selectedKeys.add(targetKey);
+            const col = (0, columnExpressions_1.evaluateExpression)(expr, this._columns, this._height);
+            evaluatedCols[i] = col;
+            targetKeys[i] = targetKey;
+            const rowMap = col && col.rowMap;
+            if (rowMap) {
+                if (activeRowMap) {
+                    const len = rowMap.length;
+                    if (len !== activeRowMap.length) {
+                        throw new exceptions_1.ShapeError(`Mismatched explode heights: Column "${targetKey}" has length ${len}, but another exploded column has length ${activeRowMap.length}`);
+                    }
+                    for (let j = 0; j < len; j++) {
+                        if (rowMap[j] !== activeRowMap[j]) {
+                            throw new exceptions_1.ShapeError(`Mismatched explode heights: Column "${targetKey}" has mismatched row lengths compared to another exploded column.`);
+                        }
+                    }
+                }
+                else {
+                    activeRowMap = rowMap;
+                }
+            }
+        }
+        let targetHeight = activeRowMap ? activeRowMap.length : this._height;
+        let shouldCollapse = numExprs > 0;
+        for (let i = 0; i < numExprs; i++) {
+            const expr = expandedExprs[i];
+            const isGlobalAgg = expr.aggFn != null && (expr.partitionBy == null || expr.partitionBy.length === 0);
+            const isLit = !!expr.isLiteral;
+            if (!isGlobalAgg && !isLit) {
+                shouldCollapse = false;
+                break;
+            }
+        }
+        for (let i = 0; i < numExprs; i++) {
+            const targetKey = targetKeys[i];
+            let col = evaluatedCols[i];
+            const colObj = col;
+            const hasRowMap = colObj && colObj.rowMap;
+            const len = (0, utils_1.isArrayOrTypedArray)(col) ? col.length : 0;
+            const expectedLen = (activeRowMap && !hasRowMap) ? this._height : targetHeight;
+            if (len !== expectedLen) {
+                throw new exceptions_1.ShapeError(`Column height mismatch: Column "${targetKey}" has length ${len}, but expected ${expectedLen}`);
+            }
+            if (activeRowMap && !hasRowMap) {
+                const mapLen = activeRowMap.length;
+                if ((0, utils_1.isTypedArray)(col)) {
+                    const newCol = new colObj.constructor(mapLen);
+                    for (let j = 0; j < mapLen; j++) {
+                        newCol[j] = colObj[activeRowMap[j]];
+                    }
+                    col = newCol;
+                }
+                else {
+                    const newCol = new Array(mapLen);
+                    for (let j = 0; j < mapLen; j++) {
+                        newCol[j] = colObj[activeRowMap[j]];
+                    }
+                    col = newCol;
+                }
+            }
+            evaluatedCols[i] = col;
+        }
+        if (shouldCollapse) {
+            targetHeight = 1;
+        }
+        for (let i = 0; i < numExprs; i++) {
+            const expr = expandedExprs[i];
+            const targetKey = targetKeys[i];
+            const col = evaluatedCols[i];
             const originalKey = expr.colName || targetKey;
-            const isPureColSelector = expr instanceof columnExpressions_1.ColumnExpr && expr.ops.length === 0 && !expr.isWindow && !expr.aggFn;
-            outSchema[targetKey] = (isPureColSelector && this._schema[originalKey])
-                ? this._schema[originalKey]
-                : (0, utils_2.inferColumnType)(newColumns[targetKey]);
+            const isPureCol = expr instanceof columnExpressions_1.ColumnExpr && expr.ops.length === 0 && !expr.isWindow && !expr.aggFn;
+            const type = (isPureCol && this._schema[originalKey]) || (0, utils_2.inferColumnType)(col);
+            outSchema[targetKey] = type;
+            newColumns[targetKey] = (0, utils_2.coerceColumn)(col, type, targetHeight);
         }
-        return new DataFrame(newColumns, outSchema, this._height);
+        return DataFrame._createDirect(newColumns, outSchema, targetHeight);
     }
     get shape() {
         return [this.height, this.width];
@@ -532,9 +650,9 @@ class DataFrame {
         const { by, descending = false, nullsLast = true, custom } = config;
         const sortKeys = (0, utils_1.toValidArray)(by);
         for (let i = 0; i < sortKeys.length; i++) {
-            const keyOrExpr = sortKeys[i];
-            if (typeof keyOrExpr === "string") {
-                (0, exceptions_1.assertColumnExists)(keyOrExpr, this._columns, "Sort key");
+            const expr = columnExpressions_1.ColumnExpr.toColExpr(sortKeys[i]);
+            if (expr.colName) {
+                (0, exceptions_1.assertColumnExists)(expr.colName, this._columns, "Sort key");
             }
         }
         const descArray = Array.isArray(descending)
@@ -546,9 +664,7 @@ class DataFrame {
             const keyOrExpr = sortKeys[i];
             const isDesc = descArray[i] ? -1 : 1;
             const customComp = (custom && typeof keyOrExpr === "string") ? custom[keyOrExpr] : null;
-            const values = keyOrExpr?.evaluate
-                ? keyOrExpr.evaluate(this._columns, this._height)
-                : (this._columns[keyOrExpr] || new Array(this._height).fill(null));
+            const values = columnExpressions_1.ColumnExpr.toColExpr(keyOrExpr).evaluate(this._columns, this._height);
             plan[i] = {
                 values,
                 isDesc,
@@ -585,33 +701,103 @@ class DataFrame {
             return 0;
         });
         const newColumns = (0, utils_2.gatherColumnsByIndices)(this._columns, indices);
-        return new DataFrame(newColumns, this._schema, this._height);
+        return DataFrame._createDirect(newColumns, this._schema, this._height);
     }
     tail(n = 10) {
         return this.limit(n, { offset: 0, from: 'end' });
     }
+    to_dict() {
+        return { ...this._columns };
+    }
+    to_dicts() {
+        return (0, utils_2.columnsToRows)(this._columns, this._height);
+    }
     to_list(nameOrExpr) {
         if (this._height === 0)
             return [];
-        const isExpr = nameOrExpr && typeof nameOrExpr !== "string" && "evaluate" in nameOrExpr;
-        let colData;
-        if (isExpr) {
-            const expr = nameOrExpr;
-            colData = expr.evaluate(this._columns, this._height);
+        if (nameOrExpr == null) {
+            return new Array(this._height).fill(null);
+        }
+        const expr = columnExpressions_1.ColumnExpr.toColExpr(nameOrExpr);
+        const colData = expr.evaluate(this._columns, this._height);
+        return Array.isArray(colData) ? colData : Array.from(colData);
+    }
+    transpose({ include_header: includeHeader = false, header_name: headerName = "column", column_names: colNamesOpt } = {}) {
+        if (this._height === 0) {
+            const cols = {};
+            const schema = {};
+            if (includeHeader) {
+                cols[headerName] = (0, utils_2.coerceColumn)([], datatypes_1.DataTypeRegistry.Utf8, 0);
+                schema[headerName] = datatypes_1.DataTypeRegistry.Utf8;
+            }
+            return DataFrame._createDirect(cols, schema, 0);
+        }
+        let dataColumns = this.columns;
+        if (typeof colNamesOpt === "string") {
+            (0, exceptions_1.assertColumnExists)(colNamesOpt, this._columns, "column_names");
+            dataColumns = dataColumns.filter(c => c !== colNamesOpt);
+        }
+        let newColNames = [];
+        if (typeof colNamesOpt === "string") {
+            const keyCol = this._columns[colNamesOpt];
+            newColNames = new Array(this._height);
+            for (let i = 0; i < this._height; i++) {
+                const val = keyCol[i];
+                if (val == null) {
+                    throw new exceptions_1.DataFrameError(`Transpose column_names column "${colNamesOpt}" contains null/undefined at index ${i}`);
+                }
+                newColNames[i] = String(val);
+            }
+        }
+        else if (colNamesOpt != null && typeof colNamesOpt !== "string" && Symbol.iterator in Object(colNamesOpt)) {
+            const colNamesArr = Array.from(colNamesOpt);
+            if (colNamesArr.length !== this._height) {
+                throw new exceptions_1.DataFrameError(`column_names length (${colNamesArr.length}) must match the height of the DataFrame (${this._height})`);
+            }
+            newColNames = new Array(this._height);
+            for (let i = 0; i < this._height; i++) {
+                newColNames[i] = String(colNamesArr[i]);
+            }
         }
         else {
-            const key = nameOrExpr;
-            if (key == null) {
-                return new Array(this._height).fill(null);
+            newColNames = new Array(this._height);
+            for (let i = 0; i < this._height; i++) {
+                newColNames[i] = `column_${i}`;
             }
-            (0, exceptions_1.assertColumnExists)(key, this._columns, "Column");
-            colData = this._columns[key];
         }
-        return Array.isArray(colData) ? colData : Array.from(colData);
+        const uniqueNames = new Set();
+        if (includeHeader) {
+            uniqueNames.add(headerName);
+        }
+        for (let i = 0; i < newColNames.length; i++) {
+            const name = newColNames[i];
+            if (uniqueNames.has(name)) {
+                throw new exceptions_1.DataFrameError(`Duplicate column name in transposed DataFrame: "${name}"`);
+            }
+            uniqueNames.add(name);
+        }
+        const numDataCols = dataColumns.length;
+        const newCols = {};
+        const newSchema = {};
+        if (includeHeader) {
+            newCols[headerName] = (0, utils_2.coerceColumn)(dataColumns, datatypes_1.DataTypeRegistry.Utf8, numDataCols);
+            newSchema[headerName] = datatypes_1.DataTypeRegistry.Utf8;
+        }
+        for (let i = 0; i < this._height; i++) {
+            const colName = newColNames[i];
+            const rawVals = new Array(numDataCols);
+            for (let j = 0; j < numDataCols; j++) {
+                rawVals[j] = this._columns[dataColumns[j]][i];
+            }
+            const type = (0, utils_2.inferColumnType)(rawVals);
+            newCols[colName] = (0, utils_2.coerceColumn)(rawVals, type, numDataCols);
+            newSchema[colName] = type;
+        }
+        return DataFrame._createDirect(newCols, newSchema, numDataCols);
     }
     unique(columns) {
         if (this._height === 0)
-            return new DataFrame({}, this._schema, 0);
+            return DataFrame._createDirect({}, this._schema, 0);
         const colsArr = (0, utils_1.toValidArray)(columns);
         const colsStr = colsArr.length === 0
             ? Object.keys(this._columns)
@@ -621,23 +807,9 @@ class DataFrame {
         }
         const seen = new Set();
         const matchingIndices = [];
-        const colsLen = colsStr.length;
         const height = this._height;
-        const singleCol = colsLen === 1 ? this._columns[colsStr[0]] : null;
         for (let i = 0; i < height; i++) {
-            let hash;
-            if (singleCol) {
-                const val = singleCol[i];
-                hash = val == null ? "" : String(val);
-            }
-            else {
-                const vals = new Array(colsLen);
-                for (let j = 0; j < colsLen; j++) {
-                    const val = this._columns[colsStr[j]][i];
-                    vals[j] = val == null ? "" : String(val);
-                }
-                hash = vals.join(constants_1.KEY_SEPARATOR);
-            }
+            const hash = (0, utils_2.computeRowHash)(this._columns, colsStr, i);
             if (!seen.has(hash)) {
                 seen.add(hash);
                 matchingIndices.push(i);
@@ -645,12 +817,12 @@ class DataFrame {
         }
         const newColumns = (0, utils_2.gatherColumnsByIndices)(this._columns, matchingIndices);
         const newHeight = matchingIndices.length;
-        return new DataFrame(newColumns, this._schema, newHeight);
+        return DataFrame._createDirect(newColumns, this._schema, newHeight);
     }
     unpivot(config) {
         const { idVars, valueVars, varName = "variable", valueName = "value" } = config;
-        const idVarsStr = (0, utils_1.toValidArray)(idVars).map(String);
-        const valueVarsStr = (0, utils_1.toValidArray)(valueVars).map(String);
+        const idVarsStr = (0, utils_1.toValidStringArray)(idVars);
+        const valueVarsStr = (0, utils_1.toValidStringArray)(valueVars);
         const idVarsLen = idVarsStr.length;
         const valueVarsLen = valueVarsStr.length;
         for (const idKey of idVarsStr) {
@@ -685,7 +857,7 @@ class DataFrame {
         }
         outSchema[varName] = datatypes_1.DataTypeRegistry.Utf8;
         outSchema[valueName] = (0, utils_2.inferColumnType)(newColumns[valueName]);
-        return new DataFrame(newColumns, outSchema, newHeight);
+        return DataFrame._createDirect(newColumns, outSchema, newHeight);
     }
     vstack(other) {
         return this.concat(other, { how: "vertical" });
@@ -700,12 +872,16 @@ class DataFrame {
             if (typeof arg === "string") {
                 exprs.push(new columnExpressions_1.ColumnExpr(arg));
             }
-            else if ((0, utils_1.isObj)(arg) && 'evaluate' in arg) {
+            else if (columnExpressions_1.ColumnExpr.isColExpr(arg)) {
                 exprs.push(arg);
             }
             else if ((0, utils_1.isObj)(arg)) {
-                for (const [key, val] of Object.entries(arg)) {
-                    if ((0, utils_1.isObj)(val) && 'evaluate' in val) {
+                const keys = Object.keys(arg);
+                const numKeys = keys.length;
+                for (let i = 0; i < numKeys; i++) {
+                    const key = keys[i];
+                    const val = arg[key];
+                    if (columnExpressions_1.ColumnExpr.isColExpr(val)) {
                         exprs.push(val.alias(key));
                     }
                     else {
@@ -719,31 +895,96 @@ class DataFrame {
         return exprs;
     }
     with_columns(...args) {
+        if (args.length === 0)
+            return this;
         const exprs = this._normalizeArgs(args);
         const allKeys = Object.keys(this._columns);
-        const expandedExprs = (0, columnExpressions_1.resolveColumnSelectors)(exprs, allKeys);
+        const expandedExprs = (0, columnExpressions_1.resolveColumnSelectors)(exprs, allKeys, undefined, this._schema, this._columns);
         const numEntries = expandedExprs.length;
-        const newColumns = { ...this._columns };
-        const outSchema = { ...this._schema };
+        if (numEntries === 0)
+            return this;
+        const overrides = new Map();
         for (let j = 0; j < numEntries; j++) {
             const expr = expandedExprs[j];
             const name = expr.outputName || expr.colName || columnExpressions_1.ALL_COLUMNS_MARKER;
-            if (expr.isWindow) {
-                newColumns[name] = (0, utils_2.resolveWindowExpr)(expr, this._columns, this._height);
+            overrides.set(name, expr);
+        }
+        const selectList = [];
+        const numKeys = allKeys.length;
+        for (let i = 0; i < numKeys; i++) {
+            const key = allKeys[i];
+            selectList.push(overrides.get(key) || new columnExpressions_1.ColumnExpr(key));
+            overrides.delete(key);
+        }
+        for (const expr of overrides.values()) {
+            selectList.push(expr);
+        }
+        return this.select(...selectList);
+    }
+    with_row_index(name = "index", offset = 0) {
+        const expr = (0, columnExpressions_1.seq_range)(offset, {
+            mode: "independent",
+            dtype: datatypes_1.DataTypeRegistry.UInt32,
+            step: 1
+        });
+        const df = this.insert_column(0, name, expr);
+        df._schema[name] = datatypes_1.DataTypeRegistry.UInt32;
+        return df;
+    }
+    write_json(file, { format = "json", replacerOptions } = {}) {
+        if (format !== "json" && format !== "ndjson") {
+            throw new TypeError(`Unsupported JSON format: "${format}". Expected "json" or "ndjson".`);
+        }
+        const safeReplacer = replacerOptions?.replacer === null
+            ? undefined
+            : (0, json_1.createSafeJsonReplacer)(replacerOptions);
+        let jsonStr;
+        if (format === "ndjson") {
+            const dicts = this.to_dicts();
+            const len = dicts.length;
+            const lines = new Array(len);
+            for (let i = 0; i < len; i++) {
+                lines[i] = JSON.stringify(dicts[i], safeReplacer);
             }
-            else {
-                newColumns[name] = expr.evaluate(this._columns, this._height);
+            jsonStr = lines.join(constants_1.NEWLINE);
+        }
+        else {
+            jsonStr = JSON.stringify(this.to_dicts(), safeReplacer);
+        }
+        (0, utils_2.writeStringToFileOrStream)(file, jsonStr);
+        return jsonStr;
+    }
+    write_csv(file, options = {}) {
+        if (file) {
+            if (typeof file === "string") {
+                const fs = require("fs");
+                const fd = fs.openSync(file, "w");
+                try {
+                    (0, utils_1.stringifyCSV)(this._columns, this._height, {
+                        ...options,
+                        onRow: (str) => {
+                            fs.writeSync(fd, str, null, "utf8");
+                        }
+                    });
+                }
+                finally {
+                    fs.closeSync(fd);
+                }
             }
-            const originalKey = expr.colName || name;
-            const isPureColSelector = expr instanceof columnExpressions_1.ColumnExpr && expr.ops.length === 0 && !expr.isWindow && !expr.aggFn;
-            if (isPureColSelector && this._schema[originalKey]) {
-                outSchema[name] = this._schema[originalKey];
+            else if (typeof file === "object" && typeof file.write === "function") {
+                (0, utils_1.stringifyCSV)(this._columns, this._height, {
+                    ...options,
+                    onRow: (str) => {
+                        file.write(str);
+                    }
+                });
             }
             else {
-                outSchema[name] = (0, utils_2.inferColumnType)(newColumns[name]);
+                throw new TypeError("Invalid file argument. Expected a file path string or a writable stream/object with a write method.");
             }
+            return "";
         }
-        return new DataFrame(newColumns, outSchema, this._height);
+        return (0, utils_1.stringifyCSV)(this._columns, this._height, options);
     }
 }
 exports.DataFrame = DataFrame;