df-script 1.4.0 → 1.5.0

package/dist/datatypes/types.js

@@ -8,7 +8,7 @@ const utils_1 = require("../utils");
 // ============================================================================
 class Int8Type extends DataType_1.SignedIntegerType {
     name = "Int8";
-    coerce(val) { return (0, utils_1.toValidInt)(val, "Int8"); }
+    coerce(val) { return (0, utils_1.toValidInt)(val, { range: "Int8" }); }
     equals(other) { return other.name === "Int8"; }
     allocate(size) { return new Int8Array(size); }
 }
@@ -16,7 +16,7 @@ exports.Int8Type = Int8Type;
 exports.Int8 = new Int8Type();
 class Int16Type extends DataType_1.SignedIntegerType {
     name = "Int16";
-    coerce(val) { return (0, utils_1.toValidInt)(val, "Int16"); }
+    coerce(val) { return (0, utils_1.toValidInt)(val, { range: "Int16" }); }
     equals(other) { return other.name === "Int16"; }
     allocate(size) { return new Int16Array(size); }
 }
@@ -24,7 +24,7 @@ exports.Int16Type = Int16Type;
 exports.Int16 = new Int16Type();
 class Int32Type extends DataType_1.SignedIntegerType {
     name = "Int32";
-    coerce(val) { return (0, utils_1.toValidInt)(val, "Int32"); }
+    coerce(val) { return (0, utils_1.toValidInt)(val, { range: "Int32" }); }
     equals(other) { return other.name === "Int32"; }
     allocate(size) { return new Int32Array(size); }
 }
@@ -32,7 +32,7 @@ exports.Int32Type = Int32Type;
 exports.Int32 = new Int32Type();
 class Int64Type extends DataType_1.SignedIntegerType {
     name = "Int64";
-    coerce(val) { return (0, utils_1.toValidBigInt)(val); }
+    coerce(val) { return (0, utils_1.toValidBigInt)(val, { truncate: true }); }
     equals(other) { return other.name === "Int64"; }
     allocate(size) { return new BigInt64Array(size); }
 }
@@ -40,7 +40,7 @@ exports.Int64Type = Int64Type;
 exports.Int64 = new Int64Type();
 class UInt8Type extends DataType_1.UnsignedIntegerType {
     name = "UInt8";
-    coerce(val) { return (0, utils_1.toValidInt)(val, "UInt8"); }
+    coerce(val) { return (0, utils_1.toValidInt)(val, { range: "UInt8" }); }
     equals(other) { return other.name === "UInt8"; }
     allocate(size) { return new Uint8Array(size); }
 }
@@ -48,7 +48,7 @@ exports.UInt8Type = UInt8Type;
 exports.UInt8 = new UInt8Type();
 class UInt16Type extends DataType_1.UnsignedIntegerType {
     name = "UInt16";
-    coerce(val) { return (0, utils_1.toValidInt)(val, "UInt16"); }
+    coerce(val) { return (0, utils_1.toValidInt)(val, { range: "UInt16" }); }
     equals(other) { return other.name === "UInt16"; }
     allocate(size) { return new Uint16Array(size); }
 }
@@ -56,7 +56,7 @@ exports.UInt16Type = UInt16Type;
 exports.UInt16 = new UInt16Type();
 class UInt32Type extends DataType_1.UnsignedIntegerType {
     name = "UInt32";
-    coerce(val) { return (0, utils_1.toValidInt)(val, "UInt32"); }
+    coerce(val) { return (0, utils_1.toValidInt)(val, { range: "UInt32" }); }
     equals(other) { return other.name === "UInt32"; }
     allocate(size) { return new Uint32Array(size); }
 }
@@ -64,7 +64,7 @@ exports.UInt32Type = UInt32Type;
 exports.UInt32 = new UInt32Type();
 class UInt64Type extends DataType_1.UnsignedIntegerType {
     name = "UInt64";
-    coerce(val) { return (0, utils_1.toValidBigInt)(val, "UInt64"); }
+    coerce(val) { return (0, utils_1.toValidBigInt)(val, { range: "UInt64" }); }
     equals(other) { return other.name === "UInt64"; }
     allocate(size) { return new BigUint64Array(size); }
 }
@@ -72,7 +72,7 @@ exports.UInt64Type = UInt64Type;
 exports.UInt64 = new UInt64Type();
 class Float32Type extends DataType_1.FloatDataType {
     name = "Float32";
-    coerce(val) { return (0, utils_1.toValidFloat)(val, "Float32"); }
+    coerce(val) { return (0, utils_1.toValidFloat)(val, { floatPrecision: "Float32" }); }
     equals(other) { return other.name === "Float32"; }
     allocate(size) { return new Float32Array(size); }
 }
@@ -80,7 +80,7 @@ exports.Float32Type = Float32Type;
 exports.Float32 = new Float32Type();
 class Float64Type extends DataType_1.FloatDataType {
     name = "Float64";
-    coerce(val) { return (0, utils_1.toValidFloat)(val, "Float64"); }
+    coerce(val) { return (0, utils_1.toValidFloat)(val, { floatPrecision: "Float64" }); }
     equals(other) { return other.name === "Float64"; }
     allocate(size) { return new Float64Array(size); }
 }
@@ -261,7 +261,11 @@ class StructType extends DataType_1.NestedDataType {
         if (!(0, utils_1.isObj)(val))
             return null;
         const res = {};
-        for (const [k, type] of globalThis.Object.entries(this.fields)) {
+        const keys = globalThis.Object.keys(this.fields);
+        const len = keys.length;
+        for (let i = 0; i < len; i++) {
+            const k = keys[i];
+            const type = this.fields[k];
             res[k] = type.coerce(val[k]);
         }
         return res;

package/dist/exceptions/index.d.ts

@@ -10,4 +10,6 @@ export declare class SchemaError extends DFScriptError {
 }
 export declare class ComputeError extends DFScriptError {
 }
+export declare class ShapeError extends DFScriptError {
+}
 export * from "./utils";

package/dist/exceptions/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ComputeError = exports.SchemaError = exports.ColumnNotFoundError = exports.DataFrameError = exports.DFScriptError = void 0;
+exports.ShapeError = exports.ComputeError = exports.SchemaError = exports.ColumnNotFoundError = exports.DataFrameError = exports.DFScriptError = void 0;
 class DFScriptError extends Error {
     constructor(message) {
         super(message);
@@ -40,4 +40,7 @@ exports.SchemaError = SchemaError;
 class ComputeError extends DFScriptError {
 }
 exports.ComputeError = ComputeError;
+class ShapeError extends DFScriptError {
+}
+exports.ShapeError = ShapeError;
 __exportStar(require("./utils"), exports);

package/dist/exceptions/utils.d.ts

@@ -1,2 +1,3 @@
 import type { ColumnDict } from "../types";
 export declare function assertColumnExists(columnName: string, columns: ColumnDict, context: string, suffix?: string): void;
+export declare function assertHeight(columns: ColumnDict, height?: number): number;

package/dist/exceptions/utils.js

@@ -1,9 +1,28 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.assertColumnExists = assertColumnExists;
+exports.assertHeight = assertHeight;
 const index_1 = require("./index");
+const utils_1 = require("../utils");
 function assertColumnExists(columnName, columns, context, suffix = "") {
     if (!(columnName in columns)) {
         throw new index_1.ColumnNotFoundError(columnName, `${context} "${columnName}" does not exist${suffix}`);
     }
 }
+function assertHeight(columns, height) {
+    const keys = Object.keys(columns);
+    const numKeys = keys.length;
+    let expectedHeight = height;
+    for (let i = 0; i < numKeys; i++) {
+        const key = keys[i];
+        const col = columns[key];
+        const len = (0, utils_1.isArrayOrTypedArray)(col) ? col.length : 0;
+        if (expectedHeight === undefined) {
+            expectedHeight = len;
+        }
+        else if (len !== expectedHeight) {
+            throw new index_1.ShapeError(`Column height mismatch: Column "${key}" has length ${len}, but DataFrame height is ${expectedHeight}`);
+        }
+    }
+    return expectedHeight === undefined ? 0 : expectedHeight;
+}

package/dist/functions/concat.d.ts

@@ -1,4 +1,3 @@
 import { DataFrame } from "../dataframe/dataframe";
 import type { ConcatOptions, ConcatItem, RowRecord } from "../types";
-export declare function normalizeToDataFrames(item: any, context: string, index: number): DataFrame<any>[];
-export declare function concat<U extends RowRecord = any>(rawItems: ConcatItem | ConcatItem[], options?: ConcatOptions): DataFrame<U>;
+export declare function concat<U extends RowRecord = any>(rawItems: ConcatItem | ConcatItem[], { how, horizontal }?: ConcatOptions): DataFrame<U>;

package/dist/functions/concat.js

@@ -1,6 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeToDataFrames = normalizeToDataFrames;
 exports.concat = concat;
 const dataframe_1 = require("../dataframe/dataframe");
 const datatypes_1 = require("../datatypes");
@@ -34,7 +33,7 @@ function normalizeToDataFrames(item, context, index) {
     }
     throw new exceptions_1.DataFrameError(`Invalid input to ${context} at index ${index}: expected DataFrame, row array, or column dictionary.`);
 }
-function concat(rawItems, options = {}) {
+function concat(rawItems, { how = 'vertical', horizontal } = {}) {
     if (rawItems == null) {
         throw new exceptions_1.DataFrameError("Invalid input to concat: rawItems cannot be null or undefined.");
     }
@@ -43,10 +42,9 @@ function concat(rawItems, options = {}) {
     for (let i = 0; i < itemsArray.length; i++) {
         items.push(...normalizeToDataFrames(itemsArray[i], "concat", i));
     }
-    const { how = 'vertical' } = options;
-    const strict = options.horizontal?.strict ?? true;
+    const strict = horizontal?.strict ?? true;
     if (items.length === 0)
-        return new dataframe_1.DataFrame({}, {}, 0);
+        return dataframe_1.DataFrame._createDirect({}, {}, 0);
     if (items.length === 1 && how !== 'horizontal')
         return items[0];
     switch (how) {
@@ -58,7 +56,7 @@ function concat(rawItems, options = {}) {
                 }
             }
             if (validItems.length === 0)
-                return new dataframe_1.DataFrame({}, {}, 0);
+                return dataframe_1.DataFrame._createDirect({}, {}, 0);
             const firstDF = validItems[0];
             const firstKeys = Object.keys(firstDF._columns);
             for (let i = 0; i < items.length; i++) {
@@ -85,7 +83,17 @@ function concat(rawItems, options = {}) {
             for (const item of items) {
                 newHeight += item.height;
             }
-            const outSchema = items.find(df => df.height > 0)?.schema || items[0]?.schema || {};
+            let outSchema = {};
+            const itemsLen = items.length;
+            for (let i = 0; i < itemsLen; i++) {
+                if (items[i].height > 0) {
+                    outSchema = items[i].schema;
+                    break;
+                }
+            }
+            if (Object.keys(outSchema).length === 0 && itemsLen > 0) {
+                outSchema = items[0].schema;
+            }
             const newColumns = {};
             for (const key of firstKeys) {
                 const type = outSchema[key];
@@ -133,7 +141,7 @@ function concat(rawItems, options = {}) {
                 }
                 offset += h;
             }
-            return new dataframe_1.DataFrame(newColumns, outSchema, newHeight);
+            return dataframe_1.DataFrame._createDirect(newColumns, outSchema, newHeight);
         }
         case 'horizontal': {
             let maxHeight = items[0].height;
@@ -146,7 +154,7 @@ function concat(rawItems, options = {}) {
             for (let idx = 0; idx < items.length; idx++) {
                 const df = items[idx];
                 if (strict && df.height !== maxHeight) {
-                    throw new exceptions_1.DataFrameError(`[Horizontal] Row count mismatch at index ${idx}. Expected ${maxHeight}, got ${df.height}. Set strict=false to allow padding.`);
+                    throw new exceptions_1.ShapeError(`[Horizontal] Row count mismatch at index ${idx}. Expected ${maxHeight}, got ${df.height}. Set strict=false to allow padding.`);
                 }
                 for (const key of Object.keys(df._columns)) {
                     if (allColNames.has(key)) {
@@ -160,7 +168,11 @@ function concat(rawItems, options = {}) {
             for (const df of items) {
                 const h = df.height;
                 Object.assign(outSchema, df.schema);
-                for (const [key, col] of Object.entries(df._columns)) {
+                const keys = Object.keys(df._columns);
+                const numKeys = keys.length;
+                for (let idxKey = 0; idxKey < numKeys; idxKey++) {
+                    const key = keys[idxKey];
+                    const col = df._columns[key];
                     if (h === maxHeight) {
                         newColumns[key] = (0, utils_1.isTypedArray)(col) ? Array.from(col) : col;
                     }
@@ -176,7 +188,7 @@ function concat(rawItems, options = {}) {
                     }
                 }
             }
-            return new dataframe_1.DataFrame(newColumns, outSchema, maxHeight);
+            return dataframe_1.DataFrame._createDirect(newColumns, outSchema, maxHeight);
         }
         case 'diagonal': {
             const allColumnsSet = new Set();
@@ -242,7 +254,7 @@ function concat(rawItems, options = {}) {
                 }
                 offset += h;
             }
-            return new dataframe_1.DataFrame(newColumns, outSchema, newHeight);
+            return dataframe_1.DataFrame._createDirect(newColumns, outSchema, newHeight);
         }
     }
 }

package/dist/functions/index.d.ts

@@ -1 +1,3 @@
-export * from "./concat";
+export { concat } from "./concat";
+export { read_json } from "./read_json";
+export { read_csv } from "./read_csv";

package/dist/functions/index.js

@@ -1,17 +1,9 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./concat"), exports);
+exports.read_csv = exports.read_json = exports.concat = void 0;
+var concat_1 = require("./concat");
+Object.defineProperty(exports, "concat", { enumerable: true, get: function () { return concat_1.concat; } });
+var read_json_1 = require("./read_json");
+Object.defineProperty(exports, "read_json", { enumerable: true, get: function () { return read_json_1.read_json; } });
+var read_csv_1 = require("./read_csv");
+Object.defineProperty(exports, "read_csv", { enumerable: true, get: function () { return read_csv_1.read_csv; } });

package/dist/functions/read_csv.d.ts

@@ -0,0 +1,8 @@
+import { DataFrame } from "../dataframe/dataframe";
+import type { ReadCSVOptions } from "../dataframe/types";
+import type { RowRecord } from "../types";
+/**
+ * Reads a CSV string and constructs a DataFrame.
+ * Automatically infers column data types unless an explicit schema is provided.
+ */
+export declare function read_csv<T extends RowRecord = any>(content: string, options?: ReadCSVOptions): DataFrame<T>;

package/dist/functions/read_csv.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.read_csv = read_csv;
+const dataframe_1 = require("../dataframe/dataframe");
+const utils_1 = require("../utils");
+/**
+ * Reads a CSV string and constructs a DataFrame.
+ * Automatically infers column data types unless an explicit schema is provided.
+ */
+function read_csv(content, options = {}) {
+    const { hasHeader = true, schema, inferSchema = true, } = options;
+    const rows = (0, utils_1.parseCSV)(content, options);
+    if (rows.length === 0) {
+        return new dataframe_1.DataFrame({});
+    }
+    let headers;
+    let dataRows;
+    if (hasHeader) {
+        headers = rows[0];
+        dataRows = rows.slice(1);
+    }
+    else {
+        headers = rows[0].map((_, i) => `column_${i}`);
+        dataRows = rows;
+    }
+    const numCols = headers.length;
+    const numRows = dataRows.length;
+    const columns = {};
+    for (let c = 0; c < numCols; c++) {
+        const colName = headers[c];
+        const colData = new Array(numRows);
+        for (let r = 0; r < numRows; r++) {
+            colData[r] = dataRows[r][c] !== undefined ? dataRows[r][c] : "";
+        }
+        columns[colName] = colData;
+    }
+    const coercedColumns = {};
+    for (let c = 0; c < numCols; c++) {
+        const colName = headers[c];
+        const rawValues = columns[colName];
+        if (schema && schema[colName]) {
+            coercedColumns[colName] = rawValues;
+        }
+        else if (inferSchema) {
+            const { values } = (0, utils_1.inferAndCoerceCSVColumn)(rawValues, options);
+            coercedColumns[colName] = values;
+        }
+        else {
+            coercedColumns[colName] = rawValues;
+        }
+    }
+    return new dataframe_1.DataFrame(coercedColumns, schema);
+}

package/dist/functions/read_json.d.ts

@@ -0,0 +1,10 @@
+import { DataFrame } from "../dataframe/dataframe";
+import type { ReadJSONOptions } from "../dataframe/types";
+/**
+ * Parses JSON content (JSON or NDJSON) and loads it into a new DataFrame.
+ *
+ * @param content The JSON or NDJSON content string.
+ * @param options Parse and configuration options.
+ * @returns A new DataFrame instance populated with the parsed records.
+ */
+export declare function read_json(content: string, { format, trimBeforeParse, schema, ...parseOpts }?: ReadJSONOptions): DataFrame<any>;

package/dist/functions/read_json.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.read_json = read_json;
+const dataframe_1 = require("../dataframe/dataframe");
+const utils_1 = require("../utils");
+const exceptions_1 = require("../exceptions");
+/**
+ * Parses JSON content (JSON or NDJSON) and loads it into a new DataFrame.
+ *
+ * @param content The JSON or NDJSON content string.
+ * @param options Parse and configuration options.
+ * @returns A new DataFrame instance populated with the parsed records.
+ */
+function read_json(content, { format = "json", trimBeforeParse = true, schema, ...parseOpts } = {}) {
+    const parsed = (0, utils_1.safeJsonParse)(content, { format, trimBeforeParse, ...parseOpts });
+    if (parsed === content) {
+        throw new exceptions_1.DataFrameError(`Invalid JSON input: must be a valid, non-empty JSON ${format} string.`);
+    }
+    const parsedData = Array.isArray(parsed) ? parsed : ((0, utils_1.isObj)(parsed) ? [parsed] : []);
+    return new dataframe_1.DataFrame(parsedData, schema);
+}

package/dist/functions/transpose.d.ts

@@ -0,0 +1,7 @@
+import { DataFrame } from "../dataframe/dataframe";
+import type { RowRecord } from "../types";
+import type { TransposeOptions } from "../dataframe/types";
+/**
+ * Transposes the rows and columns of a DataFrame.
+ */
+export declare function transpose<T extends RowRecord = any>(df: DataFrame<T>, { include_header: includeHeader, header_name: headerName, column_names: colNamesOpt }?: TransposeOptions): DataFrame<any>;

package/dist/functions/transpose.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transpose = transpose;
+const dataframe_1 = require("../dataframe/dataframe");
+const datatypes_1 = require("../datatypes");
+const utils_1 = require("../dataframe/utils");
+const exceptions_1 = require("../exceptions");
+/**
+ * Transposes the rows and columns of a DataFrame.
+ */
+function transpose(df, { include_header: includeHeader = false, header_name: headerName = "column", column_names: colNamesOpt } = {}) {
+    const height = df.height;
+    if (height === 0) {
+        const cols = {};
+        const schema = {};
+        if (includeHeader) {
+            cols[headerName] = (0, utils_1.coerceColumn)([], datatypes_1.DataTypeRegistry.Utf8, 0);
+            schema[headerName] = datatypes_1.DataTypeRegistry.Utf8;
+        }
+        return dataframe_1.DataFrame._createDirect(cols, schema, 0);
+    }
+    let dataColumns = df.columns;
+    if (typeof colNamesOpt === "string") {
+        (0, exceptions_1.assertColumnExists)(colNamesOpt, df._columns, "column_names");
+        dataColumns = dataColumns.filter(c => c !== colNamesOpt);
+    }
+    let newColNames = [];
+    if (typeof colNamesOpt === "string") {
+        const keyCol = df._columns[colNamesOpt];
+        newColNames = new Array(height);
+        for (let i = 0; i < height; i++) {
+            const val = keyCol[i];
+            if (val == null) {
+                throw new exceptions_1.DataFrameError(`Transpose column_names column "${colNamesOpt}" contains null/undefined at index ${i}`);
+            }
+            newColNames[i] = String(val);
+        }
+    }
+    else if (colNamesOpt != null && typeof colNamesOpt !== "string" && Symbol.iterator in Object(colNamesOpt)) {
+        const colNamesArr = Array.from(colNamesOpt);
+        if (colNamesArr.length !== height) {
+            throw new exceptions_1.DataFrameError(`column_names length (${colNamesArr.length}) must match the height of the DataFrame (${height})`);
+        }
+        newColNames = new Array(height);
+        for (let i = 0; i < height; i++) {
+            newColNames[i] = String(colNamesArr[i]);
+        }
+    }
+    else {
+        newColNames = new Array(height);
+        for (let i = 0; i < height; i++) {
+            newColNames[i] = `column_${i}`;
+        }
+    }
+    const uniqueNames = new Set();
+    if (includeHeader) {
+        uniqueNames.add(headerName);
+    }
+    for (let i = 0; i < newColNames.length; i++) {
+        const name = newColNames[i];
+        if (uniqueNames.has(name)) {
+            throw new exceptions_1.DataFrameError(`Duplicate column name in transposed DataFrame: "${name}"`);
+        }
+        uniqueNames.add(name);
+    }
+    const numDataCols = dataColumns.length;
+    const newCols = {};
+    const newSchema = {};
+    if (includeHeader) {
+        newCols[headerName] = (0, utils_1.coerceColumn)(dataColumns, datatypes_1.DataTypeRegistry.Utf8, numDataCols);
+        newSchema[headerName] = datatypes_1.DataTypeRegistry.Utf8;
+    }
+    for (let i = 0; i < height; i++) {
+        const colName = newColNames[i];
+        const rawVals = new Array(numDataCols);
+        for (let j = 0; j < numDataCols; j++) {
+            rawVals[j] = df._columns[dataColumns[j]][i];
+        }
+        const type = (0, utils_1.inferColumnType)(rawVals);
+        newCols[colName] = (0, utils_1.coerceColumn)(rawVals, type, numDataCols);
+        newSchema[colName] = type;
+    }
+    return dataframe_1.DataFrame._createDirect(newCols, newSchema, numDataCols);
+}