df-script 1.4.0 → 1.6.0

package/dist/dataframe/dataframe.d.ts

@@ -1,42 +1,59 @@
 import { GroupedData } from "./grouped/grouped";
-import type { IExpr, ColumnDict, DataFrameColumns, ConcatOptions, ConcatItem, HorizontalConcatOptions, RowRecord } from "../types";
-import type { LimitOptions, SortOptions, PivotOptions, JoinOptions, UnpivotOptions } from "./types";
-import { DataType } from "../datatypes";
+import type { IExpr, ColumnData, ColumnDict, DataFrameColumns, ConcatOptions, ConcatItem, HorizontalConcatOptions, RowRecord, DataFrameSchema, RegisteredDataType, ExplodeOptions, IntoExpr, FillNullOptions } from "../types";
+import type { LimitOptions, SortOptions, PivotOptions, JoinOptions, UnpivotOptions, TransposeOptions, WriteJSONOptions, WriteCSVOptions } from "./types";
 export declare class DataFrame<T extends RowRecord = any> {
     _columns: DataFrameColumns<T>;
     private _height;
     private _schema;
-    constructor(data: T[] | ColumnDict, schema?: Record<string, DataType>, height?: number);
+    static _createDirect<U extends RowRecord = any>(columns: ColumnDict, schema: DataFrameSchema, height: number): DataFrame<U>;
+    constructor(data: T[] | ColumnDict, schema?: DataFrameSchema, height?: number);
     private inferSchema;
     private applySchema;
-    getSchema(): Record<string, DataType>;
-    to_dicts(): T[];
-    to_dict(): DataFrameColumns<T>;
     get columns(): string[];
     concat<U extends RowRecord = any>(items: ConcatItem | ConcatItem[], options?: ConcatOptions): DataFrame<U>;
     drop<K extends keyof T>(...args: (K | K[])[]): DataFrame<Omit<T, K>>;
-    get dtypes(): DataType[];
+    drop_nulls(subset?: string | string[]): DataFrame<T>;
+    get dtypes(): RegisteredDataType[];
+    explode(columns: IntoExpr | IntoExpr[], options?: ExplodeOptions): DataFrame<any>;
+    fill_null(options?: FillNullOptions): DataFrame<T>;
     filter(...exprs: (IExpr | ((row: T) => any))[]): DataFrame<T>;
+    get_schema(): DataFrameSchema;
     groupby<K extends keyof T>(keys: K | K[]): GroupedData<T, K>;
     head(n?: number): DataFrame<T>;
     get height(): number;
     hstack<U extends RowRecord = any>(other: ConcatItem | ConcatItem[], options?: HorizontalConcatOptions): DataFrame<U>;
+    insert_column(index: number, name: string, expr: IntoExpr): DataFrame<any>;
+    item(row?: number, column?: number | string): any;
+    iter_columns(): Generator<ColumnData>;
+    iter_rows({ named }?: {
+        named?: boolean;
+    }): Generator<any[] | Record<string, any>>;
     join<U extends RowRecord = any, R extends RowRecord = any>(config: JoinOptions<T, U>): DataFrame<R>;
-    limit(n: number, options?: LimitOptions): DataFrame<T>;
+    limit(n: number, { offset, from }?: LimitOptions): DataFrame<T>;
     pivot<U extends RowRecord = any>(config: PivotOptions<T>): DataFrame<U>;
     rename(mapping?: Partial<Record<keyof T, string>>): DataFrame<any>;
     reverse(): DataFrame<T>;
-    get schema(): Record<string, DataType>;
+    get schema(): DataFrameSchema;
     select<U extends RowRecord = any>(...args: (string | IExpr | Record<string, any> | (string | IExpr | Record<string, any>)[])[]): DataFrame<U>;
     get shape(): [number, number];
     slice(start: number, end?: number): DataFrame<T>;
     sort(config?: SortOptions<T>): DataFrame<T>;
     tail(n?: number): DataFrame<T>;
-    to_list<K extends keyof T>(nameOrExpr: K | IExpr): any[];
+    to_dict(): DataFrameColumns<T>;
+    to_dicts(): T[];
+    to_array<K extends keyof T>(nameOrExpr: K | IExpr): any[];
+    transpose({ include_header: includeHeader, header_name: headerName, column_names: colNamesOpt }?: TransposeOptions): DataFrame<any>;
     unique<K extends keyof T>(columns?: K | K[]): DataFrame<T>;
     unpivot<U extends RowRecord = any>(config: UnpivotOptions<T>): DataFrame<U>;
     vstack<U extends RowRecord = any>(other: ConcatItem | ConcatItem[]): DataFrame<U>;
     get width(): number;
     private _normalizeArgs;
     with_columns(...args: (string | IExpr | Record<string, any> | (string | IExpr | Record<string, any>)[])[]): DataFrame<any>;
+    with_row_index(name?: string, offset?: number): DataFrame<any>;
+    write_json(file?: string | {
+        write: (str: string) => void;
+    }, { format, replacerOptions }?: WriteJSONOptions): string;
+    write_csv(file?: string | {
+        write: (str: string) => void;
+    }, options?: WriteCSVOptions): string;
 }

package/dist/dataframe/grouped/grouped.d.ts

@@ -1,7 +1,6 @@
 import { DataFrame } from "../dataframe";
 import type { GroupMap } from "../types";
-import { DataType } from "../../datatypes";
-import type { IExpr, ColumnDict, RowRecord } from "../../types";
+import type { IExpr, ColumnDict, RowRecord, DataFrameSchema } from "../../types";
 export declare class GroupedData<T, K extends keyof T> {
     private groups;
     private keys;
@@ -9,7 +8,7 @@ export declare class GroupedData<T, K extends keyof T> {
     private parentColumns;
     private parentHeight;
     private parentSchema;
-    constructor(groups: GroupMap, keys: K[], allKeys: (keyof T)[], parentColumns: ColumnDict, parentHeight: number, parentSchema: Record<string, DataType>);
+    constructor(groups: GroupMap, keys: K[], allKeys: (keyof T)[], parentColumns: ColumnDict, parentHeight: number, parentSchema: DataFrameSchema);
     to_dataframe<U extends RowRecord = any>(): DataFrame<U>;
     agg<U extends RowRecord = any>(...exprs: (IExpr | any)[]): DataFrame<U>;
 }

package/dist/dataframe/types.d.ts

@@ -1,5 +1,7 @@
-import type { IExpr, AggFn, RowRecord } from "../types";
+import type { IExpr, AggFn, RowRecord, DataFrameSchema, JSONFormat } from "../types";
 import type { DataFrame } from "./dataframe";
+import type { JSONParseOptions, SafeJsonReplacerOptions, NDJSONParseOptions } from "../utils";
+export type { JSONParseOptions, SafeJsonReplacerOptions, NDJSONParseOptions };
 export type JoinType = "inner" | "left" | "outer" | "right";
 export type LimitPosition = "start" | "end";
 export type GroupMap = Map<string, number[]>;
@@ -31,3 +33,65 @@ export interface UnpivotOptions<T> {
     varName?: string;
     valueName?: string;
 }
+export interface TransposeOptions {
+    include_header?: boolean;
+    header_name?: string;
+    column_names?: string | Iterable<string>;
+}
+export interface ReadJSONOptions extends JSONParseOptions {
+    /**
+     * Optional explicit schema mapping column names to their registered data types.
+     */
+    schema?: DataFrameSchema;
+}
+/**
+ * The `replacer` argument type extracted directly from the overloads of the built-in `JSON.stringify`.
+ * Hover or Go-to-Definition on `JSON.stringify` below to inspect the standard library signatures.
+ */
+export type JSONStringifyReplacer = typeof JSON.stringify extends {
+    (value: any, replacer?: infer R1, space?: any): string;
+    (value: any, replacer?: infer R2, space?: any): string;
+} ? R1 | R2 : never;
+export interface WriteJSONOptions {
+    /**
+     * The format of the JSON output.
+     * @default "json"
+     */
+    format?: JSONFormat;
+    /**
+     * Options for the safe JSON replacer.
+     */
+    replacerOptions?: SafeJsonReplacerOptions;
+}
+export type { WriteCSVOptions } from "../utils";
+export interface ReadCSVOptions {
+    /**
+     * Whether the CSV has a header row.
+     * @default true
+     */
+    hasHeader?: boolean;
+    /**
+     * Character that separates fields.
+     * @default ","
+     */
+    separator?: string;
+    /**
+     * The character used for quoting fields.
+     * @default '"'
+     */
+    quoteChar?: string;
+    /**
+     * String representations of null values.
+     * @default ["", "NA", "null", "NaN"]
+     */
+    nullValues?: string[];
+    /**
+     * Optional explicit schema mapping column names to their registered data types.
+     */
+    schema?: DataFrameSchema;
+    /**
+     * Try to infer types from values if no schema is provided.
+     * @default true
+     */
+    inferSchema?: boolean;
+}

package/dist/dataframe/utils.d.ts

@@ -1,5 +1,4 @@
-import type { IExpr, ColumnData, ColumnDict } from "../types";
-import { DataType } from "../datatypes";
+import type { IExpr, ColumnData, ColumnDict, RegisteredDataType } from "../types";
 export declare function resolveWindowExpr(expr: IExpr, columns: ColumnDict, height: number): ColumnData;
 export declare function rowsToColumns(rows: any[]): {
     columns: ColumnDict;
@@ -7,5 +6,14 @@ export declare function rowsToColumns(rows: any[]): {
 };
 export declare function columnsToRows(columns: ColumnDict, height: number): any[];
 export declare function getRowFromColumns(columns: ColumnDict, idx: number, keys: string[]): any;
-export declare function inferColumnType(col: ColumnData): DataType;
+export declare function inferColumnType(col: ColumnData): RegisteredDataType;
 export declare function gatherColumnsByIndices(columns: ColumnDict, indices: number[]): ColumnDict;
+/**
+ * Computes a hash string for a row at the given index, using one or more column keys.
+ * Includes a single-key fast path to avoid array allocation and join overhead.
+ */
+export declare function computeRowHash(columns: ColumnDict, keys: string[], rowIndex: number): string;
+export declare function coerceColumn(col: ColumnData, type: RegisteredDataType, height: number): ColumnData;
+export declare function writeStringToFileOrStream(file: string | {
+    write: (str: string) => void;
+} | undefined, content: string): void;

package/dist/datatypes/DataType.d.ts

@@ -1,8 +1,9 @@
 export declare abstract class DataType<T = any> {
     abstract readonly name: string;
-    abstract coerce(val: any): T;
+    abstract coerce(val: unknown): T;
     abstract equals(other: DataType): boolean;
     abstract allocate(size: number): ArrayLike<T>;
+    matches(selector: any): boolean;
     get isNumeric(): boolean;
     get isInteger(): boolean;
     get isFloat(): boolean;

package/dist/datatypes/index.d.ts

@@ -1,5 +1,5 @@
 import { DataType as BaseDataType } from "./DataType";
-import { DecimalType } from "./types";
+import { DecimalType, BooleanDataType as Boolean, DateDataType as Date, ObjectDataType as Object, ArrayDataType as Array } from "./types";
 export { BaseDataType as DataType };
 export declare const DataTypeRegistry: {
     Int8: import("./types").Int8Type;
@@ -22,8 +22,9 @@ export declare const DataTypeRegistry: {
     Duration: import("./types").DurationType;
     Object: import("./types").ObjectType;
     Null: import("./types").NullType;
-    List: <TInner>(inner: BaseDataType<TInner>) => import("./types").ListType<TInner>;
-    Struct: <TFields extends import("..").RowRecord>(fields: { [K in keyof TFields]: BaseDataType<TFields[K]>; }) => import("./types").StructType<TFields>;
+    Array: <TInner>(inner: import("./types").RegisteredDataType & BaseDataType<TInner>) => import("./types").ArrayType<TInner>;
+    Struct: <TFields extends import("..").RowRecord>(fields: { [K in keyof TFields]: import("./types").RegisteredDataType & BaseDataType<TFields[K]>; }) => import("./types").StructType<TFields>;
 };
+export { Boolean, Date, Object, Array };
 export * from "./types";
 export * from "./DataType";

package/dist/datatypes/types.d.ts

@@ -2,70 +2,70 @@ import { DataType, SignedIntegerType, UnsignedIntegerType, FloatDataType, Tempor
 import type { RowRecord } from "../types";
 export declare class Int8Type extends SignedIntegerType {
     readonly name = "Int8";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Int8Array;
 }
 export declare const Int8: Int8Type;
 export declare class Int16Type extends SignedIntegerType {
     readonly name = "Int16";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Int16Array;
 }
 export declare const Int16: Int16Type;
 export declare class Int32Type extends SignedIntegerType {
     readonly name = "Int32";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Int32Array;
 }
 export declare const Int32: Int32Type;
 export declare class Int64Type extends SignedIntegerType<bigint | null> {
     readonly name = "Int64";
-    coerce(val: any): bigint | null;
+    coerce(val: unknown): bigint | null;
     equals(other: DataType): boolean;
     allocate(size: number): BigInt64Array;
 }
 export declare const Int64: Int64Type;
 export declare class UInt8Type extends UnsignedIntegerType {
     readonly name = "UInt8";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Uint8Array;
 }
 export declare const UInt8: UInt8Type;
 export declare class UInt16Type extends UnsignedIntegerType {
     readonly name = "UInt16";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Uint16Array;
 }
 export declare const UInt16: UInt16Type;
 export declare class UInt32Type extends UnsignedIntegerType {
     readonly name = "UInt32";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Uint32Array;
 }
 export declare const UInt32: UInt32Type;
 export declare class UInt64Type extends UnsignedIntegerType<bigint | null> {
     readonly name = "UInt64";
-    coerce(val: any): bigint | null;
+    coerce(val: unknown): bigint | null;
     equals(other: DataType): boolean;
     allocate(size: number): BigUint64Array;
 }
 export declare const UInt64: UInt64Type;
 export declare class Float32Type extends FloatDataType {
     readonly name = "Float32";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Float32Array;
 }
 export declare const Float32: Float32Type;
 export declare class Float64Type extends FloatDataType {
     readonly name = "Float64";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): Float64Array;
 }
@@ -75,39 +75,39 @@ export declare class DecimalType extends NumericDataType {
     readonly scale?: number | undefined;
     readonly name: string;
     constructor(precision?: number | undefined, scale?: number | undefined);
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): (number | null)[];
 }
-export declare class BooleanType extends DataType {
+export declare class BooleanType extends DataType<boolean | null> {
     readonly name = "Boolean";
     get isBoolean(): boolean;
-    coerce(val: any): boolean | null;
+    coerce(val: unknown): boolean | null;
     equals(other: DataType): boolean;
     allocate(size: number): (boolean | null)[];
 }
-export declare const Boolean: BooleanType;
-export declare class Utf8Type extends DataType {
+export declare const BooleanDataType: BooleanType;
+export declare class Utf8Type extends DataType<string | null> {
     readonly name = "Utf8";
     get isString(): boolean;
     get isUtf8(): boolean;
-    coerce(val: any): string | null;
+    coerce(val: unknown): string | null;
     equals(other: DataType): boolean;
     allocate(size: number): (string | null)[];
 }
 export declare const Utf8: Utf8Type;
-export declare class BinaryType extends DataType {
+export declare class BinaryType extends DataType<Uint8Array | null> {
     readonly name = "Binary";
     get isBinary(): boolean;
-    coerce(val: any): Uint8Array | null;
+    coerce(val: unknown): Uint8Array | null;
     equals(other: DataType): boolean;
     allocate(size: number): (Uint8Array | null)[];
 }
 export declare const Binary: BinaryType;
-export declare class NullType extends DataType {
+export declare class NullType extends DataType<null> {
     readonly name = "Null";
     get isNull(): boolean;
-    coerce(_val: any): null;
+    coerce(_val: unknown): null;
     equals(other: DataType): boolean;
     allocate(size: number): null[];
 }
@@ -115,58 +115,59 @@ export declare const Null: NullType;
 export declare class ObjectType extends DataType {
     readonly name = "Object";
     get isObject(): boolean;
-    coerce(val: any): any;
+    coerce(val: unknown): any;
     equals(other: DataType): boolean;
     allocate(size: number): any[];
 }
-export declare const Object: ObjectType;
-export declare class DateType extends TemporalDataType {
+export declare const ObjectDataType: ObjectType;
+export declare class DateType extends TemporalDataType<Date | null> {
     readonly name = "Date";
-    coerce(val: any): Date | null;
+    coerce(val: unknown): Date | null;
     equals(other: DataType): boolean;
     allocate(size: number): (Date | null)[];
 }
-export declare const Date: DateType;
-export declare class DatetimeType extends TemporalDataType {
+export declare const DateDataType: DateType;
+export declare class DatetimeType extends TemporalDataType<Date | null> {
     readonly name = "Datetime";
-    coerce(val: any): Date | null;
+    coerce(val: unknown): Date | null;
     equals(other: DataType): boolean;
     allocate(size: number): (Date | null)[];
 }
 export declare const Datetime: DatetimeType;
-export declare class TimeType extends TemporalDataType {
+export declare class TimeType extends TemporalDataType<string | null> {
     readonly name = "Time";
-    coerce(val: any): string | null;
+    coerce(val: unknown): string | null;
     equals(other: DataType): boolean;
     allocate(size: number): (string | null)[];
 }
 export declare const Time: TimeType;
-export declare class DurationType extends TemporalDataType {
+export declare class DurationType extends TemporalDataType<number | null> {
     readonly name = "Duration";
-    coerce(val: any): number | null;
+    coerce(val: unknown): number | null;
     equals(other: DataType): boolean;
     allocate(size: number): (number | null)[];
 }
 export declare const Duration: DurationType;
-export declare class ListType<TInner = any> extends NestedDataType<TInner[] | null> {
-    readonly innerType: DataType<TInner>;
-    readonly name = "List";
-    constructor(innerType: DataType<TInner>);
-    coerce(val: any): TInner[] | null;
+export declare class ArrayType<TInner = any> extends NestedDataType<TInner[] | null> {
+    readonly innerType: RegisteredDataType & DataType<TInner>;
+    readonly name = "Array";
+    constructor(innerType: RegisteredDataType & DataType<TInner>);
+    coerce(val: unknown): TInner[] | null;
     equals(other: DataType): boolean;
     allocate(size: number): (TInner[] | null)[];
 }
-export declare const List: <TInner>(inner: DataType<TInner>) => ListType<TInner>;
+export declare const ArrayDataType: <TInner>(inner: RegisteredDataType & DataType<TInner>) => ArrayType<TInner>;
 export declare class StructType<TFields extends RowRecord = any> extends NestedDataType<TFields | null> {
     readonly fields: {
-        [K in keyof TFields]: DataType<TFields[K]>;
+        [K in keyof TFields]: RegisteredDataType & DataType<TFields[K]>;
     };
     readonly name = "Struct";
     constructor(fields: {
-        [K in keyof TFields]: DataType<TFields[K]>;
+        [K in keyof TFields]: RegisteredDataType & DataType<TFields[K]>;
     });
-    coerce(val: any): TFields | null;
+    coerce(val: unknown): TFields | null;
     equals(other: DataType): boolean;
     allocate(size: number): (TFields | null)[];
 }
-export declare const Struct: <TFields extends RowRecord>(fields: { [K in keyof TFields]: DataType<TFields[K]>; }) => StructType<TFields>;
+export declare const Struct: <TFields extends RowRecord>(fields: { [K in keyof TFields]: RegisteredDataType & DataType<TFields[K]>; }) => StructType<TFields>;
+export type RegisteredDataType = Int8Type | Int16Type | Int32Type | Int64Type | UInt8Type | UInt16Type | UInt32Type | UInt64Type | Float32Type | Float64Type | DecimalType | BooleanType | Utf8Type | BinaryType | NullType | ObjectType | DateType | DatetimeType | TimeType | DurationType | ArrayType<any> | StructType<any>;

package/dist/exceptions/index.d.ts

@@ -10,4 +10,6 @@ export declare class SchemaError extends DFScriptError {
 }
 export declare class ComputeError extends DFScriptError {
 }
+export declare class ShapeError extends DFScriptError {
+}
 export * from "./utils";

package/dist/exceptions/utils.d.ts

@@ -1,2 +1,3 @@
 import type { ColumnDict } from "../types";
 export declare function assertColumnExists(columnName: string, columns: ColumnDict, context: string, suffix?: string): void;
+export declare function assertHeight(columns: ColumnDict, height?: number): number;

package/dist/functions/concat.d.ts

@@ -1,4 +1,3 @@
 import { DataFrame } from "../dataframe/dataframe";
 import type { ConcatOptions, ConcatItem, RowRecord } from "../types";
-export declare function normalizeToDataFrames(item: any, context: string, index: number): DataFrame<any>[];
-export declare function concat<U extends RowRecord = any>(rawItems: ConcatItem | ConcatItem[], options?: ConcatOptions): DataFrame<U>;
+export declare function concat<U extends RowRecord = any>(rawItems: ConcatItem | ConcatItem[], { how, horizontal }?: ConcatOptions): DataFrame<U>;

package/dist/functions/index.d.ts

@@ -1 +1,3 @@
-export * from "./concat";
+export { concat } from "./concat";
+export { read_json } from "./read_json";
+export { read_csv } from "./read_csv";

package/dist/functions/read_csv.d.ts

@@ -0,0 +1,8 @@
+import { DataFrame } from "../dataframe/dataframe";
+import type { ReadCSVOptions } from "../dataframe/types";
+import type { RowRecord } from "../types";
+/**
+ * Reads a CSV string and constructs a DataFrame.
+ * Automatically infers column data types unless an explicit schema is provided.
+ */
+export declare function read_csv<T extends RowRecord = any>(content: string, options?: ReadCSVOptions): DataFrame<T>;

package/dist/functions/read_json.d.ts

@@ -0,0 +1,10 @@
+import { DataFrame } from "../dataframe/dataframe";
+import type { ReadJSONOptions } from "../dataframe/types";
+/**
+ * Parses JSON content (JSON or NDJSON) and loads it into a new DataFrame.
+ *
+ * @param content The JSON or NDJSON content string.
+ * @param options Parse and configuration options.
+ * @returns A new DataFrame instance populated with the parsed records.
+ */
+export declare function read_json(content: string, { format, trimBeforeParse, schema, ...parseOpts }?: ReadJSONOptions): DataFrame<any>;