@platforma-open/milaboratories.software-ptabler.schema 1.8.0 → 1.10.0

package/dist/expressions.d.ts

@@ -1,5 +1,5 @@
 import { DataType } from './common';
-export type Expression = ComparisonExpression | BinaryArithmeticExpression | UnaryArithmeticExpression | CastExpression | BooleanLogicExpression | NotExpression | NullCheckExpression | StringJoinExpression | HashExpression | ColumnReferenceExpression | ConstantValueExpression | RankExpression | CumsumExpression | ExtendedUnaryStringExpression | StringDistanceExpression | FuzzyStringFilterExpression | WhenThenOtherwiseExpression | SubstringExpression | StringReplaceExpression | StringContainsExpression | StringStartsWithExpression | StringEndsWithExpression | StringContainsAnyExpression | StringCountMatchesExpression | StringExtractExpression | MinMaxExpression | FillNaExpression | WindowExpression;
+export type Expression = ComparisonExpression | BinaryArithmeticExpression | UnaryArithmeticExpression | CastExpression | BooleanLogicExpression | NotExpression | NullCheckExpression | StringJoinExpression | HashExpression | ColumnReferenceExpression | ConstantValueExpression | RankExpression | CumsumExpression | ExtendedUnaryStringExpression | StringDistanceExpression | FuzzyStringFilterExpression | WhenThenOtherwiseExpression | SubstringExpression | StringReplaceExpression | StringContainsExpression | StringStartsWithExpression | StringEndsWithExpression | StringContainsAnyExpression | StringCountMatchesExpression | StringExtractExpression | MinMaxExpression | FillNaExpression | WindowExpression | StructFieldExpression;
 /** Represents all possible expression types in the system. */
 export type ComparisonOperator = 'gt' | 'ge' | 'eq' | 'lt' | 'le' | 'neq';
 /** Defines a comparison operation between two expressions. */
@@ -406,3 +406,19 @@ export interface WindowExpression {
     /** List of expressions to partition the data by. The aggregation is performed independently within each partition. */
     partitionBy: Expression[];
 }
+/**
+ * Represents a struct field access operation.
+ * This operation retrieves a single field from a struct (nested data structure).
+ * It corresponds to Polars' struct.field() functionality.
+ */
+export interface StructFieldExpression {
+    /** The type of operation, always 'struct_field'. */
+    type: 'struct_field';
+    /** The struct expression to extract fields from. */
+    struct: Expression;
+    /**
+     * The field name to extract from the struct.
+     * Currently only supports single field extraction due to Polars behavior limitations.
+     */
+    fields: string;
+}

package/dist/index.d.ts

@@ -1,10 +1,12 @@
-import { ReadCsvStep, WriteCsvStep } from './io';
+import { ReadCsvStep, ReadNdjsonStep, WriteCsvStep, WriteNdjsonStep, BaseFileReadStep, BaseFileWriteStep } from './io';
 import { AddColumnsStep, FilterStep, SelectStep, WithColumnsStep, WithoutColumnsStep } from './basic_steps';
 import { AggregateStep } from './aggregate';
 import { AnyJoinStep } from './join';
 import { ConcatenateStep } from './concatenate';
 import { SortStep } from './sort';
-export type PTablerStep = ReadCsvStep | WriteCsvStep | AddColumnsStep | FilterStep | AggregateStep | AnyJoinStep | ConcatenateStep | SortStep | SelectStep | WithColumnsStep | WithoutColumnsStep;
+export type PTablerStep = ReadCsvStep | ReadNdjsonStep | WriteCsvStep | WriteNdjsonStep | AddColumnsStep | FilterStep | AggregateStep | AnyJoinStep | ConcatenateStep | SortStep | SelectStep | WithColumnsStep | WithoutColumnsStep;
 export type PTablerWorkflow = {
     workflow: PTablerStep[];
 };
+export type { BaseFileReadStep, BaseFileWriteStep };
+export type { Expression, StructFieldExpression } from './expressions';

package/dist/io.d.ts

@@ -10,16 +10,15 @@ export interface ColumnSchema {
     /** Optional: A specific string to be interpreted as a null value for this column. */
     nullValue?: string;
 }
-/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
-export interface ReadCsvStep {
-    /** The type of the step, which is always 'read_csv' for this operation. */
-    type: 'read_csv';
-    /** Path to the CSV file to be read. */
+/**
+ * Base interface for file reading operations that contains common fields
+ * shared across different file format readers.
+ */
+export interface BaseFileReadStep {
+    /** Path to the file to be read. */
     file: string;
     /** The name assigned to the loaded DataFrame in the tablespace. */
     name: string;
-    /** Optional: The delimiter character used in the CSV file. */
-    delimiter?: string;
     /**
      * Optional: Provides schema information for specific columns.
      * If `infer_schema` is `true` (default), these definitions act as overrides
@@ -30,39 +29,61 @@ export interface ReadCsvStep {
      */
     schema?: ColumnSchema[];
     /**
-     * Optional: Whether to infer the schema from the CSV file using Polars'
+     * Optional: Whether to infer the schema from the file using Polars'
      * default inference mechanism (e.g., reading a certain number of rows).
      * Defaults to `true`. If set to `false`, type inference is disabled,
      * and types will rely on the `schema` field or Polars' defaults for
      * columns not specified in `schema`.
      */
-    infer_schema?: boolean;
+    inferSchema?: boolean;
+    /**
+     * Optional: Return null if parsing fails because of schema mismatches.
+     * Defaults to `false`.
+     */
+    ignoreErrors?: boolean;
+    /**
+     * Optional: Stop reading after this many rows.
+     * If not specified, all rows will be read.
+     */
+    nRows?: number;
+}
+/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
+export interface ReadCsvStep extends BaseFileReadStep {
+    /** The type of the step, which is always 'read_csv' for this operation. */
+    type: 'read_csv';
+    /** Optional: The delimiter character used in the CSV file. */
+    delimiter?: string;
+}
+/** Represents the configuration for a step that reads data from an NDJSON file into the tablespace. */
+export interface ReadNdjsonStep extends BaseFileReadStep {
+    /** The type of the step, which is always 'read_ndjson' for this operation. */
+    type: 'read_ndjson';
 }
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ * Base interface for file writing operations that contains common fields
+ * shared across different file format writers.
  */
-export interface WriteCsvStep {
-    /** The type of the step, which is always 'write_csv' for this operation. */
-    type: 'write_csv';
+export interface BaseFileWriteStep {
     /** The name of the table in the tablespace to be written. */
     table: string;
-    /** Path to the output CSV file. */
+    /** Path to the output file. */
     file: string;
-    /** Optional: A list of column names to write to the CSV. If omitted, all columns are written. */
+    /** Optional: A list of column names to write to the file. If omitted, all columns are written. */
     columns?: string[];
+}
+/**
+ * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ */
+export interface WriteCsvStep extends BaseFileWriteStep {
+    /** The type of the step, which is always 'write_csv' for this operation. */
+    type: 'write_csv';
     /** Optional: The delimiter character to use in the output CSV file. */
     delimiter?: string;
 }
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a JSON file.
+ * Represents the configuration for a step that writes a table from the tablespace to an NDJSON file.
  */
-export interface WriteJsonStep {
-    /** The type of the step, which is always 'write_json' for this operation. */
-    type: 'write_json';
-    /** The name of the table in the tablespace to be written. */
-    table: string;
-    /** Path to the output JSON file. */
-    file: string;
-    /** Optional: A list of column names to write to the JSON. If omitted, all columns are written. */
-    columns?: string[];
+export interface WriteNdjsonStep extends BaseFileWriteStep {
+    /** The type of the step, which is always 'write_ndjson' for this operation. */
+    type: 'write_ndjson';
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-open/milaboratories.software-ptabler.schema",
-  "version": "1.8.0",
+  "version": "1.10.0",
   "description": "Type definitions for PTabler",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",

package/src/expressions.ts

@@ -28,7 +28,8 @@ export type Expression =
   | StringExtractExpression
   | MinMaxExpression
   | FillNaExpression
-  | WindowExpression;
+  | WindowExpression
+  | StructFieldExpression;
 
 /** Represents all possible expression types in the system. */
 export type ComparisonOperator = 'gt' | 'ge' | 'eq' | 'lt' | 'le' | 'neq';
@@ -514,3 +515,20 @@ export interface WindowExpression {
   /** List of expressions to partition the data by. The aggregation is performed independently within each partition. */
   partitionBy: Expression[];
 }
+
+/**
+ * Represents a struct field access operation.
+ * This operation retrieves a single field from a struct (nested data structure).
+ * It corresponds to Polars' struct.field() functionality.
+ */
+export interface StructFieldExpression {
+  /** The type of operation, always 'struct_field'. */
+  type: 'struct_field';
+  /** The struct expression to extract fields from. */
+  struct: Expression;
+  /**
+   * The field name to extract from the struct.
+   * Currently only supports single field extraction due to Polars behavior limitations.
+   */
+  fields: string;
+}

package/src/index.ts

@@ -1,4 +1,4 @@
-import type { ReadCsvStep, WriteCsvStep } from './io';
+import type { ReadCsvStep, ReadNdjsonStep, WriteCsvStep, WriteNdjsonStep, BaseFileReadStep, BaseFileWriteStep } from './io';
 import type { AddColumnsStep, FilterStep, SelectStep, WithColumnsStep, WithoutColumnsStep } from './basic_steps';
 import type { AggregateStep } from './aggregate';
 import type { AnyJoinStep } from './join';
@@ -7,7 +7,9 @@ import type { SortStep } from './sort';
 
 export type PTablerStep =
   | ReadCsvStep
+  | ReadNdjsonStep
   | WriteCsvStep
+  | WriteNdjsonStep
   | AddColumnsStep
   | FilterStep
   | AggregateStep
@@ -21,3 +23,9 @@ export type PTablerStep =
 export type PTablerWorkflow = {
   workflow: PTablerStep[];
 };
+
+// Re-export base interfaces for potential external use
+export type { BaseFileReadStep, BaseFileWriteStep };
+
+// Re-export expression types for external use
+export type { Expression, StructFieldExpression } from './expressions';

package/src/io.ts

@@ -12,16 +12,15 @@ export interface ColumnSchema {
   nullValue?: string;
 }
 
-/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
-export interface ReadCsvStep {
-  /** The type of the step, which is always 'read_csv' for this operation. */
-  type: 'read_csv';
-  /** Path to the CSV file to be read. */
+/**
+ * Base interface for file reading operations that contains common fields
+ * shared across different file format readers.
+ */
+export interface BaseFileReadStep {
+  /** Path to the file to be read. */
   file: string;
   /** The name assigned to the loaded DataFrame in the tablespace. */
   name: string;
-  /** Optional: The delimiter character used in the CSV file. */
-  delimiter?: string;
   /**
    * Optional: Provides schema information for specific columns.
    * If `infer_schema` is `true` (default), these definitions act as overrides
@@ -32,41 +31,76 @@ export interface ReadCsvStep {
    */
   schema?: ColumnSchema[];
   /**
-   * Optional: Whether to infer the schema from the CSV file using Polars'
+   * Optional: Whether to infer the schema from the file using Polars'
    * default inference mechanism (e.g., reading a certain number of rows).
    * Defaults to `true`. If set to `false`, type inference is disabled,
    * and types will rely on the `schema` field or Polars' defaults for
    * columns not specified in `schema`.
    */
-  infer_schema?: boolean;
+  inferSchema?: boolean;
+  /**
+   * Optional: Return null if parsing fails because of schema mismatches.
+   * Defaults to `false`.
+   */
+  ignoreErrors?: boolean;
+  /**
+   * Optional: Stop reading after this many rows.
+   * If not specified, all rows will be read.
+   */
+  nRows?: number;
+}
+
+/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
+export interface ReadCsvStep extends BaseFileReadStep {
+  /** The type of the step, which is always 'read_csv' for this operation. */
+  type: 'read_csv';
+  /** Optional: The delimiter character used in the CSV file. */
+  delimiter?: string;
+}
+
+/** Represents the configuration for a step that reads data from an NDJSON file into the tablespace. */
+export interface ReadNdjsonStep extends BaseFileReadStep {
+  /** The type of the step, which is always 'read_ndjson' for this operation. */
+  type: 'read_ndjson';
 }
 
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ * Base interface for file writing operations that contains common fields
+ * shared across different file format writers.
  */
-export interface WriteCsvStep {
-  /** The type of the step, which is always 'write_csv' for this operation. */
-  type: 'write_csv';
+export interface BaseFileWriteStep {
   /** The name of the table in the tablespace to be written. */
   table: string;
-  /** Path to the output CSV file. */
+  /** Path to the output file. */
   file: string;
-  /** Optional: A list of column names to write to the CSV. If omitted, all columns are written. */
+  /** Optional: A list of column names to write to the file. If omitted, all columns are written. */
   columns?: string[];
+}
+
+/**
+ * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ */
+export interface WriteCsvStep extends BaseFileWriteStep {
+  /** The type of the step, which is always 'write_csv' for this operation. */
+  type: 'write_csv';
   /** Optional: The delimiter character to use in the output CSV file. */
   delimiter?: string;
 }
 
+// Not yet supported, should be a normal write_json, but we don't have a lazy sink_json, can create a workaround
+// if needed.
+// /**
+//  * Represents the configuration for a step that writes a table from the tablespace to a JSON file.
+//  */
+// export interface WriteJsonStep extends BaseFileWriteStep {
+//   /** The type of the step, which is always 'write_json' for this operation. */
+//   type: 'write_json';
+// }
+
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a JSON file.
+ * Represents the configuration for a step that writes a table from the tablespace to an NDJSON file.
  */
-export interface WriteJsonStep {
-  /** The type of the step, which is always 'write_json' for this operation. */
-  type: 'write_json';
-  /** The name of the table in the tablespace to be written. */
-  table: string;
-  /** Path to the output JSON file. */
-  file: string;
-  /** Optional: A list of column names to write to the JSON. If omitted, all columns are written. */
-  columns?: string[];
+export interface WriteNdjsonStep extends BaseFileWriteStep {
+  /** The type of the step, which is always 'write_ndjson' for this operation. */
+  type: 'write_ndjson';
 }