@platforma-open/milaboratories.software-ptabler.schema 1.7.0 → 1.9.0

package/dist/expressions.d.ts

@@ -1,5 +1,5 @@
 import { DataType } from './common';
-export type Expression = ComparisonExpression | BinaryArithmeticExpression | UnaryArithmeticExpression | CastExpression | BooleanLogicExpression | NotExpression | NullCheckExpression | StringJoinExpression | HashExpression | ColumnReferenceExpression | ConstantValueExpression | RankExpression | CumsumExpression | ExtendedUnaryStringExpression | StringDistanceExpression | FuzzyStringFilterExpression | WhenThenOtherwiseExpression | SubstringExpression | StringReplaceExpression | MinMaxExpression | FillNaExpression | WindowExpression;
+export type Expression = ComparisonExpression | BinaryArithmeticExpression | UnaryArithmeticExpression | CastExpression | BooleanLogicExpression | NotExpression | NullCheckExpression | StringJoinExpression | HashExpression | ColumnReferenceExpression | ConstantValueExpression | RankExpression | CumsumExpression | ExtendedUnaryStringExpression | StringDistanceExpression | FuzzyStringFilterExpression | WhenThenOtherwiseExpression | SubstringExpression | StringReplaceExpression | StringContainsExpression | StringStartsWithExpression | StringEndsWithExpression | StringContainsAnyExpression | StringCountMatchesExpression | StringExtractExpression | MinMaxExpression | FillNaExpression | WindowExpression;
 /** Represents all possible expression types in the system. */
 export type ComparisonOperator = 'gt' | 'ge' | 'eq' | 'lt' | 'le' | 'neq';
 /** Defines a comparison operation between two expressions. */
@@ -276,6 +276,94 @@ export interface StringReplaceExpression {
     /** If true, treat the pattern as a literal string. If false or undefined, treat it as a regex. Defaults to false. */
     literal?: boolean;
 }
+/**
+ * Represents a string contains operation.
+ * Checks if the string contains a substring that matches a pattern using regex or literal matching.
+ * Based on polars.Series.str.contains - supports both regex and literal pattern matching with optional case-insensitive flags.
+ */
+export interface StringContainsExpression {
+    /** The type of operation, always 'str_contains'. */
+    type: 'str_contains';
+    /** The input string expression to search in. */
+    value: Expression;
+    /** The pattern to search for. Can be a regex pattern (default) or literal string when literal=true. */
+    pattern: Expression | string;
+    /** If true, treat the pattern as a literal string. If false, treat it as a regex pattern. Defaults to false. */
+    literal?: boolean;
+    /** If true, raise an error if pattern is invalid regex. If false, return null for invalid patterns. Defaults to true. */
+    strict?: boolean;
+}
+/**
+ * Represents a string starts_with operation.
+ * Checks if the string starts with a specified prefix. Always uses literal matching (no regex support).
+ * Based on polars.Series.str.starts_with - only supports literal prefix matching.
+ */
+export interface StringStartsWithExpression {
+    /** The type of operation, always 'str_starts_with'. */
+    type: 'str_starts_with';
+    /** The input string expression to check. */
+    value: Expression;
+    /** The prefix to check for (always treated as literal string, no regex support). */
+    prefix: Expression | string;
+}
+/**
+ * Represents a string ends_with operation.
+ * Checks if the string ends with a specified suffix. Always uses literal matching (no regex support).
+ * Based on polars.Series.str.ends_with - only supports literal suffix matching.
+ */
+export interface StringEndsWithExpression {
+    /** The type of operation, always 'str_ends_with'. */
+    type: 'str_ends_with';
+    /** The input string expression to check. */
+    value: Expression;
+    /** The suffix to check for (always treated as literal string, no regex support). */
+    suffix: Expression | string;
+}
+/**
+ * Represents a string contains_any operation using the Aho-Corasick algorithm.
+ * Checks if the string contains any of the provided patterns using fast multi-pattern string matching.
+ * Based on polars.Series.str.contains_any - uses Aho-Corasick algorithm for efficient multi-pattern matching.
+ */
+export interface StringContainsAnyExpression {
+    /** The type of operation, always 'str_contains_any'. */
+    type: 'str_contains_any';
+    /** The input string expression to search in. */
+    value: Expression;
+    /** Array of literal string patterns to search for. Only immediate string values are supported, no expressions or regex patterns. */
+    patterns: string[];
+    /** Enable ASCII-aware case insensitive matching. When enabled, searching is performed without respect to case for ASCII letters (a-z and A-Z) only. Defaults to false. */
+    asciiCaseInsensitive?: boolean;
+}
+/**
+ * Represents a string count_matches operation.
+ * Counts the number of times a pattern occurs in the string using regex or literal matching.
+ * Based on polars.Series.str.count_matches - supports both regex and literal pattern matching.
+ */
+export interface StringCountMatchesExpression {
+    /** The type of operation, always 'str_count_matches'. */
+    type: 'str_count_matches';
+    /** The input string expression to count matches in. */
+    value: Expression;
+    /** The pattern to count occurrences of. Can be a regex pattern (default) or literal string when literal=true. */
+    pattern: Expression | string;
+    /** If true, treat the pattern as a literal string. If false, treat it as a regex pattern. Defaults to false. */
+    literal?: boolean;
+}
+/**
+ * Represents a string extract operation using regex patterns.
+ * Extracts the first match of a regex pattern from the string, optionally targeting specific capture groups.
+ * Based on polars.Series.str.extract - only supports regex patterns (no literal mode).
+ */
+export interface StringExtractExpression {
+    /** The type of operation, always 'str_extract'. */
+    type: 'str_extract';
+    /** The input string expression to extract from. */
+    value: Expression;
+    /** The regex pattern to extract. Must be a valid regex pattern - no literal string mode is supported. */
+    pattern: Expression | string;
+    /** The capture group index to extract. Group 0 is the entire match, group 1 is the first capture group, etc. Defaults to 0. */
+    groupIndex?: number;
+}
 /** Defines the supported min/max operators. */
 export type MinMaxOperator = 'min' | 'max';
 /** Represents a min or max operation on a list of expressions. */

package/dist/index.d.ts

@@ -1,10 +1,11 @@
-import { ReadCsvStep, WriteCsvStep } from './io';
+import { ReadCsvStep, ReadNdjsonStep, WriteCsvStep, WriteNdjsonStep, BaseFileReadStep, BaseFileWriteStep } from './io';
 import { AddColumnsStep, FilterStep, SelectStep, WithColumnsStep, WithoutColumnsStep } from './basic_steps';
 import { AggregateStep } from './aggregate';
 import { AnyJoinStep } from './join';
 import { ConcatenateStep } from './concatenate';
 import { SortStep } from './sort';
-export type PTablerStep = ReadCsvStep | WriteCsvStep | AddColumnsStep | FilterStep | AggregateStep | AnyJoinStep | ConcatenateStep | SortStep | SelectStep | WithColumnsStep | WithoutColumnsStep;
+export type PTablerStep = ReadCsvStep | ReadNdjsonStep | WriteCsvStep | WriteNdjsonStep | AddColumnsStep | FilterStep | AggregateStep | AnyJoinStep | ConcatenateStep | SortStep | SelectStep | WithColumnsStep | WithoutColumnsStep;
 export type PTablerWorkflow = {
     workflow: PTablerStep[];
 };
+export type { BaseFileReadStep, BaseFileWriteStep };

package/dist/io.d.ts

@@ -10,16 +10,15 @@ export interface ColumnSchema {
     /** Optional: A specific string to be interpreted as a null value for this column. */
     nullValue?: string;
 }
-/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
-export interface ReadCsvStep {
-    /** The type of the step, which is always 'read_csv' for this operation. */
-    type: 'read_csv';
-    /** Path to the CSV file to be read. */
+/**
+ * Base interface for file reading operations that contains common fields
+ * shared across different file format readers.
+ */
+export interface BaseFileReadStep {
+    /** Path to the file to be read. */
     file: string;
     /** The name assigned to the loaded DataFrame in the tablespace. */
     name: string;
-    /** Optional: The delimiter character used in the CSV file. */
-    delimiter?: string;
     /**
      * Optional: Provides schema information for specific columns.
      * If `infer_schema` is `true` (default), these definitions act as overrides
@@ -30,39 +29,61 @@ export interface ReadCsvStep {
      */
     schema?: ColumnSchema[];
     /**
-     * Optional: Whether to infer the schema from the CSV file using Polars'
+     * Optional: Whether to infer the schema from the file using Polars'
      * default inference mechanism (e.g., reading a certain number of rows).
      * Defaults to `true`. If set to `false`, type inference is disabled,
      * and types will rely on the `schema` field or Polars' defaults for
      * columns not specified in `schema`.
      */
-    infer_schema?: boolean;
+    inferSchema?: boolean;
+    /**
+     * Optional: Return null if parsing fails because of schema mismatches.
+     * Defaults to `false`.
+     */
+    ignoreErrors?: boolean;
+    /**
+     * Optional: Stop reading after this many rows.
+     * If not specified, all rows will be read.
+     */
+    nRows?: number;
+}
+/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
+export interface ReadCsvStep extends BaseFileReadStep {
+    /** The type of the step, which is always 'read_csv' for this operation. */
+    type: 'read_csv';
+    /** Optional: The delimiter character used in the CSV file. */
+    delimiter?: string;
+}
+/** Represents the configuration for a step that reads data from an NDJSON file into the tablespace. */
+export interface ReadNdjsonStep extends BaseFileReadStep {
+    /** The type of the step, which is always 'read_ndjson' for this operation. */
+    type: 'read_ndjson';
 }
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ * Base interface for file writing operations that contains common fields
+ * shared across different file format writers.
  */
-export interface WriteCsvStep {
-    /** The type of the step, which is always 'write_csv' for this operation. */
-    type: 'write_csv';
+export interface BaseFileWriteStep {
     /** The name of the table in the tablespace to be written. */
     table: string;
-    /** Path to the output CSV file. */
+    /** Path to the output file. */
     file: string;
-    /** Optional: A list of column names to write to the CSV. If omitted, all columns are written. */
+    /** Optional: A list of column names to write to the file. If omitted, all columns are written. */
     columns?: string[];
+}
+/**
+ * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ */
+export interface WriteCsvStep extends BaseFileWriteStep {
+    /** The type of the step, which is always 'write_csv' for this operation. */
+    type: 'write_csv';
     /** Optional: The delimiter character to use in the output CSV file. */
     delimiter?: string;
 }
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a JSON file.
+ * Represents the configuration for a step that writes a table from the tablespace to an NDJSON file.
  */
-export interface WriteJsonStep {
-    /** The type of the step, which is always 'write_json' for this operation. */
-    type: 'write_json';
-    /** The name of the table in the tablespace to be written. */
-    table: string;
-    /** Path to the output JSON file. */
-    file: string;
-    /** Optional: A list of column names to write to the JSON. If omitted, all columns are written. */
-    columns?: string[];
+export interface WriteNdjsonStep extends BaseFileWriteStep {
+    /** The type of the step, which is always 'write_ndjson' for this operation. */
+    type: 'write_ndjson';
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-open/milaboratories.software-ptabler.schema",
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "Type definitions for PTabler",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",

package/src/expressions.ts

@@ -20,6 +20,12 @@ export type Expression =
   | WhenThenOtherwiseExpression
   | SubstringExpression
   | StringReplaceExpression
+  | StringContainsExpression
+  | StringStartsWithExpression
+  | StringEndsWithExpression
+  | StringContainsAnyExpression
+  | StringCountMatchesExpression
+  | StringExtractExpression
   | MinMaxExpression
   | FillNaExpression
   | WindowExpression;
@@ -357,6 +363,100 @@ export interface StringReplaceExpression {
   literal?: boolean;
 }
 
+/**
+ * Represents a string contains operation.
+ * Checks if the string contains a substring that matches a pattern using regex or literal matching.
+ * Based on polars.Series.str.contains - supports both regex and literal pattern matching with optional case-insensitive flags.
+ */
+export interface StringContainsExpression {
+  /** The type of operation, always 'str_contains'. */
+  type: 'str_contains';
+  /** The input string expression to search in. */
+  value: Expression;
+  /** The pattern to search for. Can be a regex pattern (default) or literal string when literal=true. */
+  pattern: Expression | string;
+  /** If true, treat the pattern as a literal string. If false, treat it as a regex pattern. Defaults to false. */
+  literal?: boolean;
+  /** If true, raise an error if pattern is invalid regex. If false, return null for invalid patterns. Defaults to true. */
+  strict?: boolean;
+}
+
+/**
+ * Represents a string starts_with operation.
+ * Checks if the string starts with a specified prefix. Always uses literal matching (no regex support).
+ * Based on polars.Series.str.starts_with - only supports literal prefix matching.
+ */
+export interface StringStartsWithExpression {
+  /** The type of operation, always 'str_starts_with'. */
+  type: 'str_starts_with';
+  /** The input string expression to check. */
+  value: Expression;
+  /** The prefix to check for (always treated as literal string, no regex support). */
+  prefix: Expression | string;
+}
+
+/**
+ * Represents a string ends_with operation.
+ * Checks if the string ends with a specified suffix. Always uses literal matching (no regex support).
+ * Based on polars.Series.str.ends_with - only supports literal suffix matching.
+ */
+export interface StringEndsWithExpression {
+  /** The type of operation, always 'str_ends_with'. */
+  type: 'str_ends_with';
+  /** The input string expression to check. */
+  value: Expression;
+  /** The suffix to check for (always treated as literal string, no regex support). */
+  suffix: Expression | string;
+}
+
+/**
+ * Represents a string contains_any operation using the Aho-Corasick algorithm.
+ * Checks if the string contains any of the provided patterns using fast multi-pattern string matching.
+ * Based on polars.Series.str.contains_any - uses Aho-Corasick algorithm for efficient multi-pattern matching.
+ */
+export interface StringContainsAnyExpression {
+  /** The type of operation, always 'str_contains_any'. */
+  type: 'str_contains_any';
+  /** The input string expression to search in. */
+  value: Expression;
+  /** Array of literal string patterns to search for. Only immediate string values are supported, no expressions or regex patterns. */
+  patterns: string[];
+  /** Enable ASCII-aware case insensitive matching. When enabled, searching is performed without respect to case for ASCII letters (a-z and A-Z) only. Defaults to false. */
+  asciiCaseInsensitive?: boolean;
+}
+
+/**
+ * Represents a string count_matches operation.
+ * Counts the number of times a pattern occurs in the string using regex or literal matching.
+ * Based on polars.Series.str.count_matches - supports both regex and literal pattern matching.
+ */
+export interface StringCountMatchesExpression {
+  /** The type of operation, always 'str_count_matches'. */
+  type: 'str_count_matches';
+  /** The input string expression to count matches in. */
+  value: Expression;
+  /** The pattern to count occurrences of. Can be a regex pattern (default) or literal string when literal=true. */
+  pattern: Expression | string;
+  /** If true, treat the pattern as a literal string. If false, treat it as a regex pattern. Defaults to false. */
+  literal?: boolean;
+}
+
+/**
+ * Represents a string extract operation using regex patterns.
+ * Extracts the first match of a regex pattern from the string, optionally targeting specific capture groups.
+ * Based on polars.Series.str.extract - only supports regex patterns (no literal mode).
+ */
+export interface StringExtractExpression {
+  /** The type of operation, always 'str_extract'. */
+  type: 'str_extract';
+  /** The input string expression to extract from. */
+  value: Expression;
+  /** The regex pattern to extract. Must be a valid regex pattern - no literal string mode is supported. */
+  pattern: Expression | string;
+  /** The capture group index to extract. Group 0 is the entire match, group 1 is the first capture group, etc. Defaults to 0. */
+  groupIndex?: number;
+}
+
 /** Defines the supported min/max operators. */
 export type MinMaxOperator = 'min' | 'max';
 

package/src/index.ts

@@ -1,4 +1,4 @@
-import type { ReadCsvStep, WriteCsvStep } from './io';
+import type { ReadCsvStep, ReadNdjsonStep, WriteCsvStep, WriteJsonStep, WriteNdjsonStep, BaseFileReadStep, BaseFileWriteStep } from './io';
 import type { AddColumnsStep, FilterStep, SelectStep, WithColumnsStep, WithoutColumnsStep } from './basic_steps';
 import type { AggregateStep } from './aggregate';
 import type { AnyJoinStep } from './join';
@@ -7,7 +7,9 @@ import type { SortStep } from './sort';
 
 export type PTablerStep =
   | ReadCsvStep
+  | ReadNdjsonStep
   | WriteCsvStep
+  | WriteNdjsonStep
   | AddColumnsStep
   | FilterStep
   | AggregateStep
@@ -21,3 +23,6 @@ export type PTablerStep =
 export type PTablerWorkflow = {
   workflow: PTablerStep[];
 };
+
+// Re-export base interfaces for potential external use
+export type { BaseFileReadStep, BaseFileWriteStep };

package/src/io.ts

@@ -12,16 +12,15 @@ export interface ColumnSchema {
   nullValue?: string;
 }
 
-/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
-export interface ReadCsvStep {
-  /** The type of the step, which is always 'read_csv' for this operation. */
-  type: 'read_csv';
-  /** Path to the CSV file to be read. */
+/**
+ * Base interface for file reading operations that contains common fields
+ * shared across different file format readers.
+ */
+export interface BaseFileReadStep {
+  /** Path to the file to be read. */
   file: string;
   /** The name assigned to the loaded DataFrame in the tablespace. */
   name: string;
-  /** Optional: The delimiter character used in the CSV file. */
-  delimiter?: string;
   /**
    * Optional: Provides schema information for specific columns.
    * If `infer_schema` is `true` (default), these definitions act as overrides
@@ -32,41 +31,76 @@ export interface ReadCsvStep {
    */
   schema?: ColumnSchema[];
   /**
-   * Optional: Whether to infer the schema from the CSV file using Polars'
+   * Optional: Whether to infer the schema from the file using Polars'
    * default inference mechanism (e.g., reading a certain number of rows).
    * Defaults to `true`. If set to `false`, type inference is disabled,
    * and types will rely on the `schema` field or Polars' defaults for
    * columns not specified in `schema`.
    */
-  infer_schema?: boolean;
+  inferSchema?: boolean;
+  /**
+   * Optional: Return null if parsing fails because of schema mismatches.
+   * Defaults to `false`.
+   */
+  ignoreErrors?: boolean;
+  /**
+   * Optional: Stop reading after this many rows.
+   * If not specified, all rows will be read.
+   */
+  nRows?: number;
+}
+
+/** Represents the configuration for a step that reads data from a CSV file into the tablespace. */
+export interface ReadCsvStep extends BaseFileReadStep {
+  /** The type of the step, which is always 'read_csv' for this operation. */
+  type: 'read_csv';
+  /** Optional: The delimiter character used in the CSV file. */
+  delimiter?: string;
+}
+
+/** Represents the configuration for a step that reads data from an NDJSON file into the tablespace. */
+export interface ReadNdjsonStep extends BaseFileReadStep {
+  /** The type of the step, which is always 'read_ndjson' for this operation. */
+  type: 'read_ndjson';
 }
 
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ * Base interface for file writing operations that contains common fields
+ * shared across different file format writers.
  */
-export interface WriteCsvStep {
-  /** The type of the step, which is always 'write_csv' for this operation. */
-  type: 'write_csv';
+export interface BaseFileWriteStep {
   /** The name of the table in the tablespace to be written. */
   table: string;
-  /** Path to the output CSV file. */
+  /** Path to the output file. */
   file: string;
-  /** Optional: A list of column names to write to the CSV. If omitted, all columns are written. */
+  /** Optional: A list of column names to write to the file. If omitted, all columns are written. */
   columns?: string[];
+}
+
+/**
+ * Represents the configuration for a step that writes a table from the tablespace to a CSV file.
+ */
+export interface WriteCsvStep extends BaseFileWriteStep {
+  /** The type of the step, which is always 'write_csv' for this operation. */
+  type: 'write_csv';
   /** Optional: The delimiter character to use in the output CSV file. */
   delimiter?: string;
 }
 
+// Not yet supported, should be a normal write_json, but we don't have a lazy sink_json, can create a workaround
+// if needed.
+// /**
+//  * Represents the configuration for a step that writes a table from the tablespace to a JSON file.
+//  */
+// export interface WriteJsonStep extends BaseFileWriteStep {
+//   /** The type of the step, which is always 'write_json' for this operation. */
+//   type: 'write_json';
+// }
+
 /**
- * Represents the configuration for a step that writes a table from the tablespace to a JSON file.
+ * Represents the configuration for a step that writes a table from the tablespace to an NDJSON file.
  */
-export interface WriteJsonStep {
-  /** The type of the step, which is always 'write_json' for this operation. */
-  type: 'write_json';
-  /** The name of the table in the tablespace to be written. */
-  table: string;
-  /** Path to the output JSON file. */
-  file: string;
-  /** Optional: A list of column names to write to the JSON. If omitted, all columns are written. */
-  columns?: string[];
+export interface WriteNdjsonStep extends BaseFileWriteStep {
+  /** The type of the step, which is always 'write_ndjson' for this operation. */
+  type: 'write_ndjson';
 }