@platforma-open/milaboratories.software-ptabler.schema 1.12.0 → 1.12.2

package/src/expressions/pframes.ts

@@ -0,0 +1,118 @@
+import type { Expression } from './base';
+
+/**
+ * Represents a regex matching operation using ECMAScript regular expressions.
+ * Takes a string expression as input and returns a boolean indicating if the input value
+ * matches the provided ECMAScript regular expression.
+ */
+export interface MatchesEcmaRegexExpression {
+  /** The type of operation, always 'matches_ecma_regex'. */
+  type: 'matches_ecma_regex';
+  /** The string expression whose value will be compared. */
+  value: Expression;
+  /** The ECMAScript regular expression to match against. */
+  ecma_regex: string;
+}
+
+/**
+ * Represents a fuzzy string matching operation.
+ * Takes a string expression as input and returns a boolean indicating if the input value
+ * contains a close match to the provided reference string.
+ */
+export interface ContainsFuzzyMatchExpression {
+  /** The type of operation, always 'contains_fuzzy_match'. */
+  type: 'contains_fuzzy_match';
+  /** The string expression whose value will be compared. */
+  value: Expression;
+  /** The string reference to compare against. */
+  reference: string;
+  /** The maximum number of edits allowed to be considered a match. */
+  max_edits: number;
+  /** The wildcard character to use. */
+  wildcard?: string;
+  /** If true, only substitutions are allowed (deletions and insertions are also allowed by default). */
+  substitutions_only?: boolean;
+}
+
+/**
+ * Represents a regex replacement operation using ECMAScript regular expressions.
+ * Performs JavaScript String.prototype.replace() operation.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace
+ *
+ * @example
+ * Input: "result.POIS_P001_W104_PBMC_2.clns"
+ * Pattern: "^.*(P\\d+)_(W\\d+).*$"
+ * Replacement: "$2-$1"
+ * Result: "W104-P001"
+ */
+export interface ReplaceEcmaRegexExpression {
+  /** The type of operation, always 'replace_ecma_regex'. */
+  type: 'replace_ecma_regex';
+  /** The string expression whose value will be replaced. */
+  value: Expression;
+  /**
+   * String representing ECMAScript RegEx with at least one capturing group.
+   * If you need to reorder capturing groups - use RegExp matching the whole string
+   * (must start with string begin anchor ^, end with string end anchor $).
+   * Use regex playground https://regexr.com/ to test your ideas.
+   */
+  ecma_regex: string;
+  /**
+   * Replacement pattern used to construct result string from captured groups.
+   * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement
+   * Empty string as result would become NA.
+   */
+  replacement: string;
+}
+
+/**
+ * Represents a regex extraction operation using ECMAScript regular expressions.
+ * Simplified 'regexpReplace' with replacement set to $1.
+ * This means that string is replaced with first capture group value.
+ *
+ * RegEx must match the entire string, this would be enforced even when ^ and $ are skipped.
+ * If there are no matches - value would be replaced with empty string.
+ *
+ * @example
+ * // Example 1:
+ * Input: "123___abc.xlsx"
+ * Pattern: "\\d+___([a-z]+).xlsx"
+ * Result: "abc"
+ *
+ * @example
+ * // Example 2:
+ * Input: "123___abc.xlsx"
+ * Pattern: "(\\d+)___([a-z]+).xlsx"
+ * Result: "123"
+ *
+ * @example
+ * // Example 3:
+ * Input: "123___abc.xlsx"
+ * Pattern: "((\\d+)___([a-z]+)).xlsx"
+ * Result: "123___abc"
+ *
+ * @example
+ * // Wrong example (pattern doesn't match entire string):
+ * Input: "123___abc.xlsx"
+ * Pattern: "(\\d+___[a-z]+)"
+ * Result: "" (empty string, as .xlsx part is missing in pattern, so pattern was not matched)
+ *
+ * @example
+ * // Correct example:
+ * Input: "123___abc.xlsx"
+ * Pattern: "(\\d+___[a-z]+).xlsx"
+ * Result: "123___abc"
+ */
+export interface ExtractEcmaRegexExpression {
+  /** The type of operation, always 'extract_ecma_regex'. */
+  type: 'extract_ecma_regex';
+  /** The string expression whose value will be extracted. */
+  value: Expression;
+  /**
+   * String representing ECMAScript RegEx with at least one capturing group.
+   * RegEx must match the entire string, this would be enforced even when ^ and $ are skipped.
+   * If there are no matches - value would be replaced with empty string.
+   */
+  ecma_regex: string;
+}

package/src/expressions/selectors.ts

@@ -0,0 +1,203 @@
+import type { AxisSpec } from '@milaboratories/pl-model-common';
+
+/**
+ * Represents a selector for all columns.
+ * Selects all columns in the DataFrame.
+ */
+export interface AllSelectorExpression {
+  /** The type of operation, always 'all'. */
+  type: 'selector_all';
+}
+
+/**
+ * Represents a selector for string columns.
+ * Selects all columns with string data types.
+ */
+export interface StringSelectorExpression {
+  /** The type of operation, always 'string'. */
+  type: 'selector_string';
+}
+
+/**
+ * Represents a selector for numeric columns.
+ * Selects all columns with numeric data types (integers and floats).
+ */
+export interface NumericSelectorExpression {
+  /** The type of operation, always 'numeric'. */
+  type: 'selector_numeric';
+}
+
+/**
+ * Represents a selector for integer columns.
+ * Selects all columns with integer data types.
+ */
+export interface IntegerSelectorExpression {
+  /** The type of operation, always 'integer'. */
+  type: 'selector_integer';
+}
+
+/**
+ * Represents a selector for float columns.
+ * Selects all columns with floating-point data types.
+ */
+export interface FloatSelectorExpression {
+  /** The type of operation, always 'float'. */
+  type: 'selector_float';
+}
+
+/**
+ * Represents a selector for columns that start with a specific prefix.
+ * Selects columns whose names start with the specified prefix string.
+ */
+export interface StartsWithSelectorExpression {
+  /** The type of operation, always 'starts_with'. */
+  type: 'selector_starts_with';
+  /** The prefix to match column names against. */
+  prefix: string;
+}
+
+/**
+ * Represents a selector for columns that end with a specific suffix.
+ * Selects columns whose names end with the specified suffix string.
+ */
+export interface EndsWithSelectorExpression {
+  /** The type of operation, always 'ends_with'. */
+  type: 'selector_ends_with';
+  /** The suffix to match column names against. */
+  suffix: string;
+}
+
+/**
+ * Represents a selector for columns that contain a specific substring.
+ * Selects columns whose names contain the specified substring.
+ */
+export interface ContainsSelectorExpression {
+  /** The type of operation, always 'contains'. */
+  type: 'selector_contains';
+  /** The substring to match within column names. */
+  substring: string;
+}
+
+/**
+ * Represents a selector for columns that match a regex pattern.
+ * Selects columns whose names match the specified regular expression pattern.
+ */
+export interface MatchesSelectorExpression {
+  /** The type of operation, always 'matches'. */
+  type: 'selector_matches';
+  /** The regex pattern to match column names against. */
+  pattern: string;
+}
+
+/**
+ * Represents a selector that excludes specific columns by name.
+ * Selects all columns except those explicitly listed.
+ */
+export interface ExcludeSelectorExpression {
+  /** The type of operation, always 'exclude'. */
+  type: 'selector_exclude';
+  /** The list of column names to exclude from selection. */
+  columns: string[];
+}
+
+/**
+ * Represents a selector for columns by their exact names.
+ * Selects columns that match any of the specified names.
+ */
+export interface ByNameSelectorExpression {
+  /** The type of operation, always 'by_name'. */
+  type: 'selector_by_name';
+  /** The list of column names to select. */
+  names: string[];
+}
+
+/**
+ * Represents a selector for axis.
+ * Selects axis with the given spec if it exists in the result.
+ */
+export interface AxisSelectorExpression {
+  /** The type of operation, always 'axis'. */
+  type: 'selector_axis';
+  /** The axis to select. */
+  axis: AxisSpec;
+}
+
+/**
+ * Represents a selector for nested columns.
+ * Selects columns that have nested/complex data types (e.g., structs, lists).
+ */
+export interface NestedSelectorExpression {
+  /** The type of operation, always 'nested'. */
+  type: 'selector_nested';
+}
+
+/** Defines all available selector expression types. */
+export type SelectorExpression =
+  | AllSelectorExpression
+  | StringSelectorExpression
+  | NumericSelectorExpression
+  | IntegerSelectorExpression
+  | FloatSelectorExpression
+  | StartsWithSelectorExpression
+  | EndsWithSelectorExpression
+  | ContainsSelectorExpression
+  | MatchesSelectorExpression
+  | ExcludeSelectorExpression
+  | ByNameSelectorExpression
+  | AxisSelectorExpression
+  | NestedSelectorExpression;
+
+/**
+ * Represents the complement of a selector.
+ * Selects all columns that are NOT selected by the inner selector.
+ */
+export interface SelectorComplementExpression {
+  /** The type of operation, always 'selector_complement'. */
+  type: 'selector_complement';
+  /** The selector to complement. */
+  selector: SelectorExpression;
+}
+
+/**
+ * Represents the union of multiple selectors.
+ * Selects columns that match ANY of the provided selectors (logical OR).
+ */
+export interface SelectorUnionExpression {
+  /** The type of operation, always 'selector_union'. */
+  type: 'selector_union';
+  /** The list of selectors to union. */
+  selectors: SelectorExpression[];
+}
+
+/**
+ * Represents the intersection of multiple selectors.
+ * Selects columns that match ALL of the provided selectors (logical AND).
+ */
+export interface SelectorIntersectionExpression {
+  /** The type of operation, always 'selector_intersection'. */
+  type: 'selector_intersection';
+  /** The list of selectors to intersect. */
+  selectors: SelectorExpression[];
+}
+
+/**
+ * Represents the difference of multiple selectors.
+ * Selects columns from the first selector minus those selected by subsequent selectors.
+ */
+export interface SelectorDifferenceExpression {
+  /** The type of operation, always 'selector_difference'. */
+  type: 'selector_difference';
+  /** The list of selectors to apply difference operation. */
+  selectors: SelectorExpression[];
+}
+
+/**
+ * Represents the symmetric difference of multiple selectors.
+ * Selects columns that are in an odd number of the provided selectors (logical XOR).
+ */
+export interface SelectorSymmetricDifferenceExpression {
+  /** The type of operation, always 'selector_symmetric_difference'. */
+  type: 'selector_symmetric_difference';
+  /** The list of selectors to apply symmetric difference operation. */
+  selectors: SelectorExpression[];
+}

package/src/expressions/string.ts

@@ -0,0 +1,168 @@
+import type { Expression } from './base';
+
+/** Represents a string join operation on an array of expressions. */
+export interface StringJoinExpression {
+  /** The type of operation, always 'str_join'. */
+  type: 'str_join';
+  /** An array of expressions whose string representations will be joined. */
+  operands: Expression[];
+  /** An optional delimiter string to insert between joined elements. */
+  delimiter?: string;
+}
+
+/** Defines the supported unary string operators. */
+export type UnaryStringOperator = 'to_upper' | 'to_lower';
+
+/** Represents a unary string operation on a single expression. */
+export interface ExtendedUnaryStringExpression {
+  /** The type of unary string operation (e.g., 'to_upper', 'to_lower', 'str_len'). */
+  type: UnaryStringOperator | 'str_len';
+  /** The string expression to operate on. */
+  value: Expression;
+}
+
+/**
+ * Represents a substring extraction operation on an expression.
+ * Extracts a portion of the string value resulting from the 'value' expression.
+ * The substring starts at the 'start' index (0-based).
+ * - If 'length' is provided, it specifies the maximum length of the substring.
+ * - If 'end' is provided, it specifies the index *before* which the substring ends.
+ * - If neither 'length' nor 'end' is provided, the substring extends to the end of the string.
+ * - 'length' and 'end' are mutually exclusive.
+ * If the requested substring range extends beyond the actual string length,
+ * the extraction automatically stops at the end of the string.
+ */
+export interface SubstringExpression {
+  /** The type of operation, always 'substring'. */
+  type: 'substring';
+  /** The expression whose string value will be used. */
+  value: Expression;
+  /** The starting position (0-indexed). Should evaluate to a number. */
+  start: Expression;
+  /** The length of the substring. Mutually exclusive with 'end'. Should evaluate to a number. */
+  length?: Expression;
+  /** The end position of the substring (exclusive). Mutually exclusive with 'length'. Should evaluate to a number. */
+  end?: Expression;
+}
+
+/**
+ * Represents a string replacement operation.
+ * Replaces occurrences of a pattern (regex or literal) in a string expression with a replacement string.
+ * The behavior is aligned with Polars' `replace` and `replace_all` functions.
+ *
+ * - If `literal` is true, the `pattern` is treated as a literal string. Otherwise, it's treated as a regular expression.
+ * - If `replaceAll` is true, all occurrences of the pattern are replaced. Otherwise, only the first occurrence is replaced.
+ *
+ * When using regular expressions (i.e., `literal` is false or undefined):
+ * - Positional capture groups can be referenced in the `replacement` string using `$n` or `${n}` (e.g., `$1` for the first group).
+ * - Named capture groups can be referenced using `${name}`.
+ * - To include a literal dollar sign (`$`) in the replacement, it must be escaped as `$$`.
+ */
+export interface StringReplaceExpression {
+  /** The type of operation, always 'str_replace'. */
+  type: 'str_replace';
+  /** The input string expression to operate on. */
+  value: Expression;
+  /** The pattern (regex or literal string) to search for. Can be a string literal or an expression evaluating to a string. */
+  pattern: Expression | string;
+  /** The replacement string. Can be a string literal or an expression evaluating to a string. Can use $n or ${name} for captured groups if pattern is a regex. */
+  replacement: Expression | string;
+  /** If true, replace all occurrences of the pattern. If false or undefined, replace only the first. Defaults to false. */
+  replaceAll?: boolean;
+  /** If true, treat the pattern as a literal string. If false or undefined, treat it as a regex. Defaults to false. */
+  literal?: boolean;
+}
+
+/**
+ * Represents a string contains operation.
+ * Checks if the string contains a substring that matches a pattern using regex or literal matching.
+ * Based on polars.Series.str.contains - supports both regex and literal pattern matching with optional case-insensitive flags.
+ */
+export interface StringContainsExpression {
+  /** The type of operation, always 'str_contains'. */
+  type: 'str_contains';
+  /** The input string expression to search in. */
+  value: Expression;
+  /** The pattern to search for. Can be a regex pattern (default) or literal string when literal=true. */
+  pattern: Expression | string;
+  /** If true, treat the pattern as a literal string. If false, treat it as a regex pattern. Defaults to false. */
+  literal?: boolean;
+  /** If true, raise an error if pattern is invalid regex. If false, return null for invalid patterns. Defaults to true. */
+  strict?: boolean;
+}
+
+/**
+ * Represents a string starts_with operation.
+ * Checks if the string starts with a specified prefix. Always uses literal matching (no regex support).
+ * Based on polars.Series.str.starts_with - only supports literal prefix matching.
+ */
+export interface StringStartsWithExpression {
+  /** The type of operation, always 'str_starts_with'. */
+  type: 'str_starts_with';
+  /** The input string expression to check. */
+  value: Expression;
+  /** The prefix to check for (always treated as literal string, no regex support). */
+  prefix: Expression | string;
+}
+
+/**
+ * Represents a string ends_with operation.
+ * Checks if the string ends with a specified suffix. Always uses literal matching (no regex support).
+ * Based on polars.Series.str.ends_with - only supports literal suffix matching.
+ */
+export interface StringEndsWithExpression {
+  /** The type of operation, always 'str_ends_with'. */
+  type: 'str_ends_with';
+  /** The input string expression to check. */
+  value: Expression;
+  /** The suffix to check for (always treated as literal string, no regex support). */
+  suffix: Expression | string;
+}
+
+/**
+ * Represents a string contains_any operation using the Aho-Corasick algorithm.
+ * Checks if the string contains any of the provided patterns using fast multi-pattern string matching.
+ * Based on polars.Series.str.contains_any - uses Aho-Corasick algorithm for efficient multi-pattern matching.
+ */
+export interface StringContainsAnyExpression {
+  /** The type of operation, always 'str_contains_any'. */
+  type: 'str_contains_any';
+  /** The input string expression to search in. */
+  value: Expression;
+  /** Array of literal string patterns to search for. Only immediate string values are supported, no expressions or regex patterns. */
+  patterns: string[];
+  /** Enable ASCII-aware case insensitive matching. When enabled, searching is performed without respect to case for ASCII letters (a-z and A-Z) only. Defaults to false. */
+  asciiCaseInsensitive?: boolean;
+}
+
+/**
+ * Represents a string count_matches operation.
+ * Counts the number of times a pattern occurs in the string using regex or literal matching.
+ * Based on polars.Series.str.count_matches - supports both regex and literal pattern matching.
+ */
+export interface StringCountMatchesExpression {
+  /** The type of operation, always 'str_count_matches'. */
+  type: 'str_count_matches';
+  /** The input string expression to count matches in. */
+  value: Expression;
+  /** The pattern to count occurrences of. Can be a regex pattern (default) or literal string when literal=true. */
+  pattern: Expression | string;
+  /** If true, treat the pattern as a literal string. If false, treat it as a regex pattern. Defaults to false. */
+  literal?: boolean;
+}
+
+/**
+ * Represents a string extract operation using regex patterns.
+ * Extracts the first match of a regex pattern from the string, optionally targeting specific capture groups.
+ * Based on polars.Series.str.extract - only supports regex patterns (no literal mode).
+ */
+export interface StringExtractExpression {
+  /** The type of operation, always 'str_extract'. */
+  type: 'str_extract';
+  /** The input string expression to extract from. */
+  value: Expression;
+  /** The regex pattern to extract. Must be a valid regex pattern - no literal string mode is supported. */
+  pattern: Expression | string;
+  /** The capture group index to extract. Group 0 is the entire match, group 1 is the first capture group, etc. Defaults to 0. */
+  groupIndex?: number;
+}

package/src/expressions/struct.ts

@@ -0,0 +1,37 @@
+import type { DataType } from '../common';
+import type { Expression } from './base';
+
+/**
+ * Represents a struct field access operation.
+ * This operation retrieves a single field from a struct (nested data structure).
+ *
+ * Uses native Polars struct.field() functionality when possible for optimal performance,
+ * but falls back to Python UDF (map_elements) when dtype casting or default values
+ * are specified, trading performance for robust handling of missing fields and null structs.
+ *
+ * When fields is an array, the operation performs recursive field access,
+ * where each element in the array represents a level in the nested structure.
+ */
+export interface StructFieldExpression {
+  /** The type of operation, always 'struct_field'. */
+  type: 'struct_field';
+  /** The struct expression to extract fields from. */
+  struct: Expression;
+  /**
+   * The field name(s) to extract from the struct.
+   * - If a string, extracts a single field from the struct.
+   * - If an array, performs recursive field access where each element represents a level in the nested structure.
+   */
+  fields: string | string[];
+  /**
+   * Optional expected data type for the returned value.
+   * This can be used for type validation or casting of the extracted field.
+   */
+  dtype?: DataType;
+  /**
+   * Optional default value to return if the field is not found or is null.
+   * If not provided and the field is missing, the operation returns null.
+   * Only constant scalar values are supported.
+   */
+  default?: string | number | boolean | null;
+}

package/src/expressions/window.ts

@@ -0,0 +1,66 @@
+import type { Expression } from './base';
+
+/**
+ * Represents a rank function applied over a dataset partition.
+ * Calculates the rank of each row within its partition based on the specified ordering.
+ */
+export interface RankExpression {
+  /** The type of operation, always 'rank'. */
+  type: 'rank';
+  /** List of expressions to partition the data by before ranking. The output of these expressions will be used for partitioning. */
+  partitionBy: Expression[];
+  /** Defines the ordering expressions within partitions to determine the rank. */
+  orderBy: Expression[];
+  /** Whether to sort in descending order. Defaults to false (ascending). */
+  descending?: boolean;
+}
+
+/**
+ * Represents a cumulative sum function applied over a dataset partition.
+ * Calculates the cumulative sum of the 'value' expression within each partition,
+ * based on the specified ordering. Values are sorted by value and then by
+ * additional_order_by before summing.
+ */
+export interface CumsumExpression {
+  /** The type of operation, always 'cumsum'. */
+  type: 'cumsum';
+  /** The expression whose values will be cumulatively summed. */
+  value: Expression;
+  /** Defines additional ordering within partitions for the cumulative sum calculation, in addition to the ordering of the values themselves. */
+  additionalOrderBy: Expression[];
+  /** List of expressions to partition the data by before calculating the cumulative sum. The output of these expressions will be used for partitioning. */
+  partitionBy: Expression[];
+  /** Whether to sort in descending order. Defaults to false (ascending). */
+  descending?: boolean;
+}
+
+/**
+ * Defines standard aggregation functions that can be used in window expressions.
+ */
+export type AggregationType =
+  | 'sum'
+  | 'mean'
+  | 'median'
+  | 'min'
+  | 'max'
+  | 'std'
+  | 'var'
+  | 'count'
+  | 'first'
+  | 'last'
+  | 'n_unique';
+
+/**
+ * Represents a window function call.
+ * This allows applying an aggregation function over a specific partition of the data.
+ */
+export interface WindowExpression {
+  /** The type of operation, always 'aggregate'. Note: This might be confusing, consider 'window_aggregate' or similar if 'aggregate' is heavily used elsewhere for a different step type. */
+  type: 'aggregate';
+  /** The aggregation function to apply (e.g., 'sum', 'mean'). */
+  aggregation: AggregationType;
+  /** The expression to apply the aggregation function to. */
+  value: Expression;
+  /** List of expressions to partition the data by. The aggregation is performed independently within each partition. */
+  partitionBy: Expression[];
+}

package/src/index.ts

@@ -1,31 +1,61 @@
-import type { ReadCsvStep, ReadNdjsonStep, WriteCsvStep, WriteNdjsonStep, BaseFileReadStep, BaseFileWriteStep } from './io';
-import type { AddColumnsStep, FilterStep, SelectStep, WithColumnsStep, WithoutColumnsStep } from './basic_steps';
+import type {
+  ReadCsvStep,
+  ReadNdjsonStep,
+  WriteCsvStep,
+  WriteNdjsonStep,
+  BaseFileReadStep,
+  BaseFileWriteStep,
+  WriteParquetStep,
+  ReadParquetStep,
+} from './io';
+import type {
+  AddColumnsStep,
+  FilterStep,
+  LimitStep,
+  SelectStep,
+  WithColumnsStep,
+  WithoutColumnsStep,
+} from './basic_steps';
 import type { AggregateStep } from './aggregate';
 import type { AnyJoinStep } from './join';
 import type { ConcatenateStep } from './concatenate';
 import type { SortStep } from './sort';
+import type { WriteFrameStep } from './write_frame';
+import type { ReadFrameStep } from './read_frame';
 
 export type PTablerStep =
   | ReadCsvStep
   | ReadNdjsonStep
+  | ReadParquetStep
   | WriteCsvStep
   | WriteNdjsonStep
+  | WriteParquetStep
   | AddColumnsStep
   | FilterStep
+  | LimitStep
   | AggregateStep
   | AnyJoinStep
   | ConcatenateStep
   | SortStep
   | SelectStep
   | WithColumnsStep
-  | WithoutColumnsStep;
+  | WithoutColumnsStep
+  | WriteFrameStep
+  | ReadFrameStep;
 
 export type PTablerWorkflow = {
   workflow: PTablerStep[];
 };
 
 // Re-export base interfaces for potential external use
-export type { BaseFileReadStep, BaseFileWriteStep };
+export type {
+  AddColumnsStep, AggregateStep,
+  AnyJoinStep, BaseFileReadStep,
+  BaseFileWriteStep, ConcatenateStep, FilterStep, ReadCsvStep,
+  ReadNdjsonStep, SelectStep, SortStep, WithColumnsStep,
+  WithoutColumnsStep, WriteCsvStep,
+  WriteNdjsonStep,
+};
 
 // Re-export expression types for external use
-export type { Expression, StructFieldExpression } from './expressions';
+export type * from './expressions';