@platforma-open/milaboratories.software-ptabler.schema 1.12.0 → 1.12.2

package/dist/write_frame.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Axes values are used as unique key to access the column data,
+ * so they cannot be floating point numbers.
+ */
+export type AxisType = 'Int' | 'Long' | 'String';
+/** PColumn values could be of any type supported by PFrames. */
+export type ColumnType = 'Int' | 'Long' | 'Float' | 'Double' | 'String';
+export interface AxisMapping {
+    /** The name of the column in the input table. */
+    column: string;
+    /**
+     * The type of the axis to be created.
+     * A cast would be performed if the type of the column in the input table is different.
+     * Warning: cast failure will result in an error.
+     */
+    type: AxisType;
+}
+export interface ColumnMapping {
+    /** The name of the column in the input table. */
+    column: string;
+    /**
+     * The value type of the PColumn to be created.
+     * A cast would be performed if the type of the column in the input table is different.
+     * Warning: cast failure will produce null values in the PColumn.
+     */
+    type: ColumnType;
+}
+/** Defines a step that creates a PFrame from the lazy table. */
+export interface WriteFrameStep {
+    /** The type identifier for this step. */
+    type: 'write_frame';
+    /** The name of the table from the tablespace which will be written. */
+    inputTable: string;
+    /**
+     * The name of the frame to be created.
+     * A folder with this name will be created in the working directory.
+     * The folder will contain the .datainfo and .parquet files representing the frame's PColumns.
+     */
+    frameName: string;
+    /**
+     * Axes specs, the order of axes in PColumns will match the order of axes in this array.
+     * Each column in the input table could be used as axis or PColumn only once.
+     * Warning: rows with null values in axes columns of input table will be discarded.
+     */
+    axes: AxisMapping[];
+    /**
+     * PColumns specs.
+     * Each column in the input table could be used as axis or PColumn only once.
+     */
+    columns: ColumnMapping[];
+    /**
+     * Partition key length, specifies the number of axes to be split into metadata partition key.
+     * Default: 0, which means no partitioning.
+     * Must be strictly less than the number of axes.
+     */
+    partitionKeyLength: number;
+}
+//# sourceMappingURL=write_frame.d.ts.map

package/dist/write_frame.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"write_frame.d.ts","sourceRoot":"","sources":["../src/write_frame.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEjD,gEAAgE;AAChE,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAExE,MAAM,WAAW,WAAW;IAC1B,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,IAAI,EAAE,QAAQ,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,gEAAgE;AAChE,MAAM,WAAW,cAAc;IAC7B,yCAAyC;IACzC,IAAI,EAAE,aAAa,CAAC;IAEpB,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;OAIG;IACH,IAAI,EAAE,WAAW,EAAE,CAAC;IAEpB;;;OAGG;IACH,OAAO,EAAE,aAAa,EAAE,CAAC;IAEzB;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,CAAC;CAC5B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-open/milaboratories.software-ptabler.schema",
-  "version": "1.12.0",
+  "version": "1.12.2",
   "description": "Type definitions for PTabler",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -16,17 +16,19 @@
     "./dist/**/*",
     "./src/**/*"
   ],
-  "dependencies": {},
+  "dependencies": {
+    "@milaboratories/pl-model-common": "1.21.6"
+  },
   "devDependencies": {
-    "@platforma-sdk/eslint-config": "^1.0.3",
-    "vite-plugin-dts": "^4.5.4",
-    "eslint": "^9.27.0",
-    "typescript": "~5.5.4",
-    "vite": "^6.3.5"
+    "eslint": "^9.25.1",
+    "typescript": "~5.6.3",
+    "@milaboratories/ts-configs": "1.0.6",
+    "@milaboratories/ts-builder": "1.0.5",
+    "@platforma-sdk/eslint-config": "1.1.0"
   },
   "scripts": {
-    "type-check": "tsc --noEmit --composite false",
-    "build": "vite build",
+    "build": "ts-builder build --target node",
+    "type-check": "ts-builder types --target node",
     "lint": "eslint .",
     "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz"
   }

package/src/aggregate.ts

@@ -1,21 +1,5 @@
 import type { Expression, AggregationType } from './expressions';
 
-/**
- * Defines standard aggregation functions that operate on a single expression.
- */
-export type StandardAggregationType =
-  | 'sum'
-  | 'mean'
-  | 'median'
-  | 'min'
-  | 'max'
-  | 'std'
-  | 'var'
-  | 'count'
-  | 'first'
-  | 'last'
-  | 'n_unique';
-
 /**
  * Defines aggregation functions that select a value from one expression based on the min/max of another expression.
  */

package/src/basic_steps.ts

@@ -20,18 +20,7 @@ export interface AddColumnsStep {
    * An array defining the new columns to be added.
    * Each object in the array specifies the name of a new column and the expression to compute its values.
    */
-  columns: {
-    /**
-     * The name of the new column.
-     */
-    name: string;
-
-    /**
-     * An Expression object defining how to compute the column's values.
-     * The expression will be evaluated for each row to generate the values for the new column.
-     */
-    expression: Expression;
-  }[];
+  columns: Expression[];
 }
 
 /**
@@ -93,19 +82,7 @@ export interface SelectStep {
    * Each object in the array specifies the name of a column in the output table
    * and the expression to compute its values.
    */
-  columns: {
-    /**
-     * The name of the column in the output table.
-     */
-    name: string;
-
-    /**
-     * An Expression object defining how to compute the column's values.
-     * This expression will be evaluated to generate the values for this column
-     * in the output table.
-     */
-    expression: Expression;
-  }[];
+  columns: Expression[];
 }
 
 /**
@@ -138,18 +115,7 @@ export interface WithColumnsStep {
    * Each object in the array specifies the name of a column and the
    * expression to compute its values.
    */
-  columns: {
-    /**
-     * The name of the new or replacement column.
-     */
-    name: string;
-
-    /**
-     * An Expression object defining how to compute the column's values.
-     * The expression will be evaluated for each row to generate the values for the column.
-     */
-    expression: Expression;
-  }[];
+  columns: Expression[];
 }
 
 /**
@@ -180,3 +146,32 @@ export interface WithoutColumnsStep {
    */
   columns: string[];
 }
+
+/**
+ * Defines a step that limits the number of rows in a table
+ * and outputs the result to a new table in the tablespace.
+ * This operation is similar to Polars' `limit` method.
+ */
+export interface LimitStep {
+  /**
+   * The type identifier for this step.
+   * Must be 'limit'.
+   */
+  type: 'limit';
+
+  /**
+   * The name of the input table in the tablespace from which rows will be limited.
+   */
+  inputTable: string;
+
+  /**
+   * The name for the resulting table that will be added to the tablespace.
+   * This new table will contain only the first n rows from the input table.
+   */
+  outputTable: string;
+
+  /**
+   * The maximum number of rows to include in the output table.
+   */
+  n: number;
+}

package/src/expressions/base.ts

@@ -0,0 +1,5 @@
+/**
+ * Forward declaration of the Expression type.
+ * The real type is defined in [index.ts](./index.ts) file to avoid circular dependencies.
+ */
+export { type Expression } from './index';

package/src/expressions/basics.ts

@@ -0,0 +1,163 @@
+import type { AxisSpec } from '@milaboratories/pl-model-common';
+import type { DataType } from '../common';
+import type { Expression } from './base';
+
+/** Represents all possible comparison operator types. */
+export type ComparisonOperator = 'gt' | 'ge' | 'eq' | 'lt' | 'le' | 'neq';
+
+/** Defines a comparison operation between two expressions. */
+export interface ComparisonExpression {
+  /** The type of comparison (e.g., 'gt', 'eq'). */
+  type: ComparisonOperator;
+  /** The left-hand side expression. */
+  lhs: Expression;
+  /** The right-hand side expression. */
+  rhs: Expression;
+}
+
+/** Defines the supported binary arithmetic operators. */
+export type BinaryArithmeticOperator =
+  | 'plus'
+  | 'minus'
+  | 'multiply'
+  | 'truediv'
+  | 'floordiv';
+
+/** Represents a binary arithmetic operation between two expressions. */
+export interface BinaryArithmeticExpression {
+  /** The type of arithmetic operation (e.g., 'plus', 'minus'). */
+  type: BinaryArithmeticOperator;
+  /** The left-hand side expression. */
+  lhs: Expression;
+  /** The right-hand side expression. */
+  rhs: Expression;
+}
+
+/** Defines the supported unary arithmetic operators. */
+export type UnaryArithmeticOperator =
+  | 'log10'
+  | 'log'
+  | 'log2'
+  | 'abs'
+  | 'sqrt'
+  | 'negate'
+  | 'floor'
+  | 'round'
+  | 'ceil';
+
+/** Represents a unary arithmetic operation on a single expression. */
+export interface UnaryArithmeticExpression {
+  /** The type of unary operation (e.g., 'log10', 'abs'). */
+  type: UnaryArithmeticOperator;
+  /** The expression to operate on. */
+  value: Expression;
+}
+
+/**
+ * Represents a type casting operation that converts the result of an expression to a specified data type.
+ */
+export interface CastExpression {
+  /** The type of operation, always 'cast'. */
+  type: 'cast';
+  /** The expression whose result will be cast to the target data type. */
+  value: Expression;
+  /** The target data type to cast the expression result to. */
+  dtype: DataType;
+  /**
+   * Whether to use strict casting mode. If true, conversion errors and overflows will throw exceptions.
+   * If false or undefined, uses non-strict mode where failures result in null values. Defaults to false.
+   */
+  strict?: boolean;
+}
+
+/** Defines the supported boolean list operators. */
+export type BooleanListOperator = 'and' | 'or';
+
+/** Represents a boolean logic operation (AND, OR) on a list of expressions. */
+export interface BooleanLogicExpression {
+  /** The type of boolean operation ('and', 'or'). */
+  type: BooleanListOperator;
+  /** An array of boolean expressions as operands. */
+  operands: Expression[]; // Array of boolean expressions
+}
+
+/** Represents a logical NOT operation on a single boolean expression. */
+export interface NotExpression {
+  /** The type of operation, always 'not'. */
+  type: 'not';
+  /** The boolean expression to negate. */
+  value: Expression;
+}
+
+/** Defines the supported null check operators. */
+export type NullCheckOperator = 'is_na' | 'is_not_na';
+
+/** Represents a null check operation (is NA, is not NA) on an expression. */
+export interface NullCheckExpression {
+  /** The type of null check ('is_na', 'is_not_na'). */
+  type: NullCheckOperator;
+  /** The expression to check for nullity. */
+  value: Expression;
+}
+
+/** Represents a reference to a column by its name. */
+export interface ColumnReferenceExpression {
+  /** The type of operation, always 'col'. */
+  type: 'col';
+  /** The name of the column to reference. */
+  name: string;
+}
+
+/** Represents a reference to an axis by its specification (or id). */
+export interface AxisReferenceExpression {
+  /** The type of operation, always 'axis'. */
+  type: 'axis';
+  /** The axis to reference. */
+  spec: AxisSpec;
+}
+
+/** Represents a constant literal value (string, number, boolean, or null). */
+export interface ConstantValueExpression {
+  /** The type of operation, always 'const'. */
+  type: 'const';
+  /** The constant value. */
+  value: string | number | boolean | null;
+}
+
+/** Defines the supported min/max operators. */
+export type MinMaxOperator = 'min' | 'max';
+
+/** Represents a min or max operation on a list of expressions. */
+export interface MinMaxExpression {
+  /** The type of operation ('min' or 'max'). */
+  type: MinMaxOperator;
+  /** An array of expressions to find the minimum or maximum value from. */
+  operands: Expression[];
+}
+
+/**
+ * Represents an "in set" operation.
+ * Checks if the value of an expression is contained within a specified set of values.
+ * Returns true if the expression's value is found in the set, false otherwise.
+ */
+export interface InSetExpression {
+  /** The type of operation, always 'in_set'. */
+  type: 'in_set';
+  /** The expression whose value will be checked for membership in the set. */
+  value: Expression;
+  /** The set of values to check membership against. */
+  set: (string | number | boolean | null)[];
+}
+
+/**
+ * Represents an alias operation.
+ * This operation creates a new expression with a specified alias.
+ */
+export interface AliasExpression {
+  /** The type of operation, always 'alias'. */
+  type: 'alias';
+  /** The expression to alias. */
+  value: Expression;
+  /** The alias name. */
+  name: string;
+}

package/src/expressions/conditional.ts

@@ -0,0 +1,59 @@
+import type { Expression } from './base';
+
+/**
+ * Represents a single "when" condition and its corresponding "then" result expression.
+ * Used within the WhenThenOtherwiseExpression.
+ */
+export interface WhenThenClause {
+  /** The condition expression. Should evaluate to a boolean. */
+  when: Expression;
+  /** The result expression if the 'when' condition is true. */
+  then: Expression;
+}
+
+/**
+ * Represents a conditional expression that evaluates a series of "when"
+ * conditions and returns the corresponding "then" expression's value.
+ * If no "when" condition is met, it returns the value of the "otherwise" expression.
+ * This mimics Polars' when/then/otherwise functionality.
+ */
+export interface WhenThenOtherwiseExpression {
+  /** The type of operation, always 'when_then_otherwise'. */
+  type: 'when_then_otherwise';
+  /** An array of "when/then" clauses to be evaluated in order. */
+  conditions: WhenThenClause[];
+  /** The expression whose value is returned if none of the "when" conditions are met. */
+  otherwise: Expression;
+}
+
+/**
+ * Represents a fill null operation.
+ * If the 'input' expression evaluates to null, the 'fillValue' expression is used.
+ * Otherwise, the 'input' expression's value is used.
+ * This is a convenience shortcut for a common pattern often implemented with
+ * conditional expressions (e.g., when(is_null(input), fillValue).otherwise(input)).
+ */
+export interface FillNullExpression {
+  /** The type of operation, always 'fill_null'. */
+  type: 'fill_null';
+  /** The primary expression to evaluate. */
+  input: Expression;
+  /** The expression whose value is used if 'input' is null. */
+  fillValue: Expression;
+}
+
+/**
+ * Represents a fill NaN operation.
+ * If the 'input' expression evaluates to NaN, the 'fillValue' expression is used.
+ * Otherwise, the 'input' expression's value is used.
+ * This is a convenience shortcut for a common pattern often implemented with
+ * conditional expressions (e.g., when(is_nan(input), fillValue).otherwise(input)).
+ */
+export interface FillNaNExpression {
+  /** The type of operation, always 'fill_nan'. */
+  type: 'fill_nan';
+  /** The primary expression to evaluate. */
+  input: Expression;
+  /** The expression whose value is used if 'input' is NaN. */
+  fillValue: Expression;
+}

package/src/expressions/fuzzy.ts

@@ -0,0 +1,51 @@
+import type { Expression } from './base';
+
+/** Defines the supported string distance metrics. */
+export type StringDistanceMetric =
+  | 'levenshtein'
+  | 'optimal_string_alignment'
+  | 'jaro_winkler';
+
+/**
+ * Represents a string distance/similarity calculation between two expressions.
+ * Computes metrics like Levenshtein, Optimal String Alignment, or Jaro-Winkler.
+ */
+export interface StringDistanceExpression {
+  /** The type of operation, always 'string_distance'. */
+  type: 'string_distance';
+  /** The specific distance metric to use. */
+  metric: StringDistanceMetric;
+  /** The first string expression. */
+  string1: Expression;
+  /** The second string expression to compare against. */
+  string2: Expression;
+  /**
+   * If true, the expression returns a similarity score (typically normalized between 0 and 1).
+   * If false or undefined, it returns the raw edit distance (e.g., Levenshtein, OSA).
+   * Jaro-Winkler inherently returns a similarity score; this flag might be ignored or influence its normalization if applicable.
+   */
+  returnSimilarity?: boolean;
+}
+
+/** Defines the supported fuzzy string filter distance metrics. */
+export type FuzzyFilterDistanceMetric = 'levenshtein' | 'hamming';
+
+/**
+ * Represents a fuzzy string filter operation on an expression.
+ * This operation compares the string value of an expression (`value`)
+ * against another string or string expression (`pattern`) using a specified
+ * distance metric (`levenshtein` or `hamming`), returning true if the distance is
+ * within the specified `bound`.
+ */
+export interface FuzzyStringFilterExpression {
+  /** The type of operation, always 'fuzzy_string_filter'. */
+  type: 'fuzzy_string_filter';
+  /** The distance metric to use for the fuzzy comparison. */
+  metric: FuzzyFilterDistanceMetric;
+  /** The expression whose string value will be compared. */
+  value: Expression;
+  /** The expression representing the string pattern to compare against. */
+  pattern: Expression;
+  /** The maximum allowed distance for a match (inclusive). */
+  bound: number;
+}

package/src/expressions/hash.ts

@@ -0,0 +1,37 @@
+import type { Expression } from './base';
+
+/** Defines the supported hash types. Includes common cryptographic and non-cryptographic algorithms. */
+export type HashType =
+  | 'sha256' // Cryptographic
+  | 'sha512' // Cryptographic
+  | 'md5' // Cryptographic (use with caution due to vulnerabilities)
+  | 'blake3' // Cryptographic
+  | 'wyhash' // Non-cryptographic
+  | 'xxh3'; // Non-cryptographic
+
+/**
+ * Defines the encoding for the hash output.
+ * - 'hex': Standard hexadecimal encoding.
+ * - 'base64': Standard base64 encoding.
+ * - 'base64_alphanumeric': Base64 encoding with non-alphanumeric characters (e.g., '+', '/') removed.
+ * - 'base64_alphanumeric_upper': Base64 encoding with non-alphanumeric characters removed and the result converted to uppercase.
+ */
+export type HashEncoding =
+  | 'hex'
+  | 'base64'
+  | 'base64_alphanumeric'
+  | 'base64_alphanumeric_upper';
+
+/** Represents a hashing operation on an expression. */
+export interface HashExpression {
+  /** The specific type of hash algorithm to apply. */
+  type: 'hash';
+  /** The type of hash algorithm to apply. */
+  hashType: HashType;
+  /** The encoding for the output hash string. */
+  encoding: HashEncoding;
+  /** The expression whose value will be hashed. */
+  value: Expression;
+  /** Optional. Minimal number of entropy bits required. Affects encoding, truncating the result to the shortest string with the requested entropy. No error if bits exceed what the hash offers. */
+  bits?: number;
+}

package/src/expressions/index.ts

@@ -0,0 +1,147 @@
+export * from './base';
+export * from './basics';
+export * from './selectors';
+export * from './string';
+export * from './fuzzy';
+export * from './conditional';
+export * from './window';
+export * from './hash';
+export * from './struct';
+export * from './pframes';
+
+import type {
+  ComparisonExpression,
+  BinaryArithmeticExpression,
+  UnaryArithmeticExpression,
+  CastExpression,
+  BooleanLogicExpression,
+  NotExpression,
+  NullCheckExpression,
+  ColumnReferenceExpression,
+  AxisReferenceExpression,
+  ConstantValueExpression,
+  MinMaxExpression,
+  InSetExpression,
+  AliasExpression,
+} from './basics';
+
+import type {
+  AllSelectorExpression,
+  StringSelectorExpression,
+  NumericSelectorExpression,
+  IntegerSelectorExpression,
+  FloatSelectorExpression,
+  StartsWithSelectorExpression,
+  EndsWithSelectorExpression,
+  ContainsSelectorExpression,
+  MatchesSelectorExpression,
+  ExcludeSelectorExpression,
+  ByNameSelectorExpression,
+  AxisSelectorExpression,
+  NestedSelectorExpression,
+  SelectorComplementExpression,
+  SelectorUnionExpression,
+  SelectorIntersectionExpression,
+  SelectorDifferenceExpression,
+  SelectorSymmetricDifferenceExpression,
+} from './selectors';
+
+import type {
+  StringJoinExpression,
+  ExtendedUnaryStringExpression,
+  SubstringExpression,
+  StringReplaceExpression,
+  StringContainsExpression,
+  StringStartsWithExpression,
+  StringEndsWithExpression,
+  StringContainsAnyExpression,
+  StringCountMatchesExpression,
+  StringExtractExpression,
+} from './string';
+
+import type {
+  StringDistanceExpression,
+  FuzzyStringFilterExpression,
+} from './fuzzy';
+
+import type {
+  WhenThenOtherwiseExpression,
+  FillNullExpression,
+  FillNaNExpression,
+} from './conditional';
+
+import type {
+  RankExpression,
+  CumsumExpression,
+  WindowExpression,
+} from './window';
+
+import type { HashExpression } from './hash';
+import type { StructFieldExpression } from './struct';
+import type {
+  MatchesEcmaRegexExpression,
+  ContainsFuzzyMatchExpression,
+  ReplaceEcmaRegexExpression,
+  ExtractEcmaRegexExpression,
+} from './pframes';
+
+/**
+ * Represents all possible expression types in the system.
+ * This is the main union type that includes all concrete expression implementations.
+ */
+export type Expression =
+  | ComparisonExpression
+  | BinaryArithmeticExpression
+  | UnaryArithmeticExpression
+  | CastExpression
+  | BooleanLogicExpression
+  | NotExpression
+  | NullCheckExpression
+  | StringJoinExpression
+  | HashExpression
+  | ColumnReferenceExpression
+  | AxisReferenceExpression
+  | ConstantValueExpression
+  | RankExpression
+  | CumsumExpression
+  | ExtendedUnaryStringExpression
+  | StringDistanceExpression
+  | FuzzyStringFilterExpression
+  | WhenThenOtherwiseExpression
+  | SubstringExpression
+  | StringReplaceExpression
+  | StringContainsExpression
+  | StringStartsWithExpression
+  | StringEndsWithExpression
+  | StringContainsAnyExpression
+  | StringCountMatchesExpression
+  | StringExtractExpression
+  | MinMaxExpression
+  | FillNullExpression
+  | FillNaNExpression
+  | WindowExpression
+  | StructFieldExpression
+  | MatchesEcmaRegexExpression
+  | ContainsFuzzyMatchExpression
+  | ReplaceEcmaRegexExpression
+  | ExtractEcmaRegexExpression
+  | InSetExpression
+  | AliasExpression
+  | AllSelectorExpression
+  | StringSelectorExpression
+  | NumericSelectorExpression
+  | IntegerSelectorExpression
+  | FloatSelectorExpression
+  | StartsWithSelectorExpression
+  | EndsWithSelectorExpression
+  | ContainsSelectorExpression
+  | MatchesSelectorExpression
+  | ExcludeSelectorExpression
+  | ByNameSelectorExpression
+  | AxisSelectorExpression
+  | NestedSelectorExpression
+  | SelectorComplementExpression
+  | SelectorUnionExpression
+  | SelectorIntersectionExpression
+  | SelectorDifferenceExpression
+  | SelectorSymmetricDifferenceExpression;