@milaboratories/pl-model-common 1.19.4 → 1.19.6

package/dist/json.d.ts

@@ -3,14 +3,14 @@ type JsonValue = JsonPrimitive | JsonValue[] | {
     [key: string]: JsonValue;
 };
 type NotAssignableToJson = bigint | symbol | Function;
-export type JsonCompatible<T> = unknown extends T ? never : {
+export type JsonCompatible<T> = unknown extends T ? unknown : {
     [P in keyof T]: T[P] extends JsonValue ? T[P] : T[P] extends NotAssignableToJson ? never : JsonCompatible<T[P]>;
 };
-export type StringifiedJson<T> = JsonCompatible<T> extends never ? never : string & {
+export type StringifiedJson<T = unknown> = JsonCompatible<T> extends never ? never : string & {
     __json_stringified: T;
 };
 export declare function stringifyJson<T>(value: JsonCompatible<T>): StringifiedJson<T>;
-export type CanonicalizedJson<T> = JsonCompatible<T> extends never ? never : string & {
+export type CanonicalizedJson<T = unknown> = JsonCompatible<T> extends never ? never : string & {
     __json_canonicalized: T;
 };
 export declare function canonicalizeJson<T>(value: JsonCompatible<T>): CanonicalizedJson<T>;

package/dist/json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../src/json.ts"],"names":[],"mappings":"AAEA,KAAK,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;AAElE,KAAK,SAAS,GAAG,aAAa,GAAG,SAAS,EAAE,GAAG;IAC7C,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAGF,KAAK,mBAAmB,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,OAAO,SAAS,CAAC,GAAG,KAAK,GAAG;KACzD,CAAC,IAAI,MAAM,CAAC,GACb,CAAC,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,GAC3B,CAAC,CAAC,CAAC,CAAC,SAAS,mBAAmB,GAAG,KAAK,GACtC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG;IAClF,kBAAkB,EAAE,CAAC,CAAC;CACvB,CAAC;AAEF,wBAAgB,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAE7E;AAED,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG;IACpF,oBAAoB,EAAE,CAAC,CAAC;CACzB,CAAC;AAEF,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAElF;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,CAEhF"}
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../src/json.ts"],"names":[],"mappings":"AAEA,KAAK,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;AAElE,KAAK,SAAS,GAAG,aAAa,GAAG,SAAS,EAAE,GAAG;IAC7C,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,CAAC;AAGF,KAAK,mBAAmB,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,OAAO,SAAS,CAAC,GAAG,OAAO,GAAG;KAC3D,CAAC,IAAI,MAAM,CAAC,GACb,CAAC,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,GAC3B,CAAC,CAAC,CAAC,CAAC,SAAS,mBAAmB,GAAG,KAAK,GACtC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI,cAAc,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG;IAC5F,kBAAkB,EAAE,CAAC,CAAC;CACvB,CAAC;AAEF,wBAAgB,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAE7E;AAED,MAAM,MAAM,iBAAiB,CAAC,CAAC,GAAG,OAAO,IAAI,cAAc,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG;IAC9F,oBAAoB,EAAE,CAAC,CAAC;CACzB,CAAC;AAEF,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAElF;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,CAEhF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-model-common",
-  "version": "1.19.4",
+  "version": "1.19.6",
   "description": "Platforma SDK Model",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -26,8 +26,8 @@
     "typescript": "~5.6.3",
     "vite": "^6.3.5",
     "vitest": "^2.1.9",
-    "@milaboratories/build-configs": "1.0.5",
-    "@platforma-sdk/eslint-config": "1.0.3"
+    "@platforma-sdk/eslint-config": "1.0.3",
+    "@milaboratories/build-configs": "1.0.5"
   },
   "scripts": {
     "type-check": "tsc --noEmit --composite false",

package/src/drivers/pframe/data_info.ts

@@ -204,6 +204,14 @@ export function isDataInfo<Blob>(value: unknown): value is DataInfo<Blob> {
  * @param mapFn - Function to transform blobs from type B1 to type B2
  * @returns A new DataInfo object with transformed blob references
  */
+export function mapDataInfo<B1, B2>(
+  dataInfo: ParquetPartitionedDataInfo<B1>,
+  mapFn: (blob: B1) => B2,
+): ParquetPartitionedDataInfo<B2>;
+export function mapDataInfo<B1, B2>(
+  dataInfo: Exclude<DataInfo<B1>, ParquetPartitionedDataInfo<B1>>,
+  mapFn: (blob: B1) => B2,
+): Exclude<DataInfo<B2>, ParquetPartitionedDataInfo<B2>>;
 export function mapDataInfo<B1, B2>(
   dataInfo: DataInfo<B1>,
   mapFn: (blob: B1) => B2,

package/src/drivers/pframe/data_types.ts

@@ -1,8 +1,5 @@
 import type { Branded } from '../../branding';
-import {
-  type ValueType,
-  type ValueTypeBytes,
-} from './spec/spec';
+import { ValueType } from './spec/spec';
 
 export type PVectorDataInt = Int32Array;
 export type PVectorDataLong = BigInt64Array;
@@ -11,12 +8,12 @@ export type PVectorDataDouble = Float64Array;
 export type PVectorDataString = (null | string)[];
 export type PVectorDataBytes = (null | Uint8Array)[];
 export type PVectorDataTyped<DataType extends ValueType> =
-  DataType extends 'Int' ? PVectorDataInt :
-    DataType extends 'Long' ? PVectorDataLong :
-      DataType extends 'Float' ? PVectorDataFloat :
-        DataType extends 'Double' ? PVectorDataDouble :
-          DataType extends 'String' ? PVectorDataString :
-            DataType extends 'Bytes' ? PVectorDataBytes :
+  DataType extends typeof ValueType.Int ? PVectorDataInt :
+    DataType extends typeof ValueType.Long ? PVectorDataLong :
+      DataType extends typeof ValueType.Float ? PVectorDataFloat :
+        DataType extends typeof ValueType.Double ? PVectorDataDouble :
+          DataType extends typeof ValueType.String ? PVectorDataString :
+            DataType extends typeof ValueType.Bytes ? PVectorDataBytes :
               never;
 export type PVectorData = PVectorDataTyped<ValueType>;
 
@@ -62,17 +59,17 @@ function isValueNA(vector: PTableVector, row: number): boolean {
   const valueType = vector.type;
   const value = vector.data[row];
   switch (valueType) {
-    case 'Int':
+    case ValueType.Int:
       return (value as PVectorDataInt[number]) === -2147483648;
-    case 'Long':
+    case ValueType.Long:
       return (value as PVectorDataLong[number]) === -9007199254740991n;
-    case 'Float':
+    case ValueType.Float:
       return Number.isNaN((value as PVectorDataFloat[number]));
-    case 'Double':
+    case ValueType.Double:
       return Number.isNaN((value as PVectorDataDouble[number]));
-    case 'String':
+    case ValueType.String:
       return (value as PVectorDataString[number]) === null;
-    case 'Bytes':
+    case ValueType.Bytes:
       return (value as PVectorDataBytes[number]) === null;
     default:
       throw Error(`unsupported data type: ${valueType satisfies never}`);
@@ -95,7 +92,7 @@ export function isPTableNA(value: unknown): value is PTableNA {
   return value === PTableNA;
 }
 
-export type ValueTypeSupported = Exclude<ValueType, ValueTypeBytes>;
+export type ValueTypeSupported = Exclude<ValueType, typeof ValueType.Bytes>;
 
 export type PTableValueInt = number;
 export type PTableValueLong = number;
@@ -103,11 +100,11 @@ export type PTableValueFloat = number;
 export type PTableValueDouble = number;
 export type PTableValueString = string;
 export type PTableValueData<DataType extends ValueTypeSupported> =
-  DataType extends 'Int' ? PTableValueInt :
-    DataType extends 'Long' ? PTableValueLong :
-      DataType extends 'Float' ? PTableValueFloat :
-        DataType extends 'Double' ? PTableValueDouble :
-          DataType extends 'String' ? PTableValueString :
+  DataType extends typeof ValueType.Int ? PTableValueInt :
+    DataType extends typeof ValueType.Long ? PTableValueLong :
+      DataType extends typeof ValueType.Float ? PTableValueFloat :
+        DataType extends typeof ValueType.Double ? PTableValueDouble :
+          DataType extends typeof ValueType.String ? PTableValueString :
             never;
 export type PTableValueDataBranded<DataType extends ValueTypeSupported> = Branded<PTableValueData<DataType>, DataType>;
 export type PTableValue<
@@ -157,8 +154,8 @@ function pTableValueImpl<
     dataType?: DataType;
   },
 ) {
-  const valueType: ValueType = column.type;
-  if (valueType === 'Bytes') {
+  const valueType = column.type;
+  if (valueType === ValueType.Bytes) {
     throw Error('Bytes not yet supported');
   }
 
@@ -176,15 +173,15 @@ function pTableValueImpl<
 
   const value = column.data[row]!;
   switch (valueType) {
-    case 'Int':
+    case ValueType.Int:
       return value as PVectorDataInt[number];
-    case 'Long':
+    case ValueType.Long:
       return Number(value as PVectorDataLong[number]);
-    case 'Float':
+    case ValueType.Float:
       return value as PVectorDataFloat[number];
-    case 'Double':
+    case ValueType.Double:
       return value as PVectorDataDouble[number];
-    case 'String':
+    case ValueType.String:
       return (value as PVectorDataString[number])!;
   }
 }

package/src/drivers/pframe/index.ts

@@ -12,3 +12,5 @@ export * from './unique_values';
 export * from './spec';
 
 export * from './driver';
+
+export * from './linker_columns';

package/src/drivers/pframe/linker_columns.test.ts

@@ -0,0 +1,238 @@
+import {
+    Annotation,
+    AxisSpec,
+    AxisSpecNormalized,
+    getArrayFromAxisTree,
+    getAxesTree,
+    getNormalizedAxesList,
+    getSetFromAxisTree,
+    PColumnIdAndSpec,
+    ValueType,
+} from './spec/index';
+import { PObjectId } from '../../pool';
+import { stringifyJson } from '../../json'
+import {
+    describe,
+    expect,
+    test,
+} from 'vitest';
+import { LinkerMap } from './linker_columns';
+
+function makeTestAxis(params: {
+    name: string;
+    parents?: AxisSpec[];
+}): AxisSpec {
+    return {
+        type: ValueType.Int,
+        name: params.name,
+        annotations: {
+            [Annotation.Label]: `${params.name} axis`,
+            ...(params.parents && params.parents.length > 0
+                ? { [Annotation.Parents]: stringifyJson(params.parents) }
+                : {}
+            ),
+        } satisfies Annotation,
+    };
+}
+
+function makeLinkerColumn(params: {
+    name: string;
+    from: AxisSpec[];
+    to: AxisSpec[];
+}): PColumnIdAndSpec {
+    return {
+        columnId: params.name as PObjectId,
+        spec: {
+            kind: 'PColumn',
+            valueType: ValueType.String,
+            name: params.name,
+            axesSpec: [...params.from, ...params.to],
+            annotations: {
+                [Annotation.Label]: `${params.name} column`,
+                [Annotation.IsLinkerColumn]: stringifyJson(true),
+            } satisfies Annotation,
+        },
+    };
+}
+
+/** Returns all permutations of initial array */
+function allPermutations<T>(arr: T[]): T[][] {
+    switch (arr.length) {
+        case 0: return [];
+        case 1: return [arr];
+        case 2: return [arr, [arr[1], arr[0]]];
+        default: return arr.reduce(
+            (acc, item, i) => acc.concat(
+                allPermutations<T>([...arr.slice(0, i), ...arr.slice(i + 1)])
+                    .map(val => [item, ...val])
+            ),
+            [] as T[][],
+        );
+    }
+};
+
+describe('Linker columns', () => {
+    test('Search in linker columns map', () => {
+        const [axis1, axis2, axis3, axis4, axis5] = getNormalizedAxesList([
+            makeTestAxis({ name: 'id1' }),
+            makeTestAxis({ name: 'id2' }),
+            makeTestAxis({ name: 'id3' }),
+            makeTestAxis({ name: 'id4' }),
+            makeTestAxis({ name: 'id5' })
+        ]);
+        const linkerMap = LinkerMap.fromColumns([
+            makeLinkerColumn({ name: 'c12', from: [axis1], to: [axis2] }),
+            makeLinkerColumn({ name: 'c13', from: [axis1], to: [axis3] }),
+            makeLinkerColumn({ name: 'c45', from: [axis4], to: [axis5] }),
+        ]);
+
+        let testCase = (params: {
+            from: AxisSpecNormalized[];
+            to: AxisSpecNormalized[];
+            expected: string[];
+        }) => {
+            const linkers = linkerMap.getLinkerColumnsForAxes({
+                from: params.from,
+                to: params.to,
+                throwWhenNoLinkExists: false,
+            });
+            expect(linkers.map(item => item.spec.name).sort()).toEqual(params.expected);
+        }
+
+        testCase({ from: [axis2], to: [axis3], expected: ['c12', 'c13'] });
+        testCase({ from: [axis1], to: [axis2], expected: ['c12'] });
+        testCase({ from: [axis1], to: [axis4], expected: []});
+    });
+
+    test('Axis tree - without parents', () => {
+        const [axisA, axisB] = getNormalizedAxesList([
+            makeTestAxis({ name: 'a' }),
+            makeTestAxis({ name: 'b' })
+        ]);
+        const tree = getAxesTree(axisA);
+        expect(getSetFromAxisTree(tree).size).toBe(1);
+        expect(getArrayFromAxisTree(tree).length).toBe(1);
+
+        expect(LinkerMap.getAxesGroups([axisA, axisB]).length).toBe(2);
+    })
+
+    test('Axis tree - with parents', () => {
+        const axisD = makeTestAxis({ name: 'd' });
+        const axisC = makeTestAxis({ name: 'c', parents: [axisD] });
+        const axisB = makeTestAxis({ name: 'b', parents: [axisC] });
+        const axisA = makeTestAxis({ name: 'a', parents: [axisB] });
+        const [axisDn, axisCn, axisBn, axisAn] = getNormalizedAxesList([axisD, axisC, axisB, axisA])
+
+        const tree = getAxesTree(axisAn);
+        expect(getSetFromAxisTree(tree).size).toBe(4);
+        expect(getArrayFromAxisTree(tree).length).toBe(4);
+
+        for (const group of allPermutations([axisAn, axisBn, axisCn, axisDn])) {
+            expect(LinkerMap.getAxesGroups(group).length).toBe(1);
+        }
+
+        const axisD2 = makeTestAxis({ name: 'd' });
+        const axisC2 = makeTestAxis({ name: 'c', parents: [axisD2] });
+        const axisB2 = makeTestAxis({ name: 'b' });
+        const axisA2 = makeTestAxis({ name: 'a', parents: [axisB2] });
+        const normalized2 = getNormalizedAxesList([axisD2, axisC2, axisB2, axisA2])
+
+        for (const group of allPermutations(normalized2)) {
+            expect(LinkerMap.getAxesGroups(group).length).toBe(2);
+        }
+
+        const axisD3 = makeTestAxis({ name: 'd' });
+        const axisC3 = makeTestAxis({ name: 'c' });
+        const axisB3 = makeTestAxis({ name: 'b' });
+        const axisA3 = makeTestAxis({ name: 'a', parents: [axisB3] });
+        const normalized3 = getNormalizedAxesList([axisD3, axisC3, axisB3, axisA3])
+
+        for (const group of allPermutations(normalized3)) {
+            expect(LinkerMap.getAxesGroups(group).length).toBe(3);
+        }
+
+        const axisD4 = makeTestAxis({ name: 'd' });
+        const axisC4 = makeTestAxis({ name: 'c' });
+        const axisB4 = makeTestAxis({ name: 'b' });
+        const axisA4 = makeTestAxis({ name: 'a' });
+        const normalized4 = getNormalizedAxesList([axisD4, axisC4, axisB4, axisA4])
+
+        for (const group of allPermutations(normalized4)) {
+            expect(LinkerMap.getAxesGroups(group).length).toBe(4);
+        }
+    })
+
+    test('Generate partial trees', () => {
+        // Axes graph of parents (A, E - roots, C, B, D - parents) in some column:
+        // A - C
+        //  \_ B _ D
+        // E/
+        //
+        // If the column is not a linker: trees to search linkers should be:
+        // 1 C
+        // 2 D
+        // 3 B - D
+        // 4 A - C
+        //    \_ B - D
+        // 5 E - B - D
+
+        // If the axes are in a linker: trees must be in the linkers map:
+
+        // 1 A - C
+        //    \_ B _ D
+        // 2 E - B - D
+
+        const axisD = makeTestAxis({ name: 'd' });
+        const axisC = makeTestAxis({ name: 'c' });
+        const axisB = makeTestAxis({ name: 'b', parents: [axisD] });
+        const axisA = makeTestAxis({ name: 'a', parents: [axisB, axisC] });
+        const axisE = makeTestAxis({ name: 'e', parents: [axisB, axisC] });
+        const axisF = makeTestAxis({ name: 'f' });
+        const axisH = makeTestAxis({ name: 'h' });
+
+        const group1 = [axisA, axisB, axisC, axisD, axisE];
+        const group2 = [axisF];
+        const group3 = [axisH];
+        const group1Normalized = getNormalizedAxesList(group1);
+        const group2Normalized = getNormalizedAxesList(group2);
+        const [axisAn, axisBn, axisCn, axisDn, axisEn] = group1Normalized;
+
+        const linker1 = makeLinkerColumn({ name: 'linker1', from: group1, to: group2 });
+        const linker2 = makeLinkerColumn({ name: 'linker2', from: group2, to: group3 });
+
+        const roots = LinkerMap.getAxesRoots(group1Normalized);
+
+        expect(roots).toEqual([axisAn, axisEn]);
+
+        const groups = LinkerMap.getAxesGroups([...group1Normalized, ...group2Normalized]);
+        expect(groups.length).toBe(2);
+        expect(groups[0]).toEqual(group1Normalized);
+        expect(groups[1]).toEqual(group2Normalized);
+
+        const linkersMap = LinkerMap.fromColumns([linker1, linker2]);
+
+        expect(
+            new Set(linkersMap.getReachableByLinkersAxesFromAxes(group2Normalized).map(a => a.name))
+        ).toEqual(
+            new Set(([...group1, ...group3]).map(a => a.name))
+        );
+        expect(linkersMap.getReachableByLinkersAxesFromAxes([axisDn])).toEqual([]);
+        expect(linkersMap.getReachableByLinkersAxesFromAxes([axisBn])).toEqual([]);
+    });
+
+    test('Order of parents should not matter', () => {
+        const axisA = makeTestAxis({ name: 'a' });
+        const axisB = makeTestAxis({ name: 'b' });
+        const axisC1 = makeTestAxis({ name: 'c', parents: [axisA, axisB] });
+        const axisC2 = makeTestAxis({ name: 'c', parents: [axisB, axisA] });
+        const axisD = makeTestAxis({ name: 'd' });
+
+        const [a, b, c1, c2, d] = getNormalizedAxesList([axisA, axisB, axisC1, axisC2, axisD]);
+        const linkerMap = LinkerMap.fromColumns([
+            makeLinkerColumn({ name: 'linker1', from: [axisA, axisB, axisC1], to: [axisD] })
+        ]);
+
+        expect(linkerMap.getReachableByLinkersAxesFromAxes([c2])).not.toHaveLength(0);
+        expect(linkerMap.getReachableByLinkersAxesFromAxes([c1])).not.toHaveLength(0);
+    })
+});