@platforma-sdk/model 1.24.11 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/PlDataTable.d.ts +4 -4
- package/dist/components/PlDataTable.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1004 -660
- package/dist/index.mjs.map +1 -1
- package/dist/render/api.d.ts +65 -21
- package/dist/render/api.d.ts.map +1 -1
- package/dist/render/internal.d.ts +4 -3
- package/dist/render/internal.d.ts.map +1 -1
- package/dist/render/split_selectors.d.ts +14 -0
- package/dist/render/split_selectors.d.ts.map +1 -0
- package/dist/render/util/axis_filtering.d.ts +14 -0
- package/dist/render/util/axis_filtering.d.ts.map +1 -0
- package/dist/render/util/index.d.ts +2 -1
- package/dist/render/util/index.d.ts.map +1 -1
- package/dist/render/util/label.d.ts +7 -1
- package/dist/render/util/label.d.ts.map +1 -1
- package/dist/render/util/{resource_map.d.ts → pcolumn_data.d.ts} +12 -2
- package/dist/render/util/pcolumn_data.d.ts.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/components/PlDataTable.ts +6 -5
- package/src/render/api.ts +425 -126
- package/src/render/internal.ts +4 -2
- package/src/render/split_selectors.ts +15 -0
- package/src/render/util/axis_filtering.ts +120 -0
- package/src/render/util/index.ts +2 -1
- package/src/render/util/label.ts +31 -3
- package/src/render/util/pcolumn_data.ts +387 -0
- package/dist/render/util/resource_map.d.ts.map +0 -1
- package/src/render/util/resource_map.ts +0 -208
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { AAxisSelector, AnchoredPColumnSelector } from '@milaboratories/pl-model-common';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* AAxisSelector with an optional split flag
|
|
5
|
+
*/
|
|
6
|
+
export type AAxisSelectorWithSplit = AAxisSelector & {
|
|
7
|
+
split?: boolean;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* APColumnSelector with an optional split flag for each axis
|
|
12
|
+
*/
|
|
13
|
+
export type APColumnSelectorWithSplit = AnchoredPColumnSelector & {
|
|
14
|
+
axes?: AAxisSelectorWithSplit[];
|
|
15
|
+
};
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
BinaryChunk,
|
|
3
|
+
DataInfoEntries,
|
|
4
|
+
PColumnDataEntry,
|
|
5
|
+
PColumnKey,
|
|
6
|
+
PColumnValue,
|
|
7
|
+
JsonDataInfoEntries,
|
|
8
|
+
JsonPartitionedDataInfoEntries,
|
|
9
|
+
BinaryPartitionedDataInfoEntries,
|
|
10
|
+
} from '@milaboratories/pl-model-common';
|
|
11
|
+
import type { AxisFilterByIdx } from '@milaboratories/pl-model-common';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Filters DataInfoEntries using axis filters, removing specified axes from keys and
|
|
15
|
+
* only keeping entries that match the filter values.
|
|
16
|
+
*
|
|
17
|
+
* @param dataInfoEntries - The data info object to filter
|
|
18
|
+
* @param axisFilters - Array of axis filters (index, value pairs)
|
|
19
|
+
* @throws Error if any filter axis is outside the partitioning axes or data axes for Json data
|
|
20
|
+
*/
|
|
21
|
+
export function filterDataInfoEntries<Blob>(
|
|
22
|
+
dataInfoEntries: BinaryPartitionedDataInfoEntries<Blob>,
|
|
23
|
+
axisFilters: AxisFilterByIdx[],
|
|
24
|
+
): BinaryPartitionedDataInfoEntries<Blob>;
|
|
25
|
+
export function filterDataInfoEntries<Blob>(
|
|
26
|
+
dataInfoEntries: JsonPartitionedDataInfoEntries<Blob>,
|
|
27
|
+
axisFilters: AxisFilterByIdx[],
|
|
28
|
+
): JsonPartitionedDataInfoEntries<Blob>;
|
|
29
|
+
export function filterDataInfoEntries<Blob>(
|
|
30
|
+
dataInfoEntries: BinaryPartitionedDataInfoEntries<Blob> | JsonPartitionedDataInfoEntries<Blob>,
|
|
31
|
+
axisFilters: AxisFilterByIdx[],
|
|
32
|
+
): BinaryPartitionedDataInfoEntries<Blob> | JsonPartitionedDataInfoEntries<Blob>;
|
|
33
|
+
export function filterDataInfoEntries(
|
|
34
|
+
dataInfoEntries: JsonDataInfoEntries,
|
|
35
|
+
axisFilters: AxisFilterByIdx[],
|
|
36
|
+
): JsonDataInfoEntries;
|
|
37
|
+
export function filterDataInfoEntries<Blob>(
|
|
38
|
+
dataInfoEntries: DataInfoEntries<Blob>,
|
|
39
|
+
axisFilters: AxisFilterByIdx[],
|
|
40
|
+
): DataInfoEntries<Blob> {
|
|
41
|
+
// Sort filters by axis index in descending order to safely remove elements from arrays
|
|
42
|
+
const sortedFilters = [...axisFilters].sort((a, b) => b[0] - a[0]);
|
|
43
|
+
|
|
44
|
+
// Check for invalid filter axes
|
|
45
|
+
if (dataInfoEntries.type === 'JsonPartitioned' || dataInfoEntries.type === 'BinaryPartitioned') {
|
|
46
|
+
const { partitionKeyLength } = dataInfoEntries;
|
|
47
|
+
for (const [axisIdx] of axisFilters)
|
|
48
|
+
if (axisIdx >= partitionKeyLength)
|
|
49
|
+
throw new Error(`Can't filter on non-partitioned axis ${axisIdx}. Must be >= ${partitionKeyLength}`);
|
|
50
|
+
} else if (dataInfoEntries.type === 'Json') {
|
|
51
|
+
const { keyLength } = dataInfoEntries;
|
|
52
|
+
for (const [axisIdx] of axisFilters)
|
|
53
|
+
if (axisIdx >= keyLength)
|
|
54
|
+
throw new Error(`Can't filter on non-data axis ${axisIdx}. Must be >= ${keyLength}`);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const keyMatchesFilters = (key: PColumnKey): boolean => {
|
|
58
|
+
for (const [axisIdx, axisValue] of sortedFilters)
|
|
59
|
+
if (key[axisIdx] !== axisValue)
|
|
60
|
+
return false;
|
|
61
|
+
return true;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const removeFilteredAxes = (key: PColumnKey): PColumnKey => {
|
|
65
|
+
const newKey = [...key];
|
|
66
|
+
|
|
67
|
+
// Remove axes in descending order to maintain correct indices
|
|
68
|
+
for (const [axisIdx] of sortedFilters)
|
|
69
|
+
newKey.splice(axisIdx, 1);
|
|
70
|
+
|
|
71
|
+
return newKey;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
switch (dataInfoEntries.type) {
|
|
75
|
+
case 'Json': {
|
|
76
|
+
const filteredData: PColumnDataEntry<PColumnValue>[] = dataInfoEntries.data
|
|
77
|
+
.filter((entry: PColumnDataEntry<PColumnValue>) => keyMatchesFilters(entry.key))
|
|
78
|
+
.map((entry: PColumnDataEntry<PColumnValue>) => ({
|
|
79
|
+
key: removeFilteredAxes(entry.key),
|
|
80
|
+
value: entry.value,
|
|
81
|
+
}));
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
type: 'Json',
|
|
85
|
+
keyLength: dataInfoEntries.keyLength - axisFilters.length,
|
|
86
|
+
data: filteredData,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
case 'JsonPartitioned': {
|
|
91
|
+
const filteredParts = dataInfoEntries.parts
|
|
92
|
+
.filter((entry: PColumnDataEntry<Blob>) => keyMatchesFilters(entry.key))
|
|
93
|
+
.map((entry: PColumnDataEntry<Blob>) => ({
|
|
94
|
+
key: removeFilteredAxes(entry.key),
|
|
95
|
+
value: entry.value,
|
|
96
|
+
}));
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
type: 'JsonPartitioned',
|
|
100
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength - axisFilters.length,
|
|
101
|
+
parts: filteredParts,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
case 'BinaryPartitioned': {
|
|
106
|
+
const filteredParts = dataInfoEntries.parts
|
|
107
|
+
.filter((entry: PColumnDataEntry<BinaryChunk<Blob>>) => keyMatchesFilters(entry.key))
|
|
108
|
+
.map((entry: PColumnDataEntry<BinaryChunk<Blob>>) => ({
|
|
109
|
+
key: removeFilteredAxes(entry.key),
|
|
110
|
+
value: entry.value,
|
|
111
|
+
}));
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
type: 'BinaryPartitioned',
|
|
115
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength - axisFilters.length,
|
|
116
|
+
parts: filteredParts,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
package/src/render/util/index.ts
CHANGED
package/src/render/util/label.ts
CHANGED
|
@@ -31,6 +31,13 @@ export const Trace = z.array(TraceEntry);
|
|
|
31
31
|
export type Trace = z.infer<typeof Trace>;
|
|
32
32
|
type FullTrace = FullTraceEntry[];
|
|
33
33
|
|
|
34
|
+
// Define the possible return types for the specExtractor function
|
|
35
|
+
type SpecExtractorResult = PObjectSpec | {
|
|
36
|
+
spec: PObjectSpec;
|
|
37
|
+
prefixTrace?: TraceEntry[];
|
|
38
|
+
suffixTrace?: TraceEntry[];
|
|
39
|
+
};
|
|
40
|
+
|
|
34
41
|
const DistancePenalty = 0.001;
|
|
35
42
|
|
|
36
43
|
const LabelType = '__LABEL__';
|
|
@@ -38,7 +45,7 @@ const LabelTypeFull = '__LABEL__@1';
|
|
|
38
45
|
|
|
39
46
|
export function deriveLabels<T>(
|
|
40
47
|
values: T[],
|
|
41
|
-
specExtractor: (obj: T) =>
|
|
48
|
+
specExtractor: (obj: T) => SpecExtractorResult,
|
|
42
49
|
ops: LabelDerivationOps = {},
|
|
43
50
|
): RecordsWithLabel<T>[] {
|
|
44
51
|
const importances = new Map<string, number>();
|
|
@@ -47,10 +54,31 @@ export function deriveLabels<T>(
|
|
|
47
54
|
const numberOfRecordsWithType = new Map<string, number>();
|
|
48
55
|
|
|
49
56
|
const enrichedRecords = values.map((value) => {
|
|
50
|
-
const
|
|
57
|
+
const extractorResult = specExtractor(value);
|
|
58
|
+
let spec: PObjectSpec;
|
|
59
|
+
let prefixTrace: TraceEntry[] | undefined;
|
|
60
|
+
let suffixTrace: TraceEntry[] | undefined;
|
|
61
|
+
|
|
62
|
+
// Check if the result is the new structure or just PObjectSpec
|
|
63
|
+
if ('spec' in extractorResult && typeof extractorResult.spec === 'object') {
|
|
64
|
+
// It's the new structure { spec, prefixTrace?, suffixTrace? }
|
|
65
|
+
spec = extractorResult.spec;
|
|
66
|
+
prefixTrace = extractorResult.prefixTrace;
|
|
67
|
+
suffixTrace = extractorResult.suffixTrace;
|
|
68
|
+
} else {
|
|
69
|
+
// It's just PObjectSpec
|
|
70
|
+
spec = extractorResult as PObjectSpec;
|
|
71
|
+
}
|
|
72
|
+
|
|
51
73
|
const label = spec.annotations?.[PAnnotationLabel];
|
|
52
74
|
const traceStr = spec.annotations?.[PAnnotationTrace];
|
|
53
|
-
const
|
|
75
|
+
const baseTrace = (traceStr ? Trace.safeParse(JSON.parse(traceStr)).data : undefined) ?? [];
|
|
76
|
+
|
|
77
|
+
const trace = [
|
|
78
|
+
...(prefixTrace ?? []),
|
|
79
|
+
...baseTrace,
|
|
80
|
+
...(suffixTrace ?? []),
|
|
81
|
+
];
|
|
54
82
|
|
|
55
83
|
if (label) {
|
|
56
84
|
const labelEntry = { label, type: LabelType, importance: -2 };
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
2
|
+
import {
|
|
3
|
+
type BinaryChunk,
|
|
4
|
+
type BinaryPartitionedDataInfoEntries,
|
|
5
|
+
type DataInfoEntries,
|
|
6
|
+
type JsonPartitionedDataInfoEntries,
|
|
7
|
+
type PColumnDataEntry,
|
|
8
|
+
type PColumnKey,
|
|
9
|
+
} from '@milaboratories/pl-model-common';
|
|
10
|
+
import type { TreeNodeAccessor } from '../accessor';
|
|
11
|
+
|
|
12
|
+
const PCD_PREFIX = 'PColumnData/';
|
|
13
|
+
|
|
14
|
+
export const RT_RESOURCE_MAP = PCD_PREFIX + 'ResourceMap';
|
|
15
|
+
export const RT_RESOURCE_MAP_PARTITIONED = PCD_PREFIX + 'Partitioned/ResourceMap';
|
|
16
|
+
|
|
17
|
+
export const RT_JSON_PARTITIONED = PCD_PREFIX + 'JsonPartitioned';
|
|
18
|
+
export const RT_BINARY_PARTITIONED = PCD_PREFIX + 'BinaryPartitioned';
|
|
19
|
+
|
|
20
|
+
const PCD_SUP_PREFIX = PCD_PREFIX + 'Partitioned/';
|
|
21
|
+
export const RT_JSON_SUPER_PARTITIONED = PCD_SUP_PREFIX + 'JsonPartitioned';
|
|
22
|
+
export const RT_BINARY_SUPER_PARTITIONED = PCD_SUP_PREFIX + 'BinaryPartitioned';
|
|
23
|
+
|
|
24
|
+
export type PColumnResourceMapEntry<T> = {
|
|
25
|
+
key: PColumnKey;
|
|
26
|
+
value: T;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type PColumnResourceMapData<T> = {
|
|
30
|
+
isComplete: boolean;
|
|
31
|
+
data: PColumnResourceMapEntry<T>[];
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
function populateResourceMapData<T>(
|
|
35
|
+
acc: TreeNodeAccessor | undefined,
|
|
36
|
+
resourceParser: (acc: TreeNodeAccessor) => T | undefined,
|
|
37
|
+
data: PColumnResourceMapEntry<T | undefined>[],
|
|
38
|
+
keyPrefix: PColumnKey = [],
|
|
39
|
+
addEntriesWithNoData: boolean,
|
|
40
|
+
): boolean {
|
|
41
|
+
if (acc === undefined) return false;
|
|
42
|
+
switch (acc.resourceType.name) {
|
|
43
|
+
case RT_RESOURCE_MAP: {
|
|
44
|
+
let isComplete = acc.getInputsLocked();
|
|
45
|
+
for (const keyStr of acc.listInputFields()) {
|
|
46
|
+
const value = acc.resolve({ field: keyStr, assertFieldType: 'Input' });
|
|
47
|
+
const key = [...keyPrefix, ...JSON.parse(keyStr)] as PColumnKey;
|
|
48
|
+
const converted = value === undefined ? undefined : resourceParser(value);
|
|
49
|
+
if (converted === undefined) isComplete = false;
|
|
50
|
+
if (converted !== undefined || addEntriesWithNoData) data.push({ key, value: converted });
|
|
51
|
+
}
|
|
52
|
+
return isComplete;
|
|
53
|
+
}
|
|
54
|
+
case RT_RESOURCE_MAP_PARTITIONED: {
|
|
55
|
+
let isComplete = acc.getInputsLocked();
|
|
56
|
+
for (const keyStr of acc.listInputFields()) {
|
|
57
|
+
const value = acc.resolve({ field: keyStr, assertFieldType: 'Input' });
|
|
58
|
+
if (value === undefined) isComplete = false;
|
|
59
|
+
else {
|
|
60
|
+
const key = [...keyPrefix, ...JSON.parse(keyStr)] as PColumnKey;
|
|
61
|
+
const populateResult = populateResourceMapData(
|
|
62
|
+
value,
|
|
63
|
+
resourceParser,
|
|
64
|
+
data,
|
|
65
|
+
key,
|
|
66
|
+
addEntriesWithNoData,
|
|
67
|
+
);
|
|
68
|
+
isComplete = isComplete && populateResult;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return isComplete;
|
|
72
|
+
}
|
|
73
|
+
default:
|
|
74
|
+
throw new Error(`Unknown resource type: ${acc.resourceType.name}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function parseResourceMap<T>(
|
|
79
|
+
acc: TreeNodeAccessor | undefined,
|
|
80
|
+
resourceParser: (acc: TreeNodeAccessor) => T | undefined,
|
|
81
|
+
addEntriesWithNoData: false
|
|
82
|
+
): PColumnResourceMapData<NonNullable<T>>;
|
|
83
|
+
export function parseResourceMap<T>(
|
|
84
|
+
acc: TreeNodeAccessor | undefined,
|
|
85
|
+
resourceParser: (acc: TreeNodeAccessor) => T | undefined,
|
|
86
|
+
addEntriesWithNoData: true
|
|
87
|
+
): PColumnResourceMapData<T | undefined>;
|
|
88
|
+
export function parseResourceMap<T>(
|
|
89
|
+
acc: TreeNodeAccessor | undefined,
|
|
90
|
+
resourceParser: (acc: TreeNodeAccessor) => T | undefined,
|
|
91
|
+
addEntriesWithNoData: boolean = false,
|
|
92
|
+
): PColumnResourceMapData<T | undefined> {
|
|
93
|
+
const data: PColumnResourceMapEntry<T | undefined>[] = [];
|
|
94
|
+
const isComplete = populateResourceMapData(acc, resourceParser, data, [], addEntriesWithNoData);
|
|
95
|
+
return { isComplete, data };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export type PColumnKeyList = {
|
|
99
|
+
/** array of keys */
|
|
100
|
+
data: PColumnKey[];
|
|
101
|
+
/** length of partition key */
|
|
102
|
+
keyLength: number;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const removeIndexSuffix = (keyStr: string): { baseKey: string; type: 'index' | 'values' } => {
|
|
106
|
+
if (keyStr.endsWith('.index')) {
|
|
107
|
+
return { baseKey: keyStr.substring(0, keyStr.length - 6), type: 'index' };
|
|
108
|
+
} else if (keyStr.endsWith('.values')) {
|
|
109
|
+
return { baseKey: keyStr.substring(0, keyStr.length - 7), type: 'values' };
|
|
110
|
+
} else {
|
|
111
|
+
throw new Error(`key must ends on .index/.values for binary p-column, got: ${keyStr}`);
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
// @TODO define a class with various resource map operations
|
|
116
|
+
/** Returns a list of all partition keys appeared in the p-column */
|
|
117
|
+
export function getPartitionKeysList(
|
|
118
|
+
acc: TreeNodeAccessor | undefined,
|
|
119
|
+
): PColumnKeyList | undefined {
|
|
120
|
+
if (!acc) return undefined;
|
|
121
|
+
|
|
122
|
+
const rt = acc.resourceType.name;
|
|
123
|
+
const meta = acc.getDataAsJson<Record<string, number>>();
|
|
124
|
+
const data: PColumnKey[] = [];
|
|
125
|
+
|
|
126
|
+
let keyLength = 0;
|
|
127
|
+
// @TODO validate meta shape
|
|
128
|
+
switch (rt) {
|
|
129
|
+
case RT_RESOURCE_MAP:
|
|
130
|
+
keyLength = meta['keyLength'];
|
|
131
|
+
break;
|
|
132
|
+
|
|
133
|
+
case RT_RESOURCE_MAP_PARTITIONED:
|
|
134
|
+
keyLength = meta['partitionKeyLength'] + meta['keyLength'];
|
|
135
|
+
break;
|
|
136
|
+
|
|
137
|
+
case RT_JSON_PARTITIONED:
|
|
138
|
+
case RT_BINARY_PARTITIONED:
|
|
139
|
+
keyLength = meta['partitionKeyLength'];
|
|
140
|
+
break;
|
|
141
|
+
|
|
142
|
+
case RT_BINARY_SUPER_PARTITIONED:
|
|
143
|
+
case RT_JSON_SUPER_PARTITIONED:
|
|
144
|
+
keyLength = meta['superPartitionKeyLength'] + meta['partitionKeyLength'];
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
switch (rt) {
|
|
149
|
+
case RT_RESOURCE_MAP:
|
|
150
|
+
case RT_JSON_PARTITIONED:
|
|
151
|
+
case RT_BINARY_PARTITIONED:
|
|
152
|
+
for (let keyStr of acc.listInputFields()) {
|
|
153
|
+
if (rt === RT_BINARY_PARTITIONED) {
|
|
154
|
+
keyStr = removeIndexSuffix(keyStr).baseKey;
|
|
155
|
+
}
|
|
156
|
+
const key = [...JSON.parse(keyStr)] as PColumnKey;
|
|
157
|
+
data.push(key);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
break;
|
|
161
|
+
|
|
162
|
+
case RT_RESOURCE_MAP_PARTITIONED:
|
|
163
|
+
case RT_BINARY_SUPER_PARTITIONED:
|
|
164
|
+
case RT_JSON_SUPER_PARTITIONED:
|
|
165
|
+
for (const supKeyStr of acc.listInputFields()) {
|
|
166
|
+
const keyPrefix = [...JSON.parse(supKeyStr)] as PColumnKey;
|
|
167
|
+
|
|
168
|
+
const value = acc.resolve({ field: supKeyStr, assertFieldType: 'Input' });
|
|
169
|
+
if (value !== undefined) {
|
|
170
|
+
for (let keyStr of value.listInputFields()) {
|
|
171
|
+
if (rt === RT_BINARY_SUPER_PARTITIONED) {
|
|
172
|
+
keyStr = removeIndexSuffix(keyStr).baseKey;
|
|
173
|
+
}
|
|
174
|
+
const key = [...keyPrefix, ...JSON.parse(keyStr)] as PColumnKey;
|
|
175
|
+
data.push(key);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return { data, keyLength };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Returns an array of unique partition keys for each column: the i-th element in the resulting 2d array contains all unique values of i-th partition axis. */
|
|
186
|
+
// @TODO define a class with various resource map operations
|
|
187
|
+
export function getUniquePartitionKeys(
|
|
188
|
+
acc: TreeNodeAccessor | undefined,
|
|
189
|
+
): (string | number)[][] | undefined {
|
|
190
|
+
const list = getPartitionKeysList(acc);
|
|
191
|
+
if (!list) return undefined;
|
|
192
|
+
|
|
193
|
+
const { data, keyLength } = list;
|
|
194
|
+
|
|
195
|
+
const result: Set<string | number>[] = [];
|
|
196
|
+
|
|
197
|
+
for (let i = 0; i < keyLength; ++i) {
|
|
198
|
+
result.push(new Set());
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
for (const l of data) {
|
|
202
|
+
if (l.length !== keyLength) {
|
|
203
|
+
throw new Error('key length does not match partition length');
|
|
204
|
+
}
|
|
205
|
+
for (let i = 0; i < keyLength; ++i) {
|
|
206
|
+
result[i].add(l[i]);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return result.map((s) => Array.from(s.values()));
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Parses the PColumn data from a TreeNodeAccessor into a DataInfoEntries structure.
|
|
215
|
+
* Returns undefined if any required data is missing.
|
|
216
|
+
* Throws error on validation failures.
|
|
217
|
+
*
|
|
218
|
+
* @param acc - The TreeNodeAccessor containing PColumn data
|
|
219
|
+
* @param keyPrefix - Optional key prefix for recursive calls
|
|
220
|
+
* @returns DataInfoEntries representation of the PColumn data, or undefined if incomplete
|
|
221
|
+
*/
|
|
222
|
+
export function parsePColumnData(
|
|
223
|
+
acc: TreeNodeAccessor | undefined,
|
|
224
|
+
keyPrefix: PColumnKey = [],
|
|
225
|
+
): JsonPartitionedDataInfoEntries<TreeNodeAccessor> | BinaryPartitionedDataInfoEntries<TreeNodeAccessor> | undefined {
|
|
226
|
+
if (acc === undefined) return undefined;
|
|
227
|
+
|
|
228
|
+
const resourceType = acc.resourceType.name;
|
|
229
|
+
const meta = acc.getDataAsJson<Record<string, number>>();
|
|
230
|
+
|
|
231
|
+
// Prevent recursive super-partitioned resources
|
|
232
|
+
if (keyPrefix.length > 0
|
|
233
|
+
&& (resourceType === RT_JSON_SUPER_PARTITIONED || resourceType === RT_BINARY_SUPER_PARTITIONED)) {
|
|
234
|
+
throw new Error(`Unexpected nested super-partitioned resource: ${resourceType}`);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
switch (resourceType) {
|
|
238
|
+
case RT_RESOURCE_MAP:
|
|
239
|
+
case RT_RESOURCE_MAP_PARTITIONED:
|
|
240
|
+
throw new Error(`Only data columns are supported, got: ${resourceType}`);
|
|
241
|
+
|
|
242
|
+
case RT_JSON_PARTITIONED: {
|
|
243
|
+
if (typeof meta?.partitionKeyLength !== 'number') {
|
|
244
|
+
throw new Error(`Missing partitionKeyLength in metadata for ${resourceType}`);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
const parts: PColumnDataEntry<TreeNodeAccessor>[] = [];
|
|
248
|
+
for (const keyStr of acc.listInputFields()) {
|
|
249
|
+
const value = acc.resolve({ field: keyStr, assertFieldType: 'Input' });
|
|
250
|
+
if (value === undefined) return undefined;
|
|
251
|
+
|
|
252
|
+
const key = [...keyPrefix, ...JSON.parse(keyStr)];
|
|
253
|
+
parts.push({ key, value });
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
type: 'JsonPartitioned',
|
|
258
|
+
partitionKeyLength: meta.partitionKeyLength,
|
|
259
|
+
parts,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
case RT_BINARY_PARTITIONED: {
|
|
264
|
+
if (typeof meta?.partitionKeyLength !== 'number') {
|
|
265
|
+
throw new Error(`Missing partitionKeyLength in metadata for ${resourceType}`);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const parts: PColumnDataEntry<BinaryChunk<TreeNodeAccessor>>[] = [];
|
|
269
|
+
const baseKeys = new Map<string, { index?: TreeNodeAccessor; values?: TreeNodeAccessor }>();
|
|
270
|
+
|
|
271
|
+
// Group fields by base key (without .index/.values suffix)
|
|
272
|
+
for (const keyStr of acc.listInputFields()) {
|
|
273
|
+
const suffix = removeIndexSuffix(keyStr);
|
|
274
|
+
|
|
275
|
+
const value = acc.resolve({ field: keyStr, assertFieldType: 'Input' });
|
|
276
|
+
if (value === undefined) return undefined;
|
|
277
|
+
|
|
278
|
+
let entry = baseKeys.get(suffix.baseKey);
|
|
279
|
+
if (!entry) {
|
|
280
|
+
entry = {};
|
|
281
|
+
baseKeys.set(suffix.baseKey, entry);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
if (suffix.type === 'index') {
|
|
285
|
+
entry.index = value;
|
|
286
|
+
} else {
|
|
287
|
+
entry.values = value;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Process complete binary chunks only
|
|
292
|
+
for (const [baseKeyStr, entry] of baseKeys.entries()) {
|
|
293
|
+
if (!entry.index || !entry.values) return undefined;
|
|
294
|
+
|
|
295
|
+
const key = [...keyPrefix, ...JSON.parse(baseKeyStr)];
|
|
296
|
+
parts.push({
|
|
297
|
+
key,
|
|
298
|
+
value: {
|
|
299
|
+
index: entry.index,
|
|
300
|
+
values: entry.values,
|
|
301
|
+
},
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
return {
|
|
306
|
+
type: 'BinaryPartitioned',
|
|
307
|
+
partitionKeyLength: meta.partitionKeyLength,
|
|
308
|
+
parts,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
case RT_JSON_SUPER_PARTITIONED: {
|
|
313
|
+
if (typeof meta?.superPartitionKeyLength !== 'number'
|
|
314
|
+
|| typeof meta?.partitionKeyLength !== 'number') {
|
|
315
|
+
throw new Error(`Missing superPartitionKeyLength or partitionKeyLength in metadata for ${resourceType}`);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const totalKeyLength = meta.superPartitionKeyLength + meta.partitionKeyLength;
|
|
319
|
+
const parts: PColumnDataEntry<TreeNodeAccessor>[] = [];
|
|
320
|
+
|
|
321
|
+
// Process all super partitions
|
|
322
|
+
for (const supKeyStr of acc.listInputFields()) {
|
|
323
|
+
const superPartition = acc.resolve({ field: supKeyStr, assertFieldType: 'Input' });
|
|
324
|
+
if (superPartition === undefined) return undefined;
|
|
325
|
+
|
|
326
|
+
// Validate inner type
|
|
327
|
+
if (superPartition.resourceType.name !== RT_JSON_PARTITIONED) {
|
|
328
|
+
throw new Error(`Expected ${RT_JSON_PARTITIONED} inside ${resourceType}, but got ${superPartition.resourceType.name}`);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const innerResult = parsePColumnData(superPartition, JSON.parse(supKeyStr) as PColumnKey);
|
|
332
|
+
|
|
333
|
+
if (innerResult === undefined) return undefined;
|
|
334
|
+
|
|
335
|
+
if (innerResult.type !== 'JsonPartitioned')
|
|
336
|
+
throw new Error(`Unexpected inner result type for ${resourceType}: ${innerResult.type}`);
|
|
337
|
+
|
|
338
|
+
parts.push(...innerResult.parts);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
return {
|
|
342
|
+
type: 'JsonPartitioned',
|
|
343
|
+
partitionKeyLength: totalKeyLength,
|
|
344
|
+
parts,
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
case RT_BINARY_SUPER_PARTITIONED: {
|
|
349
|
+
if (typeof meta?.superPartitionKeyLength !== 'number'
|
|
350
|
+
|| typeof meta?.partitionKeyLength !== 'number') {
|
|
351
|
+
throw new Error(`Missing superPartitionKeyLength or partitionKeyLength in metadata for ${resourceType}`);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
const totalKeyLength = meta.superPartitionKeyLength + meta.partitionKeyLength;
|
|
355
|
+
const parts: PColumnDataEntry<BinaryChunk<TreeNodeAccessor>>[] = [];
|
|
356
|
+
|
|
357
|
+
// Process all super partitions
|
|
358
|
+
for (const supKeyStr of acc.listInputFields()) {
|
|
359
|
+
const superPartition = acc.resolve({ field: supKeyStr, assertFieldType: 'Input' });
|
|
360
|
+
if (superPartition === undefined) return undefined;
|
|
361
|
+
|
|
362
|
+
// Validate inner type
|
|
363
|
+
if (superPartition.resourceType.name !== RT_BINARY_PARTITIONED) {
|
|
364
|
+
throw new Error(`Expected ${RT_BINARY_PARTITIONED} inside ${resourceType}, but got ${superPartition.resourceType.name}`);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const innerResult = parsePColumnData(superPartition, JSON.parse(supKeyStr) as PColumnKey);
|
|
368
|
+
|
|
369
|
+
if (innerResult === undefined) return undefined;
|
|
370
|
+
|
|
371
|
+
if (innerResult.type !== 'BinaryPartitioned')
|
|
372
|
+
throw new Error(`Unexpected inner result type for ${resourceType}: ${innerResult.type}`);
|
|
373
|
+
|
|
374
|
+
parts.push(...innerResult.parts);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
return {
|
|
378
|
+
type: 'BinaryPartitioned',
|
|
379
|
+
partitionKeyLength: totalKeyLength,
|
|
380
|
+
parts,
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
default:
|
|
385
|
+
throw new Error(`Unknown resource type: ${resourceType}`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"resource_map.d.ts","sourceRoot":"","sources":["../../../src/render/util/resource_map.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD,eAAO,MAAM,eAAe,QAA6B,CAAC;AAC1D,eAAO,MAAM,2BAA2B,QAAyC,CAAC;AAElF,eAAO,MAAM,mBAAmB,QAAiC,CAAC;AAClE,eAAO,MAAM,qBAAqB,QAAmC,CAAC;AAGtE,eAAO,MAAM,yBAAyB,QAAqC,CAAC;AAC5E,eAAO,MAAM,2BAA2B,QAAuC,CAAC;AAEhF,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE7C,MAAM,MAAM,uBAAuB,CAAC,CAAC,IAAI;IACvC,GAAG,EAAE,UAAU,CAAC;IAChB,KAAK,EAAE,CAAC,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI;IACtC,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC,EAAE,CAAC;CACpC,CAAC;AA8CF,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,GAAG,EAAE,gBAAgB,GAAG,SAAS,EACjC,cAAc,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,CAAC,GAAG,SAAS,EACxD,oBAAoB,EAAE,KAAK,GAC1B,sBAAsB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,GAAG,EAAE,gBAAgB,GAAG,SAAS,EACjC,cAAc,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,CAAC,GAAG,SAAS,EACxD,oBAAoB,EAAE,IAAI,GACzB,sBAAsB,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;AAWzC,MAAM,MAAM,cAAc,GAAG;IAC3B,oBAAoB;IACpB,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAaF,oEAAoE;AACpE,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,gBAAgB,GAAG,SAAS,GAChC,cAAc,GAAG,SAAS,CAoE5B;AAED,+JAA+J;AAE/J,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,gBAAgB,GAAG,SAAS,GAChC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,EAAE,GAAG,SAAS,CAsBnC"}
|