@milaboratories/pl-model-common 1.25.1 → 1.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_virtual/_rolldown/runtime.cjs +29 -0
- package/dist/author_marker.d.ts +9 -6
- package/dist/base32_encode.cjs +40 -46
- package/dist/base32_encode.cjs.map +1 -1
- package/dist/base32_encode.js +40 -45
- package/dist/base32_encode.js.map +1 -1
- package/dist/base64.cjs +5 -4
- package/dist/base64.cjs.map +1 -1
- package/dist/base64.d.ts +10 -6
- package/dist/base64.js +5 -3
- package/dist/base64.js.map +1 -1
- package/dist/block_state.d.ts +21 -23
- package/dist/bmodel/block_config.cjs +9 -10
- package/dist/bmodel/block_config.cjs.map +1 -1
- package/dist/bmodel/block_config.d.ts +53 -72
- package/dist/bmodel/block_config.js +9 -9
- package/dist/bmodel/block_config.js.map +1 -1
- package/dist/bmodel/code.cjs +7 -6
- package/dist/bmodel/code.cjs.map +1 -1
- package/dist/bmodel/code.d.ts +17 -13
- package/dist/bmodel/code.js +6 -4
- package/dist/bmodel/code.js.map +1 -1
- package/dist/bmodel/container.d.ts +46 -44
- package/dist/bmodel/index.cjs +3 -0
- package/dist/bmodel/index.d.ts +5 -6
- package/dist/bmodel/index.js +3 -0
- package/dist/bmodel/normalization.cjs +90 -106
- package/dist/bmodel/normalization.cjs.map +1 -1
- package/dist/bmodel/normalization.d.ts +7 -3
- package/dist/bmodel/normalization.js +90 -105
- package/dist/bmodel/normalization.js.map +1 -1
- package/dist/bmodel/types.d.ts +4 -1
- package/dist/branding.d.ts +5 -5
- package/dist/common_types.d.ts +19 -15
- package/dist/driver_kit.d.ts +25 -17
- package/dist/drivers/ChunkedStreamReader.cjs +168 -203
- package/dist/drivers/ChunkedStreamReader.cjs.map +1 -1
- package/dist/drivers/ChunkedStreamReader.d.ts +104 -100
- package/dist/drivers/ChunkedStreamReader.js +168 -202
- package/dist/drivers/ChunkedStreamReader.js.map +1 -1
- package/dist/drivers/blob.cjs +15 -17
- package/dist/drivers/blob.cjs.map +1 -1
- package/dist/drivers/blob.d.ts +35 -35
- package/dist/drivers/blob.js +14 -15
- package/dist/drivers/blob.js.map +1 -1
- package/dist/drivers/index.cjs +18 -0
- package/dist/drivers/index.d.ts +29 -9
- package/dist/drivers/index.js +18 -0
- package/dist/drivers/interfaces.d.ts +17 -14
- package/dist/drivers/log.cjs +4 -3
- package/dist/drivers/log.cjs.map +1 -1
- package/dist/drivers/log.d.ts +52 -49
- package/dist/drivers/log.js +4 -2
- package/dist/drivers/log.js.map +1 -1
- package/dist/drivers/ls.cjs +16 -19
- package/dist/drivers/ls.cjs.map +1 -1
- package/dist/drivers/ls.d.ts +73 -76
- package/dist/drivers/ls.js +16 -17
- package/dist/drivers/ls.js.map +1 -1
- package/dist/drivers/pframe/column_filter.d.ts +19 -15
- package/dist/drivers/pframe/data_info.cjs +201 -252
- package/dist/drivers/pframe/data_info.cjs.map +1 -1
- package/dist/drivers/pframe/data_info.d.ts +84 -109
- package/dist/drivers/pframe/data_info.js +201 -250
- package/dist/drivers/pframe/data_info.js.map +1 -1
- package/dist/drivers/pframe/data_types.cjs +38 -63
- package/dist/drivers/pframe/data_types.cjs.map +1 -1
- package/dist/drivers/pframe/data_types.d.ts +91 -93
- package/dist/drivers/pframe/data_types.js +38 -61
- package/dist/drivers/pframe/data_types.js.map +1 -1
- package/dist/drivers/pframe/driver.cjs +7 -0
- package/dist/drivers/pframe/driver.cjs.map +1 -0
- package/dist/drivers/pframe/driver.d.ts +50 -45
- package/dist/drivers/pframe/driver.js +6 -0
- package/dist/drivers/pframe/driver.js.map +1 -0
- package/dist/drivers/pframe/filter_spec.d.ts +127 -123
- package/dist/drivers/pframe/find_columns.d.ts +21 -17
- package/dist/drivers/pframe/index.cjs +15 -0
- package/dist/drivers/pframe/index.d.ts +23 -16
- package/dist/drivers/pframe/index.js +15 -0
- package/dist/drivers/pframe/linker_columns.cjs +184 -238
- package/dist/drivers/pframe/linker_columns.cjs.map +1 -1
- package/dist/drivers/pframe/linker_columns.d.ts +45 -38
- package/dist/drivers/pframe/linker_columns.js +184 -236
- package/dist/drivers/pframe/linker_columns.js.map +1 -1
- package/dist/drivers/pframe/pframe.d.ts +26 -21
- package/dist/drivers/pframe/query/index.cjs +1 -0
- package/dist/drivers/pframe/query/index.d.ts +3 -4
- package/dist/drivers/pframe/query/index.js +1 -0
- package/dist/drivers/pframe/query/query_common.d.ts +205 -247
- package/dist/drivers/pframe/query/query_data.d.ts +25 -23
- package/dist/drivers/pframe/query/query_spec.d.ts +30 -29
- package/dist/drivers/pframe/query/utils.cjs +164 -183
- package/dist/drivers/pframe/query/utils.cjs.map +1 -1
- package/dist/drivers/pframe/query/utils.d.ts +13 -12
- package/dist/drivers/pframe/query/utils.js +164 -181
- package/dist/drivers/pframe/query/utils.js.map +1 -1
- package/dist/drivers/pframe/spec/anchored.cjs +178 -216
- package/dist/drivers/pframe/spec/anchored.cjs.map +1 -1
- package/dist/drivers/pframe/spec/anchored.d.ts +53 -49
- package/dist/drivers/pframe/spec/anchored.js +176 -214
- package/dist/drivers/pframe/spec/anchored.js.map +1 -1
- package/dist/drivers/pframe/spec/filtered_column.cjs +8 -7
- package/dist/drivers/pframe/spec/filtered_column.cjs.map +1 -1
- package/dist/drivers/pframe/spec/filtered_column.d.ts +20 -17
- package/dist/drivers/pframe/spec/filtered_column.js +8 -6
- package/dist/drivers/pframe/spec/filtered_column.js.map +1 -1
- package/dist/drivers/pframe/spec/ids.cjs +16 -14
- package/dist/drivers/pframe/spec/ids.cjs.map +1 -1
- package/dist/drivers/pframe/spec/ids.d.ts +13 -8
- package/dist/drivers/pframe/spec/ids.js +14 -12
- package/dist/drivers/pframe/spec/ids.js.map +1 -1
- package/dist/drivers/pframe/spec/index.cjs +6 -0
- package/dist/drivers/pframe/spec/index.d.ts +6 -7
- package/dist/drivers/pframe/spec/index.js +6 -0
- package/dist/drivers/pframe/spec/native_id.cjs +16 -16
- package/dist/drivers/pframe/spec/native_id.cjs.map +1 -1
- package/dist/drivers/pframe/spec/native_id.d.ts +9 -4
- package/dist/drivers/pframe/spec/native_id.js +13 -13
- package/dist/drivers/pframe/spec/native_id.js.map +1 -1
- package/dist/drivers/pframe/spec/selectors.cjs +72 -112
- package/dist/drivers/pframe/spec/selectors.cjs.map +1 -1
- package/dist/drivers/pframe/spec/selectors.d.ts +99 -94
- package/dist/drivers/pframe/spec/selectors.js +72 -110
- package/dist/drivers/pframe/spec/selectors.js.map +1 -1
- package/dist/drivers/pframe/spec/spec.cjs +282 -324
- package/dist/drivers/pframe/spec/spec.cjs.map +1 -1
- package/dist/drivers/pframe/spec/spec.d.ts +263 -272
- package/dist/drivers/pframe/spec/spec.js +281 -322
- package/dist/drivers/pframe/spec/spec.js.map +1 -1
- package/dist/drivers/pframe/table.d.ts +28 -24
- package/dist/drivers/pframe/table_calculate.cjs +117 -147
- package/dist/drivers/pframe/table_calculate.cjs.map +1 -1
- package/dist/drivers/pframe/table_calculate.d.ts +219 -220
- package/dist/drivers/pframe/table_calculate.js +117 -145
- package/dist/drivers/pframe/table_calculate.js.map +1 -1
- package/dist/drivers/pframe/table_common.cjs +13 -14
- package/dist/drivers/pframe/table_common.cjs.map +1 -1
- package/dist/drivers/pframe/table_common.d.ts +24 -19
- package/dist/drivers/pframe/table_common.js +13 -13
- package/dist/drivers/pframe/table_common.js.map +1 -1
- package/dist/drivers/pframe/unique_values.d.ts +23 -18
- package/dist/drivers/upload.d.ts +21 -18
- package/dist/drivers/urls.cjs +5 -6
- package/dist/drivers/urls.cjs.map +1 -1
- package/dist/drivers/urls.d.ts +12 -9
- package/dist/drivers/urls.js +5 -5
- package/dist/drivers/urls.js.map +1 -1
- package/dist/errors.cjs +65 -86
- package/dist/errors.cjs.map +1 -1
- package/dist/errors.d.ts +36 -33
- package/dist/errors.js +65 -85
- package/dist/errors.js.map +1 -1
- package/dist/flags/block_flags.cjs +7 -6
- package/dist/flags/block_flags.cjs.map +1 -1
- package/dist/flags/block_flags.d.ts +12 -8
- package/dist/flags/block_flags.js +7 -5
- package/dist/flags/block_flags.js.map +1 -1
- package/dist/flags/flag_utils.cjs +69 -108
- package/dist/flags/flag_utils.cjs.map +1 -1
- package/dist/flags/flag_utils.d.ts +40 -42
- package/dist/flags/flag_utils.js +70 -107
- package/dist/flags/flag_utils.js.map +1 -1
- package/dist/flags/type_utils.d.ts +12 -11
- package/dist/httpAuth.cjs +16 -20
- package/dist/httpAuth.cjs.map +1 -1
- package/dist/httpAuth.d.ts +10 -7
- package/dist/httpAuth.js +16 -19
- package/dist/httpAuth.js.map +1 -1
- package/dist/index.cjs +176 -179
- package/dist/index.d.ts +58 -20
- package/dist/index.js +37 -34
- package/dist/json.cjs +10 -8
- package/dist/json.cjs.map +1 -1
- package/dist/json.d.ts +18 -18
- package/dist/json.js +8 -6
- package/dist/json.js.map +1 -1
- package/dist/navigation.cjs +3 -2
- package/dist/navigation.cjs.map +1 -1
- package/dist/navigation.d.ts +18 -21
- package/dist/navigation.js +3 -1
- package/dist/navigation.js.map +1 -1
- package/dist/plid.cjs +16 -20
- package/dist/plid.cjs.map +1 -1
- package/dist/plid.d.ts +11 -7
- package/dist/plid.js +15 -18
- package/dist/plid.js.map +1 -1
- package/dist/pool/entry.d.ts +11 -11
- package/dist/pool/index.d.ts +3 -4
- package/dist/pool/query.cjs +21 -43
- package/dist/pool/query.cjs.map +1 -1
- package/dist/pool/query.d.ts +25 -20
- package/dist/pool/query.js +20 -40
- package/dist/pool/query.js.map +1 -1
- package/dist/pool/spec.cjs +40 -47
- package/dist/pool/spec.cjs.map +1 -1
- package/dist/pool/spec.d.ts +33 -31
- package/dist/pool/spec.js +40 -45
- package/dist/pool/spec.js.map +1 -1
- package/dist/ref.cjs +51 -71
- package/dist/ref.cjs.map +1 -1
- package/dist/ref.d.ts +36 -33
- package/dist/ref.js +50 -69
- package/dist/ref.js.map +1 -1
- package/dist/utag.d.ts +18 -14
- package/dist/util.cjs +8 -7
- package/dist/util.cjs.map +1 -1
- package/dist/util.d.ts +5 -2
- package/dist/util.js +8 -6
- package/dist/util.js.map +1 -1
- package/dist/value_or_error.cjs +7 -3
- package/dist/value_or_error.cjs.map +1 -1
- package/dist/value_or_error.d.ts +9 -6
- package/dist/value_or_error.js +7 -2
- package/dist/value_or_error.js.map +1 -1
- package/package.json +6 -6
- package/src/flags/block_flags.ts +2 -1
- package/src/flags/flag_utils.ts +0 -22
- package/dist/author_marker.d.ts.map +0 -1
- package/dist/base32_encode.d.ts +0 -8
- package/dist/base32_encode.d.ts.map +0 -1
- package/dist/base64.d.ts.map +0 -1
- package/dist/block_state.d.ts.map +0 -1
- package/dist/bmodel/block_config.d.ts.map +0 -1
- package/dist/bmodel/code.d.ts.map +0 -1
- package/dist/bmodel/container.d.ts.map +0 -1
- package/dist/bmodel/index.d.ts.map +0 -1
- package/dist/bmodel/normalization.d.ts.map +0 -1
- package/dist/bmodel/types.d.ts.map +0 -1
- package/dist/branding.d.ts.map +0 -1
- package/dist/common_types.d.ts.map +0 -1
- package/dist/driver_kit.d.ts.map +0 -1
- package/dist/drivers/ChunkedStreamReader.d.ts.map +0 -1
- package/dist/drivers/blob.d.ts.map +0 -1
- package/dist/drivers/index.d.ts.map +0 -1
- package/dist/drivers/interfaces.d.ts.map +0 -1
- package/dist/drivers/log.d.ts.map +0 -1
- package/dist/drivers/ls.d.ts.map +0 -1
- package/dist/drivers/pframe/column_filter.d.ts.map +0 -1
- package/dist/drivers/pframe/data_info.d.ts.map +0 -1
- package/dist/drivers/pframe/data_types.d.ts.map +0 -1
- package/dist/drivers/pframe/driver.d.ts.map +0 -1
- package/dist/drivers/pframe/filter_spec.d.ts.map +0 -1
- package/dist/drivers/pframe/find_columns.d.ts.map +0 -1
- package/dist/drivers/pframe/index.d.ts.map +0 -1
- package/dist/drivers/pframe/linker_columns.d.ts.map +0 -1
- package/dist/drivers/pframe/pframe.d.ts.map +0 -1
- package/dist/drivers/pframe/query/index.d.ts.map +0 -1
- package/dist/drivers/pframe/query/query_common.d.ts.map +0 -1
- package/dist/drivers/pframe/query/query_data.d.ts.map +0 -1
- package/dist/drivers/pframe/query/query_spec.d.ts.map +0 -1
- package/dist/drivers/pframe/query/utils.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/anchored.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/filtered_column.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/ids.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/index.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/native_id.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/selectors.d.ts.map +0 -1
- package/dist/drivers/pframe/spec/spec.d.ts.map +0 -1
- package/dist/drivers/pframe/table.d.ts.map +0 -1
- package/dist/drivers/pframe/table_calculate.d.ts.map +0 -1
- package/dist/drivers/pframe/table_common.d.ts.map +0 -1
- package/dist/drivers/pframe/type_util.d.ts +0 -5
- package/dist/drivers/pframe/type_util.d.ts.map +0 -1
- package/dist/drivers/pframe/unique_values.d.ts.map +0 -1
- package/dist/drivers/upload.d.ts.map +0 -1
- package/dist/drivers/urls.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/flags/block_flags.d.ts.map +0 -1
- package/dist/flags/flag_utils.d.ts.map +0 -1
- package/dist/flags/index.d.ts +0 -4
- package/dist/flags/index.d.ts.map +0 -1
- package/dist/flags/type_utils.d.ts.map +0 -1
- package/dist/httpAuth.d.ts.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/json.d.ts.map +0 -1
- package/dist/navigation.d.ts.map +0 -1
- package/dist/plid.d.ts.map +0 -1
- package/dist/pool/entry.d.ts.map +0 -1
- package/dist/pool/index.d.ts.map +0 -1
- package/dist/pool/query.d.ts.map +0 -1
- package/dist/pool/spec.d.ts.map +0 -1
- package/dist/ref.d.ts.map +0 -1
- package/dist/utag.d.ts.map +0 -1
- package/dist/util.d.ts.map +0 -1
- package/dist/value_or_error.d.ts.map +0 -1
|
@@ -1,278 +1,227 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
var util = require('../../util.cjs');
|
|
1
|
+
const require_util = require('../../util.cjs');
|
|
4
2
|
|
|
3
|
+
//#region src/drivers/pframe/data_info.ts
|
|
5
4
|
/**
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
* Type guard function that checks if the given value is a valid DataInfo.
|
|
6
|
+
*
|
|
7
|
+
* @param value - The value to check
|
|
8
|
+
* @returns True if the value is a valid DataInfo, false otherwise
|
|
9
|
+
*/
|
|
11
10
|
function isDataInfo(value) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
data.data !== undefined &&
|
|
23
|
-
typeof data.data === "object");
|
|
24
|
-
case "JsonPartitioned":
|
|
25
|
-
case "BinaryPartitioned":
|
|
26
|
-
case "ParquetPartitioned":
|
|
27
|
-
return (typeof data.partitionKeyLength === "number" &&
|
|
28
|
-
data.parts !== undefined &&
|
|
29
|
-
typeof data.parts === "object");
|
|
30
|
-
default:
|
|
31
|
-
return false;
|
|
32
|
-
}
|
|
11
|
+
if (!value || typeof value !== "object") return false;
|
|
12
|
+
const data = value;
|
|
13
|
+
if (!("type" in data)) return false;
|
|
14
|
+
switch (data.type) {
|
|
15
|
+
case "Json": return typeof data.keyLength === "number" && data.data !== void 0 && typeof data.data === "object";
|
|
16
|
+
case "JsonPartitioned":
|
|
17
|
+
case "BinaryPartitioned":
|
|
18
|
+
case "ParquetPartitioned": return typeof data.partitionKeyLength === "number" && data.parts !== void 0 && typeof data.parts === "object";
|
|
19
|
+
default: return false;
|
|
20
|
+
}
|
|
33
21
|
}
|
|
34
22
|
function mapDataInfo(dataInfo, mapFn) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}
|
|
67
|
-
case "ParquetPartitioned": {
|
|
68
|
-
// Map each blob in parts
|
|
69
|
-
const newParts = {};
|
|
70
|
-
for (const [key, blob] of Object.entries(dataInfo.parts)) {
|
|
71
|
-
newParts[key] = mapFn(blob);
|
|
72
|
-
}
|
|
73
|
-
return {
|
|
74
|
-
...dataInfo,
|
|
75
|
-
parts: newParts,
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
}
|
|
23
|
+
if (dataInfo === void 0) return;
|
|
24
|
+
switch (dataInfo.type) {
|
|
25
|
+
case "Json": return dataInfo;
|
|
26
|
+
case "JsonPartitioned": {
|
|
27
|
+
const newParts = {};
|
|
28
|
+
for (const [key, blob] of Object.entries(dataInfo.parts)) newParts[key] = mapFn(blob);
|
|
29
|
+
return {
|
|
30
|
+
...dataInfo,
|
|
31
|
+
parts: newParts
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
case "BinaryPartitioned": {
|
|
35
|
+
const newParts = {};
|
|
36
|
+
for (const [key, chunk] of Object.entries(dataInfo.parts)) newParts[key] = {
|
|
37
|
+
index: mapFn(chunk.index),
|
|
38
|
+
values: mapFn(chunk.values)
|
|
39
|
+
};
|
|
40
|
+
return {
|
|
41
|
+
...dataInfo,
|
|
42
|
+
parts: newParts
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
case "ParquetPartitioned": {
|
|
46
|
+
const newParts = {};
|
|
47
|
+
for (const [key, blob] of Object.entries(dataInfo.parts)) newParts[key] = mapFn(blob);
|
|
48
|
+
return {
|
|
49
|
+
...dataInfo,
|
|
50
|
+
parts: newParts
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
79
54
|
}
|
|
80
55
|
/**
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
56
|
+
* @param dataInfo - The source DataInfo object
|
|
57
|
+
* @param cb - Callback, function that have access to every blob to visit them all
|
|
58
|
+
* @returns Nothing
|
|
59
|
+
*/
|
|
85
60
|
function visitDataInfo(dataInfo, cb) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
break;
|
|
102
|
-
}
|
|
103
|
-
case "ParquetPartitioned": {
|
|
104
|
-
// Visit each blob in parts
|
|
105
|
-
Object.values(dataInfo.parts).forEach(cb);
|
|
106
|
-
break;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
61
|
+
switch (dataInfo.type) {
|
|
62
|
+
case "Json": break;
|
|
63
|
+
case "JsonPartitioned":
|
|
64
|
+
Object.values(dataInfo.parts).forEach(cb);
|
|
65
|
+
break;
|
|
66
|
+
case "BinaryPartitioned":
|
|
67
|
+
Object.values(dataInfo.parts).forEach((chunk) => {
|
|
68
|
+
cb(chunk.index);
|
|
69
|
+
cb(chunk.values);
|
|
70
|
+
});
|
|
71
|
+
break;
|
|
72
|
+
case "ParquetPartitioned":
|
|
73
|
+
Object.values(dataInfo.parts).forEach(cb);
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
109
76
|
}
|
|
110
77
|
/**
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
78
|
+
* Type guard function that checks if the given value is a valid DataInfoEntries.
|
|
79
|
+
*
|
|
80
|
+
* @param value - The value to check
|
|
81
|
+
* @returns True if the value is a valid DataInfoEntries, false otherwise
|
|
82
|
+
*/
|
|
116
83
|
function isDataInfoEntries(value) {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
case "JsonPartitioned":
|
|
128
|
-
case "BinaryPartitioned":
|
|
129
|
-
case "ParquetPartitioned":
|
|
130
|
-
return typeof data.partitionKeyLength === "number" && Array.isArray(data.parts);
|
|
131
|
-
default:
|
|
132
|
-
return false;
|
|
133
|
-
}
|
|
84
|
+
if (!value || typeof value !== "object") return false;
|
|
85
|
+
const data = value;
|
|
86
|
+
if (!("type" in data)) return false;
|
|
87
|
+
switch (data.type) {
|
|
88
|
+
case "Json": return typeof data.keyLength === "number" && Array.isArray(data.data);
|
|
89
|
+
case "JsonPartitioned":
|
|
90
|
+
case "BinaryPartitioned":
|
|
91
|
+
case "ParquetPartitioned": return typeof data.partitionKeyLength === "number" && Array.isArray(data.parts);
|
|
92
|
+
default: return false;
|
|
93
|
+
}
|
|
134
94
|
}
|
|
135
95
|
/**
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
96
|
+
* Type guard function that checks if the given value is a valid PartitionedDataInfoEntries.
|
|
97
|
+
*
|
|
98
|
+
* @template Blob - Type parameter representing the storage reference type
|
|
99
|
+
* @param value - The value to check
|
|
100
|
+
* @returns True if the value is a valid PartitionedDataInfoEntries, false otherwise
|
|
101
|
+
*/
|
|
142
102
|
function isPartitionedDataInfoEntries(value) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
default:
|
|
151
|
-
return false;
|
|
152
|
-
}
|
|
103
|
+
if (!isDataInfoEntries(value)) return false;
|
|
104
|
+
switch (value.type) {
|
|
105
|
+
case "JsonPartitioned":
|
|
106
|
+
case "BinaryPartitioned":
|
|
107
|
+
case "ParquetPartitioned": return true;
|
|
108
|
+
default: return false;
|
|
109
|
+
}
|
|
153
110
|
}
|
|
154
111
|
/**
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
112
|
+
* Converts DataInfo to DataInfoEntries
|
|
113
|
+
*
|
|
114
|
+
* @param dataInfo - The record-based DataInfo object
|
|
115
|
+
* @returns The equivalent entry-based DataInfoEntries object
|
|
116
|
+
*/
|
|
160
117
|
function dataInfoToEntries(dataInfo) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
118
|
+
switch (dataInfo.type) {
|
|
119
|
+
case "Json": return {
|
|
120
|
+
type: "Json",
|
|
121
|
+
keyLength: dataInfo.keyLength,
|
|
122
|
+
data: Object.entries(dataInfo.data).map(([keyStr, value]) => {
|
|
123
|
+
return {
|
|
124
|
+
key: JSON.parse(keyStr),
|
|
125
|
+
value
|
|
126
|
+
};
|
|
127
|
+
})
|
|
128
|
+
};
|
|
129
|
+
case "JsonPartitioned": return {
|
|
130
|
+
type: "JsonPartitioned",
|
|
131
|
+
partitionKeyLength: dataInfo.partitionKeyLength,
|
|
132
|
+
parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {
|
|
133
|
+
return {
|
|
134
|
+
key: JSON.parse(keyStr),
|
|
135
|
+
value: blob
|
|
136
|
+
};
|
|
137
|
+
})
|
|
138
|
+
};
|
|
139
|
+
case "BinaryPartitioned": return {
|
|
140
|
+
type: "BinaryPartitioned",
|
|
141
|
+
partitionKeyLength: dataInfo.partitionKeyLength,
|
|
142
|
+
parts: Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {
|
|
143
|
+
return {
|
|
144
|
+
key: JSON.parse(keyStr),
|
|
145
|
+
value: chunk
|
|
146
|
+
};
|
|
147
|
+
})
|
|
148
|
+
};
|
|
149
|
+
case "ParquetPartitioned": return {
|
|
150
|
+
type: "ParquetPartitioned",
|
|
151
|
+
partitionKeyLength: dataInfo.partitionKeyLength,
|
|
152
|
+
parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {
|
|
153
|
+
return {
|
|
154
|
+
key: JSON.parse(keyStr),
|
|
155
|
+
value: blob
|
|
156
|
+
};
|
|
157
|
+
})
|
|
158
|
+
};
|
|
159
|
+
default: require_util.assertNever(dataInfo);
|
|
160
|
+
}
|
|
201
161
|
}
|
|
202
162
|
/**
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
163
|
+
* Converts DataInfoEntries to DataInfo
|
|
164
|
+
*
|
|
165
|
+
* @param dataInfoEntries - The entry-based DataInfoEntries object
|
|
166
|
+
* @returns The equivalent record-based DataInfo object
|
|
167
|
+
*/
|
|
208
168
|
function entriesToDataInfo(dataInfoEntries) {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value])),
|
|
233
|
-
};
|
|
234
|
-
default:
|
|
235
|
-
util.assertNever(dataInfoEntries);
|
|
236
|
-
}
|
|
169
|
+
switch (dataInfoEntries.type) {
|
|
170
|
+
case "Json": return {
|
|
171
|
+
type: "Json",
|
|
172
|
+
keyLength: dataInfoEntries.keyLength,
|
|
173
|
+
data: Object.fromEntries(dataInfoEntries.data.map(({ key, value }) => [JSON.stringify(key), value]))
|
|
174
|
+
};
|
|
175
|
+
case "JsonPartitioned": return {
|
|
176
|
+
type: "JsonPartitioned",
|
|
177
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength,
|
|
178
|
+
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]))
|
|
179
|
+
};
|
|
180
|
+
case "BinaryPartitioned": return {
|
|
181
|
+
type: "BinaryPartitioned",
|
|
182
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength,
|
|
183
|
+
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]))
|
|
184
|
+
};
|
|
185
|
+
case "ParquetPartitioned": return {
|
|
186
|
+
type: "ParquetPartitioned",
|
|
187
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength,
|
|
188
|
+
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]))
|
|
189
|
+
};
|
|
190
|
+
default: require_util.assertNever(dataInfoEntries);
|
|
191
|
+
}
|
|
237
192
|
}
|
|
238
193
|
function mapDataInfoEntries(dataInfoEntries, mapFn) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
...dataInfoEntries,
|
|
268
|
-
parts: dataInfoEntries.parts.map((entry) => ({
|
|
269
|
-
key: entry.key,
|
|
270
|
-
value: mapFn(entry.value),
|
|
271
|
-
})),
|
|
272
|
-
};
|
|
273
|
-
}
|
|
194
|
+
if (dataInfoEntries === void 0) return;
|
|
195
|
+
switch (dataInfoEntries.type) {
|
|
196
|
+
case "Json": return dataInfoEntries;
|
|
197
|
+
case "JsonPartitioned": return {
|
|
198
|
+
...dataInfoEntries,
|
|
199
|
+
parts: dataInfoEntries.parts.map((entry) => ({
|
|
200
|
+
key: entry.key,
|
|
201
|
+
value: mapFn(entry.value)
|
|
202
|
+
}))
|
|
203
|
+
};
|
|
204
|
+
case "BinaryPartitioned": return {
|
|
205
|
+
...dataInfoEntries,
|
|
206
|
+
parts: dataInfoEntries.parts.map((entry) => ({
|
|
207
|
+
key: entry.key,
|
|
208
|
+
value: {
|
|
209
|
+
index: mapFn(entry.value.index),
|
|
210
|
+
values: mapFn(entry.value.values)
|
|
211
|
+
}
|
|
212
|
+
}))
|
|
213
|
+
};
|
|
214
|
+
case "ParquetPartitioned": return {
|
|
215
|
+
...dataInfoEntries,
|
|
216
|
+
parts: dataInfoEntries.parts.map((entry) => ({
|
|
217
|
+
key: entry.key,
|
|
218
|
+
value: mapFn(entry.value)
|
|
219
|
+
}))
|
|
220
|
+
};
|
|
221
|
+
}
|
|
274
222
|
}
|
|
275
223
|
|
|
224
|
+
//#endregion
|
|
276
225
|
exports.dataInfoToEntries = dataInfoToEntries;
|
|
277
226
|
exports.entriesToDataInfo = entriesToDataInfo;
|
|
278
227
|
exports.isDataInfo = isDataInfo;
|
|
@@ -281,4 +230,4 @@ exports.isPartitionedDataInfoEntries = isPartitionedDataInfoEntries;
|
|
|
281
230
|
exports.mapDataInfo = mapDataInfo;
|
|
282
231
|
exports.mapDataInfoEntries = mapDataInfoEntries;
|
|
283
232
|
exports.visitDataInfo = visitDataInfo;
|
|
284
|
-
//# sourceMappingURL=data_info.cjs.map
|
|
233
|
+
//# sourceMappingURL=data_info.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data_info.cjs","sources":["../../../src/drivers/pframe/data_info.ts"],"sourcesContent":["import { assertNever } from \"../../util\";\n\n/**\n * Represents a JavaScript representation of a value in a PColumn. Can be null, a number, or a string.\n * These are the primitive types that can be stored directly in PColumns.\n *\n * Note: Actual columns can hold more value types, which are converted to these JavaScript types\n * once they enter the JavaScript runtime.\n */\nexport type PColumnValue = null | number | string;\n\n/**\n * Represents a key for a PColumn value.\n * Can be an array of strings or numbers.\n */\nexport type PColumnKey = (number | string)[];\n\n/**\n * Represents a single entry in a PColumn's data structure.\n * Contains a key and a value.\n */\nexport type PColumnDataEntry<T> = {\n /** Key for the value */\n key: PColumnKey;\n\n /** Value / blob at the given key */\n value: T;\n};\n\n/**\n * Represents column data stored as a simple JSON structure.\n * Used for small datasets that can be efficiently stored directly in memory.\n */\nexport type JsonDataInfo = {\n /** Identifier for this data format ('Json') */\n type: \"Json\";\n\n /** Number of axes that make up the complete key (tuple length) */\n keyLength: number;\n\n /**\n * Key-value pairs where keys are stringified tuples of axis values\n * and values are the column values for those coordinates\n */\n data: Record<string, PColumnValue>;\n};\n\n/**\n * Represents column data partitioned across multiple JSON blobs.\n * Used for larger datasets that need to be split into manageable chunks.\n */\nexport type JsonPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('JsonPartitioned') */\n type: \"JsonPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to blob references */\n parts: Record<string, Blob>;\n};\n\n/**\n * Represents a binary format chunk containing index and values as separate blobs.\n * Used for efficient storage and retrieval of column data in binary format.\n */\nexport type BinaryChunk<Blob> = {\n /** Binary blob containing structured index information */\n index: Blob;\n\n /** Binary blob containing the actual values */\n values: Blob;\n};\n\n/**\n * Represents column data partitioned across multiple binary chunks.\n * Optimized for efficient storage and retrieval of large datasets.\n */\nexport type BinaryPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('BinaryPartitioned') */\n type: \"BinaryPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to binary chunks */\n parts: Record<string, BinaryChunk<Blob>>;\n};\n\nexport type ParquetChunkMappingAxis = {\n /** Data type (matches PColumn axis types) */\n type: \"Int\" | \"Long\" | \"String\";\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMappingColumn = {\n /** Data type (matches PColumn value type) */\n type: \"Int\" | \"Long\" | \"Float\" | \"Double\" | \"String\";\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMapping = {\n /** Axes mappings - Parquet file is sorted by these fields in this order */\n axes: ParquetChunkMappingAxis[];\n\n /** Column mapping */\n column: ParquetChunkMappingColumn;\n};\n\nexport type ParquetChunkStats = {\n /** Number of rows in the chunk */\n numberOfRows: number;\n /** Byte size information for storage optimization and query planning */\n size: {\n /** Byte sizes for each axis column in the same order as axes mapping */\n axes: number[];\n /** Byte size for the data column */\n column: number;\n };\n};\n\nexport type ParquetChunkMetadata = {\n /** Content hash calculated for the specific axes and data this chunk represents */\n dataDigest: string;\n\n /** Pre-computed statistics for optimization without blob download */\n stats: Partial<ParquetChunkStats>;\n};\n\nexport type ParquetChunk<Blob> = {\n /** Parquet file (PTable) containing column data */\n data: Blob;\n} & ParquetChunkMapping &\n Partial<ParquetChunkMetadata>;\n\nexport type ParquetPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('ParquetPartitioned') */\n type: \"ParquetPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to parquet files */\n parts: Record<string, Blob>;\n};\n\n/**\n * Union type representing all possible data storage formats for PColumn data.\n * The specific format used depends on data size, access patterns, and performance requirements.\n *\n * @template Blob - Type parameter representing the storage reference type (could be ResourceInfo, PFrameBlobId, etc.)\n */\nexport type DataInfo<Blob> =\n | JsonDataInfo\n | JsonPartitionedDataInfo<Blob>\n | BinaryPartitionedDataInfo<Blob>\n | ParquetPartitionedDataInfo<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfo.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfo, false otherwise\n */\nexport function isDataInfo<Blob>(value: unknown): value is DataInfo<Blob> {\n if (!value || typeof value !== \"object\") {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!(\"type\" in data)) {\n return false;\n }\n\n switch (data.type) {\n case \"Json\":\n return (\n typeof data.keyLength === \"number\" &&\n data.data !== undefined &&\n typeof data.data === \"object\"\n );\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return (\n typeof data.partitionKeyLength === \"number\" &&\n data.parts !== undefined &&\n typeof data.parts === \"object\"\n );\n default:\n return false;\n }\n}\n\n/**\n * Maps blob references in a DataInfo object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfo - The source DataInfo object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfo object with transformed blob references\n */\nexport function mapDataInfo<B1, B2>(\n dataInfo: ParquetPartitionedDataInfo<B1>,\n mapFn: (blob: B1) => B2,\n): ParquetPartitionedDataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: Exclude<DataInfo<B1>, ParquetPartitionedDataInfo<B1>>,\n mapFn: (blob: B1) => B2,\n): Exclude<DataInfo<B2>, ParquetPartitionedDataInfo<B2>>;\nexport function mapDataInfo<B1, B2>(dataInfo: DataInfo<B1>, mapFn: (blob: B1) => B2): DataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: DataInfo<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfo<B2> | undefined {\n if (dataInfo === undefined) {\n return undefined;\n }\n\n switch (dataInfo.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n return dataInfo;\n case \"JsonPartitioned\": {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case \"BinaryPartitioned\": {\n // Map each index and values blob in parts\n const newParts: Record<string, BinaryChunk<B2>> = {};\n for (const [key, chunk] of Object.entries(dataInfo.parts)) {\n newParts[key] = {\n index: mapFn(chunk.index),\n values: mapFn(chunk.values),\n };\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case \"ParquetPartitioned\": {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n }\n}\n\n/**\n * @param dataInfo - The source DataInfo object\n * @param cb - Callback, function that have access to every blob to visit them all\n * @returns Nothing\n */\nexport function visitDataInfo<B>(dataInfo: DataInfo<B>, cb: (blob: B) => void): void {\n switch (dataInfo.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n break;\n case \"JsonPartitioned\": {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n case \"BinaryPartitioned\": {\n // Visit each index and values blob in parts\n Object.values(dataInfo.parts).forEach((chunk) => {\n cb(chunk.index);\n cb(chunk.values);\n });\n break;\n }\n case \"ParquetPartitioned\": {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n }\n}\n\n//\n// Lightway representation for ExplicitJsonData\n//\n\n/**\n * Represents a single key-value entry in a column's explicit data structure.\n * Used when directly instantiating PColumns with explicit data.\n */\nexport type PColumnValuesEntry = {\n key: PColumnKey;\n val: PColumnValue;\n};\n\n/**\n * Array of key-value entries representing explicit column data.\n * Used for lightweight explicit instantiation of PColumns.\n */\nexport type PColumnValues = PColumnValuesEntry[];\n\n/**\n * Entry-based representation of JsonDataInfo\n */\nexport interface JsonDataInfoEntries {\n type: \"Json\";\n keyLength: number;\n data: PColumnDataEntry<PColumnValue>[];\n}\n\n/**\n * Entry-based representation of JsonPartitionedDataInfo\n */\nexport interface JsonPartitionedDataInfoEntries<Blob> {\n type: \"JsonPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n\n/**\n * Entry-based representation of BinaryPartitionedDataInfo\n */\nexport interface BinaryPartitionedDataInfoEntries<Blob> {\n type: \"BinaryPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<BinaryChunk<Blob>>[];\n}\n\n/**\n * Entry-based representation of ParquetPartitionedDataInfo\n */\nexport interface ParquetPartitionedDataInfoEntries<Blob> {\n type: \"ParquetPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n/**\n * Union type representing all possible entry-based partitioned data storage formats\n */\nexport type PartitionedDataInfoEntries<Blob> =\n | JsonPartitionedDataInfoEntries<Blob>\n | BinaryPartitionedDataInfoEntries<Blob>\n | ParquetPartitionedDataInfoEntries<Blob>;\n\n/**\n * Union type representing all possible entry-based data storage formats\n */\nexport type DataInfoEntries<Blob> = JsonDataInfoEntries | PartitionedDataInfoEntries<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfoEntries.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfoEntries, false otherwise\n */\nexport function isDataInfoEntries<Blob>(value: unknown): value is DataInfoEntries<Blob> {\n if (!value || typeof value !== \"object\") {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!(\"type\" in data)) {\n return false;\n }\n\n switch (data.type) {\n case \"Json\":\n return typeof data.keyLength === \"number\" && Array.isArray(data.data);\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return typeof data.partitionKeyLength === \"number\" && Array.isArray(data.parts);\n default:\n return false;\n }\n}\n\n/**\n * Type guard function that checks if the given value is a valid PartitionedDataInfoEntries.\n *\n * @template Blob - Type parameter representing the storage reference type\n * @param value - The value to check\n * @returns True if the value is a valid PartitionedDataInfoEntries, false otherwise\n */\nexport function isPartitionedDataInfoEntries<Blob>(\n value: unknown,\n): value is PartitionedDataInfoEntries<Blob> {\n if (!isDataInfoEntries(value)) return false;\n switch (value.type) {\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return true;\n default:\n return false;\n }\n}\n\n/**\n * Converts DataInfo to DataInfoEntries\n *\n * @param dataInfo - The record-based DataInfo object\n * @returns The equivalent entry-based DataInfoEntries object\n */\nexport function dataInfoToEntries<Blob>(dataInfo: DataInfo<Blob>): DataInfoEntries<Blob> {\n switch (dataInfo.type) {\n case \"Json\":\n return {\n type: \"Json\",\n keyLength: dataInfo.keyLength,\n data: Object.entries(dataInfo.data).map(([keyStr, value]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value } as PColumnDataEntry<PColumnValue>;\n }),\n };\n case \"JsonPartitioned\":\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n case \"BinaryPartitioned\":\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: chunk } as PColumnDataEntry<BinaryChunk<Blob>>;\n }),\n };\n case \"ParquetPartitioned\":\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n default:\n assertNever(dataInfo);\n }\n}\n\n/**\n * Converts DataInfoEntries to DataInfo\n *\n * @param dataInfoEntries - The entry-based DataInfoEntries object\n * @returns The equivalent record-based DataInfo object\n */\nexport function entriesToDataInfo<Blob>(dataInfoEntries: DataInfoEntries<Blob>): DataInfo<Blob> {\n switch (dataInfoEntries.type) {\n case \"Json\":\n return {\n type: \"Json\",\n keyLength: dataInfoEntries.keyLength,\n data: Object.fromEntries(\n dataInfoEntries.data.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"JsonPartitioned\":\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"BinaryPartitioned\":\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"ParquetPartitioned\":\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n default:\n assertNever(dataInfoEntries);\n }\n}\n\n/**\n * Maps blob references in a DataInfoEntries object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfoEntries - The source DataInfoEntries object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfoEntries object with transformed blob references\n */\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1>,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2>;\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2> | undefined {\n if (dataInfoEntries === undefined) {\n return undefined;\n }\n\n switch (dataInfoEntries.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n return dataInfoEntries;\n case \"JsonPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n case \"BinaryPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: {\n index: mapFn(entry.value.index),\n values: mapFn(entry.value.values),\n },\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n }\n}\n"],"names":["assertNever"],"mappings":";;;;AAkKA;;;;;AAKG;AACG,SAAU,UAAU,CAAO,KAAc,EAAA;IAC7C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACvC,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,IAAI,GAAG,KAAgC;AAC7C,IAAA,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;IACd;AAEA,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,QACE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;gBAClC,IAAI,CAAC,IAAI,KAAK,SAAS;AACvB,gBAAA,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;AAEjC,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,QACE,OAAO,IAAI,CAAC,kBAAkB,KAAK,QAAQ;gBAC3C,IAAI,CAAC,KAAK,KAAK,SAAS;AACxB,gBAAA,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;AAElC,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAoBM,SAAU,WAAW,CACzB,QAAkC,EAClC,KAAuB,EAAA;AAEvB,IAAA,IAAI,QAAQ,KAAK,SAAS,EAAE;AAC1B,QAAA,OAAO,SAAS;IAClB;AAEA,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;;AAET,YAAA,OAAO,QAAQ;QACjB,KAAK,iBAAiB,EAAE;;YAEtB,MAAM,QAAQ,GAAuB,EAAE;AACvC,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACxD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YAC7B;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;QACA,KAAK,mBAAmB,EAAE;;YAExB,MAAM,QAAQ,GAAoC,EAAE;AACpD,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACzD,QAAQ,CAAC,GAAG,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,oBAAA,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;iBAC5B;YACH;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;QACA,KAAK,oBAAoB,EAAE;;YAEzB,MAAM,QAAQ,GAAuB,EAAE;AACvC,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACxD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YAC7B;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;;AAEJ;AAEA;;;;AAIG;AACG,SAAU,aAAa,CAAI,QAAqB,EAAE,EAAqB,EAAA;AAC3E,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;;YAET;QACF,KAAK,iBAAiB,EAAE;;AAEtB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC;QACF;QACA,KAAK,mBAAmB,EAAE;;AAExB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,KAAI;AAC9C,gBAAA,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;AACf,gBAAA,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,YAAA,CAAC,CAAC;YACF;QACF;QACA,KAAK,oBAAoB,EAAE;;AAEzB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC;QACF;;AAEJ;AAqEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,KAAc,EAAA;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACvC,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,IAAI,GAAG,KAAgC;AAC7C,IAAA,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;IACd;AAEA,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACvE,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,OAAO,OAAO,IAAI,CAAC,kBAAkB,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;AACjF,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAEA;;;;;;AAMG;AACG,SAAU,4BAA4B,CAC1C,KAAc,EAAA;AAEd,IAAA,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;AAAE,QAAA,OAAO,KAAK;AAC3C,IAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,OAAO,IAAI;AACb,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,QAAwB,EAAA;AAC9D,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;YACT,OAAO;AACL,gBAAA,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,QAAQ,CAAC,SAAS;AAC7B,gBAAA,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,KAAI;oBAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAoC;AACzD,gBAAA,CAAC,CAAC;aACH;AACH,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,IAAI,EAAE,iBAAiB;gBACvB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,gBAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAI;oBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAA4B;AACvD,gBAAA,CAAC,CAAC;aACH;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,IAAI,EAAE,mBAAmB;gBACzB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,gBAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,KAAI;oBAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAyC;AACrE,gBAAA,CAAC,CAAC;aACH;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,IAAI,EAAE,oBAAoB;gBAC1B,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,gBAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAI;oBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAA4B;AACvD,gBAAA,CAAC,CAAC;aACH;AACH,QAAA;YACEA,gBAAW,CAAC,QAAQ,CAAC;;AAE3B;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,eAAsC,EAAA;AAC5E,IAAA,QAAQ,eAAe,CAAC,IAAI;AAC1B,QAAA,KAAK,MAAM;YACT,OAAO;AACL,gBAAA,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,eAAe,CAAC,SAAS;AACpC,gBAAA,IAAI,EAAE,MAAM,CAAC,WAAW,CACtB,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC3E;aACF;AACH,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,IAAI,EAAE,iBAAiB;gBACvB,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,gBAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;aACF;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,IAAI,EAAE,mBAAmB;gBACzB,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,gBAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;aACF;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,IAAI,EAAE,oBAAoB;gBAC1B,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,gBAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;aACF;AACH,QAAA;YACEA,gBAAW,CAAC,eAAe,CAAC;;AAElC;AAeM,SAAU,kBAAkB,CAChC,eAAgD,EAChD,KAAuB,EAAA;AAEvB,IAAA,IAAI,eAAe,KAAK,SAAS,EAAE;AACjC,QAAA,OAAO,SAAS;IAClB;AAEA,IAAA,QAAQ,eAAe,CAAC,IAAI;AAC1B,QAAA,KAAK,MAAM;;AAET,YAAA,OAAO,eAAe;AACxB,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,GAAG,eAAe;AAClB,gBAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;oBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AAC1B,iBAAA,CAAC,CAAC;aACJ;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,GAAG,eAAe;AAClB,gBAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;oBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE;wBACL,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;wBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;AAClC,qBAAA;AACF,iBAAA,CAAC,CAAC;aACJ;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,GAAG,eAAe;AAClB,gBAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;oBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AAC1B,iBAAA,CAAC,CAAC;aACJ;;AAEP;;;;;;;;;;;"}
|
|
1
|
+
{"version":3,"file":"data_info.cjs","names":[],"sources":["../../../src/drivers/pframe/data_info.ts"],"sourcesContent":["import { assertNever } from \"../../util\";\n\n/**\n * Represents a JavaScript representation of a value in a PColumn. Can be null, a number, or a string.\n * These are the primitive types that can be stored directly in PColumns.\n *\n * Note: Actual columns can hold more value types, which are converted to these JavaScript types\n * once they enter the JavaScript runtime.\n */\nexport type PColumnValue = null | number | string;\n\n/**\n * Represents a key for a PColumn value.\n * Can be an array of strings or numbers.\n */\nexport type PColumnKey = (number | string)[];\n\n/**\n * Represents a single entry in a PColumn's data structure.\n * Contains a key and a value.\n */\nexport type PColumnDataEntry<T> = {\n /** Key for the value */\n key: PColumnKey;\n\n /** Value / blob at the given key */\n value: T;\n};\n\n/**\n * Represents column data stored as a simple JSON structure.\n * Used for small datasets that can be efficiently stored directly in memory.\n */\nexport type JsonDataInfo = {\n /** Identifier for this data format ('Json') */\n type: \"Json\";\n\n /** Number of axes that make up the complete key (tuple length) */\n keyLength: number;\n\n /**\n * Key-value pairs where keys are stringified tuples of axis values\n * and values are the column values for those coordinates\n */\n data: Record<string, PColumnValue>;\n};\n\n/**\n * Represents column data partitioned across multiple JSON blobs.\n * Used for larger datasets that need to be split into manageable chunks.\n */\nexport type JsonPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('JsonPartitioned') */\n type: \"JsonPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to blob references */\n parts: Record<string, Blob>;\n};\n\n/**\n * Represents a binary format chunk containing index and values as separate blobs.\n * Used for efficient storage and retrieval of column data in binary format.\n */\nexport type BinaryChunk<Blob> = {\n /** Binary blob containing structured index information */\n index: Blob;\n\n /** Binary blob containing the actual values */\n values: Blob;\n};\n\n/**\n * Represents column data partitioned across multiple binary chunks.\n * Optimized for efficient storage and retrieval of large datasets.\n */\nexport type BinaryPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('BinaryPartitioned') */\n type: \"BinaryPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to binary chunks */\n parts: Record<string, BinaryChunk<Blob>>;\n};\n\nexport type ParquetChunkMappingAxis = {\n /** Data type (matches PColumn axis types) */\n type: \"Int\" | \"Long\" | \"String\";\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMappingColumn = {\n /** Data type (matches PColumn value type) */\n type: \"Int\" | \"Long\" | \"Float\" | \"Double\" | \"String\";\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMapping = {\n /** Axes mappings - Parquet file is sorted by these fields in this order */\n axes: ParquetChunkMappingAxis[];\n\n /** Column mapping */\n column: ParquetChunkMappingColumn;\n};\n\nexport type ParquetChunkStats = {\n /** Number of rows in the chunk */\n numberOfRows: number;\n /** Byte size information for storage optimization and query planning */\n size: {\n /** Byte sizes for each axis column in the same order as axes mapping */\n axes: number[];\n /** Byte size for the data column */\n column: number;\n };\n};\n\nexport type ParquetChunkMetadata = {\n /** Content hash calculated for the specific axes and data this chunk represents */\n dataDigest: string;\n\n /** Pre-computed statistics for optimization without blob download */\n stats: Partial<ParquetChunkStats>;\n};\n\nexport type ParquetChunk<Blob> = {\n /** Parquet file (PTable) containing column data */\n data: Blob;\n} & ParquetChunkMapping &\n Partial<ParquetChunkMetadata>;\n\nexport type ParquetPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('ParquetPartitioned') */\n type: \"ParquetPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to parquet files */\n parts: Record<string, Blob>;\n};\n\n/**\n * Union type representing all possible data storage formats for PColumn data.\n * The specific format used depends on data size, access patterns, and performance requirements.\n *\n * @template Blob - Type parameter representing the storage reference type (could be ResourceInfo, PFrameBlobId, etc.)\n */\nexport type DataInfo<Blob> =\n | JsonDataInfo\n | JsonPartitionedDataInfo<Blob>\n | BinaryPartitionedDataInfo<Blob>\n | ParquetPartitionedDataInfo<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfo.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfo, false otherwise\n */\nexport function isDataInfo<Blob>(value: unknown): value is DataInfo<Blob> {\n if (!value || typeof value !== \"object\") {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!(\"type\" in data)) {\n return false;\n }\n\n switch (data.type) {\n case \"Json\":\n return (\n typeof data.keyLength === \"number\" &&\n data.data !== undefined &&\n typeof data.data === \"object\"\n );\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return (\n typeof data.partitionKeyLength === \"number\" &&\n data.parts !== undefined &&\n typeof data.parts === \"object\"\n );\n default:\n return false;\n }\n}\n\n/**\n * Maps blob references in a DataInfo object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfo - The source DataInfo object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfo object with transformed blob references\n */\nexport function mapDataInfo<B1, B2>(\n dataInfo: ParquetPartitionedDataInfo<B1>,\n mapFn: (blob: B1) => B2,\n): ParquetPartitionedDataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: Exclude<DataInfo<B1>, ParquetPartitionedDataInfo<B1>>,\n mapFn: (blob: B1) => B2,\n): Exclude<DataInfo<B2>, ParquetPartitionedDataInfo<B2>>;\nexport function mapDataInfo<B1, B2>(dataInfo: DataInfo<B1>, mapFn: (blob: B1) => B2): DataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: DataInfo<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfo<B2> | undefined {\n if (dataInfo === undefined) {\n return undefined;\n }\n\n switch (dataInfo.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n return dataInfo;\n case \"JsonPartitioned\": {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case \"BinaryPartitioned\": {\n // Map each index and values blob in parts\n const newParts: Record<string, BinaryChunk<B2>> = {};\n for (const [key, chunk] of Object.entries(dataInfo.parts)) {\n newParts[key] = {\n index: mapFn(chunk.index),\n values: mapFn(chunk.values),\n };\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case \"ParquetPartitioned\": {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n }\n}\n\n/**\n * @param dataInfo - The source DataInfo object\n * @param cb - Callback, function that have access to every blob to visit them all\n * @returns Nothing\n */\nexport function visitDataInfo<B>(dataInfo: DataInfo<B>, cb: (blob: B) => void): void {\n switch (dataInfo.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n break;\n case \"JsonPartitioned\": {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n case \"BinaryPartitioned\": {\n // Visit each index and values blob in parts\n Object.values(dataInfo.parts).forEach((chunk) => {\n cb(chunk.index);\n cb(chunk.values);\n });\n break;\n }\n case \"ParquetPartitioned\": {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n }\n}\n\n//\n// Lightway representation for ExplicitJsonData\n//\n\n/**\n * Represents a single key-value entry in a column's explicit data structure.\n * Used when directly instantiating PColumns with explicit data.\n */\nexport type PColumnValuesEntry = {\n key: PColumnKey;\n val: PColumnValue;\n};\n\n/**\n * Array of key-value entries representing explicit column data.\n * Used for lightweight explicit instantiation of PColumns.\n */\nexport type PColumnValues = PColumnValuesEntry[];\n\n/**\n * Entry-based representation of JsonDataInfo\n */\nexport interface JsonDataInfoEntries {\n type: \"Json\";\n keyLength: number;\n data: PColumnDataEntry<PColumnValue>[];\n}\n\n/**\n * Entry-based representation of JsonPartitionedDataInfo\n */\nexport interface JsonPartitionedDataInfoEntries<Blob> {\n type: \"JsonPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n\n/**\n * Entry-based representation of BinaryPartitionedDataInfo\n */\nexport interface BinaryPartitionedDataInfoEntries<Blob> {\n type: \"BinaryPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<BinaryChunk<Blob>>[];\n}\n\n/**\n * Entry-based representation of ParquetPartitionedDataInfo\n */\nexport interface ParquetPartitionedDataInfoEntries<Blob> {\n type: \"ParquetPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n/**\n * Union type representing all possible entry-based partitioned data storage formats\n */\nexport type PartitionedDataInfoEntries<Blob> =\n | JsonPartitionedDataInfoEntries<Blob>\n | BinaryPartitionedDataInfoEntries<Blob>\n | ParquetPartitionedDataInfoEntries<Blob>;\n\n/**\n * Union type representing all possible entry-based data storage formats\n */\nexport type DataInfoEntries<Blob> = JsonDataInfoEntries | PartitionedDataInfoEntries<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfoEntries.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfoEntries, false otherwise\n */\nexport function isDataInfoEntries<Blob>(value: unknown): value is DataInfoEntries<Blob> {\n if (!value || typeof value !== \"object\") {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!(\"type\" in data)) {\n return false;\n }\n\n switch (data.type) {\n case \"Json\":\n return typeof data.keyLength === \"number\" && Array.isArray(data.data);\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return typeof data.partitionKeyLength === \"number\" && Array.isArray(data.parts);\n default:\n return false;\n }\n}\n\n/**\n * Type guard function that checks if the given value is a valid PartitionedDataInfoEntries.\n *\n * @template Blob - Type parameter representing the storage reference type\n * @param value - The value to check\n * @returns True if the value is a valid PartitionedDataInfoEntries, false otherwise\n */\nexport function isPartitionedDataInfoEntries<Blob>(\n value: unknown,\n): value is PartitionedDataInfoEntries<Blob> {\n if (!isDataInfoEntries(value)) return false;\n switch (value.type) {\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return true;\n default:\n return false;\n }\n}\n\n/**\n * Converts DataInfo to DataInfoEntries\n *\n * @param dataInfo - The record-based DataInfo object\n * @returns The equivalent entry-based DataInfoEntries object\n */\nexport function dataInfoToEntries<Blob>(dataInfo: DataInfo<Blob>): DataInfoEntries<Blob> {\n switch (dataInfo.type) {\n case \"Json\":\n return {\n type: \"Json\",\n keyLength: dataInfo.keyLength,\n data: Object.entries(dataInfo.data).map(([keyStr, value]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value } as PColumnDataEntry<PColumnValue>;\n }),\n };\n case \"JsonPartitioned\":\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n case \"BinaryPartitioned\":\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: chunk } as PColumnDataEntry<BinaryChunk<Blob>>;\n }),\n };\n case \"ParquetPartitioned\":\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n default:\n assertNever(dataInfo);\n }\n}\n\n/**\n * Converts DataInfoEntries to DataInfo\n *\n * @param dataInfoEntries - The entry-based DataInfoEntries object\n * @returns The equivalent record-based DataInfo object\n */\nexport function entriesToDataInfo<Blob>(dataInfoEntries: DataInfoEntries<Blob>): DataInfo<Blob> {\n switch (dataInfoEntries.type) {\n case \"Json\":\n return {\n type: \"Json\",\n keyLength: dataInfoEntries.keyLength,\n data: Object.fromEntries(\n dataInfoEntries.data.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"JsonPartitioned\":\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"BinaryPartitioned\":\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"ParquetPartitioned\":\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n default:\n assertNever(dataInfoEntries);\n }\n}\n\n/**\n * Maps blob references in a DataInfoEntries object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfoEntries - The source DataInfoEntries object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfoEntries object with transformed blob references\n */\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1>,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2>;\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2> | undefined {\n if (dataInfoEntries === undefined) {\n return undefined;\n }\n\n switch (dataInfoEntries.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n return dataInfoEntries;\n case \"JsonPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n case \"BinaryPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: {\n index: mapFn(entry.value.index),\n values: mapFn(entry.value.values),\n },\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n }\n}\n"],"mappings":";;;;;;;;;AAwKA,SAAgB,WAAiB,OAAyC;AACxE,KAAI,CAAC,SAAS,OAAO,UAAU,SAC7B,QAAO;CAGT,MAAM,OAAO;AACb,KAAI,EAAE,UAAU,MACd,QAAO;AAGT,SAAQ,KAAK,MAAb;EACE,KAAK,OACH,QACE,OAAO,KAAK,cAAc,YAC1B,KAAK,SAAS,UACd,OAAO,KAAK,SAAS;EAEzB,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QACE,OAAO,KAAK,uBAAuB,YACnC,KAAK,UAAU,UACf,OAAO,KAAK,UAAU;EAE1B,QACE,QAAO;;;AAsBb,SAAgB,YACd,UACA,OAC0B;AAC1B,KAAI,aAAa,OACf;AAGF,SAAQ,SAAS,MAAjB;EACE,KAAK,OAEH,QAAO;EACT,KAAK,mBAAmB;GAEtB,MAAM,WAA+B,EAAE;AACvC,QAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,SAAS,MAAM,CACtD,UAAS,OAAO,MAAM,KAAK;AAE7B,UAAO;IACL,GAAG;IACH,OAAO;IACR;;EAEH,KAAK,qBAAqB;GAExB,MAAM,WAA4C,EAAE;AACpD,QAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,SAAS,MAAM,CACvD,UAAS,OAAO;IACd,OAAO,MAAM,MAAM,MAAM;IACzB,QAAQ,MAAM,MAAM,OAAO;IAC5B;AAEH,UAAO;IACL,GAAG;IACH,OAAO;IACR;;EAEH,KAAK,sBAAsB;GAEzB,MAAM,WAA+B,EAAE;AACvC,QAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,SAAS,MAAM,CACtD,UAAS,OAAO,MAAM,KAAK;AAE7B,UAAO;IACL,GAAG;IACH,OAAO;IACR;;;;;;;;;AAUP,SAAgB,cAAiB,UAAuB,IAA6B;AACnF,SAAQ,SAAS,MAAjB;EACE,KAAK,OAEH;EACF,KAAK;AAEH,UAAO,OAAO,SAAS,MAAM,CAAC,QAAQ,GAAG;AACzC;EAEF,KAAK;AAEH,UAAO,OAAO,SAAS,MAAM,CAAC,SAAS,UAAU;AAC/C,OAAG,MAAM,MAAM;AACf,OAAG,MAAM,OAAO;KAChB;AACF;EAEF,KAAK;AAEH,UAAO,OAAO,SAAS,MAAM,CAAC,QAAQ,GAAG;AACzC;;;;;;;;;AA8EN,SAAgB,kBAAwB,OAAgD;AACtF,KAAI,CAAC,SAAS,OAAO,UAAU,SAC7B,QAAO;CAGT,MAAM,OAAO;AACb,KAAI,EAAE,UAAU,MACd,QAAO;AAGT,SAAQ,KAAK,MAAb;EACE,KAAK,OACH,QAAO,OAAO,KAAK,cAAc,YAAY,MAAM,QAAQ,KAAK,KAAK;EACvE,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QAAO,OAAO,KAAK,uBAAuB,YAAY,MAAM,QAAQ,KAAK,MAAM;EACjF,QACE,QAAO;;;;;;;;;;AAWb,SAAgB,6BACd,OAC2C;AAC3C,KAAI,CAAC,kBAAkB,MAAM,CAAE,QAAO;AACtC,SAAQ,MAAM,MAAd;EACE,KAAK;EACL,KAAK;EACL,KAAK,qBACH,QAAO;EACT,QACE,QAAO;;;;;;;;;AAUb,SAAgB,kBAAwB,UAAiD;AACvF,SAAQ,SAAS,MAAjB;EACE,KAAK,OACH,QAAO;GACL,MAAM;GACN,WAAW,SAAS;GACpB,MAAM,OAAO,QAAQ,SAAS,KAAK,CAAC,KAAK,CAAC,QAAQ,WAAW;AAE3D,WAAO;KAAE,KADG,KAAK,MAAM,OAAO;KAChB;KAAO;KACrB;GACH;EACH,KAAK,kBACH,QAAO;GACL,MAAM;GACN,oBAAoB,SAAS;GAC7B,OAAO,OAAO,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,QAAQ,UAAU;AAE5D,WAAO;KAAE,KADG,KAAK,MAAM,OAAO;KAChB,OAAO;KAAM;KAC3B;GACH;EACH,KAAK,oBACH,QAAO;GACL,MAAM;GACN,oBAAoB,SAAS;GAC7B,OAAO,OAAO,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW;AAE7D,WAAO;KAAE,KADG,KAAK,MAAM,OAAO;KAChB,OAAO;KAAO;KAC5B;GACH;EACH,KAAK,qBACH,QAAO;GACL,MAAM;GACN,oBAAoB,SAAS;GAC7B,OAAO,OAAO,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,QAAQ,UAAU;AAE5D,WAAO;KAAE,KADG,KAAK,MAAM,OAAO;KAChB,OAAO;KAAM;KAC3B;GACH;EACH,QACE,0BAAY,SAAS;;;;;;;;;AAU3B,SAAgB,kBAAwB,iBAAwD;AAC9F,SAAQ,gBAAgB,MAAxB;EACE,KAAK,OACH,QAAO;GACL,MAAM;GACN,WAAW,gBAAgB;GAC3B,MAAM,OAAO,YACX,gBAAgB,KAAK,KAAK,EAAE,KAAK,YAAY,CAAC,KAAK,UAAU,IAAI,EAAE,MAAM,CAAC,CAC3E;GACF;EACH,KAAK,kBACH,QAAO;GACL,MAAM;GACN,oBAAoB,gBAAgB;GACpC,OAAO,OAAO,YACZ,gBAAgB,MAAM,KAAK,EAAE,KAAK,YAAY,CAAC,KAAK,UAAU,IAAI,EAAE,MAAM,CAAC,CAC5E;GACF;EACH,KAAK,oBACH,QAAO;GACL,MAAM;GACN,oBAAoB,gBAAgB;GACpC,OAAO,OAAO,YACZ,gBAAgB,MAAM,KAAK,EAAE,KAAK,YAAY,CAAC,KAAK,UAAU,IAAI,EAAE,MAAM,CAAC,CAC5E;GACF;EACH,KAAK,qBACH,QAAO;GACL,MAAM;GACN,oBAAoB,gBAAgB;GACpC,OAAO,OAAO,YACZ,gBAAgB,MAAM,KAAK,EAAE,KAAK,YAAY,CAAC,KAAK,UAAU,IAAI,EAAE,MAAM,CAAC,CAC5E;GACF;EACH,QACE,0BAAY,gBAAgB;;;AAiBlC,SAAgB,mBACd,iBACA,OACiC;AACjC,KAAI,oBAAoB,OACtB;AAGF,SAAQ,gBAAgB,MAAxB;EACE,KAAK,OAEH,QAAO;EACT,KAAK,kBACH,QAAO;GACL,GAAG;GACH,OAAO,gBAAgB,MAAM,KAAK,WAAW;IAC3C,KAAK,MAAM;IACX,OAAO,MAAM,MAAM,MAAM;IAC1B,EAAE;GACJ;EACH,KAAK,oBACH,QAAO;GACL,GAAG;GACH,OAAO,gBAAgB,MAAM,KAAK,WAAW;IAC3C,KAAK,MAAM;IACX,OAAO;KACL,OAAO,MAAM,MAAM,MAAM,MAAM;KAC/B,QAAQ,MAAM,MAAM,MAAM,OAAO;KAClC;IACF,EAAE;GACJ;EACH,KAAK,qBACH,QAAO;GACL,GAAG;GACH,OAAO,gBAAgB,MAAM,KAAK,WAAW;IAC3C,KAAK,MAAM;IACX,OAAO,MAAM,MAAM,MAAM;IAC1B,EAAE;GACJ"}
|