@milaboratories/pl-model-common 1.24.3 → 1.24.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/base32_encode.cjs +11 -11
- package/dist/base32_encode.cjs.map +1 -1
- package/dist/base32_encode.d.ts +1 -1
- package/dist/base32_encode.js +11 -11
- package/dist/base32_encode.js.map +1 -1
- package/dist/base64.cjs +2 -2
- package/dist/base64.cjs.map +1 -1
- package/dist/base64.d.ts +1 -1
- package/dist/base64.d.ts.map +1 -1
- package/dist/base64.js +2 -2
- package/dist/base64.js.map +1 -1
- package/dist/block_state.d.ts +3 -3
- package/dist/bmodel/block_config.cjs +1 -1
- package/dist/bmodel/block_config.cjs.map +1 -1
- package/dist/bmodel/block_config.d.ts +3 -3
- package/dist/bmodel/block_config.d.ts.map +1 -1
- package/dist/bmodel/block_config.js +1 -1
- package/dist/bmodel/block_config.js.map +1 -1
- package/dist/bmodel/code.cjs +1 -1
- package/dist/bmodel/code.cjs.map +1 -1
- package/dist/bmodel/code.d.ts +2 -2
- package/dist/bmodel/code.js +1 -1
- package/dist/bmodel/code.js.map +1 -1
- package/dist/bmodel/container.d.ts +5 -5
- package/dist/bmodel/index.d.ts +5 -5
- package/dist/bmodel/normalization.cjs +17 -17
- package/dist/bmodel/normalization.cjs.map +1 -1
- package/dist/bmodel/normalization.d.ts +2 -2
- package/dist/bmodel/normalization.d.ts.map +1 -1
- package/dist/bmodel/normalization.js +17 -17
- package/dist/bmodel/normalization.js.map +1 -1
- package/dist/bmodel/types.d.ts +1 -1
- package/dist/branding.d.ts +2 -2
- package/dist/common_types.d.ts +1 -1
- package/dist/driver_kit.d.ts +1 -1
- package/dist/drivers/ChunkedStreamReader.cjs +13 -12
- package/dist/drivers/ChunkedStreamReader.cjs.map +1 -1
- package/dist/drivers/ChunkedStreamReader.d.ts +2 -2
- package/dist/drivers/ChunkedStreamReader.d.ts.map +1 -1
- package/dist/drivers/ChunkedStreamReader.js +13 -12
- package/dist/drivers/ChunkedStreamReader.js.map +1 -1
- package/dist/drivers/blob.cjs +1 -1
- package/dist/drivers/blob.cjs.map +1 -1
- package/dist/drivers/blob.d.ts +4 -4
- package/dist/drivers/blob.d.ts.map +1 -1
- package/dist/drivers/blob.js +1 -1
- package/dist/drivers/blob.js.map +1 -1
- package/dist/drivers/index.d.ts +8 -8
- package/dist/drivers/log.cjs +1 -1
- package/dist/drivers/log.cjs.map +1 -1
- package/dist/drivers/log.js +1 -1
- package/dist/drivers/log.js.map +1 -1
- package/dist/drivers/ls.cjs +3 -3
- package/dist/drivers/ls.cjs.map +1 -1
- package/dist/drivers/ls.d.ts +6 -6
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/ls.js +3 -3
- package/dist/drivers/ls.js.map +1 -1
- package/dist/drivers/pframe/column_filter.d.ts +1 -1
- package/dist/drivers/pframe/data_info.cjs +119 -110
- package/dist/drivers/pframe/data_info.cjs.map +1 -1
- package/dist/drivers/pframe/data_info.d.ts +10 -10
- package/dist/drivers/pframe/data_info.d.ts.map +1 -1
- package/dist/drivers/pframe/data_info.js +119 -110
- package/dist/drivers/pframe/data_info.js.map +1 -1
- package/dist/drivers/pframe/data_types.cjs +7 -4
- package/dist/drivers/pframe/data_types.cjs.map +1 -1
- package/dist/drivers/pframe/data_types.d.ts +2 -2
- package/dist/drivers/pframe/data_types.d.ts.map +1 -1
- package/dist/drivers/pframe/data_types.js +7 -4
- package/dist/drivers/pframe/data_types.js.map +1 -1
- package/dist/drivers/pframe/driver.d.ts +10 -10
- package/dist/drivers/pframe/driver.d.ts.map +1 -1
- package/dist/drivers/pframe/find_columns.d.ts +2 -2
- package/dist/drivers/pframe/index.d.ts +14 -14
- package/dist/drivers/pframe/linker_columns.cjs +13 -5
- package/dist/drivers/pframe/linker_columns.cjs.map +1 -1
- package/dist/drivers/pframe/linker_columns.d.ts +2 -2
- package/dist/drivers/pframe/linker_columns.d.ts.map +1 -1
- package/dist/drivers/pframe/linker_columns.js +13 -5
- package/dist/drivers/pframe/linker_columns.js.map +1 -1
- package/dist/drivers/pframe/pframe.d.ts +6 -6
- package/dist/drivers/pframe/pframe.d.ts.map +1 -1
- package/dist/drivers/pframe/query/index.d.ts +2 -2
- package/dist/drivers/pframe/query/query_common.d.ts +37 -37
- package/dist/drivers/pframe/query/query_data.d.ts +2 -2
- package/dist/drivers/pframe/query/query_data.d.ts.map +1 -1
- package/dist/drivers/pframe/query/query_spec.d.ts +3 -3
- package/dist/drivers/pframe/query/query_spec.d.ts.map +1 -1
- package/dist/drivers/pframe/spec/anchored.cjs +6 -6
- package/dist/drivers/pframe/spec/anchored.cjs.map +1 -1
- package/dist/drivers/pframe/spec/anchored.d.ts +4 -4
- package/dist/drivers/pframe/spec/anchored.d.ts.map +1 -1
- package/dist/drivers/pframe/spec/anchored.js +6 -6
- package/dist/drivers/pframe/spec/anchored.js.map +1 -1
- package/dist/drivers/pframe/spec/filtered_column.cjs +1 -1
- package/dist/drivers/pframe/spec/filtered_column.cjs.map +1 -1
- package/dist/drivers/pframe/spec/filtered_column.d.ts +1 -1
- package/dist/drivers/pframe/spec/filtered_column.js +1 -1
- package/dist/drivers/pframe/spec/filtered_column.js.map +1 -1
- package/dist/drivers/pframe/spec/ids.cjs.map +1 -1
- package/dist/drivers/pframe/spec/ids.d.ts +5 -5
- package/dist/drivers/pframe/spec/ids.js.map +1 -1
- package/dist/drivers/pframe/spec/index.d.ts +6 -6
- package/dist/drivers/pframe/spec/native_id.cjs.map +1 -1
- package/dist/drivers/pframe/spec/native_id.d.ts +3 -3
- package/dist/drivers/pframe/spec/native_id.d.ts.map +1 -1
- package/dist/drivers/pframe/spec/native_id.js.map +1 -1
- package/dist/drivers/pframe/spec/selectors.cjs +1 -1
- package/dist/drivers/pframe/spec/selectors.cjs.map +1 -1
- package/dist/drivers/pframe/spec/selectors.d.ts +4 -4
- package/dist/drivers/pframe/spec/selectors.d.ts.map +1 -1
- package/dist/drivers/pframe/spec/selectors.js +1 -1
- package/dist/drivers/pframe/spec/selectors.js.map +1 -1
- package/dist/drivers/pframe/spec/spec.cjs +64 -59
- package/dist/drivers/pframe/spec/spec.cjs.map +1 -1
- package/dist/drivers/pframe/spec/spec.d.ts +15 -15
- package/dist/drivers/pframe/spec/spec.d.ts.map +1 -1
- package/dist/drivers/pframe/spec/spec.js +64 -59
- package/dist/drivers/pframe/spec/spec.js.map +1 -1
- package/dist/drivers/pframe/table.d.ts +2 -2
- package/dist/drivers/pframe/table_calculate.cjs +11 -11
- package/dist/drivers/pframe/table_calculate.cjs.map +1 -1
- package/dist/drivers/pframe/table_calculate.d.ts +30 -30
- package/dist/drivers/pframe/table_calculate.d.ts.map +1 -1
- package/dist/drivers/pframe/table_calculate.js +11 -11
- package/dist/drivers/pframe/table_calculate.js.map +1 -1
- package/dist/drivers/pframe/table_common.cjs +4 -4
- package/dist/drivers/pframe/table_common.cjs.map +1 -1
- package/dist/drivers/pframe/table_common.d.ts +6 -6
- package/dist/drivers/pframe/table_common.js +4 -4
- package/dist/drivers/pframe/table_common.js.map +1 -1
- package/dist/drivers/pframe/type_util.d.ts.map +1 -1
- package/dist/drivers/pframe/unique_values.d.ts +4 -4
- package/dist/drivers/urls.cjs +2 -2
- package/dist/drivers/urls.cjs.map +1 -1
- package/dist/drivers/urls.d.ts +1 -1
- package/dist/drivers/urls.js +2 -2
- package/dist/drivers/urls.js.map +1 -1
- package/dist/errors.cjs +12 -12
- package/dist/errors.cjs.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +12 -12
- package/dist/errors.js.map +1 -1
- package/dist/flags/block_flags.cjs +2 -2
- package/dist/flags/block_flags.cjs.map +1 -1
- package/dist/flags/block_flags.d.ts.map +1 -1
- package/dist/flags/block_flags.js +2 -2
- package/dist/flags/block_flags.js.map +1 -1
- package/dist/flags/flag_utils.cjs +31 -29
- package/dist/flags/flag_utils.cjs.map +1 -1
- package/dist/flags/flag_utils.d.ts +33 -33
- package/dist/flags/flag_utils.d.ts.map +1 -1
- package/dist/flags/flag_utils.js +31 -29
- package/dist/flags/flag_utils.js.map +1 -1
- package/dist/flags/index.d.ts +3 -3
- package/dist/flags/type_utils.d.ts +1 -1
- package/dist/flags/type_utils.d.ts.map +1 -1
- package/dist/httpAuth.cjs +4 -5
- package/dist/httpAuth.cjs.map +1 -1
- package/dist/httpAuth.d.ts +1 -1
- package/dist/httpAuth.d.ts.map +1 -1
- package/dist/httpAuth.js +4 -5
- package/dist/httpAuth.js.map +1 -1
- package/dist/index.d.ts +19 -19
- package/dist/json.cjs +1 -1
- package/dist/json.cjs.map +1 -1
- package/dist/json.d.ts.map +1 -1
- package/dist/json.js +1 -1
- package/dist/json.js.map +1 -1
- package/dist/navigation.cjs +1 -1
- package/dist/navigation.cjs.map +1 -1
- package/dist/navigation.d.ts +3 -3
- package/dist/navigation.js +1 -1
- package/dist/navigation.js.map +1 -1
- package/dist/plid.cjs +4 -4
- package/dist/plid.cjs.map +1 -1
- package/dist/plid.d.ts +1 -1
- package/dist/plid.js +4 -4
- package/dist/plid.js.map +1 -1
- package/dist/pool/entry.d.ts +1 -1
- package/dist/pool/index.d.ts +3 -3
- package/dist/pool/query.cjs +23 -23
- package/dist/pool/query.cjs.map +1 -1
- package/dist/pool/query.d.ts +9 -9
- package/dist/pool/query.d.ts.map +1 -1
- package/dist/pool/query.js +23 -23
- package/dist/pool/query.js.map +1 -1
- package/dist/pool/spec.cjs +9 -9
- package/dist/pool/spec.cjs.map +1 -1
- package/dist/pool/spec.d.ts +4 -4
- package/dist/pool/spec.d.ts.map +1 -1
- package/dist/pool/spec.js +9 -9
- package/dist/pool/spec.js.map +1 -1
- package/dist/ref.cjs +20 -18
- package/dist/ref.cjs.map +1 -1
- package/dist/ref.d.ts +1 -1
- package/dist/ref.d.ts.map +1 -1
- package/dist/ref.js +20 -18
- package/dist/ref.js.map +1 -1
- package/dist/utag.d.ts +1 -1
- package/dist/util.cjs +1 -1
- package/dist/util.cjs.map +1 -1
- package/dist/util.js +1 -1
- package/dist/util.js.map +1 -1
- package/dist/value_or_error.cjs.map +1 -1
- package/dist/value_or_error.d.ts.map +1 -1
- package/dist/value_or_error.js.map +1 -1
- package/package.json +18 -17
- package/src/base32_encode.ts +12 -12
- package/src/base64.ts +12 -8
- package/src/block_state.ts +3 -3
- package/src/bmodel/block_config.ts +12 -6
- package/src/bmodel/code.ts +3 -3
- package/src/bmodel/container.ts +5 -5
- package/src/bmodel/index.ts +5 -5
- package/src/bmodel/normalization.ts +33 -25
- package/src/bmodel/types.ts +1 -1
- package/src/branding.ts +2 -2
- package/src/common_types.ts +1 -1
- package/src/driver_kit.ts +1 -1
- package/src/drivers/ChunkedStreamReader.ts +16 -14
- package/src/drivers/blob.ts +7 -11
- package/src/drivers/index.ts +8 -8
- package/src/drivers/log.ts +3 -3
- package/src/drivers/ls.ts +18 -18
- package/src/drivers/pframe/column_filter.ts +1 -1
- package/src/drivers/pframe/data_info.ts +146 -146
- package/src/drivers/pframe/data_types.ts +56 -36
- package/src/drivers/pframe/driver.ts +20 -36
- package/src/drivers/pframe/find_columns.ts +2 -2
- package/src/drivers/pframe/index.ts +14 -14
- package/src/drivers/pframe/linker_columns.test.ts +280 -273
- package/src/drivers/pframe/linker_columns.ts +65 -39
- package/src/drivers/pframe/pframe.ts +6 -9
- package/src/drivers/pframe/query/index.ts +2 -2
- package/src/drivers/pframe/query/query_common.ts +46 -46
- package/src/drivers/pframe/query/query_data.ts +17 -10
- package/src/drivers/pframe/query/query_spec.ts +18 -10
- package/src/drivers/pframe/spec/anchored.ts +44 -33
- package/src/drivers/pframe/spec/filtered_column.ts +2 -2
- package/src/drivers/pframe/spec/ids.ts +6 -6
- package/src/drivers/pframe/spec/index.ts +6 -6
- package/src/drivers/pframe/spec/native_id.ts +7 -9
- package/src/drivers/pframe/spec/selectors.test.ts +170 -146
- package/src/drivers/pframe/spec/selectors.ts +22 -31
- package/src/drivers/pframe/spec/spec.test.ts +163 -176
- package/src/drivers/pframe/spec/spec.ts +132 -117
- package/src/drivers/pframe/table.ts +2 -2
- package/src/drivers/pframe/table_calculate.ts +44 -50
- package/src/drivers/pframe/table_common.ts +10 -10
- package/src/drivers/pframe/type_util.ts +2 -7
- package/src/drivers/pframe/unique_values.ts +4 -4
- package/src/drivers/urls.ts +3 -3
- package/src/errors.test.ts +20 -20
- package/src/errors.ts +24 -20
- package/src/flags/block_flags.ts +8 -7
- package/src/flags/flag_utils.test.ts +94 -73
- package/src/flags/flag_utils.ts +73 -50
- package/src/flags/index.ts +3 -3
- package/src/flags/type_utils.ts +10 -7
- package/src/httpAuth.ts +5 -6
- package/src/index.ts +19 -19
- package/src/json.ts +28 -17
- package/src/navigation.ts +4 -4
- package/src/plid.ts +6 -6
- package/src/pool/entry.ts +1 -1
- package/src/pool/index.ts +3 -3
- package/src/pool/query.ts +52 -56
- package/src/pool/spec.ts +20 -17
- package/src/ref.ts +25 -19
- package/src/utag.ts +1 -1
- package/src/util.ts +1 -1
- package/src/value_or_error.ts +6 -6
|
@@ -27,7 +27,7 @@ export type PColumnDataEntry<T> = {
|
|
|
27
27
|
*/
|
|
28
28
|
export type JsonDataInfo = {
|
|
29
29
|
/** Identifier for this data format ('Json') */
|
|
30
|
-
type:
|
|
30
|
+
type: "Json";
|
|
31
31
|
/** Number of axes that make up the complete key (tuple length) */
|
|
32
32
|
keyLength: number;
|
|
33
33
|
/**
|
|
@@ -42,7 +42,7 @@ export type JsonDataInfo = {
|
|
|
42
42
|
*/
|
|
43
43
|
export type JsonPartitionedDataInfo<Blob> = {
|
|
44
44
|
/** Identifier for this data format ('JsonPartitioned') */
|
|
45
|
-
type:
|
|
45
|
+
type: "JsonPartitioned";
|
|
46
46
|
/** Number of leading axes used for partitioning */
|
|
47
47
|
partitionKeyLength: number;
|
|
48
48
|
/** Map of stringified partition keys to blob references */
|
|
@@ -64,7 +64,7 @@ export type BinaryChunk<Blob> = {
|
|
|
64
64
|
*/
|
|
65
65
|
export type BinaryPartitionedDataInfo<Blob> = {
|
|
66
66
|
/** Identifier for this data format ('BinaryPartitioned') */
|
|
67
|
-
type:
|
|
67
|
+
type: "BinaryPartitioned";
|
|
68
68
|
/** Number of leading axes used for partitioning */
|
|
69
69
|
partitionKeyLength: number;
|
|
70
70
|
/** Map of stringified partition keys to binary chunks */
|
|
@@ -72,13 +72,13 @@ export type BinaryPartitionedDataInfo<Blob> = {
|
|
|
72
72
|
};
|
|
73
73
|
export type ParquetChunkMappingAxis = {
|
|
74
74
|
/** Data type (matches PColumn axis types) */
|
|
75
|
-
type:
|
|
75
|
+
type: "Int" | "Long" | "String";
|
|
76
76
|
/** Field name in the Parquet file */
|
|
77
77
|
id: string;
|
|
78
78
|
};
|
|
79
79
|
export type ParquetChunkMappingColumn = {
|
|
80
80
|
/** Data type (matches PColumn value type) */
|
|
81
|
-
type:
|
|
81
|
+
type: "Int" | "Long" | "Float" | "Double" | "String";
|
|
82
82
|
/** Field name in the Parquet file */
|
|
83
83
|
id: string;
|
|
84
84
|
};
|
|
@@ -111,7 +111,7 @@ export type ParquetChunk<Blob> = {
|
|
|
111
111
|
} & ParquetChunkMapping & Partial<ParquetChunkMetadata>;
|
|
112
112
|
export type ParquetPartitionedDataInfo<Blob> = {
|
|
113
113
|
/** Identifier for this data format ('ParquetPartitioned') */
|
|
114
|
-
type:
|
|
114
|
+
type: "ParquetPartitioned";
|
|
115
115
|
/** Number of leading axes used for partitioning */
|
|
116
116
|
partitionKeyLength: number;
|
|
117
117
|
/** Map of stringified partition keys to parquet files */
|
|
@@ -166,7 +166,7 @@ export type PColumnValues = PColumnValuesEntry[];
|
|
|
166
166
|
* Entry-based representation of JsonDataInfo
|
|
167
167
|
*/
|
|
168
168
|
export interface JsonDataInfoEntries {
|
|
169
|
-
type:
|
|
169
|
+
type: "Json";
|
|
170
170
|
keyLength: number;
|
|
171
171
|
data: PColumnDataEntry<PColumnValue>[];
|
|
172
172
|
}
|
|
@@ -174,7 +174,7 @@ export interface JsonDataInfoEntries {
|
|
|
174
174
|
* Entry-based representation of JsonPartitionedDataInfo
|
|
175
175
|
*/
|
|
176
176
|
export interface JsonPartitionedDataInfoEntries<Blob> {
|
|
177
|
-
type:
|
|
177
|
+
type: "JsonPartitioned";
|
|
178
178
|
partitionKeyLength: number;
|
|
179
179
|
parts: PColumnDataEntry<Blob>[];
|
|
180
180
|
}
|
|
@@ -182,7 +182,7 @@ export interface JsonPartitionedDataInfoEntries<Blob> {
|
|
|
182
182
|
* Entry-based representation of BinaryPartitionedDataInfo
|
|
183
183
|
*/
|
|
184
184
|
export interface BinaryPartitionedDataInfoEntries<Blob> {
|
|
185
|
-
type:
|
|
185
|
+
type: "BinaryPartitioned";
|
|
186
186
|
partitionKeyLength: number;
|
|
187
187
|
parts: PColumnDataEntry<BinaryChunk<Blob>>[];
|
|
188
188
|
}
|
|
@@ -190,7 +190,7 @@ export interface BinaryPartitionedDataInfoEntries<Blob> {
|
|
|
190
190
|
* Entry-based representation of ParquetPartitionedDataInfo
|
|
191
191
|
*/
|
|
192
192
|
export interface ParquetPartitionedDataInfoEntries<Blob> {
|
|
193
|
-
type:
|
|
193
|
+
type: "ParquetPartitioned";
|
|
194
194
|
partitionKeyLength: number;
|
|
195
195
|
parts: PColumnDataEntry<Blob>[];
|
|
196
196
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data_info.d.ts","sourceRoot":"","sources":["../../../src/drivers/pframe/data_info.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,CAAC;AAElD;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI;IAChC,wBAAwB;IACxB,GAAG,EAAE,UAAU,CAAC;IAEhB,oCAAoC;IACpC,KAAK,EAAE,CAAC,CAAC;CACV,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,+CAA+C;IAC/C,IAAI,EAAE,MAAM,CAAC;IAEb,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACpC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,uBAAuB,CAAC,IAAI,IAAI;IAC1C,0DAA0D;IAC1D,IAAI,EAAE,iBAAiB,CAAC;IAExB,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAE3B,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,WAAW,CAAC,IAAI,IAAI;IAC9B,0DAA0D;IAC1D,KAAK,EAAE,IAAI,CAAC;IAEZ,+CAA+C;IAC/C,MAAM,EAAE,IAAI,CAAC;CACd,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,yBAAyB,CAAC,IAAI,IAAI;IAC5C,4DAA4D;IAC5D,IAAI,EAAE,mBAAmB,CAAC;IAE1B,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAE3B,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,6CAA6C;IAC7C,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IAEhC,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,6CAA6C;IAC7C,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAErD,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,IAAI,EAAE,uBAAuB,EAAE,CAAC;IAEhC,qBAAqB;IACrB,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,IAAI,EAAE;QACJ,wEAAwE;QACxE,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,oCAAoC;QACpC,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,mFAAmF;IACnF,UAAU,EAAE,MAAM,CAAC;IAEnB,qEAAqE;IACrE,KAAK,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,YAAY,CAAC,IAAI,IAAI;IAC/B,mDAAmD;IACnD,IAAI,EAAE,IAAI,CAAC;CACZ,GAAG,mBAAmB,
|
|
1
|
+
{"version":3,"file":"data_info.d.ts","sourceRoot":"","sources":["../../../src/drivers/pframe/data_info.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,CAAC;AAElD;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI;IAChC,wBAAwB;IACxB,GAAG,EAAE,UAAU,CAAC;IAEhB,oCAAoC;IACpC,KAAK,EAAE,CAAC,CAAC;CACV,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,+CAA+C;IAC/C,IAAI,EAAE,MAAM,CAAC;IAEb,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACpC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,uBAAuB,CAAC,IAAI,IAAI;IAC1C,0DAA0D;IAC1D,IAAI,EAAE,iBAAiB,CAAC;IAExB,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAE3B,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,WAAW,CAAC,IAAI,IAAI;IAC9B,0DAA0D;IAC1D,KAAK,EAAE,IAAI,CAAC;IAEZ,+CAA+C;IAC/C,MAAM,EAAE,IAAI,CAAC;CACd,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,yBAAyB,CAAC,IAAI,IAAI;IAC5C,4DAA4D;IAC5D,IAAI,EAAE,mBAAmB,CAAC;IAE1B,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAE3B,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,6CAA6C;IAC7C,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IAEhC,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,6CAA6C;IAC7C,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAErD,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,IAAI,EAAE,uBAAuB,EAAE,CAAC;IAEhC,qBAAqB;IACrB,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,IAAI,EAAE;QACJ,wEAAwE;QACxE,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,oCAAoC;QACpC,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,mFAAmF;IACnF,UAAU,EAAE,MAAM,CAAC;IAEnB,qEAAqE;IACrE,KAAK,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,YAAY,CAAC,IAAI,IAAI;IAC/B,mDAAmD;IACnD,IAAI,EAAE,IAAI,CAAC;CACZ,GAAG,mBAAmB,GACrB,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAEhC,MAAM,MAAM,0BAA0B,CAAC,IAAI,IAAI;IAC7C,6DAA6D;IAC7D,IAAI,EAAE,oBAAoB,CAAC;IAE3B,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAE3B,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;CAC7B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,CAAC,IAAI,IACrB,YAAY,GACZ,uBAAuB,CAAC,IAAI,CAAC,GAC7B,yBAAyB,CAAC,IAAI,CAAC,GAC/B,0BAA0B,CAAC,IAAI,CAAC,CAAC;AAErC;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC,CA4BxE;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,EAAE,EAChC,QAAQ,EAAE,0BAA0B,CAAC,EAAE,CAAC,EACxC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GACtB,0BAA0B,CAAC,EAAE,CAAC,CAAC;AAClC,wBAAgB,WAAW,CAAC,EAAE,EAAE,EAAE,EAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,CAAC,EAC/D,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GACtB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC;AACzD,wBAAgB,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;AAoDnG;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,GAAG,IAAI,CAwBnF;AAMD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,GAAG,EAAE,UAAU,CAAC;IAChB,GAAG,EAAE,YAAY,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,kBAAkB,EAAE,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,gBAAgB,CAAC,YAAY,CAAC,EAAE,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B,CAAC,IAAI;IAClD,IAAI,EAAE,iBAAiB,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,gCAAgC,CAAC,IAAI;IACpD,IAAI,EAAE,mBAAmB,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC,CAAC,IAAI;IACrD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;CACjC;AACD;;GAEG;AACH,MAAM,MAAM,0BAA0B,CAAC,IAAI,IACvC,8BAA8B,CAAC,IAAI,CAAC,GACpC,gCAAgC,CAAC,IAAI,CAAC,GACtC,iCAAiC,CAAC,IAAI,CAAC,CAAC;AAE5C;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,IAAI,IAAI,mBAAmB,GAAG,0BAA0B,CAAC,IAAI,CAAC,CAAC;AAE3F;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,eAAe,CAAC,IAAI,CAAC,CAoBtF;AAED;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,IAAI,EAC/C,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,0BAA0B,CAAC,IAAI,CAAC,CAU3C;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAyCvF;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAqC9F;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,EAAE,EACvC,eAAe,EAAE,eAAe,CAAC,EAAE,CAAC,EACpC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GACtB,eAAe,CAAC,EAAE,CAAC,CAAC"}
|
|
@@ -7,24 +7,24 @@ import { assertNever } from '../../util.js';
|
|
|
7
7
|
* @returns True if the value is a valid DataInfo, false otherwise
|
|
8
8
|
*/
|
|
9
9
|
function isDataInfo(value) {
|
|
10
|
-
if (!value || typeof value !==
|
|
10
|
+
if (!value || typeof value !== "object") {
|
|
11
11
|
return false;
|
|
12
12
|
}
|
|
13
13
|
const data = value;
|
|
14
|
-
if (!(
|
|
14
|
+
if (!("type" in data)) {
|
|
15
15
|
return false;
|
|
16
16
|
}
|
|
17
17
|
switch (data.type) {
|
|
18
|
-
case
|
|
19
|
-
return (typeof data.keyLength ===
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
case
|
|
23
|
-
case
|
|
24
|
-
case
|
|
25
|
-
return (typeof data.partitionKeyLength ===
|
|
26
|
-
|
|
27
|
-
|
|
18
|
+
case "Json":
|
|
19
|
+
return (typeof data.keyLength === "number" &&
|
|
20
|
+
data.data !== undefined &&
|
|
21
|
+
typeof data.data === "object");
|
|
22
|
+
case "JsonPartitioned":
|
|
23
|
+
case "BinaryPartitioned":
|
|
24
|
+
case "ParquetPartitioned":
|
|
25
|
+
return (typeof data.partitionKeyLength === "number" &&
|
|
26
|
+
data.parts !== undefined &&
|
|
27
|
+
typeof data.parts === "object");
|
|
28
28
|
default:
|
|
29
29
|
return false;
|
|
30
30
|
}
|
|
@@ -34,10 +34,10 @@ function mapDataInfo(dataInfo, mapFn) {
|
|
|
34
34
|
return undefined;
|
|
35
35
|
}
|
|
36
36
|
switch (dataInfo.type) {
|
|
37
|
-
case
|
|
37
|
+
case "Json":
|
|
38
38
|
// Json type doesn't contain blobs, so return as is
|
|
39
39
|
return dataInfo;
|
|
40
|
-
case
|
|
40
|
+
case "JsonPartitioned": {
|
|
41
41
|
// Map each blob in parts
|
|
42
42
|
const newParts = {};
|
|
43
43
|
for (const [key, blob] of Object.entries(dataInfo.parts)) {
|
|
@@ -48,7 +48,7 @@ function mapDataInfo(dataInfo, mapFn) {
|
|
|
48
48
|
parts: newParts,
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
|
-
case
|
|
51
|
+
case "BinaryPartitioned": {
|
|
52
52
|
// Map each index and values blob in parts
|
|
53
53
|
const newParts = {};
|
|
54
54
|
for (const [key, chunk] of Object.entries(dataInfo.parts)) {
|
|
@@ -62,7 +62,7 @@ function mapDataInfo(dataInfo, mapFn) {
|
|
|
62
62
|
parts: newParts,
|
|
63
63
|
};
|
|
64
64
|
}
|
|
65
|
-
case
|
|
65
|
+
case "ParquetPartitioned": {
|
|
66
66
|
// Map each blob in parts
|
|
67
67
|
const newParts = {};
|
|
68
68
|
for (const [key, blob] of Object.entries(dataInfo.parts)) {
|
|
@@ -82,15 +82,15 @@ function mapDataInfo(dataInfo, mapFn) {
|
|
|
82
82
|
*/
|
|
83
83
|
function visitDataInfo(dataInfo, cb) {
|
|
84
84
|
switch (dataInfo.type) {
|
|
85
|
-
case
|
|
85
|
+
case "Json":
|
|
86
86
|
// Json type doesn't contain blobs, so return as is
|
|
87
87
|
break;
|
|
88
|
-
case
|
|
88
|
+
case "JsonPartitioned": {
|
|
89
89
|
// Visit each blob in parts
|
|
90
90
|
Object.values(dataInfo.parts).forEach(cb);
|
|
91
91
|
break;
|
|
92
92
|
}
|
|
93
|
-
case
|
|
93
|
+
case "BinaryPartitioned": {
|
|
94
94
|
// Visit each index and values blob in parts
|
|
95
95
|
Object.values(dataInfo.parts).forEach((chunk) => {
|
|
96
96
|
cb(chunk.index);
|
|
@@ -98,7 +98,7 @@ function visitDataInfo(dataInfo, cb) {
|
|
|
98
98
|
});
|
|
99
99
|
break;
|
|
100
100
|
}
|
|
101
|
-
case
|
|
101
|
+
case "ParquetPartitioned": {
|
|
102
102
|
// Visit each blob in parts
|
|
103
103
|
Object.values(dataInfo.parts).forEach(cb);
|
|
104
104
|
break;
|
|
@@ -112,22 +112,20 @@ function visitDataInfo(dataInfo, cb) {
|
|
|
112
112
|
* @returns True if the value is a valid DataInfoEntries, false otherwise
|
|
113
113
|
*/
|
|
114
114
|
function isDataInfoEntries(value) {
|
|
115
|
-
if (!value || typeof value !==
|
|
115
|
+
if (!value || typeof value !== "object") {
|
|
116
116
|
return false;
|
|
117
117
|
}
|
|
118
118
|
const data = value;
|
|
119
|
-
if (!(
|
|
119
|
+
if (!("type" in data)) {
|
|
120
120
|
return false;
|
|
121
121
|
}
|
|
122
122
|
switch (data.type) {
|
|
123
|
-
case
|
|
124
|
-
return
|
|
125
|
-
|
|
126
|
-
case
|
|
127
|
-
case
|
|
128
|
-
|
|
129
|
-
return (typeof data.partitionKeyLength === 'number'
|
|
130
|
-
&& Array.isArray(data.parts));
|
|
123
|
+
case "Json":
|
|
124
|
+
return typeof data.keyLength === "number" && Array.isArray(data.data);
|
|
125
|
+
case "JsonPartitioned":
|
|
126
|
+
case "BinaryPartitioned":
|
|
127
|
+
case "ParquetPartitioned":
|
|
128
|
+
return typeof data.partitionKeyLength === "number" && Array.isArray(data.parts);
|
|
131
129
|
default:
|
|
132
130
|
return false;
|
|
133
131
|
}
|
|
@@ -143,9 +141,9 @@ function isPartitionedDataInfoEntries(value) {
|
|
|
143
141
|
if (!isDataInfoEntries(value))
|
|
144
142
|
return false;
|
|
145
143
|
switch (value.type) {
|
|
146
|
-
case
|
|
147
|
-
case
|
|
148
|
-
case
|
|
144
|
+
case "JsonPartitioned":
|
|
145
|
+
case "BinaryPartitioned":
|
|
146
|
+
case "ParquetPartitioned":
|
|
149
147
|
return true;
|
|
150
148
|
default:
|
|
151
149
|
return false;
|
|
@@ -159,38 +157,42 @@ function isPartitionedDataInfoEntries(value) {
|
|
|
159
157
|
*/
|
|
160
158
|
function dataInfoToEntries(dataInfo) {
|
|
161
159
|
switch (dataInfo.type) {
|
|
162
|
-
case
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
160
|
+
case "Json":
|
|
161
|
+
return {
|
|
162
|
+
type: "Json",
|
|
163
|
+
keyLength: dataInfo.keyLength,
|
|
164
|
+
data: Object.entries(dataInfo.data).map(([keyStr, value]) => {
|
|
165
|
+
const key = JSON.parse(keyStr);
|
|
166
|
+
return { key, value };
|
|
167
|
+
}),
|
|
168
|
+
};
|
|
169
|
+
case "JsonPartitioned":
|
|
170
|
+
return {
|
|
171
|
+
type: "JsonPartitioned",
|
|
172
|
+
partitionKeyLength: dataInfo.partitionKeyLength,
|
|
173
|
+
parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {
|
|
174
|
+
const key = JSON.parse(keyStr);
|
|
175
|
+
return { key, value: blob };
|
|
176
|
+
}),
|
|
177
|
+
};
|
|
178
|
+
case "BinaryPartitioned":
|
|
179
|
+
return {
|
|
180
|
+
type: "BinaryPartitioned",
|
|
181
|
+
partitionKeyLength: dataInfo.partitionKeyLength,
|
|
182
|
+
parts: Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {
|
|
183
|
+
const key = JSON.parse(keyStr);
|
|
184
|
+
return { key, value: chunk };
|
|
185
|
+
}),
|
|
186
|
+
};
|
|
187
|
+
case "ParquetPartitioned":
|
|
188
|
+
return {
|
|
189
|
+
type: "ParquetPartitioned",
|
|
190
|
+
partitionKeyLength: dataInfo.partitionKeyLength,
|
|
191
|
+
parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {
|
|
192
|
+
const key = JSON.parse(keyStr);
|
|
193
|
+
return { key, value: blob };
|
|
194
|
+
}),
|
|
195
|
+
};
|
|
194
196
|
default:
|
|
195
197
|
assertNever(dataInfo);
|
|
196
198
|
}
|
|
@@ -203,26 +205,30 @@ function dataInfoToEntries(dataInfo) {
|
|
|
203
205
|
*/
|
|
204
206
|
function entriesToDataInfo(dataInfoEntries) {
|
|
205
207
|
switch (dataInfoEntries.type) {
|
|
206
|
-
case
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
208
|
+
case "Json":
|
|
209
|
+
return {
|
|
210
|
+
type: "Json",
|
|
211
|
+
keyLength: dataInfoEntries.keyLength,
|
|
212
|
+
data: Object.fromEntries(dataInfoEntries.data.map(({ key, value }) => [JSON.stringify(key), value])),
|
|
213
|
+
};
|
|
214
|
+
case "JsonPartitioned":
|
|
215
|
+
return {
|
|
216
|
+
type: "JsonPartitioned",
|
|
217
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength,
|
|
218
|
+
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value])),
|
|
219
|
+
};
|
|
220
|
+
case "BinaryPartitioned":
|
|
221
|
+
return {
|
|
222
|
+
type: "BinaryPartitioned",
|
|
223
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength,
|
|
224
|
+
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value])),
|
|
225
|
+
};
|
|
226
|
+
case "ParquetPartitioned":
|
|
227
|
+
return {
|
|
228
|
+
type: "ParquetPartitioned",
|
|
229
|
+
partitionKeyLength: dataInfoEntries.partitionKeyLength,
|
|
230
|
+
parts: Object.fromEntries(dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value])),
|
|
231
|
+
};
|
|
226
232
|
default:
|
|
227
233
|
assertNever(dataInfoEntries);
|
|
228
234
|
}
|
|
@@ -232,33 +238,36 @@ function mapDataInfoEntries(dataInfoEntries, mapFn) {
|
|
|
232
238
|
return undefined;
|
|
233
239
|
}
|
|
234
240
|
switch (dataInfoEntries.type) {
|
|
235
|
-
case
|
|
241
|
+
case "Json":
|
|
236
242
|
// Json type doesn't contain blobs, so return as is
|
|
237
243
|
return dataInfoEntries;
|
|
238
|
-
case
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
244
|
+
case "JsonPartitioned":
|
|
245
|
+
return {
|
|
246
|
+
...dataInfoEntries,
|
|
247
|
+
parts: dataInfoEntries.parts.map((entry) => ({
|
|
248
|
+
key: entry.key,
|
|
249
|
+
value: mapFn(entry.value),
|
|
250
|
+
})),
|
|
251
|
+
};
|
|
252
|
+
case "BinaryPartitioned":
|
|
253
|
+
return {
|
|
254
|
+
...dataInfoEntries,
|
|
255
|
+
parts: dataInfoEntries.parts.map((entry) => ({
|
|
256
|
+
key: entry.key,
|
|
257
|
+
value: {
|
|
258
|
+
index: mapFn(entry.value.index),
|
|
259
|
+
values: mapFn(entry.value.values),
|
|
260
|
+
},
|
|
261
|
+
})),
|
|
262
|
+
};
|
|
263
|
+
case "ParquetPartitioned":
|
|
264
|
+
return {
|
|
265
|
+
...dataInfoEntries,
|
|
266
|
+
parts: dataInfoEntries.parts.map((entry) => ({
|
|
267
|
+
key: entry.key,
|
|
268
|
+
value: mapFn(entry.value),
|
|
269
|
+
})),
|
|
270
|
+
};
|
|
262
271
|
}
|
|
263
272
|
}
|
|
264
273
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data_info.js","sources":["../../../src/drivers/pframe/data_info.ts"],"sourcesContent":["import { assertNever } from '../../util';\n\n/**\n * Represents a JavaScript representation of a value in a PColumn. Can be null, a number, or a string.\n * These are the primitive types that can be stored directly in PColumns.\n *\n * Note: Actual columns can hold more value types, which are converted to these JavaScript types\n * once they enter the JavaScript runtime.\n */\nexport type PColumnValue = null | number | string;\n\n/**\n * Represents a key for a PColumn value.\n * Can be an array of strings or numbers.\n */\nexport type PColumnKey = (number | string)[];\n\n/**\n * Represents a single entry in a PColumn's data structure.\n * Contains a key and a value.\n */\nexport type PColumnDataEntry<T> = {\n /** Key for the value */\n key: PColumnKey;\n\n /** Value / blob at the given key */\n value: T;\n};\n\n/**\n * Represents column data stored as a simple JSON structure.\n * Used for small datasets that can be efficiently stored directly in memory.\n */\nexport type JsonDataInfo = {\n /** Identifier for this data format ('Json') */\n type: 'Json';\n\n /** Number of axes that make up the complete key (tuple length) */\n keyLength: number;\n\n /**\n * Key-value pairs where keys are stringified tuples of axis values\n * and values are the column values for those coordinates\n */\n data: Record<string, PColumnValue>;\n};\n\n/**\n * Represents column data partitioned across multiple JSON blobs.\n * Used for larger datasets that need to be split into manageable chunks.\n */\nexport type JsonPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('JsonPartitioned') */\n type: 'JsonPartitioned';\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to blob references */\n parts: Record<string, Blob>;\n};\n\n/**\n * Represents a binary format chunk containing index and values as separate blobs.\n * Used for efficient storage and retrieval of column data in binary format.\n */\nexport type BinaryChunk<Blob> = {\n /** Binary blob containing structured index information */\n index: Blob;\n\n /** Binary blob containing the actual values */\n values: Blob;\n};\n\n/**\n * Represents column data partitioned across multiple binary chunks.\n * Optimized for efficient storage and retrieval of large datasets.\n */\nexport type BinaryPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('BinaryPartitioned') */\n type: 'BinaryPartitioned';\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to binary chunks */\n parts: Record<string, BinaryChunk<Blob>>;\n};\n\nexport type ParquetChunkMappingAxis = {\n /** Data type (matches PColumn axis types) */\n type: 'Int' | 'Long' | 'String';\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMappingColumn = {\n /** Data type (matches PColumn value type) */\n type: 'Int' | 'Long' | 'Float' | 'Double' | 'String';\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMapping = {\n /** Axes mappings - Parquet file is sorted by these fields in this order */\n axes: ParquetChunkMappingAxis[];\n\n /** Column mapping */\n column: ParquetChunkMappingColumn;\n};\n\nexport type ParquetChunkStats = {\n /** Number of rows in the chunk */\n numberOfRows: number;\n /** Byte size information for storage optimization and query planning */\n size: {\n /** Byte sizes for each axis column in the same order as axes mapping */\n axes: number[];\n /** Byte size for the data column */\n column: number;\n };\n};\n\nexport type ParquetChunkMetadata = {\n /** Content hash calculated for the specific axes and data this chunk represents */\n dataDigest: string;\n\n /** Pre-computed statistics for optimization without blob download */\n stats: Partial<ParquetChunkStats>;\n};\n\nexport type ParquetChunk<Blob> = {\n /** Parquet file (PTable) containing column data */\n data: Blob;\n} & ParquetChunkMapping & Partial<ParquetChunkMetadata>;\n\nexport type ParquetPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('ParquetPartitioned') */\n type: 'ParquetPartitioned';\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to parquet files */\n parts: Record<string, Blob>;\n};\n\n/**\n * Union type representing all possible data storage formats for PColumn data.\n * The specific format used depends on data size, access patterns, and performance requirements.\n *\n * @template Blob - Type parameter representing the storage reference type (could be ResourceInfo, PFrameBlobId, etc.)\n */\nexport type DataInfo<Blob> =\n | JsonDataInfo\n | JsonPartitionedDataInfo<Blob>\n | BinaryPartitionedDataInfo<Blob>\n | ParquetPartitionedDataInfo<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfo.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfo, false otherwise\n */\nexport function isDataInfo<Blob>(value: unknown): value is DataInfo<Blob> {\n if (!value || typeof value !== 'object') {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!('type' in data)) {\n return false;\n }\n\n switch (data.type) {\n case 'Json':\n return (\n typeof data.keyLength === 'number'\n && data.data !== undefined\n && typeof data.data === 'object'\n );\n case 'JsonPartitioned':\n case 'BinaryPartitioned':\n case 'ParquetPartitioned':\n return (\n typeof data.partitionKeyLength === 'number'\n && data.parts !== undefined\n && typeof data.parts === 'object'\n );\n default:\n return false;\n }\n}\n\n/**\n * Maps blob references in a DataInfo object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfo - The source DataInfo object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfo object with transformed blob references\n */\nexport function mapDataInfo<B1, B2>(\n dataInfo: ParquetPartitionedDataInfo<B1>,\n mapFn: (blob: B1) => B2,\n): ParquetPartitionedDataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: Exclude<DataInfo<B1>, ParquetPartitionedDataInfo<B1>>,\n mapFn: (blob: B1) => B2,\n): Exclude<DataInfo<B2>, ParquetPartitionedDataInfo<B2>>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: DataInfo<B1>,\n mapFn: (blob: B1) => B2,\n): DataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: DataInfo<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfo<B2> | undefined {\n if (dataInfo === undefined) {\n return undefined;\n }\n\n switch (dataInfo.type) {\n case 'Json':\n // Json type doesn't contain blobs, so return as is\n return dataInfo;\n case 'JsonPartitioned': {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case 'BinaryPartitioned': {\n // Map each index and values blob in parts\n const newParts: Record<string, BinaryChunk<B2>> = {};\n for (const [key, chunk] of Object.entries(dataInfo.parts)) {\n newParts[key] = {\n index: mapFn(chunk.index),\n values: mapFn(chunk.values),\n };\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case 'ParquetPartitioned': {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n }\n}\n\n/**\n * @param dataInfo - The source DataInfo object\n * @param cb - Callback, function that have access to every blob to visit them all\n * @returns Nothing\n */\nexport function visitDataInfo<B>(\n dataInfo: DataInfo<B>,\n cb: (blob: B) => void,\n): void {\n switch (dataInfo.type) {\n case 'Json':\n // Json type doesn't contain blobs, so return as is\n break;\n case 'JsonPartitioned': {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n case 'BinaryPartitioned': {\n // Visit each index and values blob in parts\n Object.values(dataInfo.parts).forEach((chunk) => {\n cb(chunk.index);\n cb(chunk.values);\n });\n break;\n }\n case 'ParquetPartitioned': {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n }\n}\n\n//\n// Lightway representation for ExplicitJsonData\n//\n\n/**\n * Represents a single key-value entry in a column's explicit data structure.\n * Used when directly instantiating PColumns with explicit data.\n */\nexport type PColumnValuesEntry = {\n key: PColumnKey;\n val: PColumnValue;\n};\n\n/**\n * Array of key-value entries representing explicit column data.\n * Used for lightweight explicit instantiation of PColumns.\n */\nexport type PColumnValues = PColumnValuesEntry[];\n\n/**\n * Entry-based representation of JsonDataInfo\n */\nexport interface JsonDataInfoEntries {\n type: 'Json';\n keyLength: number;\n data: PColumnDataEntry<PColumnValue>[];\n}\n\n/**\n * Entry-based representation of JsonPartitionedDataInfo\n */\nexport interface JsonPartitionedDataInfoEntries<Blob> {\n type: 'JsonPartitioned';\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n\n/**\n * Entry-based representation of BinaryPartitionedDataInfo\n */\nexport interface BinaryPartitionedDataInfoEntries<Blob> {\n type: 'BinaryPartitioned';\n partitionKeyLength: number;\n parts: PColumnDataEntry<BinaryChunk<Blob>>[];\n}\n\n/**\n * Entry-based representation of ParquetPartitionedDataInfo\n */\nexport interface ParquetPartitionedDataInfoEntries<Blob> {\n type: 'ParquetPartitioned';\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n/**\n * Union type representing all possible entry-based partitioned data storage formats\n */\nexport type PartitionedDataInfoEntries<Blob> =\n | JsonPartitionedDataInfoEntries<Blob>\n | BinaryPartitionedDataInfoEntries<Blob>\n | ParquetPartitionedDataInfoEntries<Blob>;\n\n/**\n * Union type representing all possible entry-based data storage formats\n */\nexport type DataInfoEntries<Blob> =\n | JsonDataInfoEntries\n | PartitionedDataInfoEntries<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfoEntries.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfoEntries, false otherwise\n */\nexport function isDataInfoEntries<Blob>(value: unknown): value is DataInfoEntries<Blob> {\n if (!value || typeof value !== 'object') {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!('type' in data)) {\n return false;\n }\n\n switch (data.type) {\n case 'Json':\n return (\n typeof data.keyLength === 'number'\n && Array.isArray(data.data)\n );\n case 'JsonPartitioned':\n case 'BinaryPartitioned':\n case 'ParquetPartitioned':\n return (\n typeof data.partitionKeyLength === 'number'\n && Array.isArray(data.parts)\n );\n default:\n return false;\n }\n}\n\n/**\n * Type guard function that checks if the given value is a valid PartitionedDataInfoEntries.\n *\n * @template Blob - Type parameter representing the storage reference type\n * @param value - The value to check\n * @returns True if the value is a valid PartitionedDataInfoEntries, false otherwise\n */\nexport function isPartitionedDataInfoEntries<Blob>(value: unknown): value is PartitionedDataInfoEntries<Blob> {\n if (!isDataInfoEntries(value)) return false;\n switch (value.type) {\n case 'JsonPartitioned':\n case 'BinaryPartitioned':\n case 'ParquetPartitioned':\n return true;\n default:\n return false;\n }\n}\n\n/**\n * Converts DataInfo to DataInfoEntries\n *\n * @param dataInfo - The record-based DataInfo object\n * @returns The equivalent entry-based DataInfoEntries object\n */\nexport function dataInfoToEntries<Blob>(dataInfo: DataInfo<Blob>): DataInfoEntries<Blob> {\n switch (dataInfo.type) {\n case 'Json': return {\n type: 'Json',\n keyLength: dataInfo.keyLength,\n data: Object.entries(dataInfo.data).map(([keyStr, value]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value } as PColumnDataEntry<PColumnValue>;\n }),\n };\n case 'JsonPartitioned': return {\n type: 'JsonPartitioned',\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n case 'BinaryPartitioned': return {\n type: 'BinaryPartitioned',\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: chunk } as PColumnDataEntry<BinaryChunk<Blob>>;\n }),\n };\n case 'ParquetPartitioned': return {\n type: 'ParquetPartitioned',\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n default:\n assertNever(dataInfo);\n }\n}\n\n/**\n * Converts DataInfoEntries to DataInfo\n *\n * @param dataInfoEntries - The entry-based DataInfoEntries object\n * @returns The equivalent record-based DataInfo object\n */\nexport function entriesToDataInfo<Blob>(dataInfoEntries: DataInfoEntries<Blob>): DataInfo<Blob> {\n switch (dataInfoEntries.type) {\n case 'Json': return {\n type: 'Json',\n keyLength: dataInfoEntries.keyLength,\n data: Object.fromEntries(\n dataInfoEntries.data.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case 'JsonPartitioned': return {\n type: 'JsonPartitioned',\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case 'BinaryPartitioned': return {\n type: 'BinaryPartitioned',\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case 'ParquetPartitioned': return {\n type: 'ParquetPartitioned',\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n default:\n assertNever(dataInfoEntries);\n }\n}\n\n/**\n * Maps blob references in a DataInfoEntries object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfoEntries - The source DataInfoEntries object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfoEntries object with transformed blob references\n */\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1>,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2>;\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2> | undefined {\n if (dataInfoEntries === undefined) {\n return undefined;\n }\n\n switch (dataInfoEntries.type) {\n case 'Json':\n // Json type doesn't contain blobs, so return as is\n return dataInfoEntries;\n case 'JsonPartitioned': return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n case 'BinaryPartitioned': return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: {\n index: mapFn(entry.value.index),\n values: mapFn(entry.value.values),\n },\n })),\n };\n case 'ParquetPartitioned': return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n }\n}\n"],"names":[],"mappings":";;AAiKA;;;;;AAKG;AACG,SAAU,UAAU,CAAO,KAAc,EAAA;IAC7C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACvC,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,IAAI,GAAG,KAAgC;AAC7C,IAAA,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;IACd;AAEA,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,QACE,OAAO,IAAI,CAAC,SAAS,KAAK;mBACvB,IAAI,CAAC,IAAI,KAAK;AACd,mBAAA,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;AAEpC,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,QACE,OAAO,IAAI,CAAC,kBAAkB,KAAK;mBAChC,IAAI,CAAC,KAAK,KAAK;AACf,mBAAA,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;AAErC,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAuBM,SAAU,WAAW,CACzB,QAAkC,EAClC,KAAuB,EAAA;AAEvB,IAAA,IAAI,QAAQ,KAAK,SAAS,EAAE;AAC1B,QAAA,OAAO,SAAS;IAClB;AAEA,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;;AAET,YAAA,OAAO,QAAQ;QACjB,KAAK,iBAAiB,EAAE;;YAEtB,MAAM,QAAQ,GAAuB,EAAE;AACvC,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACxD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YAC7B;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;QACA,KAAK,mBAAmB,EAAE;;YAExB,MAAM,QAAQ,GAAoC,EAAE;AACpD,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACzD,QAAQ,CAAC,GAAG,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,oBAAA,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;iBAC5B;YACH;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;QACA,KAAK,oBAAoB,EAAE;;YAEzB,MAAM,QAAQ,GAAuB,EAAE;AACvC,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACxD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YAC7B;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;;AAEJ;AAEA;;;;AAIG;AACG,SAAU,aAAa,CAC3B,QAAqB,EACrB,EAAqB,EAAA;AAErB,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;;YAET;QACF,KAAK,iBAAiB,EAAE;;AAEtB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC;QACF;QACA,KAAK,mBAAmB,EAAE;;AAExB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,KAAI;AAC9C,gBAAA,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;AACf,gBAAA,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,YAAA,CAAC,CAAC;YACF;QACF;QACA,KAAK,oBAAoB,EAAE;;AAEzB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC;QACF;;AAEJ;AAuEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,KAAc,EAAA;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACvC,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,IAAI,GAAG,KAAgC;AAC7C,IAAA,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;IACd;AAEA,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,QACE,OAAO,IAAI,CAAC,SAAS,KAAK;mBACvB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AAE/B,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,QACE,OAAO,IAAI,CAAC,kBAAkB,KAAK;mBAChC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;AAEhC,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAEA;;;;;;AAMG;AACG,SAAU,4BAA4B,CAAO,KAAc,EAAA;AAC/D,IAAA,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;AAAE,QAAA,OAAO,KAAK;AAC3C,IAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,OAAO,IAAI;AACb,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,QAAwB,EAAA;AAC9D,IAAA,QAAQ,QAAQ,CAAC,IAAI;QACnB,KAAK,MAAM,EAAE,OAAO;AAClB,YAAA,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,QAAQ,CAAC,SAAS;AAC7B,YAAA,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,KAAI;gBAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,gBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAoC;AACzD,YAAA,CAAC,CAAC;SACH;QACD,KAAK,iBAAiB,EAAE,OAAO;AAC7B,YAAA,IAAI,EAAE,iBAAiB;YACvB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,YAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAI;gBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,gBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAA4B;AACvD,YAAA,CAAC,CAAC;SACH;QACD,KAAK,mBAAmB,EAAE,OAAO;AAC/B,YAAA,IAAI,EAAE,mBAAmB;YACzB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,YAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,KAAI;gBAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,gBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAyC;AACrE,YAAA,CAAC,CAAC;SACH;QACD,KAAK,oBAAoB,EAAE,OAAO;AAChC,YAAA,IAAI,EAAE,oBAAoB;YAC1B,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,YAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAI;gBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,gBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAA4B;AACvD,YAAA,CAAC,CAAC;SACH;AACD,QAAA;YACE,WAAW,CAAC,QAAQ,CAAC;;AAE3B;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,eAAsC,EAAA;AAC5E,IAAA,QAAQ,eAAe,CAAC,IAAI;QAC1B,KAAK,MAAM,EAAE,OAAO;AAClB,YAAA,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,eAAe,CAAC,SAAS;AACpC,YAAA,IAAI,EAAE,MAAM,CAAC,WAAW,CACtB,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC3E;SACF;QACD,KAAK,iBAAiB,EAAE,OAAO;AAC7B,YAAA,IAAI,EAAE,iBAAiB;YACvB,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,YAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;SACF;QACD,KAAK,mBAAmB,EAAE,OAAO;AAC/B,YAAA,IAAI,EAAE,mBAAmB;YACzB,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,YAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;SACF;QACD,KAAK,oBAAoB,EAAE,OAAO;AAChC,YAAA,IAAI,EAAE,oBAAoB;YAC1B,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,YAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;SACF;AACD,QAAA;YACE,WAAW,CAAC,eAAe,CAAC;;AAElC;AAeM,SAAU,kBAAkB,CAChC,eAAgD,EAChD,KAAuB,EAAA;AAEvB,IAAA,IAAI,eAAe,KAAK,SAAS,EAAE;AACjC,QAAA,OAAO,SAAS;IAClB;AAEA,IAAA,QAAQ,eAAe,CAAC,IAAI;AAC1B,QAAA,KAAK,MAAM;;AAET,YAAA,OAAO,eAAe;QACxB,KAAK,iBAAiB,EAAE,OAAO;AAC7B,YAAA,GAAG,eAAe;AAClB,YAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;gBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,gBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AAC1B,aAAA,CAAC,CAAC;SACJ;QACD,KAAK,mBAAmB,EAAE,OAAO;AAC/B,YAAA,GAAG,eAAe;AAClB,YAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;gBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,gBAAA,KAAK,EAAE;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;AAClC,iBAAA;AACF,aAAA,CAAC,CAAC;SACJ;QACD,KAAK,oBAAoB,EAAE,OAAO;AAChC,YAAA,GAAG,eAAe;AAClB,YAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;gBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,gBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AAC1B,aAAA,CAAC,CAAC;SACJ;;AAEL;;;;"}
|
|
1
|
+
{"version":3,"file":"data_info.js","sources":["../../../src/drivers/pframe/data_info.ts"],"sourcesContent":["import { assertNever } from \"../../util\";\n\n/**\n * Represents a JavaScript representation of a value in a PColumn. Can be null, a number, or a string.\n * These are the primitive types that can be stored directly in PColumns.\n *\n * Note: Actual columns can hold more value types, which are converted to these JavaScript types\n * once they enter the JavaScript runtime.\n */\nexport type PColumnValue = null | number | string;\n\n/**\n * Represents a key for a PColumn value.\n * Can be an array of strings or numbers.\n */\nexport type PColumnKey = (number | string)[];\n\n/**\n * Represents a single entry in a PColumn's data structure.\n * Contains a key and a value.\n */\nexport type PColumnDataEntry<T> = {\n /** Key for the value */\n key: PColumnKey;\n\n /** Value / blob at the given key */\n value: T;\n};\n\n/**\n * Represents column data stored as a simple JSON structure.\n * Used for small datasets that can be efficiently stored directly in memory.\n */\nexport type JsonDataInfo = {\n /** Identifier for this data format ('Json') */\n type: \"Json\";\n\n /** Number of axes that make up the complete key (tuple length) */\n keyLength: number;\n\n /**\n * Key-value pairs where keys are stringified tuples of axis values\n * and values are the column values for those coordinates\n */\n data: Record<string, PColumnValue>;\n};\n\n/**\n * Represents column data partitioned across multiple JSON blobs.\n * Used for larger datasets that need to be split into manageable chunks.\n */\nexport type JsonPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('JsonPartitioned') */\n type: \"JsonPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to blob references */\n parts: Record<string, Blob>;\n};\n\n/**\n * Represents a binary format chunk containing index and values as separate blobs.\n * Used for efficient storage and retrieval of column data in binary format.\n */\nexport type BinaryChunk<Blob> = {\n /** Binary blob containing structured index information */\n index: Blob;\n\n /** Binary blob containing the actual values */\n values: Blob;\n};\n\n/**\n * Represents column data partitioned across multiple binary chunks.\n * Optimized for efficient storage and retrieval of large datasets.\n */\nexport type BinaryPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('BinaryPartitioned') */\n type: \"BinaryPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to binary chunks */\n parts: Record<string, BinaryChunk<Blob>>;\n};\n\nexport type ParquetChunkMappingAxis = {\n /** Data type (matches PColumn axis types) */\n type: \"Int\" | \"Long\" | \"String\";\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMappingColumn = {\n /** Data type (matches PColumn value type) */\n type: \"Int\" | \"Long\" | \"Float\" | \"Double\" | \"String\";\n\n /** Field name in the Parquet file */\n id: string;\n};\n\nexport type ParquetChunkMapping = {\n /** Axes mappings - Parquet file is sorted by these fields in this order */\n axes: ParquetChunkMappingAxis[];\n\n /** Column mapping */\n column: ParquetChunkMappingColumn;\n};\n\nexport type ParquetChunkStats = {\n /** Number of rows in the chunk */\n numberOfRows: number;\n /** Byte size information for storage optimization and query planning */\n size: {\n /** Byte sizes for each axis column in the same order as axes mapping */\n axes: number[];\n /** Byte size for the data column */\n column: number;\n };\n};\n\nexport type ParquetChunkMetadata = {\n /** Content hash calculated for the specific axes and data this chunk represents */\n dataDigest: string;\n\n /** Pre-computed statistics for optimization without blob download */\n stats: Partial<ParquetChunkStats>;\n};\n\nexport type ParquetChunk<Blob> = {\n /** Parquet file (PTable) containing column data */\n data: Blob;\n} & ParquetChunkMapping &\n Partial<ParquetChunkMetadata>;\n\nexport type ParquetPartitionedDataInfo<Blob> = {\n /** Identifier for this data format ('ParquetPartitioned') */\n type: \"ParquetPartitioned\";\n\n /** Number of leading axes used for partitioning */\n partitionKeyLength: number;\n\n /** Map of stringified partition keys to parquet files */\n parts: Record<string, Blob>;\n};\n\n/**\n * Union type representing all possible data storage formats for PColumn data.\n * The specific format used depends on data size, access patterns, and performance requirements.\n *\n * @template Blob - Type parameter representing the storage reference type (could be ResourceInfo, PFrameBlobId, etc.)\n */\nexport type DataInfo<Blob> =\n | JsonDataInfo\n | JsonPartitionedDataInfo<Blob>\n | BinaryPartitionedDataInfo<Blob>\n | ParquetPartitionedDataInfo<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfo.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfo, false otherwise\n */\nexport function isDataInfo<Blob>(value: unknown): value is DataInfo<Blob> {\n if (!value || typeof value !== \"object\") {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!(\"type\" in data)) {\n return false;\n }\n\n switch (data.type) {\n case \"Json\":\n return (\n typeof data.keyLength === \"number\" &&\n data.data !== undefined &&\n typeof data.data === \"object\"\n );\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return (\n typeof data.partitionKeyLength === \"number\" &&\n data.parts !== undefined &&\n typeof data.parts === \"object\"\n );\n default:\n return false;\n }\n}\n\n/**\n * Maps blob references in a DataInfo object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfo - The source DataInfo object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfo object with transformed blob references\n */\nexport function mapDataInfo<B1, B2>(\n dataInfo: ParquetPartitionedDataInfo<B1>,\n mapFn: (blob: B1) => B2,\n): ParquetPartitionedDataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: Exclude<DataInfo<B1>, ParquetPartitionedDataInfo<B1>>,\n mapFn: (blob: B1) => B2,\n): Exclude<DataInfo<B2>, ParquetPartitionedDataInfo<B2>>;\nexport function mapDataInfo<B1, B2>(dataInfo: DataInfo<B1>, mapFn: (blob: B1) => B2): DataInfo<B2>;\nexport function mapDataInfo<B1, B2>(\n dataInfo: DataInfo<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfo<B2> | undefined {\n if (dataInfo === undefined) {\n return undefined;\n }\n\n switch (dataInfo.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n return dataInfo;\n case \"JsonPartitioned\": {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case \"BinaryPartitioned\": {\n // Map each index and values blob in parts\n const newParts: Record<string, BinaryChunk<B2>> = {};\n for (const [key, chunk] of Object.entries(dataInfo.parts)) {\n newParts[key] = {\n index: mapFn(chunk.index),\n values: mapFn(chunk.values),\n };\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n case \"ParquetPartitioned\": {\n // Map each blob in parts\n const newParts: Record<string, B2> = {};\n for (const [key, blob] of Object.entries(dataInfo.parts)) {\n newParts[key] = mapFn(blob);\n }\n return {\n ...dataInfo,\n parts: newParts,\n };\n }\n }\n}\n\n/**\n * @param dataInfo - The source DataInfo object\n * @param cb - Callback, function that have access to every blob to visit them all\n * @returns Nothing\n */\nexport function visitDataInfo<B>(dataInfo: DataInfo<B>, cb: (blob: B) => void): void {\n switch (dataInfo.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n break;\n case \"JsonPartitioned\": {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n case \"BinaryPartitioned\": {\n // Visit each index and values blob in parts\n Object.values(dataInfo.parts).forEach((chunk) => {\n cb(chunk.index);\n cb(chunk.values);\n });\n break;\n }\n case \"ParquetPartitioned\": {\n // Visit each blob in parts\n Object.values(dataInfo.parts).forEach(cb);\n break;\n }\n }\n}\n\n//\n// Lightway representation for ExplicitJsonData\n//\n\n/**\n * Represents a single key-value entry in a column's explicit data structure.\n * Used when directly instantiating PColumns with explicit data.\n */\nexport type PColumnValuesEntry = {\n key: PColumnKey;\n val: PColumnValue;\n};\n\n/**\n * Array of key-value entries representing explicit column data.\n * Used for lightweight explicit instantiation of PColumns.\n */\nexport type PColumnValues = PColumnValuesEntry[];\n\n/**\n * Entry-based representation of JsonDataInfo\n */\nexport interface JsonDataInfoEntries {\n type: \"Json\";\n keyLength: number;\n data: PColumnDataEntry<PColumnValue>[];\n}\n\n/**\n * Entry-based representation of JsonPartitionedDataInfo\n */\nexport interface JsonPartitionedDataInfoEntries<Blob> {\n type: \"JsonPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n\n/**\n * Entry-based representation of BinaryPartitionedDataInfo\n */\nexport interface BinaryPartitionedDataInfoEntries<Blob> {\n type: \"BinaryPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<BinaryChunk<Blob>>[];\n}\n\n/**\n * Entry-based representation of ParquetPartitionedDataInfo\n */\nexport interface ParquetPartitionedDataInfoEntries<Blob> {\n type: \"ParquetPartitioned\";\n partitionKeyLength: number;\n parts: PColumnDataEntry<Blob>[];\n}\n/**\n * Union type representing all possible entry-based partitioned data storage formats\n */\nexport type PartitionedDataInfoEntries<Blob> =\n | JsonPartitionedDataInfoEntries<Blob>\n | BinaryPartitionedDataInfoEntries<Blob>\n | ParquetPartitionedDataInfoEntries<Blob>;\n\n/**\n * Union type representing all possible entry-based data storage formats\n */\nexport type DataInfoEntries<Blob> = JsonDataInfoEntries | PartitionedDataInfoEntries<Blob>;\n\n/**\n * Type guard function that checks if the given value is a valid DataInfoEntries.\n *\n * @param value - The value to check\n * @returns True if the value is a valid DataInfoEntries, false otherwise\n */\nexport function isDataInfoEntries<Blob>(value: unknown): value is DataInfoEntries<Blob> {\n if (!value || typeof value !== \"object\") {\n return false;\n }\n\n const data = value as Record<string, unknown>;\n if (!(\"type\" in data)) {\n return false;\n }\n\n switch (data.type) {\n case \"Json\":\n return typeof data.keyLength === \"number\" && Array.isArray(data.data);\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return typeof data.partitionKeyLength === \"number\" && Array.isArray(data.parts);\n default:\n return false;\n }\n}\n\n/**\n * Type guard function that checks if the given value is a valid PartitionedDataInfoEntries.\n *\n * @template Blob - Type parameter representing the storage reference type\n * @param value - The value to check\n * @returns True if the value is a valid PartitionedDataInfoEntries, false otherwise\n */\nexport function isPartitionedDataInfoEntries<Blob>(\n value: unknown,\n): value is PartitionedDataInfoEntries<Blob> {\n if (!isDataInfoEntries(value)) return false;\n switch (value.type) {\n case \"JsonPartitioned\":\n case \"BinaryPartitioned\":\n case \"ParquetPartitioned\":\n return true;\n default:\n return false;\n }\n}\n\n/**\n * Converts DataInfo to DataInfoEntries\n *\n * @param dataInfo - The record-based DataInfo object\n * @returns The equivalent entry-based DataInfoEntries object\n */\nexport function dataInfoToEntries<Blob>(dataInfo: DataInfo<Blob>): DataInfoEntries<Blob> {\n switch (dataInfo.type) {\n case \"Json\":\n return {\n type: \"Json\",\n keyLength: dataInfo.keyLength,\n data: Object.entries(dataInfo.data).map(([keyStr, value]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value } as PColumnDataEntry<PColumnValue>;\n }),\n };\n case \"JsonPartitioned\":\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n case \"BinaryPartitioned\":\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: chunk } as PColumnDataEntry<BinaryChunk<Blob>>;\n }),\n };\n case \"ParquetPartitioned\":\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: dataInfo.partitionKeyLength,\n parts: Object.entries(dataInfo.parts).map(([keyStr, blob]) => {\n const key = JSON.parse(keyStr) as PColumnKey;\n return { key, value: blob } as PColumnDataEntry<Blob>;\n }),\n };\n default:\n assertNever(dataInfo);\n }\n}\n\n/**\n * Converts DataInfoEntries to DataInfo\n *\n * @param dataInfoEntries - The entry-based DataInfoEntries object\n * @returns The equivalent record-based DataInfo object\n */\nexport function entriesToDataInfo<Blob>(dataInfoEntries: DataInfoEntries<Blob>): DataInfo<Blob> {\n switch (dataInfoEntries.type) {\n case \"Json\":\n return {\n type: \"Json\",\n keyLength: dataInfoEntries.keyLength,\n data: Object.fromEntries(\n dataInfoEntries.data.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"JsonPartitioned\":\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"BinaryPartitioned\":\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n case \"ParquetPartitioned\":\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: dataInfoEntries.partitionKeyLength,\n parts: Object.fromEntries(\n dataInfoEntries.parts.map(({ key, value }) => [JSON.stringify(key), value]),\n ),\n };\n default:\n assertNever(dataInfoEntries);\n }\n}\n\n/**\n * Maps blob references in a DataInfoEntries object from one type to another using a mapping function.\n *\n * @template B1 - Source blob type\n * @template B2 - Target blob type\n * @param dataInfoEntries - The source DataInfoEntries object\n * @param mapFn - Function to transform blobs from type B1 to type B2\n * @returns A new DataInfoEntries object with transformed blob references\n */\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1>,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2>;\nexport function mapDataInfoEntries<B1, B2>(\n dataInfoEntries: DataInfoEntries<B1> | undefined,\n mapFn: (blob: B1) => B2,\n): DataInfoEntries<B2> | undefined {\n if (dataInfoEntries === undefined) {\n return undefined;\n }\n\n switch (dataInfoEntries.type) {\n case \"Json\":\n // Json type doesn't contain blobs, so return as is\n return dataInfoEntries;\n case \"JsonPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n case \"BinaryPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: {\n index: mapFn(entry.value.index),\n values: mapFn(entry.value.values),\n },\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...dataInfoEntries,\n parts: dataInfoEntries.parts.map((entry) => ({\n key: entry.key,\n value: mapFn(entry.value),\n })),\n };\n }\n}\n"],"names":[],"mappings":";;AAkKA;;;;;AAKG;AACG,SAAU,UAAU,CAAO,KAAc,EAAA;IAC7C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACvC,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,IAAI,GAAG,KAAgC;AAC7C,IAAA,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;IACd;AAEA,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,QACE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;gBAClC,IAAI,CAAC,IAAI,KAAK,SAAS;AACvB,gBAAA,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;AAEjC,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,QACE,OAAO,IAAI,CAAC,kBAAkB,KAAK,QAAQ;gBAC3C,IAAI,CAAC,KAAK,KAAK,SAAS;AACxB,gBAAA,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;AAElC,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAoBM,SAAU,WAAW,CACzB,QAAkC,EAClC,KAAuB,EAAA;AAEvB,IAAA,IAAI,QAAQ,KAAK,SAAS,EAAE;AAC1B,QAAA,OAAO,SAAS;IAClB;AAEA,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;;AAET,YAAA,OAAO,QAAQ;QACjB,KAAK,iBAAiB,EAAE;;YAEtB,MAAM,QAAQ,GAAuB,EAAE;AACvC,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACxD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YAC7B;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;QACA,KAAK,mBAAmB,EAAE;;YAExB,MAAM,QAAQ,GAAoC,EAAE;AACpD,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACzD,QAAQ,CAAC,GAAG,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,oBAAA,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;iBAC5B;YACH;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;QACA,KAAK,oBAAoB,EAAE;;YAEzB,MAAM,QAAQ,GAAuB,EAAE;AACvC,YAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBACxD,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YAC7B;YACA,OAAO;AACL,gBAAA,GAAG,QAAQ;AACX,gBAAA,KAAK,EAAE,QAAQ;aAChB;QACH;;AAEJ;AAEA;;;;AAIG;AACG,SAAU,aAAa,CAAI,QAAqB,EAAE,EAAqB,EAAA;AAC3E,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;;YAET;QACF,KAAK,iBAAiB,EAAE;;AAEtB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC;QACF;QACA,KAAK,mBAAmB,EAAE;;AAExB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,KAAI;AAC9C,gBAAA,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;AACf,gBAAA,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,YAAA,CAAC,CAAC;YACF;QACF;QACA,KAAK,oBAAoB,EAAE;;AAEzB,YAAA,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC;QACF;;AAEJ;AAqEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,KAAc,EAAA;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACvC,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,IAAI,GAAG,KAAgC;AAC7C,IAAA,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;IACd;AAEA,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACvE,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,OAAO,OAAO,IAAI,CAAC,kBAAkB,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;AACjF,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAEA;;;;;;AAMG;AACG,SAAU,4BAA4B,CAC1C,KAAc,EAAA;AAEd,IAAA,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;AAAE,QAAA,OAAO,KAAK;AAC3C,IAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,QAAA,KAAK,iBAAiB;AACtB,QAAA,KAAK,mBAAmB;AACxB,QAAA,KAAK,oBAAoB;AACvB,YAAA,OAAO,IAAI;AACb,QAAA;AACE,YAAA,OAAO,KAAK;;AAElB;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,QAAwB,EAAA;AAC9D,IAAA,QAAQ,QAAQ,CAAC,IAAI;AACnB,QAAA,KAAK,MAAM;YACT,OAAO;AACL,gBAAA,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,QAAQ,CAAC,SAAS;AAC7B,gBAAA,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,KAAI;oBAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAoC;AACzD,gBAAA,CAAC,CAAC;aACH;AACH,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,IAAI,EAAE,iBAAiB;gBACvB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,gBAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAI;oBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAA4B;AACvD,gBAAA,CAAC,CAAC;aACH;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,IAAI,EAAE,mBAAmB;gBACzB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,gBAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,KAAI;oBAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAyC;AACrE,gBAAA,CAAC,CAAC;aACH;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,IAAI,EAAE,oBAAoB;gBAC1B,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;AAC/C,gBAAA,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAI;oBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAe;AAC5C,oBAAA,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAA4B;AACvD,gBAAA,CAAC,CAAC;aACH;AACH,QAAA;YACE,WAAW,CAAC,QAAQ,CAAC;;AAE3B;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAO,eAAsC,EAAA;AAC5E,IAAA,QAAQ,eAAe,CAAC,IAAI;AAC1B,QAAA,KAAK,MAAM;YACT,OAAO;AACL,gBAAA,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,eAAe,CAAC,SAAS;AACpC,gBAAA,IAAI,EAAE,MAAM,CAAC,WAAW,CACtB,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC3E;aACF;AACH,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,IAAI,EAAE,iBAAiB;gBACvB,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,gBAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;aACF;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,IAAI,EAAE,mBAAmB;gBACzB,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,gBAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;aACF;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,IAAI,EAAE,oBAAoB;gBAC1B,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;AACtD,gBAAA,KAAK,EAAE,MAAM,CAAC,WAAW,CACvB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAC5E;aACF;AACH,QAAA;YACE,WAAW,CAAC,eAAe,CAAC;;AAElC;AAeM,SAAU,kBAAkB,CAChC,eAAgD,EAChD,KAAuB,EAAA;AAEvB,IAAA,IAAI,eAAe,KAAK,SAAS,EAAE;AACjC,QAAA,OAAO,SAAS;IAClB;AAEA,IAAA,QAAQ,eAAe,CAAC,IAAI;AAC1B,QAAA,KAAK,MAAM;;AAET,YAAA,OAAO,eAAe;AACxB,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,GAAG,eAAe;AAClB,gBAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;oBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AAC1B,iBAAA,CAAC,CAAC;aACJ;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,GAAG,eAAe;AAClB,gBAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;oBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE;wBACL,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;wBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;AAClC,qBAAA;AACF,iBAAA,CAAC,CAAC;aACJ;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,GAAG,eAAe;AAClB,gBAAA,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM;oBAC3C,GAAG,EAAE,KAAK,CAAC,GAAG;AACd,oBAAA,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;AAC1B,iBAAA,CAAC,CAAC;aACJ;;AAEP;;;;"}
|
|
@@ -33,10 +33,10 @@ function isValueNA(vector, row) {
|
|
|
33
33
|
throw Error(`unsupported data type: ${valueType}`);
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
|
-
const PTableAbsent = { type:
|
|
36
|
+
const PTableAbsent = { type: "absent" };
|
|
37
37
|
/** Type guard for absent value */
|
|
38
38
|
function isPTableAbsent(value) {
|
|
39
|
-
return typeof value ===
|
|
39
|
+
return typeof value === "object" && value !== null && "type" in value && value.type === "absent";
|
|
40
40
|
}
|
|
41
41
|
const PTableNA = null;
|
|
42
42
|
/** Type guard for NA value */
|
|
@@ -49,9 +49,12 @@ function isPTableValueAxis(value, isNA) {
|
|
|
49
49
|
function pTableValueImpl(column, row, options) {
|
|
50
50
|
const valueType = column.type;
|
|
51
51
|
if (valueType === spec.ValueType.Bytes) {
|
|
52
|
-
throw Error(
|
|
52
|
+
throw Error("Bytes not yet supported");
|
|
53
53
|
}
|
|
54
|
-
if (options &&
|
|
54
|
+
if (options &&
|
|
55
|
+
"dataType" in options &&
|
|
56
|
+
options.dataType !== undefined &&
|
|
57
|
+
options.dataType !== valueType) {
|
|
55
58
|
throw Error(`expected column of type ${options.dataType}, got ${valueType}`);
|
|
56
59
|
}
|
|
57
60
|
if (isValueAbsent(column, row)) {
|