@milaboratories/pl-model-common 1.19.1 → 1.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -85,6 +85,70 @@ export type BinaryPartitionedDataInfo<Blob> = {
|
|
|
85
85
|
parts: Record<string, BinaryChunk<Blob>>;
|
|
86
86
|
};
|
|
87
87
|
|
|
88
|
+
type ParquetPartitionInfoMappingAxis = {
|
|
89
|
+
/** Data type (matches PColumn axis types) */
|
|
90
|
+
type: 'Int' | 'Long' | 'String';
|
|
91
|
+
|
|
92
|
+
/** Field name in the Parquet file */
|
|
93
|
+
id: string;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
type ParquetPartitionInfoMappingColumn = {
|
|
97
|
+
/** Data type (matches PColumn value type) */
|
|
98
|
+
type: 'Int' | 'Long' | 'Float' | 'Double' | 'String';
|
|
99
|
+
|
|
100
|
+
/** Field name in the Parquet file */
|
|
101
|
+
id: string;
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
type ParquetPartitionInfoMapping = {
|
|
105
|
+
/** Axes mappings - Parquet file is sorted by these fields in this order */
|
|
106
|
+
axes: ParquetPartitionInfoMappingAxis[];
|
|
107
|
+
|
|
108
|
+
/** Column mapping */
|
|
109
|
+
column: ParquetPartitionInfoMappingColumn;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
type ParquetPartitionInfoData<Blob> = {
|
|
113
|
+
/** Parquet file (PTable) containing column data */
|
|
114
|
+
data: Blob;
|
|
115
|
+
|
|
116
|
+
/** Content hash calculated for the specific axes and data this partition represents */
|
|
117
|
+
dataDigest?: string;
|
|
118
|
+
|
|
119
|
+
/** Pre-computed statistics for optimization without blob download */
|
|
120
|
+
stats?: {
|
|
121
|
+
/** Number of rows in the column */
|
|
122
|
+
numberOfRows?: number;
|
|
123
|
+
/** Byte size information for storage optimization and query planning */
|
|
124
|
+
numberOfBytes?: {
|
|
125
|
+
/** Byte sizes for each axis column in the same order as axes mapping */
|
|
126
|
+
axes: number[];
|
|
127
|
+
/** Byte size for the data column */
|
|
128
|
+
column: number;
|
|
129
|
+
};
|
|
130
|
+
};
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
export type ParquetChunk<Blob> = {
|
|
134
|
+
/** Mapping of column names to their data */
|
|
135
|
+
mapping: ParquetPartitionInfoMapping;
|
|
136
|
+
|
|
137
|
+
/** Data for this partition */
|
|
138
|
+
data: ParquetPartitionInfoData<Blob>;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
export type ParquetPartitionedDataInfo<Blob> = {
|
|
142
|
+
/** Identifier for this data format ('ParquetPartitioned') */
|
|
143
|
+
type: 'ParquetPartitioned';
|
|
144
|
+
|
|
145
|
+
/** Number of leading axes used for partitioning */
|
|
146
|
+
partitionKeyLength: number;
|
|
147
|
+
|
|
148
|
+
/** Map of stringified partition keys to parquet files */
|
|
149
|
+
parts: Record<string, ParquetChunk<Blob>>;
|
|
150
|
+
};
|
|
151
|
+
|
|
88
152
|
/**
|
|
89
153
|
* Union type representing all possible data storage formats for PColumn data.
|
|
90
154
|
* The specific format used depends on data size, access patterns, and performance requirements.
|