@milaboratories/pl-middle-layer 1.41.8 → 1.41.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1110 -1029
- package/dist/index.mjs.map +1 -1
- package/dist/pool/data.d.ts +2 -0
- package/dist/pool/data.d.ts.map +1 -1
- package/dist/pool/driver.d.ts.map +1 -1
- package/package.json +12 -12
- package/src/pool/data.ts +107 -13
- package/src/pool/driver.ts +21 -2
package/dist/pool/data.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ export declare const PColumnDataJsonPartitioned: ResourceType;
|
|
|
7
7
|
export declare const PColumnDataJsonSuperPartitioned: ResourceType;
|
|
8
8
|
export declare const PColumnDataBinaryPartitioned: ResourceType;
|
|
9
9
|
export declare const PColumnDataBinarySuperPartitioned: ResourceType;
|
|
10
|
+
export declare const PColumnDataParquetPartitioned: ResourceType;
|
|
11
|
+
export declare const PColumnDataParquetSuperPartitioned: ResourceType;
|
|
10
12
|
export declare const PColumnDataJson: ResourceType;
|
|
11
13
|
export type PColumnDataJsonResourceValue = {
|
|
12
14
|
keyLength: number;
|
package/dist/pool/data.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../src/pool/data.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../src/pool/data.ts"],"names":[],"mappings":";AAAA,OAAO,EAGL,KAAK,QAAQ,EACb,KAAK,YAAY,EAIjB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,aAAa,EAElB,KAAK,SAAS,EACd,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAYhF,wBAAiB,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAmB5D;AAYD,wBAAgB,QAAQ,CAAC,EAAE,EAAE,EAAE,EAC7B,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC,EAClB,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GACxB,QAAQ,CAAC,EAAE,CAAC,CAyBd;AAED,eAAO,MAAM,0BAA0B,kDAAmD,CAAC;AAC3F,eAAO,MAAM,+BAA+B,kDAG3C,CAAC;AACF,eAAO,MAAM,4BAA4B,kDAAqD,CAAC;AAC/F,eAAO,MAAM,iCAAiC,kDAG7C,CAAC;AACF,eAAO,MAAM,6BAA6B,kDAAsD,CAAC;AACjG,eAAO,MAAM,kCAAkC,kDAG9C,CAAC;AACF,eAAO,MAAM,eAAe,kDAAwC,CAAC;AAErE,MAAM,MAAM,4BAA4B,GAAG;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mCAAmC,GAAG;IAChD,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,wCAAwC,GAAG;IACrD,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,YAAY,CAAC,CAkNxB;AAED,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,WAAW,EACjB,IAAI,EAAE,aAAa,GAClB,YAAY,CAcd;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,kBAAkB,GAAG,SAAS,CAK5F;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAEpF;AAED,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAEzF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,kBAAkB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,4BAA4B,CAAC;AACzF,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EAEnB,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,IAAI,eAAe,EAC/B,OAAO,EACP,SAAS,EAET,SAAS,EAIT,aAAa,EACb,QAAQ,EAET,MAAM,sBAAsB,CAAC;AAiB9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAQ3D,KAAK,oBAAoB,GAAG,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,kBAAkB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,4BAA4B,CAAC;AACzF,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EAEnB,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,IAAI,eAAe,EAC/B,OAAO,EACP,SAAS,EAET,SAAS,EAIT,aAAa,EACb,QAAQ,EAET,MAAM,sBAAsB,CAAC;AAiB9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAQ3D,KAAK,oBAAoB,GAAG,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC;AAmQ9F,MAAM,MAAM,eAAe,GAAG;IAE5B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,mBAAmB,EAAE,MAAM,CAAC;IAI5B,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D;;;;;;OAMG;IACH,SAAS,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IAEjC,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,oBAAoB,CAAC,EACpC,GAAG,EAAE,aAAa,GACjB,YAAY,CAAC;IAEhB,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,EAC7C,GAAG,EAAE,aAAa,GACjB,YAAY,CAAC;IAEhB,mFAAmF;IACnF,kBAAkB,CAChB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAEvC,yFAAyF;IACzF,eAAe,CACb,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC,0BAA0B;IAC1B,QAAQ,CACN,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,WAAW,CAAC,CAAC;IAExB;;;;;;;SAOK;IACL,OAAO,CACL,MAAM,EAAE,YAAY,EACpB,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CAC5B;AAED,qBAAa,YAAa,YAAW,oBAAoB;IAuBrD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAxB5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAA8B;IACtE,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAA8B;IAEzD,SAAS,IAAI,OAAO,CAAC,UAAU,CAAC;WAIzB,IAAI,CACtB,UAAU,EAAE,cAAc,EAC1B,MAAM,EAAE,QAAQ,EAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,eAAe,GACnB,OAAO,CAAC,YAAY,CAAC;IAMxB,OAAO;IA6HA,YAAY,CACjB,GAAG,EAAE,SAAS,CAAC,oBAAoB,CAAC,EACpC,GAAG,EAAE,aAAa,GACjB,YAAY;IAiBR,YAAY,CACjB,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,EAChD,GAAG,EAAE,aAAa,GACjB,YAAY;IAoBF,WAAW,CACtB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,mBAAmB,CAAC;IA6BlB,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAM9E,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAM9D,kBAAkB,CAC7B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC;IAwCzB,eAAe,CAC1B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC;IAyBnB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAM1D,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAY1E,OAAO,CAClB,MAAM,EAAE,YAAY,EACpB,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,EAAE,CAAC;CAY3B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.41.
|
|
3
|
+
"version": "1.41.10",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20.16.0"
|
|
@@ -35,19 +35,19 @@
|
|
|
35
35
|
"@milaboratories/pl-http": "^1.1.4",
|
|
36
36
|
"@milaboratories/computable": "^2.6.3",
|
|
37
37
|
"@milaboratories/resolve-helper": "^1.1.0",
|
|
38
|
-
"@milaboratories/pl-model-common": "~1.19.3",
|
|
39
|
-
"@platforma-sdk/block-tools": "^2.5.80",
|
|
40
|
-
"@milaboratories/pl-drivers": "^1.8.2",
|
|
41
|
-
"@milaboratories/pl-model-middle-layer": "~1.8.6",
|
|
42
38
|
"@milaboratories/pl-client": "^2.11.6",
|
|
39
|
+
"@platforma-sdk/block-tools": "^2.5.81",
|
|
40
|
+
"@milaboratories/pl-model-common": "~1.19.4",
|
|
41
|
+
"@milaboratories/pl-drivers": "^1.8.3",
|
|
42
|
+
"@milaboratories/pl-model-middle-layer": "~1.8.7",
|
|
43
43
|
"@milaboratories/pl-tree": "~1.7.5",
|
|
44
|
-
"@
|
|
45
|
-
"@platforma-sdk/model": "~1.42.10",
|
|
46
|
-
"@platforma-sdk/workflow-tengo": "4.17.0",
|
|
44
|
+
"@platforma-sdk/model": "~1.42.15",
|
|
47
45
|
"@milaboratories/pl-model-backend": "~1.1.2",
|
|
48
|
-
"@milaboratories/
|
|
46
|
+
"@milaboratories/ts-helpers": "^1.4.3",
|
|
49
47
|
"@milaboratories/pl-errors": "^1.1.13",
|
|
50
|
-
"@milaboratories/pl-
|
|
48
|
+
"@milaboratories/pl-config": "^1.6.3",
|
|
49
|
+
"@milaboratories/pl-deployments": "^2.4.7",
|
|
50
|
+
"@platforma-sdk/workflow-tengo": "4.17.1"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"semver": "^7.7.2",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
"jest": "^29.7.0",
|
|
61
61
|
"@jest/globals": "^29.7.0",
|
|
62
62
|
"ts-jest": "^29.2.6",
|
|
63
|
-
"@milaboratories/
|
|
64
|
-
"@milaboratories/
|
|
63
|
+
"@milaboratories/build-configs": "1.0.5",
|
|
64
|
+
"@milaboratories/eslint-config": "^1.0.4"
|
|
65
65
|
},
|
|
66
66
|
"scripts": {
|
|
67
67
|
"type-check": "tsc --noEmit --composite false",
|
package/src/pool/data.ts
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
|
-
import
|
|
1
|
+
import {
|
|
2
|
+
PFrameDriverError,
|
|
3
|
+
type BinaryChunk,
|
|
4
|
+
type DataInfo,
|
|
5
|
+
type JsonDataInfo,
|
|
6
|
+
type ParquetChunk,
|
|
7
|
+
type ParquetChunkMapping,
|
|
8
|
+
type ParquetChunkMetadata,
|
|
9
|
+
type PColumnSpec,
|
|
10
|
+
type PColumnValue,
|
|
11
|
+
type PColumnValues,
|
|
12
|
+
type PlRef,
|
|
13
|
+
type PObjectId,
|
|
14
|
+
type PObjectSpec,
|
|
15
|
+
} from '@platforma-sdk/model';
|
|
2
16
|
import type { PlTreeNodeAccessor, ResourceInfo } from '@milaboratories/pl-tree';
|
|
3
17
|
import { assertNever } from '@milaboratories/ts-helpers';
|
|
4
18
|
import canonicalize from 'canonicalize';
|
|
@@ -24,6 +38,9 @@ export function* allBlobs<B>(data: DataInfo<B>): Generator<B> {
|
|
|
24
38
|
yield values;
|
|
25
39
|
}
|
|
26
40
|
return;
|
|
41
|
+
case 'ParquetPartitioned':
|
|
42
|
+
for (const [, blob] of Object.entries(data.parts)) yield blob;
|
|
43
|
+
return;
|
|
27
44
|
default:
|
|
28
45
|
assertNever(data);
|
|
29
46
|
}
|
|
@@ -47,7 +64,10 @@ export function mapBlobs<B1, B2>(
|
|
|
47
64
|
case 'Json':
|
|
48
65
|
return { ...data };
|
|
49
66
|
case 'JsonPartitioned':
|
|
50
|
-
return {
|
|
67
|
+
return {
|
|
68
|
+
...data,
|
|
69
|
+
parts: mapValues(data.parts, mapping),
|
|
70
|
+
};
|
|
51
71
|
case 'BinaryPartitioned':
|
|
52
72
|
return {
|
|
53
73
|
...data,
|
|
@@ -56,6 +76,11 @@ export function mapBlobs<B1, B2>(
|
|
|
56
76
|
values: mapping(v.values),
|
|
57
77
|
})),
|
|
58
78
|
};
|
|
79
|
+
case 'ParquetPartitioned':
|
|
80
|
+
return {
|
|
81
|
+
...data,
|
|
82
|
+
parts: mapValues(data.parts, mapping),
|
|
83
|
+
};
|
|
59
84
|
default:
|
|
60
85
|
assertNever(data);
|
|
61
86
|
}
|
|
@@ -71,6 +96,11 @@ export const PColumnDataBinarySuperPartitioned = resourceType(
|
|
|
71
96
|
'PColumnData/Partitioned/BinaryPartitioned',
|
|
72
97
|
'1',
|
|
73
98
|
);
|
|
99
|
+
export const PColumnDataParquetPartitioned = resourceType('PColumnData/ParquetPartitioned', '1');
|
|
100
|
+
export const PColumnDataParquetSuperPartitioned = resourceType(
|
|
101
|
+
'PColumnData/Partitioned/ParquetPartitioned',
|
|
102
|
+
'1',
|
|
103
|
+
);
|
|
74
104
|
export const PColumnDataJson = resourceType('PColumnData/Json', '1');
|
|
75
105
|
|
|
76
106
|
export type PColumnDataJsonResourceValue = {
|
|
@@ -89,12 +119,12 @@ export type PColumnDataSuperPartitionedResourceValue = {
|
|
|
89
119
|
|
|
90
120
|
export function parseDataInfoResource(
|
|
91
121
|
data: PlTreeNodeAccessor,
|
|
92
|
-
): DataInfo<ResourceInfo> {
|
|
93
|
-
if (!data.getIsReadyOrError()) throw new
|
|
122
|
+
): DataInfo<ResourceInfo> { // TODO: PFrameInternal.DataInfo<ResourceInfo>
|
|
123
|
+
if (!data.getIsReadyOrError()) throw new PFrameDriverError('Data not ready.');
|
|
94
124
|
|
|
95
125
|
const resourceData = data.getDataAsJson();
|
|
96
126
|
if (resourceData === undefined)
|
|
97
|
-
throw new
|
|
127
|
+
throw new PFrameDriverError('unexpected data info structure, no resource data');
|
|
98
128
|
|
|
99
129
|
if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {
|
|
100
130
|
const dataContent = resourceData as PColumnDataJsonResourceValue;
|
|
@@ -125,7 +155,7 @@ export function parseDataInfoResource(
|
|
|
125
155
|
for (const superKey of data.listInputFields()) {
|
|
126
156
|
const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });
|
|
127
157
|
const keys = superPart.listInputFields();
|
|
128
|
-
if (keys === undefined) throw new
|
|
158
|
+
if (keys === undefined) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);
|
|
129
159
|
|
|
130
160
|
for (const key of keys) {
|
|
131
161
|
const partKey = JSON.stringify([
|
|
@@ -166,13 +196,13 @@ export function parseDataInfoResource(
|
|
|
166
196
|
parts[partKey] = part;
|
|
167
197
|
}
|
|
168
198
|
part.values = data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo;
|
|
169
|
-
} else throw new
|
|
199
|
+
} else throw new PFrameDriverError(`unrecognized part field name: ${field}`);
|
|
170
200
|
}
|
|
171
201
|
|
|
172
202
|
// structure validation
|
|
173
203
|
for (const [key, part] of Object.entries(parts)) {
|
|
174
|
-
if (part.index === undefined) throw new
|
|
175
|
-
if (part.values === undefined) throw new
|
|
204
|
+
if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);
|
|
205
|
+
if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);
|
|
176
206
|
}
|
|
177
207
|
|
|
178
208
|
return {
|
|
@@ -190,7 +220,7 @@ export function parseDataInfoResource(
|
|
|
190
220
|
for (const superKey of data.listInputFields()) {
|
|
191
221
|
const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });
|
|
192
222
|
const keys = superData.listInputFields();
|
|
193
|
-
if (keys === undefined) throw new
|
|
223
|
+
if (keys === undefined) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);
|
|
194
224
|
|
|
195
225
|
for (const field of keys) {
|
|
196
226
|
if (field.endsWith('.index')) {
|
|
@@ -223,7 +253,7 @@ export function parseDataInfoResource(
|
|
|
223
253
|
field,
|
|
224
254
|
errorIfFieldNotSet: true,
|
|
225
255
|
}).resourceInfo;
|
|
226
|
-
} else throw new
|
|
256
|
+
} else throw new PFrameDriverError(`unrecognized part field name: ${field}`);
|
|
227
257
|
}
|
|
228
258
|
}
|
|
229
259
|
|
|
@@ -232,9 +262,73 @@ export function parseDataInfoResource(
|
|
|
232
262
|
partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,
|
|
233
263
|
parts: parts as Record<string, BinaryChunk<ResourceInfo>>,
|
|
234
264
|
};
|
|
265
|
+
} else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {
|
|
266
|
+
const meta = resourceData as PColumnDataPartitionedResourceValue;
|
|
267
|
+
|
|
268
|
+
const parts: Record<string, ParquetChunk<ResourceInfo>> = {};
|
|
269
|
+
for (const key of data.listInputFields()) {
|
|
270
|
+
const resource = data.traverse({ field: key, assertFieldType: 'Input', errorIfFieldNotSet: true });
|
|
271
|
+
|
|
272
|
+
const blob = resource.traverse(
|
|
273
|
+
{ field: 'blob', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
274
|
+
).resourceInfo;
|
|
275
|
+
const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;
|
|
276
|
+
const mapping = resource.traverse(
|
|
277
|
+
{ field: 'mapping', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
278
|
+
).getDataAsJson() as ParquetChunkMapping;
|
|
279
|
+
|
|
280
|
+
parts[key] = {
|
|
281
|
+
data: blob,
|
|
282
|
+
...partInfo,
|
|
283
|
+
...mapping,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
type: 'ParquetPartitioned',
|
|
289
|
+
partitionKeyLength: meta.partitionKeyLength,
|
|
290
|
+
parts: parts as unknown as Record<string, ResourceInfo>, // TODO: remove cast
|
|
291
|
+
};
|
|
292
|
+
} else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {
|
|
293
|
+
const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
|
|
294
|
+
|
|
295
|
+
const parts: Record<string, ParquetChunk<ResourceInfo>> = {};
|
|
296
|
+
for (const superKey of data.listInputFields()) {
|
|
297
|
+
const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });
|
|
298
|
+
const keys = superPart.listInputFields();
|
|
299
|
+
if (keys === undefined) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);
|
|
300
|
+
|
|
301
|
+
for (const key of keys) {
|
|
302
|
+
const resource = data.traverse({ field: key, errorIfFieldNotSet: true });
|
|
303
|
+
|
|
304
|
+
const blob = resource.traverse(
|
|
305
|
+
{ field: 'blob', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
306
|
+
).resourceInfo;
|
|
307
|
+
const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;
|
|
308
|
+
const mapping = resource.traverse(
|
|
309
|
+
{ field: 'mapping', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
310
|
+
).getDataAsJson() as ParquetChunkMapping;
|
|
311
|
+
|
|
312
|
+
const partKey = JSON.stringify([
|
|
313
|
+
...JSON.parse(superKey) as PColumnValue[],
|
|
314
|
+
...JSON.parse(key) as PColumnValue[],
|
|
315
|
+
]);
|
|
316
|
+
parts[partKey] = {
|
|
317
|
+
data: blob,
|
|
318
|
+
...partInfo,
|
|
319
|
+
...mapping,
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
return {
|
|
325
|
+
type: 'ParquetPartitioned',
|
|
326
|
+
partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,
|
|
327
|
+
parts: parts as unknown as Record<string, ResourceInfo>, // TODO: remove cast
|
|
328
|
+
};
|
|
235
329
|
}
|
|
236
330
|
|
|
237
|
-
throw new
|
|
331
|
+
throw new PFrameDriverError(`unsupported resource type: ${resourceTypeToString(data.resourceType)}`);
|
|
238
332
|
}
|
|
239
333
|
|
|
240
334
|
export function makeDataInfoFromPColumnValues(
|
|
@@ -245,7 +339,7 @@ export function makeDataInfoFromPColumnValues(
|
|
|
245
339
|
const jsonData: Record<string, PColumnValue> = {};
|
|
246
340
|
for (const { key, val } of data) {
|
|
247
341
|
if (key.length !== keyLength)
|
|
248
|
-
throw new
|
|
342
|
+
throw new PFrameDriverError(`inline column key length ${key.length} differs from axes count ${keyLength}`);
|
|
249
343
|
jsonData[JSON.stringify(key)] = val;
|
|
250
344
|
}
|
|
251
345
|
|
package/src/pool/driver.ts
CHANGED
|
@@ -62,7 +62,7 @@ function blobKey(res: ResourceInfo): string {
|
|
|
62
62
|
return String(res.id);
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
type InternalPFrameData = PFrameDef<DataInfo<ResourceInfo>>;
|
|
65
|
+
type InternalPFrameData = PFrameDef<DataInfo<ResourceInfo>>; // TODO: PFrameInternal.DataInfo<ResourceInfo>
|
|
66
66
|
|
|
67
67
|
const valueTypes: ValueType[] = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'] as const;
|
|
68
68
|
|
|
@@ -209,6 +209,11 @@ class PFrameHolder implements PFrameInternal.PFrameDataSource, AsyncDisposable {
|
|
|
209
209
|
const promises: Promise<void>[] = [];
|
|
210
210
|
for (const column of distinctСolumns) {
|
|
211
211
|
pFrame.addColumnSpec(column.id, column.spec);
|
|
212
|
+
if (column.data.type === 'ParquetPartitioned') { // TODO: remove
|
|
213
|
+
throw new PFrameDriverError(
|
|
214
|
+
`ParquetPartitioned data is not supported yet, column: ${JSON.stringify(column)}, data: ${JSON.stringify(column.data)}`,
|
|
215
|
+
);
|
|
216
|
+
}
|
|
212
217
|
promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));
|
|
213
218
|
}
|
|
214
219
|
this.pFramePromise = Promise.all(promises)
|
|
@@ -782,7 +787,7 @@ function stableKeyFromFullPTableDef(data: FullPTableDef): string {
|
|
|
782
787
|
return hash.digest().toString('hex');
|
|
783
788
|
}
|
|
784
789
|
|
|
785
|
-
function stableKeyFromPFrameData(data: PColumn<DataInfo<ResourceInfo>>[]): string {
|
|
790
|
+
function stableKeyFromPFrameData(data: PColumn<DataInfo<ResourceInfo>>[]): string { // TODO: PFrameInternal.DataInfo<ResourceInfo>
|
|
786
791
|
const orderedData = [...data].map((column) =>
|
|
787
792
|
mapPObjectData(column, (r) => {
|
|
788
793
|
let result: {
|
|
@@ -825,6 +830,20 @@ function stableKeyFromPFrameData(data: PColumn<DataInfo<ResourceInfo>>[]): strin
|
|
|
825
830
|
})),
|
|
826
831
|
};
|
|
827
832
|
break;
|
|
833
|
+
case 'ParquetPartitioned':
|
|
834
|
+
throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type)}`); // TODO: remove
|
|
835
|
+
// result = {
|
|
836
|
+
// type: r.type,
|
|
837
|
+
// keyLength: r.partitionKeyLength,
|
|
838
|
+
// payload: Object.entries(r.parts).map(([part, info]) => ({
|
|
839
|
+
// key: part,
|
|
840
|
+
// value: info.dataDigest || [
|
|
841
|
+
// blobKey(info.data),
|
|
842
|
+
// JSON.stringify({ axes: info.axes, column: info.column }),
|
|
843
|
+
// ] as const,
|
|
844
|
+
// })),
|
|
845
|
+
// };
|
|
846
|
+
// break;
|
|
828
847
|
default:
|
|
829
848
|
throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type satisfies never)}`);
|
|
830
849
|
}
|