@milaboratories/pl-middle-layer 1.41.10 → 1.41.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +960 -970
- package/dist/index.mjs.map +1 -1
- package/dist/pool/data.d.ts +6 -4
- package/dist/pool/data.d.ts.map +1 -1
- package/dist/pool/driver.d.ts.map +1 -1
- package/package.json +13 -13
- package/src/pool/data.ts +32 -37
- package/src/pool/driver.ts +19 -23
package/dist/pool/data.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { JsonDataInfo, ParquetChunk, PColumnSpec, PColumnValue, PColumnValues, PObjectId, PObjectSpec } from '@platforma-sdk/model';
|
|
2
2
|
import { PlTreeNodeAccessor, ResourceInfo } from '@milaboratories/pl-tree';
|
|
3
|
+
import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
|
|
3
4
|
import { ResourceType } from '@milaboratories/pl-client';
|
|
4
|
-
export declare function allBlobs<B>(data: DataInfo<B>): Generator<B>;
|
|
5
|
-
export declare function mapBlobs<B1, B2>(data: DataInfo<B1>, mapping: (blob: B1) => B2): DataInfo<B2>;
|
|
5
|
+
export declare function allBlobs<B>(data: PFrameInternal.DataInfo<B>): Generator<B>;
|
|
6
|
+
export declare function mapBlobs<B1, B2>(data: PFrameInternal.DataInfo<B1>, mapping: (blob: B1) => B2): PFrameInternal.DataInfo<B2>;
|
|
6
7
|
export declare const PColumnDataJsonPartitioned: ResourceType;
|
|
7
8
|
export declare const PColumnDataJsonSuperPartitioned: ResourceType;
|
|
8
9
|
export declare const PColumnDataBinaryPartitioned: ResourceType;
|
|
@@ -21,7 +22,8 @@ export type PColumnDataSuperPartitionedResourceValue = {
|
|
|
21
22
|
superPartitionKeyLength: number;
|
|
22
23
|
partitionKeyLength: number;
|
|
23
24
|
};
|
|
24
|
-
export declare function parseDataInfoResource(data: PlTreeNodeAccessor): DataInfo<ResourceInfo>;
|
|
25
|
+
export declare function parseDataInfoResource(data: PlTreeNodeAccessor): PFrameInternal.DataInfo<ResourceInfo>;
|
|
26
|
+
export declare function traverseParquetPartitionedResource(resource: PlTreeNodeAccessor): ParquetChunk<ResourceInfo>;
|
|
25
27
|
export declare function makeDataInfoFromPColumnValues(spec: PColumnSpec, data: PColumnValues): JsonDataInfo;
|
|
26
28
|
export declare function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId;
|
|
27
29
|
export declare function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId;
|
package/dist/pool/data.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../src/pool/data.ts"],"names":[],"mappings":";AAAA,OAAO,EAGL,KAAK,
|
|
1
|
+
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../src/pool/data.ts"],"names":[],"mappings":";AAAA,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,YAAY,EAGjB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,aAAa,EAElB,KAAK,SAAS,EACd,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAWhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AAE5E,wBAAiB,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAmB3E;AAYD,wBAAgB,QAAQ,CAAC,EAAE,EAAE,EAAE,EAC7B,IAAI,EAAE,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,EACjC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GACxB,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CA4B7B;AAED,eAAO,MAAM,0BAA0B,kDAAmD,CAAC;AAC3F,eAAO,MAAM,+BAA+B,kDAG3C,CAAC;AACF,eAAO,MAAM,4BAA4B,kDAAqD,CAAC;AAC/F,eAAO,MAAM,iCAAiC,kDAG7C,CAAC;AACF,eAAO,MAAM,6BAA6B,kDAAsD,CAAC;AACjG,eAAO,MAAM,kCAAkC,kDAG9C,CAAC;AACF,eAAO,MAAM,eAAe,kDAAwC,CAAC;AAErE,MAAM,MAAM,4BAA4B,GAAG;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mCAAmC,GAAG;IAChD,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,wCAAwC,GAAG;IACrD,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,kBAAkB,GACvB,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,CA0LvC;AAED,wBAAgB,kCAAkC,CAAC,QAAQ,EAAE,kBAAkB,GAAG,YAAY,CAAC,YAAY,CAAC,CAc3G;AAED,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,WAAW,EACjB,IAAI,EAAE,aAAa,GAClB,YAAY,CAcd;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,kBAAkB,GAAG,SAAS,CAK5F;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAEpF;AAED,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAEzF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,kBAAkB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,4BAA4B,CAAC;AACzF,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EAEnB,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,IAAI,eAAe,EAC/B,OAAO,EACP,SAAS,EAET,SAAS,EAIT,aAAa,EACb,QAAQ,EAET,MAAM,sBAAsB,CAAC;AAiB9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAQ3D,KAAK,oBAAoB,GAAG,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,kBAAkB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,4BAA4B,CAAC;AACzF,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EAEnB,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,IAAI,eAAe,EAC/B,OAAO,EACP,SAAS,EAET,SAAS,EAIT,aAAa,EACb,QAAQ,EAET,MAAM,sBAAsB,CAAC;AAiB9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAQ3D,KAAK,oBAAoB,GAAG,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,GAAG,aAAa,CAAC;AA8P9F,MAAM,MAAM,eAAe,GAAG;IAE5B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,mBAAmB,EAAE,MAAM,CAAC;IAI5B,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D;;;;;;OAMG;IACH,SAAS,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IAEjC,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,oBAAoB,CAAC,EACpC,GAAG,EAAE,aAAa,GACjB,YAAY,CAAC;IAEhB,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,EAC7C,GAAG,EAAE,aAAa,GACjB,YAAY,CAAC;IAEhB,mFAAmF;IACnF,kBAAkB,CAChB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAEvC,yFAAyF;IACzF,eAAe,CACb,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC,0BAA0B;IAC1B,QAAQ,CACN,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,WAAW,CAAC,CAAC;IAExB;;;;;;;SAOK;IACL,OAAO,CACL,MAAM,EAAE,YAAY,EACpB,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CAC5B;AAED,qBAAa,YAAa,YAAW,oBAAoB;IAuBrD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAxB5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAA8B;IACtE,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAA8B;IAEzD,SAAS,IAAI,OAAO,CAAC,UAAU,CAAC;WAIzB,IAAI,CACtB,UAAU,EAAE,cAAc,EAC1B,MAAM,EAAE,QAAQ,EAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,eAAe,GACnB,OAAO,CAAC,YAAY,CAAC;IAMxB,OAAO;IA6HA,YAAY,CACjB,GAAG,EAAE,SAAS,CAAC,oBAAoB,CAAC,EACpC,GAAG,EAAE,aAAa,GACjB,YAAY;IAmBR,YAAY,CACjB,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,EAChD,GAAG,EAAE,aAAa,GACjB,YAAY;IAoBF,WAAW,CACtB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,mBAAmB,CAAC;IA6BlB,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAM9E,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAM9D,kBAAkB,CAC7B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC;IAwCzB,eAAe,CAC1B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC;IAyBnB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAM1D,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAY1E,OAAO,CAClB,MAAM,EAAE,YAAY,EACpB,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,EAAE,CAAC;CAY3B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.41.
|
|
3
|
+
"version": "1.41.11",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20.16.0"
|
|
@@ -22,32 +22,32 @@
|
|
|
22
22
|
"keywords": [],
|
|
23
23
|
"license": "UNLICENSED",
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@milaboratories/pframes-rs-node": "1.0.
|
|
25
|
+
"@milaboratories/pframes-rs-node": "1.0.57",
|
|
26
26
|
"canonicalize": "~2.1.0",
|
|
27
27
|
"denque": "^2.1.0",
|
|
28
28
|
"lru-cache": "^11.1.0",
|
|
29
29
|
"quickjs-emscripten": "0.31.0",
|
|
30
|
-
"undici": "~7.
|
|
30
|
+
"undici": "~7.13.0",
|
|
31
31
|
"utility-types": "^3.11.0",
|
|
32
|
-
"yaml": "^2.
|
|
32
|
+
"yaml": "^2.8.0",
|
|
33
33
|
"zod": "~3.23.8",
|
|
34
|
-
"remeda": "^2.
|
|
35
|
-
"@milaboratories/pl-http": "^1.1.4",
|
|
34
|
+
"remeda": "^2.28.0",
|
|
36
35
|
"@milaboratories/computable": "^2.6.3",
|
|
36
|
+
"@milaboratories/pl-http": "^1.1.5",
|
|
37
37
|
"@milaboratories/resolve-helper": "^1.1.0",
|
|
38
|
-
"@milaboratories/pl-client": "^2.11.
|
|
39
|
-
"@
|
|
40
|
-
"@milaboratories/pl-model-common": "~1.19.4",
|
|
41
|
-
"@milaboratories/pl-drivers": "^1.8.3",
|
|
38
|
+
"@milaboratories/pl-client": "^2.11.7",
|
|
39
|
+
"@milaboratories/pl-drivers": "^1.9.0",
|
|
42
40
|
"@milaboratories/pl-model-middle-layer": "~1.8.7",
|
|
41
|
+
"@milaboratories/pl-model-common": "~1.19.5",
|
|
42
|
+
"@platforma-sdk/block-tools": "^2.5.81",
|
|
43
43
|
"@milaboratories/pl-tree": "~1.7.5",
|
|
44
44
|
"@platforma-sdk/model": "~1.42.15",
|
|
45
45
|
"@milaboratories/pl-model-backend": "~1.1.2",
|
|
46
|
+
"@platforma-sdk/workflow-tengo": "4.17.1",
|
|
46
47
|
"@milaboratories/ts-helpers": "^1.4.3",
|
|
47
|
-
"@milaboratories/pl-errors": "^1.1.
|
|
48
|
+
"@milaboratories/pl-errors": "^1.1.14",
|
|
48
49
|
"@milaboratories/pl-config": "^1.6.3",
|
|
49
|
-
"@milaboratories/pl-deployments": "^2.4.7"
|
|
50
|
-
"@platforma-sdk/workflow-tengo": "4.17.1"
|
|
50
|
+
"@milaboratories/pl-deployments": "^2.4.7"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"semver": "^7.7.2",
|
package/src/pool/data.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
PFrameDriverError,
|
|
3
3
|
type BinaryChunk,
|
|
4
|
-
type DataInfo,
|
|
5
4
|
type JsonDataInfo,
|
|
6
5
|
type ParquetChunk,
|
|
7
6
|
type ParquetChunkMapping,
|
|
@@ -24,22 +23,23 @@ import {
|
|
|
24
23
|
} from '@milaboratories/pl-client';
|
|
25
24
|
import type { Writable } from 'utility-types';
|
|
26
25
|
import { createHash } from 'node:crypto';
|
|
26
|
+
import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
|
|
27
27
|
|
|
28
|
-
export function* allBlobs<B>(data: DataInfo<B>): Generator<B> {
|
|
28
|
+
export function* allBlobs<B>(data: PFrameInternal.DataInfo<B>): Generator<B> {
|
|
29
29
|
switch (data.type) {
|
|
30
30
|
case 'Json':
|
|
31
31
|
return;
|
|
32
32
|
case 'JsonPartitioned':
|
|
33
|
-
for (const
|
|
33
|
+
for (const blob of Object.values(data.parts)) yield blob;
|
|
34
34
|
return;
|
|
35
35
|
case 'BinaryPartitioned':
|
|
36
|
-
for (const
|
|
36
|
+
for (const { index, values } of Object.values(data.parts)) {
|
|
37
37
|
yield index;
|
|
38
38
|
yield values;
|
|
39
39
|
}
|
|
40
40
|
return;
|
|
41
41
|
case 'ParquetPartitioned':
|
|
42
|
-
for (const
|
|
42
|
+
for (const { data: blob } of Object.values(data.parts)) yield blob;
|
|
43
43
|
return;
|
|
44
44
|
default:
|
|
45
45
|
assertNever(data);
|
|
@@ -57,9 +57,9 @@ function mapValues<T extends object, TResult>(
|
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
export function mapBlobs<B1, B2>(
|
|
60
|
-
data: DataInfo<B1>,
|
|
60
|
+
data: PFrameInternal.DataInfo<B1>,
|
|
61
61
|
mapping: (blob: B1) => B2,
|
|
62
|
-
): DataInfo<B2> {
|
|
62
|
+
): PFrameInternal.DataInfo<B2> {
|
|
63
63
|
switch (data.type) {
|
|
64
64
|
case 'Json':
|
|
65
65
|
return { ...data };
|
|
@@ -79,7 +79,10 @@ export function mapBlobs<B1, B2>(
|
|
|
79
79
|
case 'ParquetPartitioned':
|
|
80
80
|
return {
|
|
81
81
|
...data,
|
|
82
|
-
parts: mapValues(data.parts,
|
|
82
|
+
parts: mapValues(data.parts, (v) => ({
|
|
83
|
+
...v,
|
|
84
|
+
data: mapping(v.data),
|
|
85
|
+
})),
|
|
83
86
|
};
|
|
84
87
|
default:
|
|
85
88
|
assertNever(data);
|
|
@@ -119,7 +122,7 @@ export type PColumnDataSuperPartitionedResourceValue = {
|
|
|
119
122
|
|
|
120
123
|
export function parseDataInfoResource(
|
|
121
124
|
data: PlTreeNodeAccessor,
|
|
122
|
-
): DataInfo<ResourceInfo> {
|
|
125
|
+
): PFrameInternal.DataInfo<ResourceInfo> {
|
|
123
126
|
if (!data.getIsReadyOrError()) throw new PFrameDriverError('Data not ready.');
|
|
124
127
|
|
|
125
128
|
const resourceData = data.getDataAsJson();
|
|
@@ -269,25 +272,13 @@ export function parseDataInfoResource(
|
|
|
269
272
|
for (const key of data.listInputFields()) {
|
|
270
273
|
const resource = data.traverse({ field: key, assertFieldType: 'Input', errorIfFieldNotSet: true });
|
|
271
274
|
|
|
272
|
-
|
|
273
|
-
{ field: 'blob', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
274
|
-
).resourceInfo;
|
|
275
|
-
const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;
|
|
276
|
-
const mapping = resource.traverse(
|
|
277
|
-
{ field: 'mapping', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
278
|
-
).getDataAsJson() as ParquetChunkMapping;
|
|
279
|
-
|
|
280
|
-
parts[key] = {
|
|
281
|
-
data: blob,
|
|
282
|
-
...partInfo,
|
|
283
|
-
...mapping,
|
|
284
|
-
};
|
|
275
|
+
parts[key] = traverseParquetPartitionedResource(resource);
|
|
285
276
|
}
|
|
286
277
|
|
|
287
278
|
return {
|
|
288
279
|
type: 'ParquetPartitioned',
|
|
289
280
|
partitionKeyLength: meta.partitionKeyLength,
|
|
290
|
-
parts
|
|
281
|
+
parts,
|
|
291
282
|
};
|
|
292
283
|
} else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {
|
|
293
284
|
const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
|
|
@@ -301,36 +292,40 @@ export function parseDataInfoResource(
|
|
|
301
292
|
for (const key of keys) {
|
|
302
293
|
const resource = data.traverse({ field: key, errorIfFieldNotSet: true });
|
|
303
294
|
|
|
304
|
-
const blob = resource.traverse(
|
|
305
|
-
{ field: 'blob', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
306
|
-
).resourceInfo;
|
|
307
|
-
const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;
|
|
308
|
-
const mapping = resource.traverse(
|
|
309
|
-
{ field: 'mapping', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
310
|
-
).getDataAsJson() as ParquetChunkMapping;
|
|
311
|
-
|
|
312
295
|
const partKey = JSON.stringify([
|
|
313
296
|
...JSON.parse(superKey) as PColumnValue[],
|
|
314
297
|
...JSON.parse(key) as PColumnValue[],
|
|
315
298
|
]);
|
|
316
|
-
parts[partKey] =
|
|
317
|
-
data: blob,
|
|
318
|
-
...partInfo,
|
|
319
|
-
...mapping,
|
|
320
|
-
};
|
|
299
|
+
parts[partKey] = traverseParquetPartitionedResource(resource);
|
|
321
300
|
}
|
|
322
301
|
}
|
|
323
302
|
|
|
324
303
|
return {
|
|
325
304
|
type: 'ParquetPartitioned',
|
|
326
305
|
partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,
|
|
327
|
-
parts
|
|
306
|
+
parts,
|
|
328
307
|
};
|
|
329
308
|
}
|
|
330
309
|
|
|
331
310
|
throw new PFrameDriverError(`unsupported resource type: ${resourceTypeToString(data.resourceType)}`);
|
|
332
311
|
}
|
|
333
312
|
|
|
313
|
+
export function traverseParquetPartitionedResource(resource: PlTreeNodeAccessor): ParquetChunk<ResourceInfo> {
|
|
314
|
+
const blob = resource.traverse(
|
|
315
|
+
{ field: 'blob', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
316
|
+
).resourceInfo;
|
|
317
|
+
const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;
|
|
318
|
+
const mapping = resource.traverse(
|
|
319
|
+
{ field: 'mapping', assertFieldType: 'Service', errorIfFieldNotSet: true },
|
|
320
|
+
).getDataAsJson() as ParquetChunkMapping;
|
|
321
|
+
|
|
322
|
+
return {
|
|
323
|
+
data: blob,
|
|
324
|
+
...partInfo,
|
|
325
|
+
...mapping,
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
334
329
|
export function makeDataInfoFromPColumnValues(
|
|
335
330
|
spec: PColumnSpec,
|
|
336
331
|
data: PColumnValues,
|
package/src/pool/driver.ts
CHANGED
|
@@ -46,7 +46,7 @@ import {
|
|
|
46
46
|
import { LRUCache } from 'lru-cache';
|
|
47
47
|
import type { PollResource } from './ref_count_pool';
|
|
48
48
|
import { RefCountResourcePool } from './ref_count_pool';
|
|
49
|
-
import { allBlobs, makeDataInfoFromPColumnValues, mapBlobs, parseDataInfoResource } from './data';
|
|
49
|
+
import { allBlobs, makeDataInfoFromPColumnValues, mapBlobs, parseDataInfoResource, traverseParquetPartitionedResource } from './data';
|
|
50
50
|
import { createHash } from 'node:crypto';
|
|
51
51
|
import type { MiLogger } from '@milaboratories/ts-helpers';
|
|
52
52
|
import { assertNever, emptyDir, ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
|
|
@@ -62,7 +62,7 @@ function blobKey(res: ResourceInfo): string {
|
|
|
62
62
|
return String(res.id);
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
type InternalPFrameData = PFrameDef<
|
|
65
|
+
type InternalPFrameData = PFrameDef<PFrameInternal.DataInfo<ResourceInfo>>;
|
|
66
66
|
|
|
67
67
|
const valueTypes: ValueType[] = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'] as const;
|
|
68
68
|
|
|
@@ -209,11 +209,6 @@ class PFrameHolder implements PFrameInternal.PFrameDataSource, AsyncDisposable {
|
|
|
209
209
|
const promises: Promise<void>[] = [];
|
|
210
210
|
for (const column of distinctСolumns) {
|
|
211
211
|
pFrame.addColumnSpec(column.id, column.spec);
|
|
212
|
-
if (column.data.type === 'ParquetPartitioned') { // TODO: remove
|
|
213
|
-
throw new PFrameDriverError(
|
|
214
|
-
`ParquetPartitioned data is not supported yet, column: ${JSON.stringify(column)}, data: ${JSON.stringify(column.data)}`,
|
|
215
|
-
);
|
|
216
|
-
}
|
|
217
212
|
promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));
|
|
218
213
|
}
|
|
219
214
|
this.pFramePromise = Promise.all(promises)
|
|
@@ -542,14 +537,16 @@ export class PFrameDriver implements InternalPFrameDriver {
|
|
|
542
537
|
def: PFrameDef<PColumnDataUniversal>,
|
|
543
538
|
ctx: ComputableCtx,
|
|
544
539
|
): PFrameHandle {
|
|
545
|
-
const internalData = def
|
|
540
|
+
const internalData: InternalPFrameData = def
|
|
546
541
|
.filter((c) => valueTypes.find((t) => t === c.spec.valueType))
|
|
547
542
|
.map((c) =>
|
|
548
543
|
mapPObjectData(c, (d) =>
|
|
549
544
|
isPlTreeNodeAccessor(d)
|
|
550
545
|
? parseDataInfoResource(d)
|
|
551
546
|
: isDataInfo(d)
|
|
552
|
-
?
|
|
547
|
+
? d.type === 'ParquetPartitioned'
|
|
548
|
+
? mapDataInfo(d, (a) => traverseParquetPartitionedResource(a))
|
|
549
|
+
: mapDataInfo(d, (a) => a.resourceInfo)
|
|
553
550
|
: makeDataInfoFromPColumnValues(c.spec, d),
|
|
554
551
|
),
|
|
555
552
|
);
|
|
@@ -787,7 +784,7 @@ function stableKeyFromFullPTableDef(data: FullPTableDef): string {
|
|
|
787
784
|
return hash.digest().toString('hex');
|
|
788
785
|
}
|
|
789
786
|
|
|
790
|
-
function stableKeyFromPFrameData(data: PColumn<DataInfo<ResourceInfo>>[]): string {
|
|
787
|
+
function stableKeyFromPFrameData(data: PColumn<PFrameInternal.DataInfo<ResourceInfo>>[]): string {
|
|
791
788
|
const orderedData = [...data].map((column) =>
|
|
792
789
|
mapPObjectData(column, (r) => {
|
|
793
790
|
let result: {
|
|
@@ -831,19 +828,18 @@ function stableKeyFromPFrameData(data: PColumn<DataInfo<ResourceInfo>>[]): strin
|
|
|
831
828
|
};
|
|
832
829
|
break;
|
|
833
830
|
case 'ParquetPartitioned':
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
// break;
|
|
831
|
+
result = {
|
|
832
|
+
type: r.type,
|
|
833
|
+
keyLength: r.partitionKeyLength,
|
|
834
|
+
payload: Object.entries(r.parts).map(([part, info]) => ({
|
|
835
|
+
key: part,
|
|
836
|
+
value: info.dataDigest || [
|
|
837
|
+
blobKey(info.data),
|
|
838
|
+
JSON.stringify({ axes: info.axes, column: info.column }),
|
|
839
|
+
] as const,
|
|
840
|
+
})),
|
|
841
|
+
};
|
|
842
|
+
break;
|
|
847
843
|
default:
|
|
848
844
|
throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type satisfies never)}`);
|
|
849
845
|
}
|