@milaboratories/pf-driver 1.0.39 → 1.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/data_info_helpers.cjs +1 -1
- package/dist/data_info_helpers.cjs.map +1 -1
- package/dist/data_info_helpers.d.ts +1 -1
- package/dist/data_info_helpers.d.ts.map +1 -1
- package/dist/data_info_helpers.js +1 -1
- package/dist/data_info_helpers.js.map +1 -1
- package/dist/driver_decl.d.ts +2 -2
- package/dist/driver_decl.d.ts.map +1 -1
- package/dist/driver_double.cjs +1 -1
- package/dist/driver_double.cjs.map +1 -1
- package/dist/driver_double.d.ts +6 -6
- package/dist/driver_double.d.ts.map +1 -1
- package/dist/driver_double.js +1 -1
- package/dist/driver_double.js.map +1 -1
- package/dist/driver_impl.cjs +35 -32
- package/dist/driver_impl.cjs.map +1 -1
- package/dist/driver_impl.d.ts +7 -7
- package/dist/driver_impl.d.ts.map +1 -1
- package/dist/driver_impl.js +35 -32
- package/dist/driver_impl.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/pframe_pool.cjs +21 -19
- package/dist/pframe_pool.cjs.map +1 -1
- package/dist/pframe_pool.d.ts +4 -4
- package/dist/pframe_pool.d.ts.map +1 -1
- package/dist/pframe_pool.js +21 -19
- package/dist/pframe_pool.js.map +1 -1
- package/dist/ptable_cache_per_frame.cjs +6 -6
- package/dist/ptable_cache_per_frame.cjs.map +1 -1
- package/dist/ptable_cache_per_frame.d.ts +4 -4
- package/dist/ptable_cache_per_frame.d.ts.map +1 -1
- package/dist/ptable_cache_per_frame.js +6 -6
- package/dist/ptable_cache_per_frame.js.map +1 -1
- package/dist/ptable_cache_plain.cjs +5 -5
- package/dist/ptable_cache_plain.cjs.map +1 -1
- package/dist/ptable_cache_plain.d.ts +4 -4
- package/dist/ptable_cache_plain.d.ts.map +1 -1
- package/dist/ptable_cache_plain.js +5 -5
- package/dist/ptable_cache_plain.js.map +1 -1
- package/dist/ptable_def_pool.cjs +2 -2
- package/dist/ptable_def_pool.cjs.map +1 -1
- package/dist/ptable_def_pool.d.ts +4 -4
- package/dist/ptable_def_pool.js +2 -2
- package/dist/ptable_def_pool.js.map +1 -1
- package/dist/ptable_pool.cjs +30 -26
- package/dist/ptable_pool.cjs.map +1 -1
- package/dist/ptable_pool.d.ts +6 -6
- package/dist/ptable_pool.d.ts.map +1 -1
- package/dist/ptable_pool.js +30 -26
- package/dist/ptable_pool.js.map +1 -1
- package/dist/ptable_shared.cjs.map +1 -1
- package/dist/ptable_shared.d.ts +1 -1
- package/dist/ptable_shared.js.map +1 -1
- package/package.json +29 -29
- package/src/data_info_helpers.ts +6 -7
- package/src/driver_decl.ts +8 -16
- package/src/driver_double.test.ts +89 -83
- package/src/driver_double.ts +23 -19
- package/src/driver_impl.ts +115 -80
- package/src/index.ts +4 -4
- package/src/pframe_pool.ts +42 -37
- package/src/ptable_cache_per_frame.ts +16 -13
- package/src/ptable_cache_plain.ts +16 -12
- package/src/ptable_def_pool.ts +7 -7
- package/src/ptable_pool.ts +62 -45
- package/src/ptable_shared.ts +2 -2
package/src/driver_impl.ts
CHANGED
|
@@ -29,50 +29,50 @@ import {
|
|
|
29
29
|
type PTableRecordSingleValueFilterV2,
|
|
30
30
|
type PTableRecordFilter,
|
|
31
31
|
type JsonSerializable,
|
|
32
|
-
} from
|
|
33
|
-
import type { PFrameInternal } from
|
|
32
|
+
} from "@platforma-sdk/model";
|
|
33
|
+
import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
34
34
|
import {
|
|
35
35
|
assertNever,
|
|
36
36
|
ConcurrencyLimitingExecutor,
|
|
37
37
|
type PoolEntry,
|
|
38
|
-
} from
|
|
39
|
-
import { PFrameFactory } from
|
|
40
|
-
import { tmpdir } from
|
|
41
|
-
import type {
|
|
42
|
-
|
|
43
|
-
} from './driver_decl';
|
|
44
|
-
import { logPFrames } from './logging';
|
|
38
|
+
} from "@milaboratories/ts-helpers";
|
|
39
|
+
import { PFrameFactory } from "@milaboratories/pframes-rs-node";
|
|
40
|
+
import { tmpdir } from "node:os";
|
|
41
|
+
import type { AbstractInternalPFrameDriver } from "./driver_decl";
|
|
42
|
+
import { logPFrames } from "./logging";
|
|
45
43
|
import {
|
|
46
44
|
PFramePool,
|
|
47
45
|
type LocalBlobProvider as PoolLocalBlobProvider,
|
|
48
46
|
type RemoteBlobProvider as PoolRemoteBlobProvider,
|
|
49
|
-
} from
|
|
50
|
-
import { PTableDefPool } from
|
|
51
|
-
import { PTablePool } from
|
|
47
|
+
} from "./pframe_pool";
|
|
48
|
+
import { PTableDefPool } from "./ptable_def_pool";
|
|
49
|
+
import { PTablePool } from "./ptable_pool";
|
|
52
50
|
import {
|
|
53
51
|
PTableCachePerFrame,
|
|
54
52
|
PTableCachePerFrameOpsDefaults,
|
|
55
53
|
type PTableCachePerFrameOps,
|
|
56
|
-
} from
|
|
54
|
+
} from "./ptable_cache_per_frame";
|
|
57
55
|
import {
|
|
58
56
|
PTableCachePlain,
|
|
59
57
|
PTableCachePlainOpsDefaults,
|
|
60
58
|
type PTableCachePlainOps,
|
|
61
|
-
} from
|
|
59
|
+
} from "./ptable_cache_plain";
|
|
62
60
|
|
|
63
61
|
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
|
|
64
|
-
export interface LocalBlobProvider<
|
|
65
|
-
|
|
62
|
+
export interface LocalBlobProvider<
|
|
63
|
+
TreeEntry extends JsonSerializable,
|
|
64
|
+
> extends PoolLocalBlobProvider<TreeEntry> {}
|
|
66
65
|
|
|
67
66
|
export interface RemoteBlobProvider<TreeEntry extends JsonSerializable>
|
|
68
67
|
extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}
|
|
69
68
|
|
|
70
|
-
export type AbstractPFrameDriverOps = PTableCachePerFrameOps &
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
69
|
+
export type AbstractPFrameDriverOps = PTableCachePerFrameOps &
|
|
70
|
+
PTableCachePlainOps & {
|
|
71
|
+
/** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */
|
|
72
|
+
pFrameConcurrency: number;
|
|
73
|
+
/** Concurrency limits for `getShape` and `getData` requests */
|
|
74
|
+
pTableConcurrency: number;
|
|
75
|
+
};
|
|
76
76
|
|
|
77
77
|
export const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {
|
|
78
78
|
...PTableCachePerFrameOpsDefaults,
|
|
@@ -86,8 +86,10 @@ export type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> =
|
|
|
86
86
|
data: PColumnData,
|
|
87
87
|
) => PFrameInternal.DataInfo<TreeEntry>;
|
|
88
88
|
|
|
89
|
-
export class AbstractPFrameDriver<
|
|
90
|
-
|
|
89
|
+
export class AbstractPFrameDriver<
|
|
90
|
+
PColumnData,
|
|
91
|
+
TreeEntry extends JsonSerializable,
|
|
92
|
+
> implements AbstractInternalPFrameDriver<PColumnData> {
|
|
91
93
|
private readonly logger: PFrameInternal.Logger;
|
|
92
94
|
|
|
93
95
|
private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;
|
|
@@ -134,7 +136,12 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
134
136
|
this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);
|
|
135
137
|
this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);
|
|
136
138
|
|
|
137
|
-
this.pFrames = new PFramePool(
|
|
139
|
+
this.pFrames = new PFramePool(
|
|
140
|
+
this.localBlobProvider,
|
|
141
|
+
this.remoteBlobProvider,
|
|
142
|
+
this.logger,
|
|
143
|
+
spillPath,
|
|
144
|
+
);
|
|
138
145
|
this.pTableDefs = new PTableDefPool(this.logger);
|
|
139
146
|
this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);
|
|
140
147
|
|
|
@@ -154,27 +161,33 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
154
161
|
// Internal / Config API Methods
|
|
155
162
|
//
|
|
156
163
|
|
|
157
|
-
public createPFrame(
|
|
158
|
-
def: PFrameDef<PColumn<PColumnData>>,
|
|
159
|
-
): PoolEntry<PFrameHandle> {
|
|
164
|
+
public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {
|
|
160
165
|
const ValueTypes = new Set(Object.values(ValueType));
|
|
161
166
|
|
|
162
167
|
const supportedColumns = def.filter((column) => ValueTypes.has(column.spec.valueType));
|
|
163
168
|
const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);
|
|
164
|
-
const columns = uniqueColumns.map((c) =>
|
|
169
|
+
const columns = uniqueColumns.map((c) =>
|
|
170
|
+
mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),
|
|
171
|
+
);
|
|
165
172
|
|
|
166
173
|
return this.pFrames.acquire(columns);
|
|
167
174
|
}
|
|
168
175
|
|
|
169
|
-
public createPTable(
|
|
170
|
-
rawDef: PTableDef<PColumn<PColumnData>>,
|
|
171
|
-
): PoolEntry<PTableHandle> {
|
|
176
|
+
public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {
|
|
172
177
|
const pFrameEntry = this.createPFrame(extractAllColumns(rawDef.src));
|
|
173
|
-
const sortedDef = sortPTableDef(
|
|
178
|
+
const sortedDef = sortPTableDef(
|
|
179
|
+
migratePTableFilters(
|
|
180
|
+
mapPTableDef(rawDef, (c) => c.id),
|
|
181
|
+
this.logger,
|
|
182
|
+
),
|
|
183
|
+
);
|
|
174
184
|
|
|
175
185
|
const pTableEntry = this.pTableDefs.acquire({ def: sortedDef, pFrameHandle: pFrameEntry.key });
|
|
176
186
|
if (logPFrames()) {
|
|
177
|
-
this.logger(
|
|
187
|
+
this.logger(
|
|
188
|
+
"info",
|
|
189
|
+
`Create PTable call (pFrameHandle = ${pFrameEntry.key}; pTableHandle = ${pTableEntry.key})`,
|
|
190
|
+
);
|
|
178
191
|
}
|
|
179
192
|
|
|
180
193
|
const unref = () => {
|
|
@@ -201,14 +214,16 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
201
214
|
...request,
|
|
202
215
|
compatibleWith:
|
|
203
216
|
request.compatibleWith.length !== 0
|
|
204
|
-
? [
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
217
|
+
? [
|
|
218
|
+
{
|
|
219
|
+
axesSpec: [
|
|
220
|
+
...new Map(
|
|
221
|
+
request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),
|
|
222
|
+
).values(),
|
|
223
|
+
],
|
|
224
|
+
qualifications: [],
|
|
225
|
+
},
|
|
226
|
+
]
|
|
212
227
|
: [],
|
|
213
228
|
};
|
|
214
229
|
|
|
@@ -218,16 +233,24 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
218
233
|
const response = await pFrame.findColumns(iRequest);
|
|
219
234
|
return {
|
|
220
235
|
hits: response.hits
|
|
221
|
-
.filter(
|
|
222
|
-
h
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
236
|
+
.filter(
|
|
237
|
+
(h) =>
|
|
238
|
+
// only exactly matching columns
|
|
239
|
+
h.mappingVariants.length === 0 ||
|
|
240
|
+
h.mappingVariants.some(
|
|
241
|
+
(v) =>
|
|
242
|
+
v.qualifications.forHit.length === 0 &&
|
|
243
|
+
v.qualifications.forQueries.every((q) => q.length === 0),
|
|
244
|
+
),
|
|
245
|
+
)
|
|
226
246
|
.map((h) => h.hit),
|
|
227
247
|
};
|
|
228
248
|
}
|
|
229
249
|
|
|
230
|
-
public async getColumnSpec(
|
|
250
|
+
public async getColumnSpec(
|
|
251
|
+
handle: PFrameHandle,
|
|
252
|
+
columnId: PObjectId,
|
|
253
|
+
): Promise<PColumnSpec | null> {
|
|
231
254
|
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
232
255
|
const pFrame = await pFramePromise;
|
|
233
256
|
return await pFrame.getColumnSpec(columnId);
|
|
@@ -246,7 +269,8 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
246
269
|
signal?: AbortSignal,
|
|
247
270
|
): Promise<CalculateTableDataResponse> {
|
|
248
271
|
if (logPFrames()) {
|
|
249
|
-
this.logger(
|
|
272
|
+
this.logger(
|
|
273
|
+
"info",
|
|
250
274
|
`Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,
|
|
251
275
|
);
|
|
252
276
|
}
|
|
@@ -292,7 +316,8 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
292
316
|
signal?: AbortSignal,
|
|
293
317
|
): Promise<UniqueValuesResponse> {
|
|
294
318
|
if (logPFrames()) {
|
|
295
|
-
this.logger(
|
|
319
|
+
this.logger(
|
|
320
|
+
"info",
|
|
296
321
|
`Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,
|
|
297
322
|
);
|
|
298
323
|
}
|
|
@@ -302,12 +327,15 @@ implements AbstractInternalPFrameDriver<PColumnData> {
|
|
|
302
327
|
|
|
303
328
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
304
329
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
305
|
-
return await pFrame.getUniqueValues(
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
330
|
+
return await pFrame.getUniqueValues(
|
|
331
|
+
{
|
|
332
|
+
...request,
|
|
333
|
+
filters: migrateFilters(request.filters, this.logger),
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
signal: combinedSignal,
|
|
337
|
+
},
|
|
338
|
+
);
|
|
311
339
|
});
|
|
312
340
|
}
|
|
313
341
|
|
|
@@ -387,16 +415,16 @@ function sortPTableDef(def: PTableDef<PObjectId>): PTableDef<PObjectId> {
|
|
|
387
415
|
}
|
|
388
416
|
const type = lhs.type;
|
|
389
417
|
switch (type) {
|
|
390
|
-
case
|
|
418
|
+
case "column":
|
|
391
419
|
return lhs.column < (rhs as typeof lhs).column ? -1 : 1;
|
|
392
|
-
case
|
|
393
|
-
case
|
|
420
|
+
case "slicedColumn":
|
|
421
|
+
case "artificialColumn":
|
|
394
422
|
return lhs.newId < (rhs as typeof lhs).newId ? -1 : 1;
|
|
395
|
-
case
|
|
423
|
+
case "inlineColumn": {
|
|
396
424
|
return lhs.column.id < (rhs as typeof lhs).column.id ? -1 : 1;
|
|
397
425
|
}
|
|
398
|
-
case
|
|
399
|
-
case
|
|
426
|
+
case "inner":
|
|
427
|
+
case "full": {
|
|
400
428
|
const rhsInner = rhs as typeof lhs;
|
|
401
429
|
if (lhs.entries.length !== rhsInner.entries.length) {
|
|
402
430
|
return lhs.entries.length - rhsInner.entries.length;
|
|
@@ -409,7 +437,7 @@ function sortPTableDef(def: PTableDef<PObjectId>): PTableDef<PObjectId> {
|
|
|
409
437
|
}
|
|
410
438
|
return 0;
|
|
411
439
|
}
|
|
412
|
-
case
|
|
440
|
+
case "outer": {
|
|
413
441
|
const rhsOuter = rhs as typeof lhs;
|
|
414
442
|
const cmp = cmpJoinEntries(lhs.primary, rhsOuter.primary);
|
|
415
443
|
if (cmp !== 0) {
|
|
@@ -432,19 +460,19 @@ function sortPTableDef(def: PTableDef<PObjectId>): PTableDef<PObjectId> {
|
|
|
432
460
|
}
|
|
433
461
|
function sortJoinEntry(entry: JoinEntry<PObjectId>): JoinEntry<PObjectId> {
|
|
434
462
|
switch (entry.type) {
|
|
435
|
-
case
|
|
436
|
-
case
|
|
437
|
-
case
|
|
463
|
+
case "column":
|
|
464
|
+
case "slicedColumn":
|
|
465
|
+
case "inlineColumn":
|
|
438
466
|
return entry;
|
|
439
|
-
case
|
|
467
|
+
case "artificialColumn": {
|
|
440
468
|
const sortedAxesIndices = entry.axesIndices.toSorted((lhs, rhs) => lhs - rhs);
|
|
441
469
|
return {
|
|
442
470
|
...entry,
|
|
443
471
|
axesIndices: sortedAxesIndices,
|
|
444
472
|
};
|
|
445
473
|
}
|
|
446
|
-
case
|
|
447
|
-
case
|
|
474
|
+
case "inner":
|
|
475
|
+
case "full": {
|
|
448
476
|
const sortedEntries = entry.entries.map(sortJoinEntry);
|
|
449
477
|
sortedEntries.sort(cmpJoinEntries);
|
|
450
478
|
return {
|
|
@@ -452,7 +480,7 @@ function sortPTableDef(def: PTableDef<PObjectId>): PTableDef<PObjectId> {
|
|
|
452
480
|
entries: sortedEntries,
|
|
453
481
|
};
|
|
454
482
|
}
|
|
455
|
-
case
|
|
483
|
+
case "outer": {
|
|
456
484
|
const sortedSecondary = entry.secondary.map(sortJoinEntry);
|
|
457
485
|
sortedSecondary.sort(cmpJoinEntries);
|
|
458
486
|
return {
|
|
@@ -467,14 +495,14 @@ function sortPTableDef(def: PTableDef<PObjectId>): PTableDef<PObjectId> {
|
|
|
467
495
|
}
|
|
468
496
|
function sortFilters(filters: PTableRecordFilter[]): PTableRecordFilter[] {
|
|
469
497
|
return filters.toSorted((lhs, rhs) => {
|
|
470
|
-
if (lhs.column.type ===
|
|
498
|
+
if (lhs.column.type === "axis" && rhs.column.type === "axis") {
|
|
471
499
|
const lhsId = canonicalizeJson(getAxisId(lhs.column.id));
|
|
472
500
|
const rhsId = canonicalizeJson(getAxisId(rhs.column.id));
|
|
473
501
|
return lhsId < rhsId ? -1 : 1;
|
|
474
|
-
} else if (lhs.column.type ===
|
|
502
|
+
} else if (lhs.column.type === "column" && rhs.column.type === "column") {
|
|
475
503
|
return lhs.column.id < rhs.column.id ? -1 : 1;
|
|
476
504
|
} else {
|
|
477
|
-
return lhs.column.type ===
|
|
505
|
+
return lhs.column.type === "axis" ? -1 : 1;
|
|
478
506
|
}
|
|
479
507
|
});
|
|
480
508
|
}
|
|
@@ -493,11 +521,11 @@ function migrateFilters(
|
|
|
493
521
|
const filtersV1 = [];
|
|
494
522
|
const filtersV2: PTableRecordSingleValueFilterV2[] = [];
|
|
495
523
|
for (const filter of filters) {
|
|
496
|
-
if ((filter.type as unknown) ===
|
|
524
|
+
if ((filter.type as unknown) === "bySingleColumn") {
|
|
497
525
|
filtersV1.push(filter);
|
|
498
526
|
filtersV2.push({
|
|
499
527
|
...filter,
|
|
500
|
-
type:
|
|
528
|
+
type: "bySingleColumnV2",
|
|
501
529
|
});
|
|
502
530
|
} else {
|
|
503
531
|
filtersV2.push(filter);
|
|
@@ -505,7 +533,8 @@ function migrateFilters(
|
|
|
505
533
|
}
|
|
506
534
|
if (filtersV1.length > 0) {
|
|
507
535
|
const filtersV1Json = JSON.stringify(filtersV1);
|
|
508
|
-
logger(
|
|
536
|
+
logger(
|
|
537
|
+
"warn",
|
|
509
538
|
`type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,
|
|
510
539
|
);
|
|
511
540
|
}
|
|
@@ -513,15 +542,21 @@ function migrateFilters(
|
|
|
513
542
|
}
|
|
514
543
|
|
|
515
544
|
function migratePTableFilters<T>(
|
|
516
|
-
def: Omit<PTableDef<T>,
|
|
545
|
+
def: Omit<PTableDef<T>, "partitionFilters"> | PTableDef<T>,
|
|
517
546
|
logger: PFrameInternal.Logger,
|
|
518
547
|
): PTableDef<T> {
|
|
519
|
-
if (!(
|
|
548
|
+
if (!("partitionFilters" in def)) {
|
|
520
549
|
// For old blocks assume all axes filters to be partition filters
|
|
521
550
|
return {
|
|
522
551
|
...def,
|
|
523
|
-
partitionFilters: migrateFilters(
|
|
524
|
-
|
|
552
|
+
partitionFilters: migrateFilters(
|
|
553
|
+
def.filters.filter((f) => f.column.type === "axis"),
|
|
554
|
+
logger,
|
|
555
|
+
),
|
|
556
|
+
filters: migrateFilters(
|
|
557
|
+
def.filters.filter((f) => f.column.type === "column"),
|
|
558
|
+
logger,
|
|
559
|
+
),
|
|
525
560
|
};
|
|
526
561
|
}
|
|
527
562
|
return {
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from
|
|
2
|
-
export * from
|
|
3
|
-
export * from
|
|
4
|
-
export * from
|
|
1
|
+
export * from "./driver_decl";
|
|
2
|
+
export * from "./driver_impl";
|
|
3
|
+
export * from "./data_info_helpers";
|
|
4
|
+
export * from "./driver_double";
|
package/src/pframe_pool.ts
CHANGED
|
@@ -8,19 +8,16 @@ import {
|
|
|
8
8
|
type JsonSerializable,
|
|
9
9
|
type PColumn,
|
|
10
10
|
type PFrameHandle,
|
|
11
|
-
} from
|
|
12
|
-
import { hashJson, PFrameInternal } from
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
} from
|
|
17
|
-
import { PFrameFactory } from '@milaboratories/pframes-rs-node';
|
|
18
|
-
import { mapValues } from 'es-toolkit';
|
|
19
|
-
import { logPFrames } from './logging';
|
|
11
|
+
} from "@platforma-sdk/model";
|
|
12
|
+
import { hashJson, PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
13
|
+
import { RefCountPoolBase, type PoolEntry } from "@milaboratories/ts-helpers";
|
|
14
|
+
import { PFrameFactory } from "@milaboratories/pframes-rs-node";
|
|
15
|
+
import { mapValues } from "es-toolkit";
|
|
16
|
+
import { logPFrames } from "./logging";
|
|
20
17
|
|
|
21
18
|
export interface LocalBlobProvider<TreeEntry extends JsonSerializable> {
|
|
22
19
|
acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;
|
|
23
|
-
makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2,
|
|
20
|
+
makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, "parquetServer">;
|
|
24
21
|
}
|
|
25
22
|
|
|
26
23
|
export interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {
|
|
@@ -59,14 +56,14 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
59
56
|
data: PFrameInternal.DataInfo<TreeEntry>,
|
|
60
57
|
): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {
|
|
61
58
|
switch (data.type) {
|
|
62
|
-
case
|
|
59
|
+
case "Json":
|
|
63
60
|
return { ...data };
|
|
64
|
-
case
|
|
61
|
+
case "JsonPartitioned":
|
|
65
62
|
return {
|
|
66
63
|
...data,
|
|
67
64
|
parts: mapValues(data.parts, makeLocalBlobId),
|
|
68
65
|
};
|
|
69
|
-
case
|
|
66
|
+
case "BinaryPartitioned":
|
|
70
67
|
return {
|
|
71
68
|
...data,
|
|
72
69
|
parts: mapValues(data.parts, (v) => ({
|
|
@@ -74,7 +71,7 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
74
71
|
values: makeLocalBlobId(v.values),
|
|
75
72
|
})),
|
|
76
73
|
};
|
|
77
|
-
case
|
|
74
|
+
case "ParquetPartitioned":
|
|
78
75
|
return {
|
|
79
76
|
...data,
|
|
80
77
|
parts: mapValues(data.parts, (v) => ({
|
|
@@ -110,7 +107,7 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
110
107
|
.catch((err) => {
|
|
111
108
|
this.dispose();
|
|
112
109
|
pFrame.dispose();
|
|
113
|
-
const error = new PFrameDriverError(
|
|
110
|
+
const error = new PFrameDriverError("PFrame creation failed asynchronously");
|
|
114
111
|
error.cause = new Error(
|
|
115
112
|
`PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,
|
|
116
113
|
{ cause: ensureError(err) },
|
|
@@ -118,7 +115,7 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
118
115
|
throw error;
|
|
119
116
|
});
|
|
120
117
|
} catch (err: unknown) {
|
|
121
|
-
const error = new PFrameDriverError(
|
|
118
|
+
const error = new PFrameDriverError("PFrame creation failed synchronously");
|
|
122
119
|
error.cause = new Error(
|
|
123
120
|
`PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,
|
|
124
121
|
{ cause: ensureError(err) },
|
|
@@ -141,15 +138,17 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
141
138
|
this.dispose();
|
|
142
139
|
void this.pFramePromise
|
|
143
140
|
.then((pFrame) => pFrame.dispose())
|
|
144
|
-
.catch(() => {
|
|
141
|
+
.catch(() => {
|
|
142
|
+
/* mute error */
|
|
143
|
+
});
|
|
145
144
|
}
|
|
146
145
|
}
|
|
147
146
|
|
|
148
|
-
export class PFramePool<TreeEntry extends JsonSerializable>
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
147
|
+
export class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<
|
|
148
|
+
PColumn<PFrameInternal.DataInfo<TreeEntry>>[],
|
|
149
|
+
PFrameHandle,
|
|
150
|
+
PFrameHolder<TreeEntry>
|
|
151
|
+
> {
|
|
153
152
|
constructor(
|
|
154
153
|
private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,
|
|
155
154
|
private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,
|
|
@@ -159,7 +158,9 @@ export class PFramePool<TreeEntry extends JsonSerializable>
|
|
|
159
158
|
super();
|
|
160
159
|
}
|
|
161
160
|
|
|
162
|
-
protected calculateParamsKey(
|
|
161
|
+
protected calculateParamsKey(
|
|
162
|
+
params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],
|
|
163
|
+
): PFrameHandle {
|
|
163
164
|
return stableKeyFromPFrameData(params);
|
|
164
165
|
}
|
|
165
166
|
|
|
@@ -168,9 +169,9 @@ export class PFramePool<TreeEntry extends JsonSerializable>
|
|
|
168
169
|
key: PFrameHandle,
|
|
169
170
|
): PFrameHolder<TreeEntry> {
|
|
170
171
|
if (logPFrames()) {
|
|
171
|
-
this.logger(
|
|
172
|
-
|
|
173
|
-
+ `${JSON.stringify(params, bigintReplacer)}`,
|
|
172
|
+
this.logger(
|
|
173
|
+
"info",
|
|
174
|
+
`PFrame creation (pFrameHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,
|
|
174
175
|
);
|
|
175
176
|
}
|
|
176
177
|
return new PFrameHolder(
|
|
@@ -209,7 +210,7 @@ function stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(
|
|
|
209
210
|
};
|
|
210
211
|
const type = r.type;
|
|
211
212
|
switch (type) {
|
|
212
|
-
case
|
|
213
|
+
case "Json":
|
|
213
214
|
result = {
|
|
214
215
|
type: r.type,
|
|
215
216
|
keyLength: r.keyLength,
|
|
@@ -219,7 +220,7 @@ function stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(
|
|
|
219
220
|
})),
|
|
220
221
|
};
|
|
221
222
|
break;
|
|
222
|
-
case
|
|
223
|
+
case "JsonPartitioned":
|
|
223
224
|
result = {
|
|
224
225
|
type: r.type,
|
|
225
226
|
keyLength: r.partitionKeyLength,
|
|
@@ -229,7 +230,7 @@ function stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(
|
|
|
229
230
|
})),
|
|
230
231
|
};
|
|
231
232
|
break;
|
|
232
|
-
case
|
|
233
|
+
case "BinaryPartitioned":
|
|
233
234
|
result = {
|
|
234
235
|
type: r.type,
|
|
235
236
|
keyLength: r.partitionKeyLength,
|
|
@@ -239,26 +240,30 @@ function stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(
|
|
|
239
240
|
})),
|
|
240
241
|
};
|
|
241
242
|
break;
|
|
242
|
-
case
|
|
243
|
+
case "ParquetPartitioned":
|
|
243
244
|
result = {
|
|
244
245
|
type: r.type,
|
|
245
246
|
keyLength: r.partitionKeyLength,
|
|
246
247
|
payload: Object.entries(r.parts).map(([part, info]) => ({
|
|
247
248
|
key: part,
|
|
248
|
-
value:
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
249
|
+
value:
|
|
250
|
+
info.dataDigest ||
|
|
251
|
+
([
|
|
252
|
+
canonicalizeJson(info.data),
|
|
253
|
+
JSON.stringify({ axes: info.axes, column: info.column }),
|
|
254
|
+
] as const),
|
|
252
255
|
})),
|
|
253
256
|
};
|
|
254
257
|
break;
|
|
255
258
|
default:
|
|
256
|
-
throw new PFrameDriverError(
|
|
259
|
+
throw new PFrameDriverError(
|
|
260
|
+
`unsupported resource type: ${JSON.stringify(type satisfies never)}`,
|
|
261
|
+
);
|
|
257
262
|
}
|
|
258
|
-
result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);
|
|
263
|
+
result.payload.sort((lhs, rhs) => (lhs.key < rhs.key ? -1 : 1));
|
|
259
264
|
return result;
|
|
260
265
|
}),
|
|
261
266
|
);
|
|
262
|
-
orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);
|
|
267
|
+
orderedData.sort((lhs, rhs) => (lhs.id < rhs.id ? -1 : 1));
|
|
263
268
|
return hashJson(orderedData) as string as PFrameHandle;
|
|
264
269
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type { PFrameHandle, PTableHandle } from
|
|
2
|
-
import type { PFrameInternal } from
|
|
3
|
-
import type { PoolEntry } from
|
|
4
|
-
import { LRUCache } from
|
|
5
|
-
import { logPFrames } from
|
|
6
|
-
import type { PTableHolder } from
|
|
1
|
+
import type { PFrameHandle, PTableHandle } from "@platforma-sdk/model";
|
|
2
|
+
import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
3
|
+
import type { PoolEntry } from "@milaboratories/ts-helpers";
|
|
4
|
+
import { LRUCache } from "lru-cache";
|
|
5
|
+
import { logPFrames } from "./logging";
|
|
6
|
+
import type { PTableHolder } from "./ptable_pool";
|
|
7
7
|
|
|
8
8
|
export type PTableCachePerFrameOps = {
|
|
9
9
|
/** Maximum number of `calculateTableData` results cached for each PFrame */
|
|
@@ -22,7 +22,10 @@ export const PTableCachePerFrameOpsDefaults: PTableCachePerFrameOps = {
|
|
|
22
22
|
};
|
|
23
23
|
|
|
24
24
|
export class PTableCachePerFrame {
|
|
25
|
-
private readonly perFrame = new Map<
|
|
25
|
+
private readonly perFrame = new Map<
|
|
26
|
+
PFrameHandle,
|
|
27
|
+
LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>
|
|
28
|
+
>();
|
|
26
29
|
private readonly global: LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>;
|
|
27
30
|
private readonly disposeListeners = new Set<PTableHandle>();
|
|
28
31
|
|
|
@@ -33,7 +36,7 @@ export class PTableCachePerFrame {
|
|
|
33
36
|
this.global = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({
|
|
34
37
|
maxSize: this.ops.pFramesCacheMaxSize,
|
|
35
38
|
dispose: (resource, key, reason) => {
|
|
36
|
-
if (reason ===
|
|
39
|
+
if (reason === "evict") {
|
|
37
40
|
this.perFrame.get(resource.resource.pFrame)?.delete(key);
|
|
38
41
|
}
|
|
39
42
|
|
|
@@ -43,7 +46,7 @@ export class PTableCachePerFrame {
|
|
|
43
46
|
|
|
44
47
|
resource.unref();
|
|
45
48
|
if (logPFrames()) {
|
|
46
|
-
logger(
|
|
49
|
+
logger("info", `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
|
|
47
50
|
}
|
|
48
51
|
},
|
|
49
52
|
});
|
|
@@ -52,7 +55,7 @@ export class PTableCachePerFrame {
|
|
|
52
55
|
public cache(resource: PoolEntry<PTableHandle, PTableHolder>, size: number): void {
|
|
53
56
|
const key = resource.key;
|
|
54
57
|
if (logPFrames()) {
|
|
55
|
-
this.logger(
|
|
58
|
+
this.logger("info", `calculateTableData cache - added PTable ${key} with size ${size}`);
|
|
56
59
|
}
|
|
57
60
|
|
|
58
61
|
this.global.set(key, resource, { size: Math.max(size, 1) }); // 1 is minimum size to avoid cache evictions
|
|
@@ -62,7 +65,7 @@ export class PTableCachePerFrame {
|
|
|
62
65
|
perFrame = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({
|
|
63
66
|
max: this.ops.pFrameCacheMaxCount,
|
|
64
67
|
dispose: (_resource, key, reason) => {
|
|
65
|
-
if (reason ===
|
|
68
|
+
if (reason === "evict") {
|
|
66
69
|
this.global.delete(key);
|
|
67
70
|
}
|
|
68
71
|
},
|
|
@@ -77,10 +80,10 @@ export class PTableCachePerFrame {
|
|
|
77
80
|
this.global.delete(key);
|
|
78
81
|
|
|
79
82
|
this.disposeListeners.delete(key);
|
|
80
|
-
resource.resource.disposeSignal.removeEventListener(
|
|
83
|
+
resource.resource.disposeSignal.removeEventListener("abort", disposeListener);
|
|
81
84
|
};
|
|
82
85
|
this.disposeListeners.add(key);
|
|
83
|
-
resource.resource.disposeSignal.addEventListener(
|
|
86
|
+
resource.resource.disposeSignal.addEventListener("abort", disposeListener);
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
89
|
}
|