@milaboratories/pl-middle-layer 1.43.12 → 1.43.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/middle_layer/ops.cjs +2 -1
- package/dist/middle_layer/ops.cjs.map +1 -1
- package/dist/middle_layer/ops.d.ts.map +1 -1
- package/dist/middle_layer/ops.js +2 -1
- package/dist/middle_layer/ops.js.map +1 -1
- package/dist/pool/driver.cjs +286 -150
- package/dist/pool/driver.cjs.map +1 -1
- package/dist/pool/driver.d.ts +9 -1
- package/dist/pool/driver.d.ts.map +1 -1
- package/dist/pool/driver.js +287 -151
- package/dist/pool/driver.js.map +1 -1
- package/package.json +17 -17
- package/src/middle_layer/ops.ts +2 -1
- package/src/pool/driver.ts +340 -185
package/dist/pool/driver.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { __addDisposableResource, __disposeResources } from '../__external/.pnpm/@rollup_plugin-typescript@12.1.4_rollup@4.45.1_tslib@2.7.0_typescript@5.6.3/__external/tslib/tslib.es6.js';
|
|
1
2
|
import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
|
|
2
3
|
import { isPlTreeNodeAccessor } from '@milaboratories/pl-tree';
|
|
3
|
-
import { ensureError, isAbortError,
|
|
4
|
+
import { ensureError, isAbortError, mapPObjectData, mapDataInfo, isDataInfo, uniqueBy, extractAllColumns, mapPTableDef, PFrameDriverError, isPFrameDriverError } from '@platforma-sdk/model';
|
|
4
5
|
import { LRUCache } from 'lru-cache';
|
|
5
6
|
import { parseDataInfoResource, makeDataInfoFromPColumnValues, traverseParquetChunkResource } from './data.js';
|
|
6
7
|
import { createHash } from 'node:crypto';
|
|
@@ -24,7 +25,7 @@ class LocalBlobPool extends RefCountResourcePool {
|
|
|
24
25
|
calculateParamsKey(params) {
|
|
25
26
|
return makeBlobId(params);
|
|
26
27
|
}
|
|
27
|
-
createNewResource(params) {
|
|
28
|
+
createNewResource(params, _key) {
|
|
28
29
|
return this.blobDriver.getDownloadedBlob(params);
|
|
29
30
|
}
|
|
30
31
|
getByKey(blobId) {
|
|
@@ -59,7 +60,7 @@ class RemoteBlobPool extends RefCountResourcePool {
|
|
|
59
60
|
calculateParamsKey(params) {
|
|
60
61
|
return String(params.rid);
|
|
61
62
|
}
|
|
62
|
-
createNewResource(params) {
|
|
63
|
+
createNewResource(params, _key) {
|
|
63
64
|
return this.blobDriver.getOnDemandBlob(params);
|
|
64
65
|
}
|
|
65
66
|
async withContent(handle, options) {
|
|
@@ -116,7 +117,8 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
|
|
|
116
117
|
range: translatedRange,
|
|
117
118
|
});
|
|
118
119
|
}
|
|
119
|
-
this.logger('info', `PFrames blob store requesting content for ${blobId},
|
|
120
|
+
this.logger('info', `PFrames blob store requesting content for ${blobId}, `
|
|
121
|
+
+ `range [${translatedRange.start}..=${translatedRange.end}]`);
|
|
120
122
|
return await this.remoteBlobPool.withContent(blob.handle, {
|
|
121
123
|
range: translatedRange,
|
|
122
124
|
signal: params.signal,
|
|
@@ -140,7 +142,7 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
|
|
|
140
142
|
}
|
|
141
143
|
}
|
|
142
144
|
const valueTypes = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'];
|
|
143
|
-
function migrateFilters(filters) {
|
|
145
|
+
function migrateFilters(filters, logger) {
|
|
144
146
|
const filtersV1 = [];
|
|
145
147
|
const filtersV2 = [];
|
|
146
148
|
for (const filter of filters) {
|
|
@@ -157,23 +159,23 @@ function migrateFilters(filters) {
|
|
|
157
159
|
}
|
|
158
160
|
if (filtersV1.length > 0) {
|
|
159
161
|
const filtersV1Json = JSON.stringify(filtersV1);
|
|
160
|
-
|
|
162
|
+
logger('warn', `type overriten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
|
|
161
163
|
}
|
|
162
164
|
return filtersV2;
|
|
163
165
|
}
|
|
164
|
-
function migratePTableFilters(def) {
|
|
166
|
+
function migratePTableFilters(def, logger) {
|
|
165
167
|
if (!('partitionFilters' in def)) {
|
|
166
168
|
// For old blocks assume all axes filters to be partition filters
|
|
167
169
|
return {
|
|
168
170
|
...def,
|
|
169
|
-
partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis')),
|
|
170
|
-
filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column')),
|
|
171
|
+
partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis'), logger),
|
|
172
|
+
filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column'), logger),
|
|
171
173
|
};
|
|
172
174
|
}
|
|
173
175
|
return {
|
|
174
176
|
...def,
|
|
175
|
-
partitionFilters: migrateFilters(def.partitionFilters),
|
|
176
|
-
filters: migrateFilters(def.filters),
|
|
177
|
+
partitionFilters: migrateFilters(def.partitionFilters, logger),
|
|
178
|
+
filters: migrateFilters(def.filters, logger),
|
|
177
179
|
};
|
|
178
180
|
}
|
|
179
181
|
function hasArtificialColumns(entry) {
|
|
@@ -194,12 +196,134 @@ function hasArtificialColumns(entry) {
|
|
|
194
196
|
}
|
|
195
197
|
}
|
|
196
198
|
const bigintReplacer = (_, v) => (typeof v === 'bigint' ? v.toString() : v);
|
|
197
|
-
class
|
|
199
|
+
class PFramePool extends RefCountResourcePool {
|
|
200
|
+
parquetServer;
|
|
201
|
+
localBlobPool;
|
|
202
|
+
remoteBlobPool;
|
|
203
|
+
logger;
|
|
204
|
+
spillPath;
|
|
205
|
+
constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
|
|
206
|
+
super();
|
|
207
|
+
this.parquetServer = parquetServer;
|
|
208
|
+
this.localBlobPool = localBlobPool;
|
|
209
|
+
this.remoteBlobPool = remoteBlobPool;
|
|
210
|
+
this.logger = logger;
|
|
211
|
+
this.spillPath = spillPath;
|
|
212
|
+
}
|
|
213
|
+
calculateParamsKey(params) {
|
|
214
|
+
try {
|
|
215
|
+
return stableKeyFromPFrameData(params);
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
if (isPFrameDriverError(err))
|
|
219
|
+
throw err;
|
|
220
|
+
throw new PFrameDriverError(`PFrame handle calculation failed, `
|
|
221
|
+
+ `request: ${JSON.stringify(params, bigintReplacer)}, `
|
|
222
|
+
+ `error: ${ensureError(err)}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
createNewResource(params, key) {
|
|
226
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
227
|
+
this.logger('info', `PFrame creation (pFrameHandle = ${key}): `
|
|
228
|
+
+ `${JSON.stringify(params, bigintReplacer)}`);
|
|
229
|
+
}
|
|
230
|
+
return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
|
|
231
|
+
}
|
|
232
|
+
getByKey(key) {
|
|
233
|
+
const resource = super.tryGetByKey(key);
|
|
234
|
+
if (!resource)
|
|
235
|
+
throw new PFrameDriverError(`PFrame not found, handle = ${key}`);
|
|
236
|
+
return resource;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
class PTableDefPool extends RefCountResourcePool {
|
|
240
|
+
logger;
|
|
241
|
+
constructor(logger) {
|
|
242
|
+
super();
|
|
243
|
+
this.logger = logger;
|
|
244
|
+
}
|
|
245
|
+
calculateParamsKey(params) {
|
|
246
|
+
return stableKeyFromFullPTableDef(params);
|
|
247
|
+
}
|
|
248
|
+
createNewResource(params, key) {
|
|
249
|
+
return new PTableDefHolder(params, key, this.logger);
|
|
250
|
+
}
|
|
251
|
+
getByKey(key) {
|
|
252
|
+
const resource = super.tryGetByKey(key);
|
|
253
|
+
if (!resource)
|
|
254
|
+
throw new PFrameDriverError(`PTable definition not found, handle = ${key}`);
|
|
255
|
+
return resource;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
class PTablePool extends RefCountResourcePool {
|
|
259
|
+
pFrames;
|
|
260
|
+
pTableDefs;
|
|
261
|
+
logger;
|
|
262
|
+
constructor(pFrames, pTableDefs, logger) {
|
|
263
|
+
super();
|
|
264
|
+
this.pFrames = pFrames;
|
|
265
|
+
this.pTableDefs = pTableDefs;
|
|
266
|
+
this.logger = logger;
|
|
267
|
+
}
|
|
268
|
+
calculateParamsKey(params) {
|
|
269
|
+
return stableKeyFromFullPTableDef(params);
|
|
270
|
+
}
|
|
271
|
+
createNewResource(params, key) {
|
|
272
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
273
|
+
this.logger('info', `PTable creation (pTableHandle = ${key}): `
|
|
274
|
+
+ `${JSON.stringify(params, bigintReplacer)}`);
|
|
275
|
+
}
|
|
276
|
+
const handle = params.pFrameHandle;
|
|
277
|
+
const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
278
|
+
const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
|
|
279
|
+
const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
|
|
280
|
+
// 3. Sort
|
|
281
|
+
if (params.def.sorting.length > 0) {
|
|
282
|
+
const predecessor = this.acquire({
|
|
283
|
+
...params,
|
|
284
|
+
def: {
|
|
285
|
+
...params.def,
|
|
286
|
+
sorting: [],
|
|
287
|
+
},
|
|
288
|
+
});
|
|
289
|
+
const { resource: { pTablePromise } } = predecessor;
|
|
290
|
+
const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
|
|
291
|
+
return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
|
|
292
|
+
}
|
|
293
|
+
// 2. Filter (except the case with artificial columns where cartesian creates too many rows)
|
|
294
|
+
if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
|
|
295
|
+
const predecessor = this.acquire({
|
|
296
|
+
...params,
|
|
297
|
+
def: {
|
|
298
|
+
...params.def,
|
|
299
|
+
filters: [],
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
const { resource: { pTablePromise } } = predecessor;
|
|
303
|
+
const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
|
|
304
|
+
return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
|
|
305
|
+
}
|
|
306
|
+
// 1. Join
|
|
307
|
+
const table = pFramePromise.then((pFrame) => pFrame.createTable({
|
|
308
|
+
src: joinEntryToInternal(params.def.src),
|
|
309
|
+
// `params.def.filters` would be non-empty only when join has artificial columns
|
|
310
|
+
filters: [...params.def.partitionFilters, ...params.def.filters],
|
|
311
|
+
}));
|
|
312
|
+
return new PTableHolder(handle, combinedSignal, table);
|
|
313
|
+
}
|
|
314
|
+
getByKey(key) {
|
|
315
|
+
const resource = super.tryGetByKey(key);
|
|
316
|
+
if (!resource)
|
|
317
|
+
throw new PFrameDriverError(`PTable not found, handle = ${key}`);
|
|
318
|
+
return resource;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
class PTableCacheUi {
|
|
198
322
|
logger;
|
|
199
323
|
ops;
|
|
200
324
|
perFrame = new Map();
|
|
201
325
|
global;
|
|
202
|
-
disposeListeners = new
|
|
326
|
+
disposeListeners = new Set();
|
|
203
327
|
constructor(logger, ops) {
|
|
204
328
|
this.logger = logger;
|
|
205
329
|
this.ops = ops;
|
|
@@ -212,12 +336,9 @@ class PTableCache {
|
|
|
212
336
|
if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {
|
|
213
337
|
this.perFrame.delete(resource.resource.pFrame);
|
|
214
338
|
}
|
|
215
|
-
const disposeListener = this.disposeListeners.get(key);
|
|
216
|
-
this.disposeListeners.delete(key);
|
|
217
|
-
resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
|
|
218
339
|
resource.unref();
|
|
219
340
|
if (getDebugFlags().logPFrameRequests) {
|
|
220
|
-
|
|
341
|
+
logger('info', `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
|
|
221
342
|
}
|
|
222
343
|
},
|
|
223
344
|
});
|
|
@@ -241,12 +362,58 @@ class PTableCache {
|
|
|
241
362
|
this.perFrame.set(resource.resource.pFrame, perFrame);
|
|
242
363
|
}
|
|
243
364
|
perFrame.set(key, resource);
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
365
|
+
if (!this.disposeListeners.has(key)) {
|
|
366
|
+
const disposeListener = () => {
|
|
367
|
+
this.perFrame.get(resource.resource.pFrame)?.delete(key);
|
|
368
|
+
this.global.delete(key);
|
|
369
|
+
this.disposeListeners.delete(key);
|
|
370
|
+
resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
|
|
371
|
+
};
|
|
372
|
+
this.disposeListeners.add(key);
|
|
373
|
+
resource.resource.disposeSignal.addEventListener('abort', disposeListener);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
class PTableCacheModel {
|
|
378
|
+
logger;
|
|
379
|
+
global;
|
|
380
|
+
disposeListeners = new Set();
|
|
381
|
+
constructor(logger, ops) {
|
|
382
|
+
this.logger = logger;
|
|
383
|
+
this.global = new LRUCache({
|
|
384
|
+
maxSize: ops.pTablesCacheMaxSize,
|
|
385
|
+
dispose: (resource, key, reason) => {
|
|
386
|
+
resource.unref();
|
|
387
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
388
|
+
logger('info', `createPTable cache - removed PTable ${key} (reason: ${reason})`);
|
|
389
|
+
}
|
|
390
|
+
},
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
cache(resource, size, defDisposeSignal) {
|
|
394
|
+
const key = resource.key;
|
|
395
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
396
|
+
this.logger('info', `createPTable cache - added PTable ${key} with size ${size}`);
|
|
397
|
+
}
|
|
398
|
+
const status = {};
|
|
399
|
+
this.global.set(key, resource, { size: Math.max(size, 1), status }); // 1 is minimum size to avoid cache evictions
|
|
400
|
+
if (status.maxEntrySizeExceeded) {
|
|
401
|
+
resource.unref();
|
|
402
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
403
|
+
this.logger('info', `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
if (!this.disposeListeners.has(key)) {
|
|
408
|
+
const disposeListener = () => {
|
|
409
|
+
this.global.delete(key);
|
|
410
|
+
this.disposeListeners.delete(key);
|
|
411
|
+
defDisposeSignal.removeEventListener('abort', disposeListener);
|
|
412
|
+
};
|
|
413
|
+
this.disposeListeners.add(key);
|
|
414
|
+
defDisposeSignal.addEventListener('abort', disposeListener);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
250
417
|
}
|
|
251
418
|
}
|
|
252
419
|
class PFrameHolder {
|
|
@@ -319,11 +486,15 @@ class PFrameHolder {
|
|
|
319
486
|
.catch((err) => {
|
|
320
487
|
this.dispose();
|
|
321
488
|
pFrame.dispose();
|
|
322
|
-
throw new PFrameDriverError(`PFrame creation failed asynchronously,
|
|
489
|
+
throw new PFrameDriverError(`PFrame creation failed asynchronously, `
|
|
490
|
+
+ `columns: ${JSON.stringify(jsonifiedColumns)}, `
|
|
491
|
+
+ `error: ${ensureError(err)}`);
|
|
323
492
|
});
|
|
324
493
|
}
|
|
325
494
|
catch (err) {
|
|
326
|
-
throw new PFrameDriverError(`PFrame creation failed synchronously,
|
|
495
|
+
throw new PFrameDriverError(`PFrame creation failed synchronously, `
|
|
496
|
+
+ `columns: ${JSON.stringify(jsonifiedColumns)}, `
|
|
497
|
+
+ `error: ${ensureError(err)}`);
|
|
327
498
|
}
|
|
328
499
|
}
|
|
329
500
|
preloadBlob = async (blobIds) => {
|
|
@@ -347,6 +518,29 @@ class PFrameHolder {
|
|
|
347
518
|
.catch(() => { });
|
|
348
519
|
}
|
|
349
520
|
}
|
|
521
|
+
class PTableDefHolder {
|
|
522
|
+
def;
|
|
523
|
+
pTableHandle;
|
|
524
|
+
logger;
|
|
525
|
+
abortController = new AbortController();
|
|
526
|
+
constructor(def, pTableHandle, logger) {
|
|
527
|
+
this.def = def;
|
|
528
|
+
this.pTableHandle = pTableHandle;
|
|
529
|
+
this.logger = logger;
|
|
530
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
531
|
+
this.logger('info', `PTable definition saved (pTableHandle = ${this.pTableHandle})`);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
get disposeSignal() {
|
|
535
|
+
return this.abortController.signal;
|
|
536
|
+
}
|
|
537
|
+
[Symbol.dispose]() {
|
|
538
|
+
this.abortController.abort();
|
|
539
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
540
|
+
this.logger('info', `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
}
|
|
350
544
|
class PTableHolder {
|
|
351
545
|
pFrame;
|
|
352
546
|
pTablePromise;
|
|
@@ -374,8 +568,10 @@ class PFrameDriver {
|
|
|
374
568
|
logger;
|
|
375
569
|
server;
|
|
376
570
|
pFrames;
|
|
571
|
+
pTableDefs;
|
|
377
572
|
pTables;
|
|
378
|
-
|
|
573
|
+
pTableCacheUi;
|
|
574
|
+
pTableCacheModel;
|
|
379
575
|
frameConcurrencyLimiter;
|
|
380
576
|
tableConcurrencyLimiter;
|
|
381
577
|
async pprofDump() {
|
|
@@ -398,109 +594,11 @@ class PFrameDriver {
|
|
|
398
594
|
const concurrencyLimiter = new ConcurrencyLimitingExecutor(ops.pFrameConcurrency);
|
|
399
595
|
this.frameConcurrencyLimiter = concurrencyLimiter;
|
|
400
596
|
this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(ops.pTableConcurrency);
|
|
401
|
-
this.
|
|
402
|
-
this.
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
logger;
|
|
407
|
-
spillPath;
|
|
408
|
-
constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
|
|
409
|
-
super();
|
|
410
|
-
this.parquetServer = parquetServer;
|
|
411
|
-
this.localBlobPool = localBlobPool;
|
|
412
|
-
this.remoteBlobPool = remoteBlobPool;
|
|
413
|
-
this.logger = logger;
|
|
414
|
-
this.spillPath = spillPath;
|
|
415
|
-
}
|
|
416
|
-
acquire(params) {
|
|
417
|
-
return super.acquire(params);
|
|
418
|
-
}
|
|
419
|
-
getByKey(key) {
|
|
420
|
-
const resource = super.tryGetByKey(key);
|
|
421
|
-
if (!resource)
|
|
422
|
-
throw new PFrameDriverError(`PFrame not found, handle = ${key}`);
|
|
423
|
-
return resource;
|
|
424
|
-
}
|
|
425
|
-
createNewResource(params) {
|
|
426
|
-
if (getDebugFlags().logPFrameRequests)
|
|
427
|
-
this.logger('info', `PFrame creation (pFrameHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
|
|
428
|
-
return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
|
|
429
|
-
}
|
|
430
|
-
calculateParamsKey(params) {
|
|
431
|
-
try {
|
|
432
|
-
return stableKeyFromPFrameData(params);
|
|
433
|
-
}
|
|
434
|
-
catch (err) {
|
|
435
|
-
if (isPFrameDriverError(err))
|
|
436
|
-
throw err;
|
|
437
|
-
throw new PFrameDriverError(`PFrame handle calculation failed, request: ${JSON.stringify(params, bigintReplacer)}, error: ${ensureError(err)}`);
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
})(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
|
|
441
|
-
this.pTables = new (class extends RefCountResourcePool {
|
|
442
|
-
pFrames;
|
|
443
|
-
logger;
|
|
444
|
-
constructor(pFrames, logger) {
|
|
445
|
-
super();
|
|
446
|
-
this.pFrames = pFrames;
|
|
447
|
-
this.logger = logger;
|
|
448
|
-
}
|
|
449
|
-
getByKey(key) {
|
|
450
|
-
const resource = super.tryGetByKey(key);
|
|
451
|
-
if (!resource)
|
|
452
|
-
throw new PFrameDriverError(`PTable not found, handle = ${key}`);
|
|
453
|
-
return resource;
|
|
454
|
-
}
|
|
455
|
-
createNewResource(params) {
|
|
456
|
-
if (getDebugFlags().logPFrameRequests) {
|
|
457
|
-
this.logger('info', `PTable creation (pTableHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
|
|
458
|
-
}
|
|
459
|
-
const handle = params.pFrameHandle;
|
|
460
|
-
const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
461
|
-
// 3. Sort
|
|
462
|
-
if (params.def.sorting.length > 0) {
|
|
463
|
-
const predecessor = this.acquire({
|
|
464
|
-
...params,
|
|
465
|
-
def: {
|
|
466
|
-
...params.def,
|
|
467
|
-
sorting: [],
|
|
468
|
-
},
|
|
469
|
-
});
|
|
470
|
-
const { resource: { pTablePromise } } = predecessor;
|
|
471
|
-
const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
|
|
472
|
-
return new PTableHolder(handle, disposeSignal, sortedTable, predecessor);
|
|
473
|
-
}
|
|
474
|
-
// 2. Filter (except the case with artificial columns where cartesian creates too many rows)
|
|
475
|
-
if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
|
|
476
|
-
const predecessor = this.acquire({
|
|
477
|
-
...params,
|
|
478
|
-
def: {
|
|
479
|
-
...params.def,
|
|
480
|
-
filters: [],
|
|
481
|
-
},
|
|
482
|
-
});
|
|
483
|
-
const { resource: { pTablePromise } } = predecessor;
|
|
484
|
-
const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
|
|
485
|
-
return new PTableHolder(handle, disposeSignal, filteredTable, predecessor);
|
|
486
|
-
}
|
|
487
|
-
// 1. Join
|
|
488
|
-
const table = pFramePromise.then((pFrame) => pFrame.createTable({
|
|
489
|
-
src: joinEntryToInternal(params.def.src),
|
|
490
|
-
// `params.def.filters` would be non-empty only when join has artificial columns
|
|
491
|
-
filters: [...params.def.partitionFilters, ...params.def.filters],
|
|
492
|
-
}));
|
|
493
|
-
return new PTableHolder(handle, disposeSignal, table);
|
|
494
|
-
}
|
|
495
|
-
calculateParamsKey(params) {
|
|
496
|
-
try {
|
|
497
|
-
return stableKeyFromFullPTableDef(params);
|
|
498
|
-
}
|
|
499
|
-
catch (err) {
|
|
500
|
-
throw new PFrameDriverError(`PTable handle calculation failed, request: ${JSON.stringify(params)}, error: ${ensureError(err)}`);
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
})(this.pFrames, logger);
|
|
597
|
+
this.pFrames = new PFramePool(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
|
|
598
|
+
this.pTableDefs = new PTableDefPool(logger);
|
|
599
|
+
this.pTables = new PTablePool(this.pFrames, this.pTableDefs, logger);
|
|
600
|
+
this.pTableCacheUi = new PTableCacheUi(logger, ops);
|
|
601
|
+
this.pTableCacheModel = new PTableCacheModel(logger, ops);
|
|
504
602
|
}
|
|
505
603
|
async dispose() {
|
|
506
604
|
return await this.server.stop();
|
|
@@ -527,14 +625,15 @@ class PFrameDriver {
|
|
|
527
625
|
return res.key;
|
|
528
626
|
}
|
|
529
627
|
createPTable(rawDef, ctx) {
|
|
530
|
-
const def = migratePTableFilters(rawDef);
|
|
628
|
+
const def = migratePTableFilters(rawDef, this.logger);
|
|
531
629
|
const pFrameHandle = this.createPFrame(extractAllColumns(def.src), ctx);
|
|
532
630
|
const defIds = mapPTableDef(def, (c) => c.id);
|
|
533
|
-
const
|
|
534
|
-
if (getDebugFlags().logPFrameRequests)
|
|
535
|
-
this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${
|
|
536
|
-
|
|
537
|
-
|
|
631
|
+
const { key, unref } = this.pTableDefs.acquire({ def: defIds, pFrameHandle });
|
|
632
|
+
if (getDebugFlags().logPFrameRequests) {
|
|
633
|
+
this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${key})`);
|
|
634
|
+
}
|
|
635
|
+
ctx.addOnDestroy(unref); // in addition to pframe unref added in createPFrame above
|
|
636
|
+
return key;
|
|
538
637
|
}
|
|
539
638
|
//
|
|
540
639
|
// PFrame istance methods
|
|
@@ -579,9 +678,9 @@ class PFrameDriver {
|
|
|
579
678
|
}
|
|
580
679
|
const table = this.pTables.acquire({
|
|
581
680
|
pFrameHandle: handle,
|
|
582
|
-
def: migratePTableFilters(request),
|
|
681
|
+
def: migratePTableFilters(request, this.logger),
|
|
583
682
|
});
|
|
584
|
-
const {
|
|
683
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
585
684
|
const pTable = await pTablePromise;
|
|
586
685
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
587
686
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
@@ -602,7 +701,7 @@ class PFrameDriver {
|
|
|
602
701
|
withPredecessors: true,
|
|
603
702
|
signal: combinedSignal,
|
|
604
703
|
});
|
|
605
|
-
this.
|
|
704
|
+
this.pTableCacheUi.cache(table, overallSize);
|
|
606
705
|
return spec.map((spec, i) => ({
|
|
607
706
|
spec: spec,
|
|
608
707
|
data: data[i],
|
|
@@ -624,7 +723,7 @@ class PFrameDriver {
|
|
|
624
723
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
625
724
|
return await pFrame.getUniqueValues({
|
|
626
725
|
...request,
|
|
627
|
-
filters: migrateFilters(request.filters),
|
|
726
|
+
filters: migrateFilters(request.filters, this.logger),
|
|
628
727
|
}, {
|
|
629
728
|
signal: combinedSignal,
|
|
630
729
|
});
|
|
@@ -634,30 +733,60 @@ class PFrameDriver {
|
|
|
634
733
|
// PTable istance methods
|
|
635
734
|
//
|
|
636
735
|
async getSpec(handle) {
|
|
637
|
-
const {
|
|
638
|
-
|
|
639
|
-
|
|
736
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
737
|
+
try {
|
|
738
|
+
const { def } = this.pTableDefs.getByKey(handle);
|
|
739
|
+
const table = __addDisposableResource(env_1, this.pTables.acquire(def), false);
|
|
740
|
+
const { pTablePromise } = table.resource;
|
|
741
|
+
const pTable = await pTablePromise;
|
|
742
|
+
return pTable.getSpec();
|
|
743
|
+
}
|
|
744
|
+
catch (e_1) {
|
|
745
|
+
env_1.error = e_1;
|
|
746
|
+
env_1.hasError = true;
|
|
747
|
+
}
|
|
748
|
+
finally {
|
|
749
|
+
__disposeResources(env_1);
|
|
750
|
+
}
|
|
640
751
|
}
|
|
641
752
|
async getShape(handle, signal) {
|
|
642
|
-
const {
|
|
753
|
+
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
754
|
+
const table = this.pTables.acquire(def);
|
|
755
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
643
756
|
const pTable = await pTablePromise;
|
|
644
757
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
645
|
-
|
|
646
|
-
|
|
758
|
+
const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
759
|
+
const shape = await pTable.getShape({
|
|
647
760
|
signal: combinedSignal,
|
|
648
761
|
});
|
|
762
|
+
const overallSize = await pTable.getFootprint({
|
|
763
|
+
withPredecessors: true,
|
|
764
|
+
signal: combinedSignal,
|
|
765
|
+
});
|
|
766
|
+
return { shape, overallSize };
|
|
649
767
|
});
|
|
768
|
+
this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
|
|
769
|
+
return shape;
|
|
650
770
|
}
|
|
651
771
|
async getData(handle, columnIndices, range, signal) {
|
|
652
|
-
const {
|
|
772
|
+
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
773
|
+
const table = this.pTables.acquire(def);
|
|
774
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
653
775
|
const pTable = await pTablePromise;
|
|
654
776
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
655
|
-
|
|
656
|
-
|
|
777
|
+
const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
778
|
+
const data = await pTable.getData(columnIndices, {
|
|
657
779
|
range,
|
|
658
780
|
signal: combinedSignal,
|
|
659
781
|
});
|
|
782
|
+
const overallSize = await pTable.getFootprint({
|
|
783
|
+
withPredecessors: true,
|
|
784
|
+
signal: combinedSignal,
|
|
785
|
+
});
|
|
786
|
+
return { data, overallSize };
|
|
660
787
|
});
|
|
788
|
+
this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
|
|
789
|
+
return data;
|
|
661
790
|
}
|
|
662
791
|
}
|
|
663
792
|
function joinEntryToInternal(entry) {
|
|
@@ -713,9 +842,16 @@ function joinEntryToInternal(entry) {
|
|
|
713
842
|
}
|
|
714
843
|
}
|
|
715
844
|
function stableKeyFromFullPTableDef(data) {
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
845
|
+
try {
|
|
846
|
+
const hash = createHash('sha256');
|
|
847
|
+
hash.update(canonicalize(data));
|
|
848
|
+
return hash.digest().toString('hex');
|
|
849
|
+
}
|
|
850
|
+
catch (err) {
|
|
851
|
+
throw new PFrameDriverError(`PTable handle calculation failed, `
|
|
852
|
+
+ `request: ${JSON.stringify(data)}, `
|
|
853
|
+
+ `error: ${ensureError(err)}`);
|
|
854
|
+
}
|
|
719
855
|
}
|
|
720
856
|
function stableKeyFromPFrameData(data) {
|
|
721
857
|
const orderedData = [...data].map((column) => mapPObjectData(column, (r) => {
|