@milaboratories/pl-middle-layer 1.43.12 → 1.43.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/middle_layer/ops.cjs +2 -1
- package/dist/middle_layer/ops.cjs.map +1 -1
- package/dist/middle_layer/ops.d.ts.map +1 -1
- package/dist/middle_layer/ops.js +2 -1
- package/dist/middle_layer/ops.js.map +1 -1
- package/dist/pool/driver.cjs +286 -150
- package/dist/pool/driver.cjs.map +1 -1
- package/dist/pool/driver.d.ts +9 -1
- package/dist/pool/driver.d.ts.map +1 -1
- package/dist/pool/driver.js +287 -151
- package/dist/pool/driver.js.map +1 -1
- package/package.json +17 -17
- package/src/middle_layer/ops.ts +2 -1
- package/src/pool/driver.ts +340 -185
package/dist/pool/driver.cjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
var tslib_es6 = require('../__external/.pnpm/@rollup_plugin-typescript@12.1.4_rollup@4.45.1_tslib@2.7.0_typescript@5.6.3/__external/tslib/tslib.es6.cjs');
|
|
3
4
|
var plModelMiddleLayer = require('@milaboratories/pl-model-middle-layer');
|
|
4
5
|
var plTree = require('@milaboratories/pl-tree');
|
|
5
6
|
var model = require('@platforma-sdk/model');
|
|
@@ -26,7 +27,7 @@ class LocalBlobPool extends tsHelpers.RefCountResourcePool {
|
|
|
26
27
|
calculateParamsKey(params) {
|
|
27
28
|
return makeBlobId(params);
|
|
28
29
|
}
|
|
29
|
-
createNewResource(params) {
|
|
30
|
+
createNewResource(params, _key) {
|
|
30
31
|
return this.blobDriver.getDownloadedBlob(params);
|
|
31
32
|
}
|
|
32
33
|
getByKey(blobId) {
|
|
@@ -61,7 +62,7 @@ class RemoteBlobPool extends tsHelpers.RefCountResourcePool {
|
|
|
61
62
|
calculateParamsKey(params) {
|
|
62
63
|
return String(params.rid);
|
|
63
64
|
}
|
|
64
|
-
createNewResource(params) {
|
|
65
|
+
createNewResource(params, _key) {
|
|
65
66
|
return this.blobDriver.getOnDemandBlob(params);
|
|
66
67
|
}
|
|
67
68
|
async withContent(handle, options) {
|
|
@@ -118,7 +119,8 @@ class BlobStore extends plModelMiddleLayer.PFrameInternal.BaseObjectStore {
|
|
|
118
119
|
range: translatedRange,
|
|
119
120
|
});
|
|
120
121
|
}
|
|
121
|
-
this.logger('info', `PFrames blob store requesting content for ${blobId},
|
|
122
|
+
this.logger('info', `PFrames blob store requesting content for ${blobId}, `
|
|
123
|
+
+ `range [${translatedRange.start}..=${translatedRange.end}]`);
|
|
122
124
|
return await this.remoteBlobPool.withContent(blob.handle, {
|
|
123
125
|
range: translatedRange,
|
|
124
126
|
signal: params.signal,
|
|
@@ -142,7 +144,7 @@ class BlobStore extends plModelMiddleLayer.PFrameInternal.BaseObjectStore {
|
|
|
142
144
|
}
|
|
143
145
|
}
|
|
144
146
|
const valueTypes = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'];
|
|
145
|
-
function migrateFilters(filters) {
|
|
147
|
+
function migrateFilters(filters, logger) {
|
|
146
148
|
const filtersV1 = [];
|
|
147
149
|
const filtersV2 = [];
|
|
148
150
|
for (const filter of filters) {
|
|
@@ -159,23 +161,23 @@ function migrateFilters(filters) {
|
|
|
159
161
|
}
|
|
160
162
|
if (filtersV1.length > 0) {
|
|
161
163
|
const filtersV1Json = JSON.stringify(filtersV1);
|
|
162
|
-
|
|
164
|
+
logger('warn', `type overriten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
|
|
163
165
|
}
|
|
164
166
|
return filtersV2;
|
|
165
167
|
}
|
|
166
|
-
function migratePTableFilters(def) {
|
|
168
|
+
function migratePTableFilters(def, logger) {
|
|
167
169
|
if (!('partitionFilters' in def)) {
|
|
168
170
|
// For old blocks assume all axes filters to be partition filters
|
|
169
171
|
return {
|
|
170
172
|
...def,
|
|
171
|
-
partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis')),
|
|
172
|
-
filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column')),
|
|
173
|
+
partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis'), logger),
|
|
174
|
+
filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column'), logger),
|
|
173
175
|
};
|
|
174
176
|
}
|
|
175
177
|
return {
|
|
176
178
|
...def,
|
|
177
|
-
partitionFilters: migrateFilters(def.partitionFilters),
|
|
178
|
-
filters: migrateFilters(def.filters),
|
|
179
|
+
partitionFilters: migrateFilters(def.partitionFilters, logger),
|
|
180
|
+
filters: migrateFilters(def.filters, logger),
|
|
179
181
|
};
|
|
180
182
|
}
|
|
181
183
|
function hasArtificialColumns(entry) {
|
|
@@ -196,12 +198,134 @@ function hasArtificialColumns(entry) {
|
|
|
196
198
|
}
|
|
197
199
|
}
|
|
198
200
|
const bigintReplacer = (_, v) => (typeof v === 'bigint' ? v.toString() : v);
|
|
199
|
-
class
|
|
201
|
+
class PFramePool extends tsHelpers.RefCountResourcePool {
|
|
202
|
+
parquetServer;
|
|
203
|
+
localBlobPool;
|
|
204
|
+
remoteBlobPool;
|
|
205
|
+
logger;
|
|
206
|
+
spillPath;
|
|
207
|
+
constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
|
|
208
|
+
super();
|
|
209
|
+
this.parquetServer = parquetServer;
|
|
210
|
+
this.localBlobPool = localBlobPool;
|
|
211
|
+
this.remoteBlobPool = remoteBlobPool;
|
|
212
|
+
this.logger = logger;
|
|
213
|
+
this.spillPath = spillPath;
|
|
214
|
+
}
|
|
215
|
+
calculateParamsKey(params) {
|
|
216
|
+
try {
|
|
217
|
+
return stableKeyFromPFrameData(params);
|
|
218
|
+
}
|
|
219
|
+
catch (err) {
|
|
220
|
+
if (model.isPFrameDriverError(err))
|
|
221
|
+
throw err;
|
|
222
|
+
throw new model.PFrameDriverError(`PFrame handle calculation failed, `
|
|
223
|
+
+ `request: ${JSON.stringify(params, bigintReplacer)}, `
|
|
224
|
+
+ `error: ${model.ensureError(err)}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
createNewResource(params, key) {
|
|
228
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
229
|
+
this.logger('info', `PFrame creation (pFrameHandle = ${key}): `
|
|
230
|
+
+ `${JSON.stringify(params, bigintReplacer)}`);
|
|
231
|
+
}
|
|
232
|
+
return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
|
|
233
|
+
}
|
|
234
|
+
getByKey(key) {
|
|
235
|
+
const resource = super.tryGetByKey(key);
|
|
236
|
+
if (!resource)
|
|
237
|
+
throw new model.PFrameDriverError(`PFrame not found, handle = ${key}`);
|
|
238
|
+
return resource;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
class PTableDefPool extends tsHelpers.RefCountResourcePool {
|
|
242
|
+
logger;
|
|
243
|
+
constructor(logger) {
|
|
244
|
+
super();
|
|
245
|
+
this.logger = logger;
|
|
246
|
+
}
|
|
247
|
+
calculateParamsKey(params) {
|
|
248
|
+
return stableKeyFromFullPTableDef(params);
|
|
249
|
+
}
|
|
250
|
+
createNewResource(params, key) {
|
|
251
|
+
return new PTableDefHolder(params, key, this.logger);
|
|
252
|
+
}
|
|
253
|
+
getByKey(key) {
|
|
254
|
+
const resource = super.tryGetByKey(key);
|
|
255
|
+
if (!resource)
|
|
256
|
+
throw new model.PFrameDriverError(`PTable definition not found, handle = ${key}`);
|
|
257
|
+
return resource;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
class PTablePool extends tsHelpers.RefCountResourcePool {
|
|
261
|
+
pFrames;
|
|
262
|
+
pTableDefs;
|
|
263
|
+
logger;
|
|
264
|
+
constructor(pFrames, pTableDefs, logger) {
|
|
265
|
+
super();
|
|
266
|
+
this.pFrames = pFrames;
|
|
267
|
+
this.pTableDefs = pTableDefs;
|
|
268
|
+
this.logger = logger;
|
|
269
|
+
}
|
|
270
|
+
calculateParamsKey(params) {
|
|
271
|
+
return stableKeyFromFullPTableDef(params);
|
|
272
|
+
}
|
|
273
|
+
createNewResource(params, key) {
|
|
274
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
275
|
+
this.logger('info', `PTable creation (pTableHandle = ${key}): `
|
|
276
|
+
+ `${JSON.stringify(params, bigintReplacer)}`);
|
|
277
|
+
}
|
|
278
|
+
const handle = params.pFrameHandle;
|
|
279
|
+
const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
280
|
+
const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
|
|
281
|
+
const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
|
|
282
|
+
// 3. Sort
|
|
283
|
+
if (params.def.sorting.length > 0) {
|
|
284
|
+
const predecessor = this.acquire({
|
|
285
|
+
...params,
|
|
286
|
+
def: {
|
|
287
|
+
...params.def,
|
|
288
|
+
sorting: [],
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
const { resource: { pTablePromise } } = predecessor;
|
|
292
|
+
const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
|
|
293
|
+
return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
|
|
294
|
+
}
|
|
295
|
+
// 2. Filter (except the case with artificial columns where cartesian creates too many rows)
|
|
296
|
+
if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
|
|
297
|
+
const predecessor = this.acquire({
|
|
298
|
+
...params,
|
|
299
|
+
def: {
|
|
300
|
+
...params.def,
|
|
301
|
+
filters: [],
|
|
302
|
+
},
|
|
303
|
+
});
|
|
304
|
+
const { resource: { pTablePromise } } = predecessor;
|
|
305
|
+
const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
|
|
306
|
+
return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
|
|
307
|
+
}
|
|
308
|
+
// 1. Join
|
|
309
|
+
const table = pFramePromise.then((pFrame) => pFrame.createTable({
|
|
310
|
+
src: joinEntryToInternal(params.def.src),
|
|
311
|
+
// `params.def.filters` would be non-empty only when join has artificial columns
|
|
312
|
+
filters: [...params.def.partitionFilters, ...params.def.filters],
|
|
313
|
+
}));
|
|
314
|
+
return new PTableHolder(handle, combinedSignal, table);
|
|
315
|
+
}
|
|
316
|
+
getByKey(key) {
|
|
317
|
+
const resource = super.tryGetByKey(key);
|
|
318
|
+
if (!resource)
|
|
319
|
+
throw new model.PFrameDriverError(`PTable not found, handle = ${key}`);
|
|
320
|
+
return resource;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
class PTableCacheUi {
|
|
200
324
|
logger;
|
|
201
325
|
ops;
|
|
202
326
|
perFrame = new Map();
|
|
203
327
|
global;
|
|
204
|
-
disposeListeners = new
|
|
328
|
+
disposeListeners = new Set();
|
|
205
329
|
constructor(logger, ops) {
|
|
206
330
|
this.logger = logger;
|
|
207
331
|
this.ops = ops;
|
|
@@ -214,12 +338,9 @@ class PTableCache {
|
|
|
214
338
|
if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {
|
|
215
339
|
this.perFrame.delete(resource.resource.pFrame);
|
|
216
340
|
}
|
|
217
|
-
const disposeListener = this.disposeListeners.get(key);
|
|
218
|
-
this.disposeListeners.delete(key);
|
|
219
|
-
resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
|
|
220
341
|
resource.unref();
|
|
221
342
|
if (index.getDebugFlags().logPFrameRequests) {
|
|
222
|
-
|
|
343
|
+
logger('info', `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
|
|
223
344
|
}
|
|
224
345
|
},
|
|
225
346
|
});
|
|
@@ -243,12 +364,58 @@ class PTableCache {
|
|
|
243
364
|
this.perFrame.set(resource.resource.pFrame, perFrame);
|
|
244
365
|
}
|
|
245
366
|
perFrame.set(key, resource);
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
367
|
+
if (!this.disposeListeners.has(key)) {
|
|
368
|
+
const disposeListener = () => {
|
|
369
|
+
this.perFrame.get(resource.resource.pFrame)?.delete(key);
|
|
370
|
+
this.global.delete(key);
|
|
371
|
+
this.disposeListeners.delete(key);
|
|
372
|
+
resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
|
|
373
|
+
};
|
|
374
|
+
this.disposeListeners.add(key);
|
|
375
|
+
resource.resource.disposeSignal.addEventListener('abort', disposeListener);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
class PTableCacheModel {
|
|
380
|
+
logger;
|
|
381
|
+
global;
|
|
382
|
+
disposeListeners = new Set();
|
|
383
|
+
constructor(logger, ops) {
|
|
384
|
+
this.logger = logger;
|
|
385
|
+
this.global = new lruCache.LRUCache({
|
|
386
|
+
maxSize: ops.pTablesCacheMaxSize,
|
|
387
|
+
dispose: (resource, key, reason) => {
|
|
388
|
+
resource.unref();
|
|
389
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
390
|
+
logger('info', `createPTable cache - removed PTable ${key} (reason: ${reason})`);
|
|
391
|
+
}
|
|
392
|
+
},
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
cache(resource, size, defDisposeSignal) {
|
|
396
|
+
const key = resource.key;
|
|
397
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
398
|
+
this.logger('info', `createPTable cache - added PTable ${key} with size ${size}`);
|
|
399
|
+
}
|
|
400
|
+
const status = {};
|
|
401
|
+
this.global.set(key, resource, { size: Math.max(size, 1), status }); // 1 is minimum size to avoid cache evictions
|
|
402
|
+
if (status.maxEntrySizeExceeded) {
|
|
403
|
+
resource.unref();
|
|
404
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
405
|
+
this.logger('info', `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
else {
|
|
409
|
+
if (!this.disposeListeners.has(key)) {
|
|
410
|
+
const disposeListener = () => {
|
|
411
|
+
this.global.delete(key);
|
|
412
|
+
this.disposeListeners.delete(key);
|
|
413
|
+
defDisposeSignal.removeEventListener('abort', disposeListener);
|
|
414
|
+
};
|
|
415
|
+
this.disposeListeners.add(key);
|
|
416
|
+
defDisposeSignal.addEventListener('abort', disposeListener);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
252
419
|
}
|
|
253
420
|
}
|
|
254
421
|
class PFrameHolder {
|
|
@@ -321,11 +488,15 @@ class PFrameHolder {
|
|
|
321
488
|
.catch((err) => {
|
|
322
489
|
this.dispose();
|
|
323
490
|
pFrame.dispose();
|
|
324
|
-
throw new model.PFrameDriverError(`PFrame creation failed asynchronously,
|
|
491
|
+
throw new model.PFrameDriverError(`PFrame creation failed asynchronously, `
|
|
492
|
+
+ `columns: ${JSON.stringify(jsonifiedColumns)}, `
|
|
493
|
+
+ `error: ${model.ensureError(err)}`);
|
|
325
494
|
});
|
|
326
495
|
}
|
|
327
496
|
catch (err) {
|
|
328
|
-
throw new model.PFrameDriverError(`PFrame creation failed synchronously,
|
|
497
|
+
throw new model.PFrameDriverError(`PFrame creation failed synchronously, `
|
|
498
|
+
+ `columns: ${JSON.stringify(jsonifiedColumns)}, `
|
|
499
|
+
+ `error: ${model.ensureError(err)}`);
|
|
329
500
|
}
|
|
330
501
|
}
|
|
331
502
|
preloadBlob = async (blobIds) => {
|
|
@@ -349,6 +520,29 @@ class PFrameHolder {
|
|
|
349
520
|
.catch(() => { });
|
|
350
521
|
}
|
|
351
522
|
}
|
|
523
|
+
class PTableDefHolder {
|
|
524
|
+
def;
|
|
525
|
+
pTableHandle;
|
|
526
|
+
logger;
|
|
527
|
+
abortController = new AbortController();
|
|
528
|
+
constructor(def, pTableHandle, logger) {
|
|
529
|
+
this.def = def;
|
|
530
|
+
this.pTableHandle = pTableHandle;
|
|
531
|
+
this.logger = logger;
|
|
532
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
533
|
+
this.logger('info', `PTable definition saved (pTableHandle = ${this.pTableHandle})`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
get disposeSignal() {
|
|
537
|
+
return this.abortController.signal;
|
|
538
|
+
}
|
|
539
|
+
[Symbol.dispose]() {
|
|
540
|
+
this.abortController.abort();
|
|
541
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
542
|
+
this.logger('info', `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
}
|
|
352
546
|
class PTableHolder {
|
|
353
547
|
pFrame;
|
|
354
548
|
pTablePromise;
|
|
@@ -376,8 +570,10 @@ class PFrameDriver {
|
|
|
376
570
|
logger;
|
|
377
571
|
server;
|
|
378
572
|
pFrames;
|
|
573
|
+
pTableDefs;
|
|
379
574
|
pTables;
|
|
380
|
-
|
|
575
|
+
pTableCacheUi;
|
|
576
|
+
pTableCacheModel;
|
|
381
577
|
frameConcurrencyLimiter;
|
|
382
578
|
tableConcurrencyLimiter;
|
|
383
579
|
async pprofDump() {
|
|
@@ -400,109 +596,11 @@ class PFrameDriver {
|
|
|
400
596
|
const concurrencyLimiter = new tsHelpers.ConcurrencyLimitingExecutor(ops.pFrameConcurrency);
|
|
401
597
|
this.frameConcurrencyLimiter = concurrencyLimiter;
|
|
402
598
|
this.tableConcurrencyLimiter = new tsHelpers.ConcurrencyLimitingExecutor(ops.pTableConcurrency);
|
|
403
|
-
this.
|
|
404
|
-
this.
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
logger;
|
|
409
|
-
spillPath;
|
|
410
|
-
constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
|
|
411
|
-
super();
|
|
412
|
-
this.parquetServer = parquetServer;
|
|
413
|
-
this.localBlobPool = localBlobPool;
|
|
414
|
-
this.remoteBlobPool = remoteBlobPool;
|
|
415
|
-
this.logger = logger;
|
|
416
|
-
this.spillPath = spillPath;
|
|
417
|
-
}
|
|
418
|
-
acquire(params) {
|
|
419
|
-
return super.acquire(params);
|
|
420
|
-
}
|
|
421
|
-
getByKey(key) {
|
|
422
|
-
const resource = super.tryGetByKey(key);
|
|
423
|
-
if (!resource)
|
|
424
|
-
throw new model.PFrameDriverError(`PFrame not found, handle = ${key}`);
|
|
425
|
-
return resource;
|
|
426
|
-
}
|
|
427
|
-
createNewResource(params) {
|
|
428
|
-
if (index.getDebugFlags().logPFrameRequests)
|
|
429
|
-
this.logger('info', `PFrame creation (pFrameHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
|
|
430
|
-
return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
|
|
431
|
-
}
|
|
432
|
-
calculateParamsKey(params) {
|
|
433
|
-
try {
|
|
434
|
-
return stableKeyFromPFrameData(params);
|
|
435
|
-
}
|
|
436
|
-
catch (err) {
|
|
437
|
-
if (model.isPFrameDriverError(err))
|
|
438
|
-
throw err;
|
|
439
|
-
throw new model.PFrameDriverError(`PFrame handle calculation failed, request: ${JSON.stringify(params, bigintReplacer)}, error: ${model.ensureError(err)}`);
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
})(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
|
|
443
|
-
this.pTables = new (class extends tsHelpers.RefCountResourcePool {
|
|
444
|
-
pFrames;
|
|
445
|
-
logger;
|
|
446
|
-
constructor(pFrames, logger) {
|
|
447
|
-
super();
|
|
448
|
-
this.pFrames = pFrames;
|
|
449
|
-
this.logger = logger;
|
|
450
|
-
}
|
|
451
|
-
getByKey(key) {
|
|
452
|
-
const resource = super.tryGetByKey(key);
|
|
453
|
-
if (!resource)
|
|
454
|
-
throw new model.PFrameDriverError(`PTable not found, handle = ${key}`);
|
|
455
|
-
return resource;
|
|
456
|
-
}
|
|
457
|
-
createNewResource(params) {
|
|
458
|
-
if (index.getDebugFlags().logPFrameRequests) {
|
|
459
|
-
this.logger('info', `PTable creation (pTableHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
|
|
460
|
-
}
|
|
461
|
-
const handle = params.pFrameHandle;
|
|
462
|
-
const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
463
|
-
// 3. Sort
|
|
464
|
-
if (params.def.sorting.length > 0) {
|
|
465
|
-
const predecessor = this.acquire({
|
|
466
|
-
...params,
|
|
467
|
-
def: {
|
|
468
|
-
...params.def,
|
|
469
|
-
sorting: [],
|
|
470
|
-
},
|
|
471
|
-
});
|
|
472
|
-
const { resource: { pTablePromise } } = predecessor;
|
|
473
|
-
const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
|
|
474
|
-
return new PTableHolder(handle, disposeSignal, sortedTable, predecessor);
|
|
475
|
-
}
|
|
476
|
-
// 2. Filter (except the case with artificial columns where cartesian creates too many rows)
|
|
477
|
-
if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
|
|
478
|
-
const predecessor = this.acquire({
|
|
479
|
-
...params,
|
|
480
|
-
def: {
|
|
481
|
-
...params.def,
|
|
482
|
-
filters: [],
|
|
483
|
-
},
|
|
484
|
-
});
|
|
485
|
-
const { resource: { pTablePromise } } = predecessor;
|
|
486
|
-
const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
|
|
487
|
-
return new PTableHolder(handle, disposeSignal, filteredTable, predecessor);
|
|
488
|
-
}
|
|
489
|
-
// 1. Join
|
|
490
|
-
const table = pFramePromise.then((pFrame) => pFrame.createTable({
|
|
491
|
-
src: joinEntryToInternal(params.def.src),
|
|
492
|
-
// `params.def.filters` would be non-empty only when join has artificial columns
|
|
493
|
-
filters: [...params.def.partitionFilters, ...params.def.filters],
|
|
494
|
-
}));
|
|
495
|
-
return new PTableHolder(handle, disposeSignal, table);
|
|
496
|
-
}
|
|
497
|
-
calculateParamsKey(params) {
|
|
498
|
-
try {
|
|
499
|
-
return stableKeyFromFullPTableDef(params);
|
|
500
|
-
}
|
|
501
|
-
catch (err) {
|
|
502
|
-
throw new model.PFrameDriverError(`PTable handle calculation failed, request: ${JSON.stringify(params)}, error: ${model.ensureError(err)}`);
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
})(this.pFrames, logger);
|
|
599
|
+
this.pFrames = new PFramePool(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
|
|
600
|
+
this.pTableDefs = new PTableDefPool(logger);
|
|
601
|
+
this.pTables = new PTablePool(this.pFrames, this.pTableDefs, logger);
|
|
602
|
+
this.pTableCacheUi = new PTableCacheUi(logger, ops);
|
|
603
|
+
this.pTableCacheModel = new PTableCacheModel(logger, ops);
|
|
506
604
|
}
|
|
507
605
|
async dispose() {
|
|
508
606
|
return await this.server.stop();
|
|
@@ -529,14 +627,15 @@ class PFrameDriver {
|
|
|
529
627
|
return res.key;
|
|
530
628
|
}
|
|
531
629
|
createPTable(rawDef, ctx) {
|
|
532
|
-
const def = migratePTableFilters(rawDef);
|
|
630
|
+
const def = migratePTableFilters(rawDef, this.logger);
|
|
533
631
|
const pFrameHandle = this.createPFrame(model.extractAllColumns(def.src), ctx);
|
|
534
632
|
const defIds = model.mapPTableDef(def, (c) => c.id);
|
|
535
|
-
const
|
|
536
|
-
if (index.getDebugFlags().logPFrameRequests)
|
|
537
|
-
this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${
|
|
538
|
-
|
|
539
|
-
|
|
633
|
+
const { key, unref } = this.pTableDefs.acquire({ def: defIds, pFrameHandle });
|
|
634
|
+
if (index.getDebugFlags().logPFrameRequests) {
|
|
635
|
+
this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${key})`);
|
|
636
|
+
}
|
|
637
|
+
ctx.addOnDestroy(unref); // in addition to pframe unref added in createPFrame above
|
|
638
|
+
return key;
|
|
540
639
|
}
|
|
541
640
|
//
|
|
542
641
|
// PFrame istance methods
|
|
@@ -581,9 +680,9 @@ class PFrameDriver {
|
|
|
581
680
|
}
|
|
582
681
|
const table = this.pTables.acquire({
|
|
583
682
|
pFrameHandle: handle,
|
|
584
|
-
def: migratePTableFilters(request),
|
|
683
|
+
def: migratePTableFilters(request, this.logger),
|
|
585
684
|
});
|
|
586
|
-
const {
|
|
685
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
587
686
|
const pTable = await pTablePromise;
|
|
588
687
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
589
688
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
@@ -604,7 +703,7 @@ class PFrameDriver {
|
|
|
604
703
|
withPredecessors: true,
|
|
605
704
|
signal: combinedSignal,
|
|
606
705
|
});
|
|
607
|
-
this.
|
|
706
|
+
this.pTableCacheUi.cache(table, overallSize);
|
|
608
707
|
return spec.map((spec, i) => ({
|
|
609
708
|
spec: spec,
|
|
610
709
|
data: data[i],
|
|
@@ -626,7 +725,7 @@ class PFrameDriver {
|
|
|
626
725
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
627
726
|
return await pFrame.getUniqueValues({
|
|
628
727
|
...request,
|
|
629
|
-
filters: migrateFilters(request.filters),
|
|
728
|
+
filters: migrateFilters(request.filters, this.logger),
|
|
630
729
|
}, {
|
|
631
730
|
signal: combinedSignal,
|
|
632
731
|
});
|
|
@@ -636,30 +735,60 @@ class PFrameDriver {
|
|
|
636
735
|
// PTable istance methods
|
|
637
736
|
//
|
|
638
737
|
async getSpec(handle) {
|
|
639
|
-
const {
|
|
640
|
-
|
|
641
|
-
|
|
738
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
739
|
+
try {
|
|
740
|
+
const { def } = this.pTableDefs.getByKey(handle);
|
|
741
|
+
const table = tslib_es6.__addDisposableResource(env_1, this.pTables.acquire(def), false);
|
|
742
|
+
const { pTablePromise } = table.resource;
|
|
743
|
+
const pTable = await pTablePromise;
|
|
744
|
+
return pTable.getSpec();
|
|
745
|
+
}
|
|
746
|
+
catch (e_1) {
|
|
747
|
+
env_1.error = e_1;
|
|
748
|
+
env_1.hasError = true;
|
|
749
|
+
}
|
|
750
|
+
finally {
|
|
751
|
+
tslib_es6.__disposeResources(env_1);
|
|
752
|
+
}
|
|
642
753
|
}
|
|
643
754
|
async getShape(handle, signal) {
|
|
644
|
-
const {
|
|
755
|
+
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
756
|
+
const table = this.pTables.acquire(def);
|
|
757
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
645
758
|
const pTable = await pTablePromise;
|
|
646
759
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
647
|
-
|
|
648
|
-
|
|
760
|
+
const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
761
|
+
const shape = await pTable.getShape({
|
|
649
762
|
signal: combinedSignal,
|
|
650
763
|
});
|
|
764
|
+
const overallSize = await pTable.getFootprint({
|
|
765
|
+
withPredecessors: true,
|
|
766
|
+
signal: combinedSignal,
|
|
767
|
+
});
|
|
768
|
+
return { shape, overallSize };
|
|
651
769
|
});
|
|
770
|
+
this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
|
|
771
|
+
return shape;
|
|
652
772
|
}
|
|
653
773
|
async getData(handle, columnIndices, range, signal) {
|
|
654
|
-
const {
|
|
774
|
+
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
775
|
+
const table = this.pTables.acquire(def);
|
|
776
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
655
777
|
const pTable = await pTablePromise;
|
|
656
778
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
657
|
-
|
|
658
|
-
|
|
779
|
+
const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
780
|
+
const data = await pTable.getData(columnIndices, {
|
|
659
781
|
range,
|
|
660
782
|
signal: combinedSignal,
|
|
661
783
|
});
|
|
784
|
+
const overallSize = await pTable.getFootprint({
|
|
785
|
+
withPredecessors: true,
|
|
786
|
+
signal: combinedSignal,
|
|
787
|
+
});
|
|
788
|
+
return { data, overallSize };
|
|
662
789
|
});
|
|
790
|
+
this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
|
|
791
|
+
return data;
|
|
663
792
|
}
|
|
664
793
|
}
|
|
665
794
|
function joinEntryToInternal(entry) {
|
|
@@ -715,9 +844,16 @@ function joinEntryToInternal(entry) {
|
|
|
715
844
|
}
|
|
716
845
|
}
|
|
717
846
|
function stableKeyFromFullPTableDef(data) {
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
847
|
+
try {
|
|
848
|
+
const hash = node_crypto.createHash('sha256');
|
|
849
|
+
hash.update(canonicalize(data));
|
|
850
|
+
return hash.digest().toString('hex');
|
|
851
|
+
}
|
|
852
|
+
catch (err) {
|
|
853
|
+
throw new model.PFrameDriverError(`PTable handle calculation failed, `
|
|
854
|
+
+ `request: ${JSON.stringify(data)}, `
|
|
855
|
+
+ `error: ${model.ensureError(err)}`);
|
|
856
|
+
}
|
|
721
857
|
}
|
|
722
858
|
function stableKeyFromPFrameData(data) {
|
|
723
859
|
const orderedData = [...data].map((column) => model.mapPObjectData(column, (r) => {
|