@milaboratories/pl-middle-layer 1.43.12 → 1.43.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  'use strict';
2
2
 
3
+ var tslib_es6 = require('../__external/.pnpm/@rollup_plugin-typescript@12.1.4_rollup@4.45.1_tslib@2.7.0_typescript@5.6.3/__external/tslib/tslib.es6.cjs');
3
4
  var plModelMiddleLayer = require('@milaboratories/pl-model-middle-layer');
4
5
  var plTree = require('@milaboratories/pl-tree');
5
6
  var model = require('@platforma-sdk/model');
@@ -26,7 +27,7 @@ class LocalBlobPool extends tsHelpers.RefCountResourcePool {
26
27
  calculateParamsKey(params) {
27
28
  return makeBlobId(params);
28
29
  }
29
- createNewResource(params) {
30
+ createNewResource(params, _key) {
30
31
  return this.blobDriver.getDownloadedBlob(params);
31
32
  }
32
33
  getByKey(blobId) {
@@ -61,7 +62,7 @@ class RemoteBlobPool extends tsHelpers.RefCountResourcePool {
61
62
  calculateParamsKey(params) {
62
63
  return String(params.rid);
63
64
  }
64
- createNewResource(params) {
65
+ createNewResource(params, _key) {
65
66
  return this.blobDriver.getOnDemandBlob(params);
66
67
  }
67
68
  async withContent(handle, options) {
@@ -118,7 +119,8 @@ class BlobStore extends plModelMiddleLayer.PFrameInternal.BaseObjectStore {
118
119
  range: translatedRange,
119
120
  });
120
121
  }
121
- this.logger('info', `PFrames blob store requesting content for ${blobId}, range [${translatedRange.start}..=${translatedRange.end}]`);
122
+ this.logger('info', `PFrames blob store requesting content for ${blobId}, `
123
+ + `range [${translatedRange.start}..=${translatedRange.end}]`);
122
124
  return await this.remoteBlobPool.withContent(blob.handle, {
123
125
  range: translatedRange,
124
126
  signal: params.signal,
@@ -142,7 +144,7 @@ class BlobStore extends plModelMiddleLayer.PFrameInternal.BaseObjectStore {
142
144
  }
143
145
  }
144
146
  const valueTypes = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'];
145
- function migrateFilters(filters) {
147
+ function migrateFilters(filters, logger) {
146
148
  const filtersV1 = [];
147
149
  const filtersV2 = [];
148
150
  for (const filter of filters) {
@@ -159,23 +161,23 @@ function migrateFilters(filters) {
159
161
  }
160
162
  if (filtersV1.length > 0) {
161
163
  const filtersV1Json = JSON.stringify(filtersV1);
162
- console.warn(`type overriten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
164
+ logger('warn', `type overriten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
163
165
  }
164
166
  return filtersV2;
165
167
  }
166
- function migratePTableFilters(def) {
168
+ function migratePTableFilters(def, logger) {
167
169
  if (!('partitionFilters' in def)) {
168
170
  // For old blocks assume all axes filters to be partition filters
169
171
  return {
170
172
  ...def,
171
- partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis')),
172
- filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column')),
173
+ partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis'), logger),
174
+ filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column'), logger),
173
175
  };
174
176
  }
175
177
  return {
176
178
  ...def,
177
- partitionFilters: migrateFilters(def.partitionFilters),
178
- filters: migrateFilters(def.filters),
179
+ partitionFilters: migrateFilters(def.partitionFilters, logger),
180
+ filters: migrateFilters(def.filters, logger),
179
181
  };
180
182
  }
181
183
  function hasArtificialColumns(entry) {
@@ -196,12 +198,134 @@ function hasArtificialColumns(entry) {
196
198
  }
197
199
  }
198
200
  const bigintReplacer = (_, v) => (typeof v === 'bigint' ? v.toString() : v);
199
- class PTableCache {
201
+ class PFramePool extends tsHelpers.RefCountResourcePool {
202
+ parquetServer;
203
+ localBlobPool;
204
+ remoteBlobPool;
205
+ logger;
206
+ spillPath;
207
+ constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
208
+ super();
209
+ this.parquetServer = parquetServer;
210
+ this.localBlobPool = localBlobPool;
211
+ this.remoteBlobPool = remoteBlobPool;
212
+ this.logger = logger;
213
+ this.spillPath = spillPath;
214
+ }
215
+ calculateParamsKey(params) {
216
+ try {
217
+ return stableKeyFromPFrameData(params);
218
+ }
219
+ catch (err) {
220
+ if (model.isPFrameDriverError(err))
221
+ throw err;
222
+ throw new model.PFrameDriverError(`PFrame handle calculation failed, `
223
+ + `request: ${JSON.stringify(params, bigintReplacer)}, `
224
+ + `error: ${model.ensureError(err)}`);
225
+ }
226
+ }
227
+ createNewResource(params, key) {
228
+ if (index.getDebugFlags().logPFrameRequests) {
229
+ this.logger('info', `PFrame creation (pFrameHandle = ${key}): `
230
+ + `${JSON.stringify(params, bigintReplacer)}`);
231
+ }
232
+ return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
233
+ }
234
+ getByKey(key) {
235
+ const resource = super.tryGetByKey(key);
236
+ if (!resource)
237
+ throw new model.PFrameDriverError(`PFrame not found, handle = ${key}`);
238
+ return resource;
239
+ }
240
+ }
241
+ class PTableDefPool extends tsHelpers.RefCountResourcePool {
242
+ logger;
243
+ constructor(logger) {
244
+ super();
245
+ this.logger = logger;
246
+ }
247
+ calculateParamsKey(params) {
248
+ return stableKeyFromFullPTableDef(params);
249
+ }
250
+ createNewResource(params, key) {
251
+ return new PTableDefHolder(params, key, this.logger);
252
+ }
253
+ getByKey(key) {
254
+ const resource = super.tryGetByKey(key);
255
+ if (!resource)
256
+ throw new model.PFrameDriverError(`PTable definition not found, handle = ${key}`);
257
+ return resource;
258
+ }
259
+ }
260
+ class PTablePool extends tsHelpers.RefCountResourcePool {
261
+ pFrames;
262
+ pTableDefs;
263
+ logger;
264
+ constructor(pFrames, pTableDefs, logger) {
265
+ super();
266
+ this.pFrames = pFrames;
267
+ this.pTableDefs = pTableDefs;
268
+ this.logger = logger;
269
+ }
270
+ calculateParamsKey(params) {
271
+ return stableKeyFromFullPTableDef(params);
272
+ }
273
+ createNewResource(params, key) {
274
+ if (index.getDebugFlags().logPFrameRequests) {
275
+ this.logger('info', `PTable creation (pTableHandle = ${key}): `
276
+ + `${JSON.stringify(params, bigintReplacer)}`);
277
+ }
278
+ const handle = params.pFrameHandle;
279
+ const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
280
+ const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
281
+ const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
282
+ // 3. Sort
283
+ if (params.def.sorting.length > 0) {
284
+ const predecessor = this.acquire({
285
+ ...params,
286
+ def: {
287
+ ...params.def,
288
+ sorting: [],
289
+ },
290
+ });
291
+ const { resource: { pTablePromise } } = predecessor;
292
+ const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
293
+ return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
294
+ }
295
+ // 2. Filter (except the case with artificial columns where cartesian creates too many rows)
296
+ if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
297
+ const predecessor = this.acquire({
298
+ ...params,
299
+ def: {
300
+ ...params.def,
301
+ filters: [],
302
+ },
303
+ });
304
+ const { resource: { pTablePromise } } = predecessor;
305
+ const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
306
+ return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
307
+ }
308
+ // 1. Join
309
+ const table = pFramePromise.then((pFrame) => pFrame.createTable({
310
+ src: joinEntryToInternal(params.def.src),
311
+ // `params.def.filters` would be non-empty only when join has artificial columns
312
+ filters: [...params.def.partitionFilters, ...params.def.filters],
313
+ }));
314
+ return new PTableHolder(handle, combinedSignal, table);
315
+ }
316
+ getByKey(key) {
317
+ const resource = super.tryGetByKey(key);
318
+ if (!resource)
319
+ throw new model.PFrameDriverError(`PTable not found, handle = ${key}`);
320
+ return resource;
321
+ }
322
+ }
323
+ class PTableCacheUi {
200
324
  logger;
201
325
  ops;
202
326
  perFrame = new Map();
203
327
  global;
204
- disposeListeners = new Map();
328
+ disposeListeners = new Set();
205
329
  constructor(logger, ops) {
206
330
  this.logger = logger;
207
331
  this.ops = ops;
@@ -214,12 +338,9 @@ class PTableCache {
214
338
  if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {
215
339
  this.perFrame.delete(resource.resource.pFrame);
216
340
  }
217
- const disposeListener = this.disposeListeners.get(key);
218
- this.disposeListeners.delete(key);
219
- resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
220
341
  resource.unref();
221
342
  if (index.getDebugFlags().logPFrameRequests) {
222
- this.logger('info', `calculateTableData cache - removed PTable ${key}`);
343
+ logger('info', `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
223
344
  }
224
345
  },
225
346
  });
@@ -243,12 +364,58 @@ class PTableCache {
243
364
  this.perFrame.set(resource.resource.pFrame, perFrame);
244
365
  }
245
366
  perFrame.set(key, resource);
246
- const disposeListener = () => {
247
- this.perFrame.get(resource.resource.pFrame)?.delete(key);
248
- this.global.delete(key);
249
- };
250
- this.disposeListeners.set(key, disposeListener);
251
- resource.resource.disposeSignal.addEventListener('abort', disposeListener);
367
+ if (!this.disposeListeners.has(key)) {
368
+ const disposeListener = () => {
369
+ this.perFrame.get(resource.resource.pFrame)?.delete(key);
370
+ this.global.delete(key);
371
+ this.disposeListeners.delete(key);
372
+ resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
373
+ };
374
+ this.disposeListeners.add(key);
375
+ resource.resource.disposeSignal.addEventListener('abort', disposeListener);
376
+ }
377
+ }
378
+ }
379
+ class PTableCacheModel {
380
+ logger;
381
+ global;
382
+ disposeListeners = new Set();
383
+ constructor(logger, ops) {
384
+ this.logger = logger;
385
+ this.global = new lruCache.LRUCache({
386
+ maxSize: ops.pTablesCacheMaxSize,
387
+ dispose: (resource, key, reason) => {
388
+ resource.unref();
389
+ if (index.getDebugFlags().logPFrameRequests) {
390
+ logger('info', `createPTable cache - removed PTable ${key} (reason: ${reason})`);
391
+ }
392
+ },
393
+ });
394
+ }
395
+ cache(resource, size, defDisposeSignal) {
396
+ const key = resource.key;
397
+ if (index.getDebugFlags().logPFrameRequests) {
398
+ this.logger('info', `createPTable cache - added PTable ${key} with size ${size}`);
399
+ }
400
+ const status = {};
401
+ this.global.set(key, resource, { size: Math.max(size, 1), status }); // 1 is minimum size to avoid cache evictions
402
+ if (status.maxEntrySizeExceeded) {
403
+ resource.unref();
404
+ if (index.getDebugFlags().logPFrameRequests) {
405
+ this.logger('info', `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);
406
+ }
407
+ }
408
+ else {
409
+ if (!this.disposeListeners.has(key)) {
410
+ const disposeListener = () => {
411
+ this.global.delete(key);
412
+ this.disposeListeners.delete(key);
413
+ defDisposeSignal.removeEventListener('abort', disposeListener);
414
+ };
415
+ this.disposeListeners.add(key);
416
+ defDisposeSignal.addEventListener('abort', disposeListener);
417
+ }
418
+ }
252
419
  }
253
420
  }
254
421
  class PFrameHolder {
@@ -321,11 +488,15 @@ class PFrameHolder {
321
488
  .catch((err) => {
322
489
  this.dispose();
323
490
  pFrame.dispose();
324
- throw new model.PFrameDriverError(`PFrame creation failed asynchronously, columns: ${JSON.stringify(jsonifiedColumns)}, error: ${model.ensureError(err)}`);
491
+ throw new model.PFrameDriverError(`PFrame creation failed asynchronously, `
492
+ + `columns: ${JSON.stringify(jsonifiedColumns)}, `
493
+ + `error: ${model.ensureError(err)}`);
325
494
  });
326
495
  }
327
496
  catch (err) {
328
- throw new model.PFrameDriverError(`PFrame creation failed synchronously, columns: ${JSON.stringify(jsonifiedColumns)}, error: ${model.ensureError(err)}`);
497
+ throw new model.PFrameDriverError(`PFrame creation failed synchronously, `
498
+ + `columns: ${JSON.stringify(jsonifiedColumns)}, `
499
+ + `error: ${model.ensureError(err)}`);
329
500
  }
330
501
  }
331
502
  preloadBlob = async (blobIds) => {
@@ -349,6 +520,29 @@ class PFrameHolder {
349
520
  .catch(() => { });
350
521
  }
351
522
  }
523
+ class PTableDefHolder {
524
+ def;
525
+ pTableHandle;
526
+ logger;
527
+ abortController = new AbortController();
528
+ constructor(def, pTableHandle, logger) {
529
+ this.def = def;
530
+ this.pTableHandle = pTableHandle;
531
+ this.logger = logger;
532
+ if (index.getDebugFlags().logPFrameRequests) {
533
+ this.logger('info', `PTable definition saved (pTableHandle = ${this.pTableHandle})`);
534
+ }
535
+ }
536
+ get disposeSignal() {
537
+ return this.abortController.signal;
538
+ }
539
+ [Symbol.dispose]() {
540
+ this.abortController.abort();
541
+ if (index.getDebugFlags().logPFrameRequests) {
542
+ this.logger('info', `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);
543
+ }
544
+ }
545
+ }
352
546
  class PTableHolder {
353
547
  pFrame;
354
548
  pTablePromise;
@@ -376,8 +570,10 @@ class PFrameDriver {
376
570
  logger;
377
571
  server;
378
572
  pFrames;
573
+ pTableDefs;
379
574
  pTables;
380
- pTableCache;
575
+ pTableCacheUi;
576
+ pTableCacheModel;
381
577
  frameConcurrencyLimiter;
382
578
  tableConcurrencyLimiter;
383
579
  async pprofDump() {
@@ -400,109 +596,11 @@ class PFrameDriver {
400
596
  const concurrencyLimiter = new tsHelpers.ConcurrencyLimitingExecutor(ops.pFrameConcurrency);
401
597
  this.frameConcurrencyLimiter = concurrencyLimiter;
402
598
  this.tableConcurrencyLimiter = new tsHelpers.ConcurrencyLimitingExecutor(ops.pTableConcurrency);
403
- this.pTableCache = new PTableCache(logger, ops);
404
- this.pFrames = new (class extends tsHelpers.RefCountResourcePool {
405
- parquetServer;
406
- localBlobPool;
407
- remoteBlobPool;
408
- logger;
409
- spillPath;
410
- constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
411
- super();
412
- this.parquetServer = parquetServer;
413
- this.localBlobPool = localBlobPool;
414
- this.remoteBlobPool = remoteBlobPool;
415
- this.logger = logger;
416
- this.spillPath = spillPath;
417
- }
418
- acquire(params) {
419
- return super.acquire(params);
420
- }
421
- getByKey(key) {
422
- const resource = super.tryGetByKey(key);
423
- if (!resource)
424
- throw new model.PFrameDriverError(`PFrame not found, handle = ${key}`);
425
- return resource;
426
- }
427
- createNewResource(params) {
428
- if (index.getDebugFlags().logPFrameRequests)
429
- this.logger('info', `PFrame creation (pFrameHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
430
- return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
431
- }
432
- calculateParamsKey(params) {
433
- try {
434
- return stableKeyFromPFrameData(params);
435
- }
436
- catch (err) {
437
- if (model.isPFrameDriverError(err))
438
- throw err;
439
- throw new model.PFrameDriverError(`PFrame handle calculation failed, request: ${JSON.stringify(params, bigintReplacer)}, error: ${model.ensureError(err)}`);
440
- }
441
- }
442
- })(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
443
- this.pTables = new (class extends tsHelpers.RefCountResourcePool {
444
- pFrames;
445
- logger;
446
- constructor(pFrames, logger) {
447
- super();
448
- this.pFrames = pFrames;
449
- this.logger = logger;
450
- }
451
- getByKey(key) {
452
- const resource = super.tryGetByKey(key);
453
- if (!resource)
454
- throw new model.PFrameDriverError(`PTable not found, handle = ${key}`);
455
- return resource;
456
- }
457
- createNewResource(params) {
458
- if (index.getDebugFlags().logPFrameRequests) {
459
- this.logger('info', `PTable creation (pTableHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
460
- }
461
- const handle = params.pFrameHandle;
462
- const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
463
- // 3. Sort
464
- if (params.def.sorting.length > 0) {
465
- const predecessor = this.acquire({
466
- ...params,
467
- def: {
468
- ...params.def,
469
- sorting: [],
470
- },
471
- });
472
- const { resource: { pTablePromise } } = predecessor;
473
- const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
474
- return new PTableHolder(handle, disposeSignal, sortedTable, predecessor);
475
- }
476
- // 2. Filter (except the case with artificial columns where cartesian creates too many rows)
477
- if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
478
- const predecessor = this.acquire({
479
- ...params,
480
- def: {
481
- ...params.def,
482
- filters: [],
483
- },
484
- });
485
- const { resource: { pTablePromise } } = predecessor;
486
- const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
487
- return new PTableHolder(handle, disposeSignal, filteredTable, predecessor);
488
- }
489
- // 1. Join
490
- const table = pFramePromise.then((pFrame) => pFrame.createTable({
491
- src: joinEntryToInternal(params.def.src),
492
- // `params.def.filters` would be non-empty only when join has artificial columns
493
- filters: [...params.def.partitionFilters, ...params.def.filters],
494
- }));
495
- return new PTableHolder(handle, disposeSignal, table);
496
- }
497
- calculateParamsKey(params) {
498
- try {
499
- return stableKeyFromFullPTableDef(params);
500
- }
501
- catch (err) {
502
- throw new model.PFrameDriverError(`PTable handle calculation failed, request: ${JSON.stringify(params)}, error: ${model.ensureError(err)}`);
503
- }
504
- }
505
- })(this.pFrames, logger);
599
+ this.pFrames = new PFramePool(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
600
+ this.pTableDefs = new PTableDefPool(logger);
601
+ this.pTables = new PTablePool(this.pFrames, this.pTableDefs, logger);
602
+ this.pTableCacheUi = new PTableCacheUi(logger, ops);
603
+ this.pTableCacheModel = new PTableCacheModel(logger, ops);
506
604
  }
507
605
  async dispose() {
508
606
  return await this.server.stop();
@@ -529,14 +627,15 @@ class PFrameDriver {
529
627
  return res.key;
530
628
  }
531
629
  createPTable(rawDef, ctx) {
532
- const def = migratePTableFilters(rawDef);
630
+ const def = migratePTableFilters(rawDef, this.logger);
533
631
  const pFrameHandle = this.createPFrame(model.extractAllColumns(def.src), ctx);
534
632
  const defIds = model.mapPTableDef(def, (c) => c.id);
535
- const res = this.pTables.acquire({ def: defIds, pFrameHandle });
536
- if (index.getDebugFlags().logPFrameRequests)
537
- this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${JSON.stringify(res)}): ${JSON.stringify(model.mapPTableDef(def, (c) => c.spec), bigintReplacer)}`);
538
- ctx.addOnDestroy(res.unref); // in addition to pframe unref added in createPFrame above
539
- return res.key;
633
+ const { key, unref } = this.pTableDefs.acquire({ def: defIds, pFrameHandle });
634
+ if (index.getDebugFlags().logPFrameRequests) {
635
+ this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${key})`);
636
+ }
637
+ ctx.addOnDestroy(unref); // in addition to pframe unref added in createPFrame above
638
+ return key;
540
639
  }
541
640
  //
542
641
  // PFrame istance methods
@@ -581,9 +680,9 @@ class PFrameDriver {
581
680
  }
582
681
  const table = this.pTables.acquire({
583
682
  pFrameHandle: handle,
584
- def: migratePTableFilters(request),
683
+ def: migratePTableFilters(request, this.logger),
585
684
  });
586
- const { resource: { pTablePromise, disposeSignal } } = table;
685
+ const { pTablePromise, disposeSignal } = table.resource;
587
686
  const pTable = await pTablePromise;
588
687
  const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
589
688
  return await this.frameConcurrencyLimiter.run(async () => {
@@ -604,7 +703,7 @@ class PFrameDriver {
604
703
  withPredecessors: true,
605
704
  signal: combinedSignal,
606
705
  });
607
- this.pTableCache.cache(table, overallSize);
706
+ this.pTableCacheUi.cache(table, overallSize);
608
707
  return spec.map((spec, i) => ({
609
708
  spec: spec,
610
709
  data: data[i],
@@ -626,7 +725,7 @@ class PFrameDriver {
626
725
  return await this.frameConcurrencyLimiter.run(async () => {
627
726
  return await pFrame.getUniqueValues({
628
727
  ...request,
629
- filters: migrateFilters(request.filters),
728
+ filters: migrateFilters(request.filters, this.logger),
630
729
  }, {
631
730
  signal: combinedSignal,
632
731
  });
@@ -636,30 +735,60 @@ class PFrameDriver {
636
735
  // PTable istance methods
637
736
  //
638
737
  async getSpec(handle) {
639
- const { pTablePromise } = this.pTables.getByKey(handle);
640
- const pTable = await pTablePromise;
641
- return pTable.getSpec();
738
+ const env_1 = { stack: [], error: void 0, hasError: false };
739
+ try {
740
+ const { def } = this.pTableDefs.getByKey(handle);
741
+ const table = tslib_es6.__addDisposableResource(env_1, this.pTables.acquire(def), false);
742
+ const { pTablePromise } = table.resource;
743
+ const pTable = await pTablePromise;
744
+ return pTable.getSpec();
745
+ }
746
+ catch (e_1) {
747
+ env_1.error = e_1;
748
+ env_1.hasError = true;
749
+ }
750
+ finally {
751
+ tslib_es6.__disposeResources(env_1);
752
+ }
642
753
  }
643
754
  async getShape(handle, signal) {
644
- const { pTablePromise, disposeSignal } = this.pTables.getByKey(handle);
755
+ const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
756
+ const table = this.pTables.acquire(def);
757
+ const { pTablePromise, disposeSignal } = table.resource;
645
758
  const pTable = await pTablePromise;
646
759
  const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
647
- return await this.tableConcurrencyLimiter.run(async () => {
648
- return await pTable.getShape({
760
+ const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
761
+ const shape = await pTable.getShape({
649
762
  signal: combinedSignal,
650
763
  });
764
+ const overallSize = await pTable.getFootprint({
765
+ withPredecessors: true,
766
+ signal: combinedSignal,
767
+ });
768
+ return { shape, overallSize };
651
769
  });
770
+ this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
771
+ return shape;
652
772
  }
653
773
  async getData(handle, columnIndices, range, signal) {
654
- const { pTablePromise, disposeSignal } = this.pTables.getByKey(handle);
774
+ const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
775
+ const table = this.pTables.acquire(def);
776
+ const { pTablePromise, disposeSignal } = table.resource;
655
777
  const pTable = await pTablePromise;
656
778
  const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
657
- return await this.tableConcurrencyLimiter.run(async () => {
658
- return await pTable.getData(columnIndices, {
779
+ const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
780
+ const data = await pTable.getData(columnIndices, {
659
781
  range,
660
782
  signal: combinedSignal,
661
783
  });
784
+ const overallSize = await pTable.getFootprint({
785
+ withPredecessors: true,
786
+ signal: combinedSignal,
787
+ });
788
+ return { data, overallSize };
662
789
  });
790
+ this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
791
+ return data;
663
792
  }
664
793
  }
665
794
  function joinEntryToInternal(entry) {
@@ -715,9 +844,16 @@ function joinEntryToInternal(entry) {
715
844
  }
716
845
  }
717
846
  function stableKeyFromFullPTableDef(data) {
718
- const hash = node_crypto.createHash('sha256');
719
- hash.update(canonicalize(data));
720
- return hash.digest().toString('hex');
847
+ try {
848
+ const hash = node_crypto.createHash('sha256');
849
+ hash.update(canonicalize(data));
850
+ return hash.digest().toString('hex');
851
+ }
852
+ catch (err) {
853
+ throw new model.PFrameDriverError(`PTable handle calculation failed, `
854
+ + `request: ${JSON.stringify(data)}, `
855
+ + `error: ${model.ensureError(err)}`);
856
+ }
721
857
  }
722
858
  function stableKeyFromPFrameData(data) {
723
859
  const orderedData = [...data].map((column) => model.mapPObjectData(column, (r) => {