@milaboratories/pl-middle-layer 1.43.59 → 1.43.60

package/dist/pool/driver.js

@@ -1,26 +1,23 @@
-import { __addDisposableResource, __disposeResources } from '../__external/.pnpm/@rollup_plugin-typescript@12.1.4_rollup@4.52.4_tslib@2.7.0_typescript@5.6.3/__external/tslib/tslib.es6.js';
+import { ensureError, isAbortError, PFrameDriverError, mapDataInfo, isDataInfo } from '@platforma-sdk/model';
 import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import { emptyDir, RefCountPoolBase } from '@milaboratories/ts-helpers';
 import { isPlTreeNodeAccessor } from '@milaboratories/pl-tree';
-import { ensureError, isAbortError, mapPObjectData, mapDataInfo, isDataInfo, uniqueBy, extractAllColumns, mapPTableDef, PFrameDriverError, isPFrameDriverError, canonicalizeJson, getAxisId } from '@platforma-sdk/model';
-import { LRUCache } from 'lru-cache';
-import { parseDataInfoResource, makeDataInfoFromPColumnValues, traverseParquetChunkResource } from './data.js';
-import { createHash } from 'node:crypto';
-import { mapValues } from 'es-toolkit';
-import { emptyDir, ConcurrencyLimitingExecutor, RefCountResourcePool, assertNever } from '@milaboratories/ts-helpers';
-import canonicalize from 'canonicalize';
-import { PFrameFactory, HttpHelpers } from '@milaboratories/pframes-rs-node';
+import { AbstractPFrameDriverOpsDefaults, AbstractPFrameDriver, makeJsonDataInfo } from '@milaboratories/pf-driver';
+import { HttpHelpers } from '@milaboratories/pframes-rs-node';
 import path from 'node:path';
-import { getDebugFlags } from '../debug/index.js';
 import { Readable } from 'node:stream';
+import { parseDataInfoResource, traverseParquetChunkResource } from './data.js';
 
 function makeBlobId(res) {
     return String(res.rid);
 }
-class LocalBlobPool extends RefCountResourcePool {
+class LocalBlobProviderImpl extends RefCountPoolBase {
     blobDriver;
-    constructor(blobDriver) {
+    logger;
+    constructor(blobDriver, logger) {
         super();
         this.blobDriver = blobDriver;
+        this.logger = logger;
     }
     calculateParamsKey(params) {
         return makeBlobId(params);
@@ -31,38 +28,48 @@ class LocalBlobPool extends RefCountResourcePool {
     getByKey(blobId) {
         const resource = super.tryGetByKey(blobId);
         if (!resource)
-            throw new PFrameDriverError(`Blob with id ${blobId} not found.`);
+            throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);
         return resource;
     }
-    async preloadBlob(blobIds, signal) {
-        try {
-            await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));
-        }
-        catch (err) {
-            if (!isAbortError(err))
-                throw err;
-        }
-    }
-    ;
-    async resolveBlobContent(blobId, signal) {
-        const computable = this.getByKey(blobId);
-        const blob = await computable.awaitStableValue(signal);
-        return await this.blobDriver.getContent(blob.handle, { signal });
+    makeDataSource(signal) {
+        return {
+            preloadBlob: async (blobIds) => {
+                try {
+                    await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));
+                }
+                catch (err) {
+                    if (!isAbortError(err))
+                        throw err;
+                }
+            },
+            resolveBlobContent: async (blobId) => {
+                const computable = this.getByKey(blobId);
+                const blob = await computable.awaitStableValue(signal);
+                return await this.blobDriver.getContent(blob.handle, { signal });
+            },
+        };
     }
-    ;
 }
-class RemoteBlobPool extends RefCountResourcePool {
+class RemoteBlobPool extends RefCountPoolBase {
     blobDriver;
-    constructor(blobDriver) {
+    logger;
+    constructor(blobDriver, logger) {
         super();
         this.blobDriver = blobDriver;
+        this.logger = logger;
     }
     calculateParamsKey(params) {
-        return String(params.rid);
+        return makeBlobId(params);
     }
     createNewResource(params, _key) {
         return this.blobDriver.getOnDemandBlob(params);
     }
+    getByKey(blobId) {
+        const resource = super.tryGetByKey(blobId);
+        if (!resource)
+            throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);
+        return resource;
+    }
     async withContent(handle, options) {
         return await this.blobDriver.withContent(handle, {
             range: {
@@ -75,10 +82,10 @@ class RemoteBlobPool extends RefCountResourcePool {
     }
 }
 class BlobStore extends PFrameInternal.BaseObjectStore {
-    remoteBlobPool;
+    remoteBlobProvider;
     constructor(options) {
         super(options);
-        this.remoteBlobPool = options.remoteBlobPool;
+        this.remoteBlobProvider = options.remoteBlobProvider;
     }
     async request(filename, params) {
         const blobId = filename.slice(0, -PFrameInternal.ParquetExtension.length);
@@ -91,7 +98,7 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
             }
         };
         try {
-            const computable = this.remoteBlobPool.tryGetByKey(blobId);
+            const computable = this.remoteBlobProvider.tryGetByKey(blobId);
             if (!computable)
                 return await respond({ type: 'NotFound' });
             let blob;
@@ -119,7 +126,7 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
             }
             this.logger('info', `PFrames blob store requesting content for ${blobId}, `
                 + `range [${translatedRange.start}..=${translatedRange.end}]`);
-            return await this.remoteBlobPool.withContent(blob.handle, {
+            return await this.remoteBlobProvider.withContent(blob.handle, {
                 range: translatedRange,
                 signal: params.signal,
                 handler: async (data) => {
@@ -141,885 +148,59 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
         }
     }
 }
-const valueTypes = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'];
-function migrateFilters(filters, logger) {
-    const filtersV1 = [];
-    const filtersV2 = [];
-    for (const filter of filters) {
-        if (filter.type === 'bySingleColumn') {
-            filtersV1.push(filter);
-            filtersV2.push({
-                ...filter,
-                type: 'bySingleColumnV2',
-            });
-        }
-        else {
-            filtersV2.push(filter);
-        }
-    }
-    if (filtersV1.length > 0) {
-        const filtersV1Json = JSON.stringify(filtersV1);
-        logger('warn', `type overriten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
-    }
-    return filtersV2;
-}
-function migratePTableFilters(def, logger) {
-    if (!('partitionFilters' in def)) {
-        // For old blocks assume all axes filters to be partition filters
-        return {
-            ...def,
-            partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis'), logger),
-            filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column'), logger),
-        };
-    }
-    return {
-        ...def,
-        partitionFilters: migrateFilters(def.partitionFilters, logger),
-        filters: migrateFilters(def.filters, logger),
-    };
-}
-function hasArtificialColumns(entry) {
-    switch (entry.type) {
-        case 'column':
-        case 'slicedColumn':
-        case 'inlineColumn':
-            return false;
-        case 'artificialColumn':
-            return true;
-        case 'full':
-        case 'inner':
-            return entry.entries.some(hasArtificialColumns);
-        case 'outer':
-            return hasArtificialColumns(entry.primary) || entry.secondary.some(hasArtificialColumns);
-        default:
-            assertNever(entry);
-    }
-}
-const bigintReplacer = (_, v) => (typeof v === 'bigint' ? v.toString() : v);
-class PFramePool extends RefCountResourcePool {
-    parquetServer;
-    localBlobPool;
-    remoteBlobPool;
-    logger;
-    spillPath;
-    constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath) {
-        super();
-        this.parquetServer = parquetServer;
-        this.localBlobPool = localBlobPool;
-        this.remoteBlobPool = remoteBlobPool;
-        this.logger = logger;
-        this.spillPath = spillPath;
-    }
-    calculateParamsKey(params) {
-        try {
-            return stableKeyFromPFrameData(params);
-        }
-        catch (err) {
-            if (isPFrameDriverError(err))
-                throw err;
-            throw new PFrameDriverError(`PFrame handle calculation failed, `
-                + `request: ${JSON.stringify(params, bigintReplacer)}, `
-                + `error: ${ensureError(err)}`);
-        }
-    }
-    createNewResource(params, key) {
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `PFrame creation (pFrameHandle = ${key}): `
-                + `${JSON.stringify(params, bigintReplacer)}`);
-        }
-        return new PFrameHolder(this.parquetServer, this.localBlobPool, this.remoteBlobPool, this.logger, this.spillPath, params);
-    }
-    getByKey(key) {
-        const resource = super.tryGetByKey(key);
-        if (!resource)
-            throw new PFrameDriverError(`PFrame not found, handle = ${key}`);
-        return resource;
-    }
-}
-class PTableDefPool extends RefCountResourcePool {
-    logger;
-    constructor(logger) {
-        super();
-        this.logger = logger;
-    }
-    calculateParamsKey(params) {
-        return stableKeyFromFullPTableDef(params);
-    }
-    createNewResource(params, key) {
-        return new PTableDefHolder(params, key, this.logger);
-    }
-    getByKey(key) {
-        const resource = super.tryGetByKey(key);
-        if (!resource)
-            throw new PFrameDriverError(`PTable definition not found, handle = ${key}`);
-        return resource;
-    }
-}
-class PTablePool extends RefCountResourcePool {
-    pFrames;
-    pTableDefs;
-    logger;
-    constructor(pFrames, pTableDefs, logger) {
-        super();
-        this.pFrames = pFrames;
-        this.pTableDefs = pTableDefs;
-        this.logger = logger;
-    }
-    calculateParamsKey(params) {
-        return stableKeyFromFullPTableDef(params);
-    }
-    createNewResource(params, key) {
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `PTable creation (pTableHandle = ${key}): `
-                + `${JSON.stringify(params, bigintReplacer)}`);
-        }
-        const handle = params.pFrameHandle;
-        const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
-        const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
-        const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
-        // 3. Sort
-        if (params.def.sorting.length > 0) {
-            const predecessor = this.acquire({
-                ...params,
-                def: {
-                    ...params.def,
-                    sorting: [],
-                },
-            });
-            const { resource: { pTablePromise } } = predecessor;
-            const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
-            return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
-        }
-        // 2. Filter (except the case with artificial columns where cartesian creates too many rows)
-        if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
-            const predecessor = this.acquire({
-                ...params,
-                def: {
-                    ...params.def,
-                    filters: [],
-                },
-            });
-            const { resource: { pTablePromise } } = predecessor;
-            const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
-            return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
-        }
-        // 1. Join
-        const table = pFramePromise.then((pFrame) => pFrame.createTable({
-            src: joinEntryToInternal(params.def.src),
-            // `params.def.filters` would be non-empty only when join has artificial columns
-            filters: [...params.def.partitionFilters, ...params.def.filters],
-        }));
-        return new PTableHolder(handle, combinedSignal, table);
-    }
-    getByKey(key) {
-        const resource = super.tryGetByKey(key);
-        if (!resource)
-            throw new PFrameDriverError(`PTable not found, handle = ${key}`);
-        return resource;
-    }
-}
-class PTableCacheUi {
-    logger;
-    ops;
-    perFrame = new Map();
-    global;
-    disposeListeners = new Set();
-    constructor(logger, ops) {
-        this.logger = logger;
-        this.ops = ops;
-        this.global = new LRUCache({
-            maxSize: this.ops.pFramesCacheMaxSize,
-            dispose: (resource, key, reason) => {
-                if (reason === 'evict') {
-                    this.perFrame.get(resource.resource.pFrame)?.delete(key);
-                }
-                if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {
-                    this.perFrame.delete(resource.resource.pFrame);
-                }
-                resource.unref();
-                if (getDebugFlags().logPFrameRequests) {
-                    logger('info', `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
-                }
-            },
-        });
-    }
-    cache(resource, size) {
-        const key = resource.key;
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `calculateTableData cache - added PTable ${key} with size ${size}`);
-        }
-        this.global.set(key, resource, { size: Math.max(size, 1) }); // 1 is minimum size to avoid cache evictions
-        let perFrame = this.perFrame.get(resource.resource.pFrame);
-        if (!perFrame) {
-            perFrame = new LRUCache({
-                max: this.ops.pFrameCacheMaxCount,
-                dispose: (_resource, key, reason) => {
-                    if (reason === 'evict') {
-                        this.global.delete(key);
-                    }
-                },
-            });
-            this.perFrame.set(resource.resource.pFrame, perFrame);
-        }
-        perFrame.set(key, resource);
-        if (!this.disposeListeners.has(key)) {
-            const disposeListener = () => {
-                this.perFrame.get(resource.resource.pFrame)?.delete(key);
-                this.global.delete(key);
-                this.disposeListeners.delete(key);
-                resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
-            };
-            this.disposeListeners.add(key);
-            resource.resource.disposeSignal.addEventListener('abort', disposeListener);
-        }
-    }
-}
-class PTableCacheModel {
-    logger;
-    global;
-    disposeListeners = new Set();
-    constructor(logger, ops) {
-        this.logger = logger;
-        this.global = new LRUCache({
-            maxSize: ops.pTablesCacheMaxSize,
-            dispose: (resource, key, reason) => {
-                resource.unref();
-                if (getDebugFlags().logPFrameRequests) {
-                    logger('info', `createPTable cache - removed PTable ${key} (reason: ${reason})`);
-                }
-            },
-        });
-    }
-    cache(resource, size, defDisposeSignal) {
-        const key = resource.key;
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `createPTable cache - added PTable ${key} with size ${size}`);
-        }
-        const status = {};
-        this.global.set(key, resource, { size: Math.max(size, 1), status }); // 1 is minimum size to avoid cache evictions
-        if (status.maxEntrySizeExceeded) {
-            resource.unref();
-            if (getDebugFlags().logPFrameRequests) {
-                this.logger('info', `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);
-            }
-        }
-        else {
-            if (!this.disposeListeners.has(key)) {
-                const disposeListener = () => {
-                    this.global.delete(key);
-                    this.disposeListeners.delete(key);
-                    defDisposeSignal.removeEventListener('abort', disposeListener);
-                };
-                this.disposeListeners.add(key);
-                defDisposeSignal.addEventListener('abort', disposeListener);
-            }
-        }
-    }
-}
-class PFrameHolder {
-    parquetServer;
-    localBlobPool;
-    remoteBlobPool;
-    spillPath;
-    pFramePromise;
-    abortController = new AbortController();
-    localBlobs = [];
-    remoteBlobs = [];
-    constructor(parquetServer, localBlobPool, remoteBlobPool, logger, spillPath, columns) {
-        this.parquetServer = parquetServer;
-        this.localBlobPool = localBlobPool;
-        this.remoteBlobPool = remoteBlobPool;
-        this.spillPath = spillPath;
-        const makeLocalBlobId = (blob) => {
-            const localBlob = this.localBlobPool.acquire(blob);
-            this.localBlobs.push(localBlob);
-            return localBlob.key;
-        };
-        const makeRemoteBlobId = (blob) => {
-            const remoteBlob = this.remoteBlobPool.acquire(blob);
-            this.remoteBlobs.push(remoteBlob);
-            return remoteBlob.key + PFrameInternal.ParquetExtension;
-        };
-        const mapColumnData = (data) => {
-            switch (data.type) {
-                case 'Json':
-                    return { ...data };
-                case 'JsonPartitioned':
-                    return {
-                        ...data,
-                        parts: mapValues(data.parts, makeLocalBlobId),
-                    };
-                case 'BinaryPartitioned':
-                    return {
-                        ...data,
-                        parts: mapValues(data.parts, (v) => ({
-                            index: makeLocalBlobId(v.index),
-                            values: makeLocalBlobId(v.values),
-                        })),
-                    };
-                case 'ParquetPartitioned':
-                    return {
-                        ...data,
-                        parts: mapValues(data.parts, (v) => ({
-                            ...v,
-                            data: makeRemoteBlobId(v.data),
-                        })),
-                    };
-                default:
-                    assertNever(data);
-            }
-        };
-        const jsonifiedColumns = columns.map((column) => ({
-            ...column,
-            data: mapColumnData(column.data),
-        }));
-        try {
-            const pFrame = PFrameFactory.createPFrame({ spillPath: this.spillPath, logger });
-            pFrame.setDataSource(this);
-            const promises = [];
-            for (const column of jsonifiedColumns) {
-                pFrame.addColumnSpec(column.id, column.spec);
-                promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));
-            }
-            this.pFramePromise = Promise.all(promises)
-                .then(() => pFrame)
-                .catch((err) => {
-                this.dispose();
-                pFrame.dispose();
-                throw new PFrameDriverError(`PFrame creation failed asynchronously, `
-                    + `columns: ${JSON.stringify(jsonifiedColumns)}, `
-                    + `error: ${ensureError(err)}`);
-            });
-        }
-        catch (err) {
-            throw new PFrameDriverError(`PFrame creation failed synchronously, `
-                + `columns: ${JSON.stringify(jsonifiedColumns)}, `
-                + `error: ${ensureError(err)}`);
-        }
-    }
-    preloadBlob = async (blobIds) => {
-        return await this.localBlobPool.preloadBlob(blobIds, this.disposeSignal);
-    };
-    resolveBlobContent = async (blobId) => {
-        return await this.localBlobPool.resolveBlobContent(blobId, this.disposeSignal);
-    };
-    get disposeSignal() {
-        return this.abortController.signal;
-    }
-    dispose() {
-        this.abortController.abort();
-        this.localBlobs.forEach((entry) => entry.unref());
-        this.remoteBlobs.forEach((entry) => entry.unref());
-    }
-    async [Symbol.asyncDispose]() {
-        this.dispose();
-        await this.pFramePromise
-            .then((pFrame) => pFrame.dispose())
-            .catch(() => { });
-    }
-}
-class PTableDefHolder {
-    def;
-    pTableHandle;
-    logger;
-    abortController = new AbortController();
-    constructor(def, pTableHandle, logger) {
-        this.def = def;
-        this.pTableHandle = pTableHandle;
-        this.logger = logger;
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `PTable definition saved (pTableHandle = ${this.pTableHandle})`);
-        }
-    }
-    get disposeSignal() {
-        return this.abortController.signal;
-    }
-    [Symbol.dispose]() {
-        this.abortController.abort();
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);
-        }
-    }
-}
-class PTableHolder {
-    pFrame;
-    pTablePromise;
-    predecessor;
-    abortController = new AbortController();
-    combinedDisposeSignal;
-    constructor(pFrame, pFrameDisposeSignal, pTablePromise, predecessor) {
-        this.pFrame = pFrame;
-        this.pTablePromise = pTablePromise;
-        this.predecessor = predecessor;
-        this.combinedDisposeSignal = AbortSignal.any([pFrameDisposeSignal, this.abortController.signal]);
-    }
-    get disposeSignal() {
-        return this.combinedDisposeSignal;
-    }
-    async [Symbol.asyncDispose]() {
-        this.abortController.abort();
-        await this.pTablePromise
-            .then((pTable) => pTable.dispose())
-            .catch(() => { });
-        this.predecessor?.unref();
-    }
-}
-class PFrameDriver {
-    logger;
+class RemoteBlobProviderImpl {
+    pool;
     server;
-    pFrames;
-    pTableDefs;
-    pTables;
-    pTableCacheUi;
-    pTableCacheModel;
-    frameConcurrencyLimiter;
-    tableConcurrencyLimiter;
-    async pprofDump() {
-        return await PFrameFactory.pprofDump();
-    }
-    static async init(blobDriver, miLogger, spillPath, ops) {
-        const resolvedSpillPath = path.resolve(spillPath);
-        await emptyDir(resolvedSpillPath);
-        const logger = (level, message) => miLogger[level](message);
-        const localBlobPool = new LocalBlobPool(blobDriver);
-        const remoteBlobPool = new RemoteBlobPool(blobDriver);
-        const store = new BlobStore({ remoteBlobPool, logger });
-        const handler = HttpHelpers.createRequestHandler({ store: store });
-        const server = await HttpHelpers.createHttpServer({ handler, port: ops.parquetServerPort });
-        return new PFrameDriver(logger, server, localBlobPool, remoteBlobPool, resolvedSpillPath, ops);
-    }
-    constructor(logger, server, localBlobPool, remoteBlobPool, spillPath, ops) {
-        this.logger = logger;
+    constructor(pool, server) {
+        this.pool = pool;
         this.server = server;
-        const concurrencyLimiter = new ConcurrencyLimitingExecutor(ops.pFrameConcurrency);
-        this.frameConcurrencyLimiter = concurrencyLimiter;
-        this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(ops.pTableConcurrency);
-        this.pFrames = new PFramePool(server.info, localBlobPool, remoteBlobPool, logger, spillPath);
-        this.pTableDefs = new PTableDefPool(logger);
-        this.pTables = new PTablePool(this.pFrames, this.pTableDefs, logger);
-        this.pTableCacheUi = new PTableCacheUi(logger, ops);
-        this.pTableCacheModel = new PTableCacheModel(logger, ops);
-    }
-    async dispose() {
-        return await this.server.stop();
-    }
-    async [Symbol.asyncDispose]() {
-        return await this.dispose();
-    }
-    //
-    // Internal / Config API Methods
-    //
-    createPFrame(def, ctx) {
-        const columns = def
-            .filter((c) => valueTypes.find((t) => t === c.spec.valueType))
-            .map((c) => mapPObjectData(c, (d) => isPlTreeNodeAccessor(d)
-            ? parseDataInfoResource(d)
-            : isDataInfo(d)
-                ? d.type === 'ParquetPartitioned'
-                    ? mapDataInfo(d, (a) => traverseParquetChunkResource(a))
-                    : mapDataInfo(d, (a) => a.persist())
-                : makeDataInfoFromPColumnValues(c.spec, d)));
-        const distinctColumns = uniqueBy(columns, (column) => column.id);
-        const res = this.pFrames.acquire(distinctColumns);
-        ctx.addOnDestroy(res.unref);
-        return res.key;
     }
-    createPTable(rawDef, ctx) {
-        const def = migratePTableFilters(rawDef, this.logger);
-        const pFrameHandle = this.createPFrame(extractAllColumns(def.src), ctx);
-        const defIds = mapPTableDef(def, (c) => c.id);
-        const sortedDef = sortPTableDef(defIds);
-        const { key, unref } = this.pTableDefs.acquire({ def: sortedDef, pFrameHandle });
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${key})`);
-        }
-        ctx.addOnDestroy(unref); // in addition to pframe unref added in createPFrame above
-        return key;
+    static async init(blobDriver, logger, serverOptions) {
+        const pool = new RemoteBlobPool(blobDriver, logger);
+        const store = new BlobStore({ remoteBlobProvider: pool, logger });
+        const handler = HttpHelpers.createRequestHandler({ store });
+        const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });
+        logger('info', `PFrames HTTP server started on ${server.info.url}`);
+        return new RemoteBlobProviderImpl(pool, server);
     }
-    //
-    // PFrame istance methods
-    //
-    async findColumns(handle, request) {
-        const iRequest = {
-            ...request,
-            compatibleWith: request.compatibleWith.length !== 0
-                ? [{
-                        axesSpec: [
-                            ...new Map(request.compatibleWith.map((item) => [canonicalize(item), item])).values(),
-                        ],
-                        qualifications: [],
-                    }]
-                : [],
-        };
-        const { pFramePromise } = this.pFrames.getByKey(handle);
-        const pFrame = await pFramePromise;
-        const responce = await pFrame.findColumns(iRequest);
-        return {
-            hits: responce.hits
-                .filter((h) => // only exactly matching columns
-             h.mappingVariants.length === 0
-                || h.mappingVariants.some((v) => v.qualifications.forHit.length === 0
-                    && v.qualifications.forQueries.every((q) => q.length === 0)))
-                .map((h) => h.hit),
-        };
+    acquire(params) {
+        return this.pool.acquire(params);
     }
-    async getColumnSpec(handle, columnId) {
-        const { pFramePromise } = this.pFrames.getByKey(handle);
-        const pFrame = await pFramePromise;
-        return await pFrame.getColumnSpec(columnId);
+    httpServerInfo() {
+        return this.server.info;
     }
-    async listColumns(handle) {
-        const { pFramePromise } = this.pFrames.getByKey(handle);
-        const pFrame = await pFramePromise;
-        return await pFrame.listColumns();
-    }
-    async calculateTableData(handle, request, range, signal) {
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
-        }
-        const table = this.pTables.acquire({
-            pFrameHandle: handle,
-            def: sortPTableDef(migratePTableFilters(request, this.logger)),
-        });
-        const { pTablePromise, disposeSignal } = table.resource;
-        const pTable = await pTablePromise;
-        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
-        return await this.frameConcurrencyLimiter.run(async () => {
-            try {
-                const spec = pTable.getSpec();
-                const data = await pTable.getData([...spec.keys()], {
-                    range,
-                    signal: combinedSignal,
-                });
-                const resultSize = await pTable.getFootprint({
-                    withPredecessors: false,
-                    signal: combinedSignal,
-                });
-                if (resultSize >= 2 * 1024 * 1024 * 1024) {
-                    throw new PFrameDriverError(`Join results exceed 2GB, please add filters to shrink the result size`);
-                }
-                const overallSize = await pTable.getFootprint({
-                    withPredecessors: true,
-                    signal: combinedSignal,
-                });
-                this.pTableCacheUi.cache(table, overallSize);
-                return spec.map((spec, i) => ({
-                    spec: spec,
-                    data: data[i],
-                }));
-            }
-            catch (err) {
-                table.unref();
-                throw err;
-            }
-        });
-    }
-    async getUniqueValues(handle, request, signal) {
-        if (getDebugFlags().logPFrameRequests) {
-            this.logger('info', `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
-        }
-        const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
-        const pFrame = await pFramePromise;
-        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
-        return await this.frameConcurrencyLimiter.run(async () => {
-            return await pFrame.getUniqueValues({
-                ...request,
-                filters: migrateFilters(request.filters, this.logger),
-            }, {
-                signal: combinedSignal,
-            });
-        });
-    }
-    //
-    // PTable istance methods
-    //
-    async getSpec(handle) {
-        const env_1 = { stack: [], error: void 0, hasError: false };
-        try {
-            const { def } = this.pTableDefs.getByKey(handle);
-            const table = __addDisposableResource(env_1, this.pTables.acquire(def), false);
-            const { pTablePromise } = table.resource;
-            const pTable = await pTablePromise;
-            return pTable.getSpec();
-        }
-        catch (e_1) {
-            env_1.error = e_1;
-            env_1.hasError = true;
-        }
-        finally {
-            __disposeResources(env_1);
-        }
-    }
-    async getShape(handle, signal) {
-        const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
-        const table = this.pTables.acquire(def);
-        const { pTablePromise, disposeSignal } = table.resource;
-        const pTable = await pTablePromise;
-        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
-        const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
-            const shape = await pTable.getShape({
-                signal: combinedSignal,
-            });
-            const overallSize = await pTable.getFootprint({
-                withPredecessors: true,
-                signal: combinedSignal,
-            });
-            return { shape, overallSize };
-        });
-        this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
-        return shape;
-    }
-    async getData(handle, columnIndices, range, signal) {
-        const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
-        const table = this.pTables.acquire(def);
-        const { pTablePromise, disposeSignal } = table.resource;
-        const pTable = await pTablePromise;
-        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
-        const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
-            const data = await pTable.getData(columnIndices, {
-                range,
-                signal: combinedSignal,
-            });
-            const overallSize = await pTable.getFootprint({
-                withPredecessors: true,
-                signal: combinedSignal,
-            });
-            return { data, overallSize };
-        });
-        this.pTableCacheModel.cache(table, overallSize, defDisposeSignal);
-        return data;
-    }
-}
-function joinEntryToInternal(entry) {
-    const type = entry.type;
-    switch (type) {
-        case 'column':
-            return {
-                type: 'column',
-                columnId: entry.column,
-            };
-        case 'slicedColumn':
-            return {
-                type: 'slicedColumn',
-                columnId: entry.column,
-                newId: entry.newId,
-                axisFilters: entry.axisFilters,
-            };
-        case 'artificialColumn':
-            return {
-                type: 'artificialColumn',
-                columnId: entry.column,
-                newId: entry.newId,
-                axesIndices: entry.axesIndices,
-            };
-        case 'inlineColumn':
-            return {
-                type: 'inlineColumn',
-                newId: entry.column.id,
-                spec: entry.column.spec,
-                dataInfo: {
-                    type: 'Json',
-                    keyLength: entry.column.spec.axesSpec.length,
-                    data: entry.column.data.reduce((acc, row) => {
-                        acc[JSON.stringify(row.key)] = row.val;
-                        return acc;
-                    }, {}),
-                },
-            };
-        case 'inner':
-        case 'full':
-            return {
-                type: entry.type,
-                entries: entry.entries.map((col) => joinEntryToInternal(col)),
-            };
-        case 'outer':
-            return {
-                type: 'outer',
-                primary: joinEntryToInternal(entry.primary),
-                secondary: entry.secondary.map((col) => joinEntryToInternal(col)),
-            };
-        default:
-            throw new PFrameDriverError(`unsupported PFrame join entry type: ${type}`);
-    }
-}
-function sortPTableDef(def) {
-    function cmpJoinEntries(lhs, rhs) {
-        if (lhs.type !== rhs.type) {
-            return lhs.type < rhs.type ? -1 : 1;
-        }
-        const type = lhs.type;
-        switch (type) {
-            case 'column':
-                return lhs.column < rhs.column ? -1 : 1;
-            case 'slicedColumn':
-            case 'artificialColumn':
-                return lhs.newId < rhs.newId ? -1 : 1;
-            case 'inlineColumn': {
-                return lhs.column.id < rhs.column.id ? -1 : 1;
-            }
-            case 'inner':
-            case 'full': {
-                const rhsInner = rhs;
-                if (lhs.entries.length !== rhsInner.entries.length) {
-                    return lhs.entries.length - rhsInner.entries.length;
-                }
-                for (let i = 0; i < lhs.entries.length; i++) {
-                    const cmp = cmpJoinEntries(lhs.entries[i], rhsInner.entries[i]);
-                    if (cmp !== 0) {
-                        return cmp;
-                    }
-                }
-                return 0;
-            }
-            case 'outer': {
-                const rhsOuter = rhs;
-                const cmp = cmpJoinEntries(lhs.primary, rhsOuter.primary);
-                if (cmp !== 0) {
-                    return cmp;
-                }
-                if (lhs.secondary.length !== rhsOuter.secondary.length) {
-                    return lhs.secondary.length - rhsOuter.secondary.length;
-                }
-                for (let i = 0; i < lhs.secondary.length; i++) {
-                    const cmp = cmpJoinEntries(lhs.secondary[i], rhsOuter.secondary[i]);
-                    if (cmp !== 0) {
-                        return cmp;
-                    }
-                }
-                return 0;
-            }
-            default:
-                assertNever(type);
-        }
-    }
-    function sortJoinEntry(entry) {
-        switch (entry.type) {
-            case 'column':
-            case 'slicedColumn':
-            case 'inlineColumn':
-                return entry;
-            case 'artificialColumn': {
-                const sortedAxesIndices = entry.axesIndices.toSorted((lhs, rhs) => lhs - rhs);
-                return {
-                    ...entry,
-                    axesIndices: sortedAxesIndices,
-                };
-            }
-            case 'inner':
-            case 'full': {
-                const sortedEntries = entry.entries.map(sortJoinEntry);
-                sortedEntries.sort(cmpJoinEntries);
-                return {
-                    ...entry,
-                    entries: sortedEntries,
-                };
-            }
-            case 'outer': {
-                const sortedSecondary = entry.secondary.map(sortJoinEntry);
-                sortedSecondary.sort(cmpJoinEntries);
-                return {
-                    ...entry,
-                    primary: sortJoinEntry(entry.primary),
-                    secondary: sortedSecondary,
-                };
-            }
-            default:
-                assertNever(entry);
-        }
-    }
-    function sortFilters(filters) {
-        return filters.toSorted((lhs, rhs) => {
-            if (lhs.column.type === 'axis' && rhs.column.type === 'axis') {
-                const lhsId = canonicalizeJson(getAxisId(lhs.column.id));
-                const rhsId = canonicalizeJson(getAxisId(rhs.column.id));
-                return lhsId < rhsId ? -1 : 1;
-            }
-            else if (lhs.column.type === 'column' && rhs.column.type === 'column') {
-                return lhs.column.id < rhs.column.id ? -1 : 1;
-            }
-            else {
-                return lhs.column.type === 'axis' ? -1 : 1;
-            }
-        });
-    }
-    return {
-        src: sortJoinEntry(def.src),
-        partitionFilters: sortFilters(def.partitionFilters),
-        filters: sortFilters(def.filters),
-        sorting: def.sorting,
+    async [Symbol.asyncDispose]() {
+        await this.server.stop();
+    }
+}
+const PFrameDriverOpsDefaults = {
+    ...AbstractPFrameDriverOpsDefaults,
+    parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS
+};
+async function createPFrameDriver(params) {
+    const resolvedSpillPath = path.resolve(params.spillPath);
+    await emptyDir(resolvedSpillPath);
+    const logger = (level, message) => params.logger[level](message);
+    const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
+    const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
+    const resolveDataInfo = (spec, data) => {
+        return isPlTreeNodeAccessor(data)
+            ? parseDataInfoResource(data)
+            : isDataInfo(data)
+                ? data.type === 'ParquetPartitioned'
+                    ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))
+                    : mapDataInfo(data, (a) => a.persist())
+                : makeJsonDataInfo(spec, data);
     };
-}
-function stableKeyFromFullPTableDef(data) {
-    try {
-        const hash = createHash('sha256');
-        hash.update(canonicalize(data));
-        return hash.digest().toString('hex');
-    }
-    catch (err) {
-        throw new PFrameDriverError(`PTable handle calculation failed, `
-            + `request: ${JSON.stringify(data)}, `
-            + `error: ${ensureError(err)}`);
-    }
-}
-function stableKeyFromPFrameData(data) {
-    const orderedData = [...data].map((column) => mapPObjectData(column, (r) => {
-        let result;
-        const type = r.type;
-        switch (type) {
-            case 'Json':
-                result = {
-                    type: r.type,
-                    keyLength: r.keyLength,
-                    payload: Object.entries(r.data).map(([part, value]) => ({
-                        key: part,
-                        value,
-                    })),
-                };
-                break;
-            case 'JsonPartitioned':
-                result = {
-                    type: r.type,
-                    keyLength: r.partitionKeyLength,
-                    payload: Object.entries(r.parts).map(([part, info]) => ({
-                        key: part,
-                        value: makeBlobId(info),
-                    })),
-                };
-                break;
-            case 'BinaryPartitioned':
-                result = {
-                    type: r.type,
-                    keyLength: r.partitionKeyLength,
-                    payload: Object.entries(r.parts).map(([part, info]) => ({
-                        key: part,
-                        value: [makeBlobId(info.index), makeBlobId(info.values)],
-                    })),
-                };
-                break;
-            case 'ParquetPartitioned':
-                result = {
-                    type: r.type,
-                    keyLength: r.partitionKeyLength,
-                    payload: Object.entries(r.parts).map(([part, info]) => ({
-                        key: part,
-                        value: info.dataDigest || [
-                            makeBlobId(info.data),
-                            JSON.stringify({ axes: info.axes, column: info.column }),
-                        ],
-                    })),
-                };
-                break;
-            default:
-                throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type)}`);
-        }
-        result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);
-        return result;
-    }));
-    orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);
-    const hash = createHash('sha256');
-    hash.update(canonicalize(orderedData));
-    return hash.digest().toString('hex');
+    return new AbstractPFrameDriver({
+        logger,
+        localBlobProvider,
+        remoteBlobProvider,
+        spillPath: resolvedSpillPath,
+        options: params.options,
+        resolveDataInfo,
+    });
 }
 
-export { PFrameDriver };
+export { PFrameDriverOpsDefaults, createPFrameDriver };
 //# sourceMappingURL=driver.js.map