@milaboratories/pf-driver 1.7.7 → 1.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/dist/driver_impl.cjs +2 -1
  2. package/dist/driver_impl.cjs.map +1 -1
  3. package/dist/driver_impl.d.ts.map +1 -1
  4. package/dist/driver_impl.js +2 -1
  5. package/dist/driver_impl.js.map +1 -1
  6. package/dist/ptable_shared.cjs +28 -1
  7. package/dist/ptable_shared.cjs.map +1 -1
  8. package/dist/ptable_shared.js +28 -2
  9. package/dist/ptable_shared.js.map +1 -1
  10. package/package.json +6 -6
  11. package/src/driver_impl.ts +4 -1
  12. package/src/ptable_shared.test.ts +105 -0
  13. package/src/ptable_shared.ts +26 -1
package/dist/driver_impl.cjs CHANGED
@@ -2,6 +2,7 @@ const require_runtime = require("./_virtual/_rolldown/runtime.cjs");
2
2
  const require_csv_writer = require("./csv_writer.cjs");
3
3
  const require_logging = require("./logging.cjs");
4
4
  const require_pframe_pool = require("./pframe_pool.cjs");
5
+ const require_ptable_shared = require("./ptable_shared.cjs");
5
6
  const require_ptable_def_pool = require("./ptable_def_pool.cjs");
6
7
  const require_ptable_pool = require("./ptable_pool.cjs");
7
8
  const require_ptable_cache_per_frame = require("./ptable_cache_per_frame.cjs");
@@ -112,7 +113,7 @@ var AbstractPFrameDriver = class {
112
113
  const pFrameGuard = _usingCtx3.u(new _milaboratories_helpers.PoolEntryGuard(this.createPFrame(columns)));
113
114
  const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;
114
115
  const sortedQuery = (0, _milaboratories_pl_model_common.sortSpecQuery)((0, _milaboratories_pl_model_common.mapSpecQueryColumns)(def.query, (c) => c.id));
115
- const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);
116
+ const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(require_ptable_shared.embedInlineColumnTypeSpecs(sortedQuery));
116
117
  const pTableGuard = _usingCtx3.u(new _milaboratories_helpers.PoolEntryGuard(this.pTableDefs.acquire({
117
118
  pFrameHandle: pFrameGuard.key,
118
119
  tableSpec,
package/dist/driver_impl.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"driver_impl.cjs","names":["PTableCachePerFrameOpsDefaults","PTableCachePlainOpsDefaults","PFrameFactory","ConcurrencyLimitingExecutor","PFramePool","PTableDefPool","PTablePool","PTableCachePerFrame","PTableCachePlain","ValueType","PoolEntryGuard","logPFrames","streamPTableRows","fs","Readable","zlib","columnLabel","XLSX_MAX_ROWS_PER_SHEET","PFrameDriverError","bigintReplacer"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n PFrameDriverError,\n XLSX_MAX_ROWS_PER_SHEET,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { expandAxes, findTableColumn, rewriteLegacyFilters } from \"@milaboratories/pframes-rs-wasm\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { columnLabel, streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n ExportPTableOptions,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n // `combinedSignal` is passed twice on purpose: into the native calls for cooperative\n // cancellation, and into `run` so the concurrency slot is released on abort even if a\n // native call fails to unwind. Dropping the `run` signal reintroduces the queue wedge.\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { result, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { result: { path: options.path, rowsWritten, bytesWritten }, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return result;\n }\n\n public async exportPTable(\n handle: PTableHandle,\n options: ExportPTableOptions,\n signal?: AbortSignal,\n ): Promise<void> {\n const { path, columnIndices } = options;\n\n this.logger(\"info\", `[ExportPTable] ENTER (handle = ${handle}, path = ${path})`);\n const startTime = performance.now();\n\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n const headers: Record<number, string> = {};\n for (const index of columnIndices) {\n headers[index] = columnLabel(def.tableSpec[index]);\n }\n\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const overallSize = await this.tableConcurrencyLimiter.run(async () => {\n // Cap data rows per xlsx sheet below Excel's hard limit of 1,048,576.\n if (path.toLowerCase().endsWith(\".xlsx\")) {\n const shape = await pTable.getShape({ signal: combinedSignal });\n if (shape.rows > XLSX_MAX_ROWS_PER_SHEET) {\n const error = new PFrameDriverError(`exportPTable failed`);\n error.cause = new Error(\n `xlsx export rejected: ${shape.rows} rows exceed the per-sheet ` +\n `limit of ${XLSX_MAX_ROWS_PER_SHEET} rows`,\n );\n throw error;\n }\n }\n\n await pTable.export(path, headers, { signal: combinedSignal });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[ExportPTable] complete (handle = ${handle}, path = ${path}), ` +\n `duration = ${durationMs}ms`,\n );\n }\n\n return overallSize;\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { columnSpecs } = this.pFrames.getByKey(handle);\n return columnSpecs[columnId] ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { data, overallSize } = await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 200k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, columnSpecs, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const columnSpec = columnSpecs[request.columnId];\n if (!columnSpec) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);\n throw error;\n }\n\n const axisIds = expandAxes(columnSpec.axesSpec);\n const tableSpec: PTableColumnSpec[] = [\n ...columnSpec.axesSpec.map(\n (axisSpec, i): PTableColumnSpec => ({\n type: \"axis\",\n id: axisIds[i],\n spec: axisSpec,\n }),\n ),\n { type: \"column\", id: request.columnId, spec: columnSpec },\n ];\n\n let axisIndex: number | undefined;\n if (request.axis !== undefined) {\n const index = findTableColumn(tableSpec, { type: \"axis\", id: request.axis });\n if (index < 0) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(\n `axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,\n );\n throw error;\n }\n axisIndex = index;\n }\n\n const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {\n columnId: request.columnId,\n axisIndex,\n filters: rewriteLegacyFilters({\n tableSpec,\n filters: migrateFilters(request.filters, this.logger),\n }),\n limit: request.limit,\n };\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });\n }, combinedSignal);\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { shape, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA4FA,MAAa,kCAA2D;CACtE,GAAGA,+BAAAA;CACH,GAAGC,2BAAAA;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAMC,gCAAAA,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,aAAA,GAAA,QAAA,SAAoB,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAIC,2BAAAA,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAIA,2BAAAA,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAIC,oBAAAA,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAIC,wBAAAA,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAIC,oBAAAA,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAIC,+BAAAA,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAIC,2BAAAA,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOC,gCAAAA,UAAU,CAAC;EAKpD,MAAM,WAAA,GAAA,gCAAA,UAJmB,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,OAAA,GAAA,gCAAA,0BAA+B,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,OAAA,GAAA,gCAAA,gBAClB,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,mBAAqC,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAIC,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAA,GAAA,gCAAA,eACJ,oBAAA,GAAA,gCAAA,cACe,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,yBAA2C,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAID,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,eAAA,GAAA,gCAAA,gBAAA,GAAA,gCAAA,qBAAgD,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAAc,YAAY;GAEtE,MAAM,cAAA,WAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAID,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,EAAA,GAAA,wBAAA,OAAO,EAAE,CAAC,CAC3E;GAQD,MAAM,EAAE,QAAQ,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACjF,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAWE,mBAAAA,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,WAAA,GAAA,2BAAA,sBAA2B,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcC,QAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAASC,YAAAA,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,QAAA,GAAA,qBAAA,UAAe,QADFC,UAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,QAAA,GAAA,qBAAA,UAAe,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,WAAW,SAAS;IAI9C,MAAM,cAAc,aAAa;AAEjC,QAAIJ,gBAAAA,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,QAAQ;MAAE,MAAM,QAAQ;MAAM;MAAa;MAAc;KAAE;KAAa;MAChF,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,aACX,QACA,SACA,QACe;;;GACf,MAAM,EAAE,MAAM,kBAAkB;AAEhC,QAAK,OAAO,QAAQ,kCAAkC,OAAO,WAAW,KAAK,GAAG;GAChF,MAAM,YAAY,YAAY,KAAK;GAEnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAID,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAwB,EAAA,GAAA,wBAAA,OAAO,EAAE,CAAC,CACnE;GAED,MAAM,UAAkC,EAAE;AAC1C,QAAK,MAAM,SAAS,cAClB,SAAQ,SAASM,mBAAAA,YAAY,IAAI,UAAU,OAAO;GAMpD,MAAM,cAAc,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAErE,QAAI,KAAK,aAAa,CAAC,SAAS,QAAQ,EAAE;KACxC,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;AAC/D,SAAI,MAAM,OAAOC,gCAAAA,yBAAyB;MACxC,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,sBAAsB;AAC1D,YAAM,wBAAQ,IAAI,MAChB,yBAAyB,MAAM,KAAK,sCACtBD,gCAAAA,wBAAwB,OACvC;AACD,YAAM;;;AAIV,UAAM,OAAO,OAAO,MAAM,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAE9D,MAAM,cAAc,MAAM,WAAW,SAAS;AAE9C,QAAIN,gBAAAA,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,qCAAqC,OAAO,WAAW,KAAK,gBAC5C,WAAW,IAC5B;;AAGH,WAAO;MACN,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;;;;;;;CAO/E,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,EAAA,GAAA,gCAAA,kBAAkB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,gBAAgB,KAAK,QAAQ,SAAS,OAAO;AACrD,SAAO,YAAY,aAAa;;CAGlC,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAIA,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAASQ,gCAAAA,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,aAAA,GAAA,gCAAA,eAA0B,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAIT,wBAAAA,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAIlF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MANI,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;MACvD;MACA,QAAQ;MACT,CAAC;KAGa,aADK,MAAM,WAAW,SAAS;KAClB;MAC3B,eAAe;AAElB,QAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAC9D,UAAO,UAAU,KAAK,MAAM,OAAO;IAC3B;IACN,MAAM,KAAK;IACZ,EAAE;;;;;;;CAGL,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAASQ,gCAAAA,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,aAAa,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EACvF,MAAM,aAAa,MAAM;EAEzB,MAAM,aAAa,YAAY,QAAQ;AACvC,MAAI,CAAC,YAAY;GACf,MAAM,QAAQ,IAAID,gCAAAA,kBAAkB,yBAAyB;AAC7D,SAAM,wBAAQ,IAAI,MAAM,UAAU,QAAQ,SAAS,kCAAkC;AACrF,SAAM;;EAGR,MAAM,WAAA,GAAA,gCAAA,YAAqB,WAAW,SAAS;EAC/C,MAAM,YAAgC,CACpC,GAAG,WAAW,SAAS,KACpB,UAAU,OAAyB;GAClC,MAAM;GACN,IAAI,QAAQ;GACZ,MAAM;GACP,EACF,EACD;GAAE,MAAM;GAAU,IAAI,QAAQ;GAAU,MAAM;GAAY,CAC3D;EAED,IAAI;AACJ,MAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,MAAM,SAAA,GAAA,gCAAA,iBAAwB,WAAW;IAAE,MAAM;IAAQ,IAAI,QAAQ;IAAM,CAAC;AAC5E,OAAI,QAAQ,GAAG;IACb,MAAM,QAAQ,IAAIA,gCAAAA,kBAAkB,yBAAyB;AAC7D,UAAM,wBAAQ,IAAI,MAChB,QAAQ,KAAK,UAAU,QAAQ,KAAK,CAAC,kCAAkC,QAAQ,WAChF;AACD,UAAM;;AAER,eAAY;;EAGd,MAAM,kBAAwD;GAC5D,UAAU,QAAQ;GAClB;GACA,UAAA,GAAA,gCAAA,sBAA8B;IAC5B;IACA,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,CAAC;GACF,OAAO,QAAQ;GAChB;EAED,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBAAgB,iBAAiB,EAAE,QAAQ,gBAAgB,CAAC;KACnF,eAAe;;CAOpB,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIR,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAOhF,WAAO;KAAE,OANK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAIc,aAFI,MAAM,WAAW,SAAS;KAEjB;MAC5B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIA,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAQ/E,WAAO;KAAE,MAPI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAIa,aAFK,MAAM,WAAW,SAAS;KAElB;MAC3B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,MAAA,GAAA,wBAAA,OAAU,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
1
+ {"version":3,"file":"driver_impl.cjs","names":["PTableCachePerFrameOpsDefaults","PTableCachePlainOpsDefaults","PFrameFactory","ConcurrencyLimitingExecutor","PFramePool","PTableDefPool","PTablePool","PTableCachePerFrame","PTableCachePlain","ValueType","PoolEntryGuard","logPFrames","embedInlineColumnTypeSpecs","streamPTableRows","fs","Readable","zlib","columnLabel","XLSX_MAX_ROWS_PER_SHEET","PFrameDriverError","bigintReplacer"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n PFrameDriverError,\n XLSX_MAX_ROWS_PER_SHEET,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { expandAxes, findTableColumn, rewriteLegacyFilters } from \"@milaboratories/pframes-rs-wasm\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { columnLabel, streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n ExportPTableOptions,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport { embedInlineColumnTypeSpecs } from \"./ptable_shared\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(\n embedInlineColumnTypeSpecs(sortedQuery),\n );\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n // `combinedSignal` is passed twice on purpose: into the native calls for cooperative\n // cancellation, and into `run` so the concurrency slot is released on abort even if a\n // native call fails to unwind. Dropping the `run` signal reintroduces the queue wedge.\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { result, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { result: { path: options.path, rowsWritten, bytesWritten }, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return result;\n }\n\n public async exportPTable(\n handle: PTableHandle,\n options: ExportPTableOptions,\n signal?: AbortSignal,\n ): Promise<void> {\n const { path, columnIndices } = options;\n\n this.logger(\"info\", `[ExportPTable] ENTER (handle = ${handle}, path = ${path})`);\n const startTime = performance.now();\n\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n const headers: Record<number, string> = {};\n for (const index of columnIndices) {\n headers[index] = columnLabel(def.tableSpec[index]);\n }\n\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const overallSize = await this.tableConcurrencyLimiter.run(async () => {\n // Cap data rows per xlsx sheet below Excel's hard limit of 1,048,576.\n if (path.toLowerCase().endsWith(\".xlsx\")) {\n const shape = await pTable.getShape({ signal: combinedSignal });\n if (shape.rows > XLSX_MAX_ROWS_PER_SHEET) {\n const error = new PFrameDriverError(`exportPTable failed`);\n error.cause = new Error(\n `xlsx export rejected: ${shape.rows} rows exceed the per-sheet ` +\n `limit of ${XLSX_MAX_ROWS_PER_SHEET} rows`,\n );\n throw error;\n }\n }\n\n await pTable.export(path, headers, { signal: combinedSignal });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[ExportPTable] complete (handle = ${handle}, path = ${path}), ` +\n `duration = ${durationMs}ms`,\n );\n }\n\n return overallSize;\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { columnSpecs } = this.pFrames.getByKey(handle);\n return columnSpecs[columnId] ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { data, overallSize } = await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 200k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, columnSpecs, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const columnSpec = columnSpecs[request.columnId];\n if (!columnSpec) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);\n throw error;\n }\n\n const axisIds = expandAxes(columnSpec.axesSpec);\n const tableSpec: PTableColumnSpec[] = [\n ...columnSpec.axesSpec.map(\n (axisSpec, i): PTableColumnSpec => ({\n type: \"axis\",\n id: axisIds[i],\n spec: axisSpec,\n }),\n ),\n { type: \"column\", id: request.columnId, spec: columnSpec },\n ];\n\n let axisIndex: number | undefined;\n if (request.axis !== undefined) {\n const index = findTableColumn(tableSpec, { type: \"axis\", id: request.axis });\n if (index < 0) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(\n `axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,\n );\n throw error;\n }\n axisIndex = index;\n }\n\n const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {\n columnId: request.columnId,\n axisIndex,\n filters: rewriteLegacyFilters({\n tableSpec,\n filters: migrateFilters(request.filters, this.logger),\n }),\n limit: request.limit,\n };\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });\n }, combinedSignal);\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { shape, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AA6FA,MAAa,kCAA2D;CACtE,GAAGA,+BAAAA;CACH,GAAGC,2BAAAA;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAMC,gCAAAA,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,aAAA,GAAA,QAAA,SAAoB,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAIC,2BAAAA,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAIA,2BAAAA,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAIC,oBAAAA,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAIC,wBAAAA,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAIC,oBAAAA,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAIC,+BAAAA,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAIC,2BAAAA,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOC,gCAAAA,UAAU,CAAC;EAKpD,MAAM,WAAA,GAAA,gCAAA,UAJmB,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,OAAA,GAAA,gCAAA,0BAA+B,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,OAAA,GAAA,gCAAA,gBAClB,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,mBAAqC,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAIC,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAA,GAAA,gCAAA,eACJ,oBAAA,GAAA,gCAAA,cACe,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,yBAA2C,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAID,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,eAAA,GAAA,gCAAA,gBAAA,GAAA,gCAAA,qBAAgD,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAC1CE,sBAAAA,2BAA2B,YAAY,CACxC;GAED,MAAM,cAAA,WAAA,EAAc,IAAIF,wBAAAA,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAID,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,EAAA,GAAA,wBAAA,OAAO,EAAE,CAAC,CAC3E;GAQD,MAAM,EAAE,QAAQ,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACjF,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAWG,mBAAAA,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,WAAA,GAAA,2BAAA,sBAA2B,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcC,QAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAASC,YAAAA,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,QAAA,GAAA,qBAAA,UAAe,QADFC,UAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,QAAA,GAAA,qBAAA,UAAe,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,WAAW,SAAS;IAI9C,MAAM,cAAc,aAAa;AAEjC,QAAIL,gBAAAA,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,QAAQ;MAAE,MAAM,QAAQ;MAAM;MAAa;MAAc;KAAE;KAAa;MAChF,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,aACX,QACA,SACA,QACe;;;GACf,MAAM,EAAE,MAAM,kBAAkB;AAEhC,QAAK,OAAO,QAAQ,kCAAkC,OAAO,WAAW,KAAK,GAAG;GAChF,MAAM,YAAY,YAAY,KAAK;GAEnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAID,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAwB,EAAA,GAAA,wBAAA,OAAO,EAAE,CAAC,CACnE;GAED,MAAM,UAAkC,EAAE;AAC1C,QAAK,MAAM,SAAS,cAClB,SAAQ,SAASO,mBAAAA,YAAY,IAAI,UAAU,OAAO;GAMpD,MAAM,cAAc,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAErE,QAAI,KAAK,aAAa,CAAC,SAAS,QAAQ,EAAE;KACxC,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;AAC/D,SAAI,MAAM,OAAOC,gCAAAA,yBAAyB;MACxC,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,sBAAsB;AAC1D,YAAM,wBAAQ,IAAI,MAChB,yBAAyB,MAAM,KAAK,sCACtBD,gCAAAA,wBAAwB,OACvC;AACD,YAAM;;;AAIV,UAAM,OAAO,OAAO,MAAM,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAE9D,MAAM,cAAc,MAAM,WAAW,SAAS;AAE9C,QAAIP,gBAAAA,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,qCAAqC,OAAO,WAAW,KAAK,gBAC5C,WAAW,IAC5B;;AAGH,WAAO;MACN,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;;;;;;;CAO/E,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,EAAA,GAAA,gCAAA,kBAAkB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,gBAAgB,KAAK,QAAQ,SAAS,OAAO;AACrD,SAAO,YAAY,aAAa;;CAGlC,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAIA,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAASS,gCAAAA,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,aAAA,GAAA,gCAAA,eAA0B,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAIV,wBAAAA,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAIlF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MANI,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;MACvD;MACA,QAAQ;MACT,CAAC;KAGa,aADK,MAAM,WAAW,SAAS;KAClB;MAC3B,eAAe;AAElB,QAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAC9D,UAAO,UAAU,KAAK,MAAM,OAAO;IAC3B;IACN,MAAM,KAAK;IACZ,EAAE;;;;;;;CAGL,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAASS,gCAAAA,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,aAAa,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EACvF,MAAM,aAAa,MAAM;EAEzB,MAAM,aAAa,YAAY,QAAQ;AACvC,MAAI,CAAC,YAAY;GACf,MAAM,QAAQ,IAAID,gCAAAA,kBAAkB,yBAAyB;AAC7D,SAAM,wBAAQ,IAAI,MAAM,UAAU,QAAQ,SAAS,kCAAkC;AACrF,SAAM;;EAGR,MAAM,WAAA,GAAA,gCAAA,YAAqB,WAAW,SAAS;EAC/C,MAAM,YAAgC,CACpC,GAAG,WAAW,SAAS,KACpB,UAAU,OAAyB;GAClC,MAAM;GACN,IAAI,QAAQ;GACZ,MAAM;GACP,EACF,EACD;GAAE,MAAM;GAAU,IAAI,QAAQ;GAAU,MAAM;GAAY,CAC3D;EAED,IAAI;AACJ,MAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,MAAM,SAAA,GAAA,gCAAA,iBAAwB,WAAW;IAAE,MAAM;IAAQ,IAAI,QAAQ;IAAM,CAAC;AAC5E,OAAI,QAAQ,GAAG;IACb,MAAM,QAAQ,IAAIA,gCAAAA,kBAAkB,yBAAyB;AAC7D,UAAM,wBAAQ,IAAI,MAChB,QAAQ,KAAK,UAAU,QAAQ,KAAK,CAAC,kCAAkC,QAAQ,WAChF;AACD,UAAM;;AAER,eAAY;;EAGd,MAAM,kBAAwD;GAC5D,UAAU,QAAQ;GAClB;GACA,UAAA,GAAA,gCAAA,sBAA8B;IAC5B;IACA,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,CAAC;GACF,OAAO,QAAQ;GAChB;EAED,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBAAgB,iBAAiB,EAAE,QAAQ,gBAAgB,CAAC;KACnF,eAAe;;CAOpB,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIT,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAOhF,WAAO;KAAE,OANK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAIc,aAFI,MAAM,WAAW,SAAS;KAEjB;MAC5B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIA,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAQ/E,WAAO;KAAE,MAPI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAIa,aAFK,MAAM,WAAW,SAAS;KAElB;MAC3B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,MAAA,GAAA,wBAAA,OAAU,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
package/dist/driver_impl.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"driver_impl.d.ts","names":[],"sources":["../src/driver_impl.ts"],"mappings":";;;;;;;;;UA8EiB,iBAAA,mBAAoC,gBAAA,UAC3C,mBAAA,CAAsB,SAAA,GAAY,eAAA;AAAA,UAE3B,kBAAA,mBAAqC,gBAAA,UAC5C,oBAAA,CAAuB,SAAA,GAAY,eAAA;AAAA,KAEjC,uBAAA,GAA0B,sBAAA,GACpC,mBAAA;EAPmD,iFASjD,iBAAA,UARM;EAUN,iBAAA;AAAA;AAAA,cAGS,+BAAA,EAAiC,uBAAA;AAAA,KAOlC,gBAAA,gCAAgD,gBAAA,KAC1D,IAAA,EAAM,WAAA,EACN,IAAA,EAAM,WAAA,KACH,cAAA,CAAe,QAAA,CAAS,SAAA;AAAA,cAEhB,oBAAA,gCAEO,gBAAA,aACP,4BAAA,CAA6B,WAAA;EAAA,iBACvB,MAAA;EAAA,iBAEA,iBAAA;EAAA,iBACA,kBAAA;EAAA,iBAEA,eAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,UAAA;EAAA,iBACA,OAAA;EAAA,iBAEA,mBAAA;EAAA,iBACA,gBAAA;EAAA,iBAEA,uBAAA;EAAA,iBACA,uBAAA;EAEJ,SAAA,CAAA,GAAa,OAAA,CAAQ,UAAA;;IAKhC,MAAA;IACA,iBAAA;IACA,kBAAA;IACA,SAAA;IACA,OAAA;IACA;EAAA;IAEA,MAAA,GAAS,cAAA,CAAe,MAAA;IACxB,iBAAA,EAAmB,iBAAA,CAAkB,SAAA;IACrC,kBAAA,EAAoB,kBAAA,CAAmB,SAAA;IACvC,SAAA;IACA,OAAA,GAAU,uBAAA;IACV,eAAA,EAAiB,gBAAA,CAAiB,WAAA,EAAa,SAAA;EAAA;EAyB3C,OAAA,CAAA,GAAW,OAAA;EAAA,CAUV,MAAA,CAAO,YAAA,KAAiB,OAAA;EAQxB,YAAA,CAAa,GAAA,EAAK,SAAA,CAAU,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAa9D,YAAA,CAAa,MAAA,EAAQ,SAAA,CAAU,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAsCjE,cAAA,CAAe,GAAA,EAAK,WAAA,CAAY,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAmC5D,eAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,wBAAA,GACR,OAAA,CAAQ,uBAAA;EA+EE,YAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,qBAAA,EACT,MAAA,GAAS,WAAA,GACR,OAAA;EA4DU,WAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,kBAAA,GACR,OAAA,CAAQ,mBAAA;EAoCE,aAAA,CACX,MAAA,EAAQ,YAAA,EACR,QAAA,EAAU,SAAA,GACT,OAAA,CAAQ,WAAA;EAKE,WAAA,CAAY,MAAA,EAAQ,YAAA,GAAe,OAAA,CAAQ,gBAAA;EAK3C,kBAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,yBAAA,CAA0B,SAAA,GACnC,KAAA,EAAO,UAAA,cACP,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,0BAAA;EA4CE,eAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,mBAAA,EACT,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,oBAAA;EA+DE,OAAA,CAAQ,MAAA,EAAQ,YAAA,GAAe,OAAA,CAAQ,gBAAA;EAKvC,QAAA,CAAS,MAAA,EAAQ,YAAA,EAAc,MAAA,GAAS,WAAA,GAAc,OAAA,CAAQ,WAAA;EAsB9D,OAAA,CACX,MAAA,EAAQ,YAAA,EACR,aAAA,YACA,KAAA,EAAO,UAAA,cACP,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,YAAA;AAAA"}
1
+ {"version":3,"file":"driver_impl.d.ts","names":[],"sources":["../src/driver_impl.ts"],"mappings":";;;;;;;;;UA+EiB,iBAAA,mBAAoC,gBAAA,UAC3C,mBAAA,CAAsB,SAAA,GAAY,eAAA;AAAA,UAE3B,kBAAA,mBAAqC,gBAAA,UAC5C,oBAAA,CAAuB,SAAA,GAAY,eAAA;AAAA,KAEjC,uBAAA,GAA0B,sBAAA,GACpC,mBAAA;EAPmD,iFASjD,iBAAA,UARM;EAUN,iBAAA;AAAA;AAAA,cAGS,+BAAA,EAAiC,uBAAA;AAAA,KAOlC,gBAAA,gCAAgD,gBAAA,KAC1D,IAAA,EAAM,WAAA,EACN,IAAA,EAAM,WAAA,KACH,cAAA,CAAe,QAAA,CAAS,SAAA;AAAA,cAEhB,oBAAA,gCAEO,gBAAA,aACP,4BAAA,CAA6B,WAAA;EAAA,iBACvB,MAAA;EAAA,iBAEA,iBAAA;EAAA,iBACA,kBAAA;EAAA,iBAEA,eAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,UAAA;EAAA,iBACA,OAAA;EAAA,iBAEA,mBAAA;EAAA,iBACA,gBAAA;EAAA,iBAEA,uBAAA;EAAA,iBACA,uBAAA;EAEJ,SAAA,CAAA,GAAa,OAAA,CAAQ,UAAA;;IAKhC,MAAA;IACA,iBAAA;IACA,kBAAA;IACA,SAAA;IACA,OAAA;IACA;EAAA;IAEA,MAAA,GAAS,cAAA,CAAe,MAAA;IACxB,iBAAA,EAAmB,iBAAA,CAAkB,SAAA;IACrC,kBAAA,EAAoB,kBAAA,CAAmB,SAAA;IACvC,SAAA;IACA,OAAA,GAAU,uBAAA;IACV,eAAA,EAAiB,gBAAA,CAAiB,WAAA,EAAa,SAAA;EAAA;EAyB3C,OAAA,CAAA,GAAW,OAAA;EAAA,CAUV,MAAA,CAAO,YAAA,KAAiB,OAAA;EAQxB,YAAA,CAAa,GAAA,EAAK,SAAA,CAAU,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAa9D,YAAA,CAAa,MAAA,EAAQ,SAAA,CAAU,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAsCjE,cAAA,CAAe,GAAA,EAAK,WAAA,CAAY,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAqC5D,eAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,wBAAA,GACR,OAAA,CAAQ,uBAAA;EA+EE,YAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,qBAAA,EACT,MAAA,GAAS,WAAA,GACR,OAAA;EA4DU,WAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,kBAAA,GACR,OAAA,CAAQ,mBAAA;EAoCE,aAAA,CACX,MAAA,EAAQ,YAAA,EACR,QAAA,EAAU,SAAA,GACT,OAAA,CAAQ,WAAA;EAKE,WAAA,CAAY,MAAA,EAAQ,YAAA,GAAe,OAAA,CAAQ,gBAAA;EAK3C,kBAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,yBAAA,CAA0B,SAAA,GACnC,KAAA,EAAO,UAAA,cACP,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,0BAAA;EA4CE,eAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,mBAAA,EACT,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,oBAAA;EA+DE,OAAA,CAAQ,MAAA,EAAQ,YAAA,GAAe,OAAA,CAAQ,gBAAA;EAKvC,QAAA,CAAS,MAAA,EAAQ,YAAA,EAAc,MAAA,GAAS,WAAA,GAAc,OAAA,CAAQ,WAAA;EAsB9D,OAAA,CACX,MAAA,EAAQ,YAAA,EACR,aAAA,YACA,KAAA,EAAO,UAAA,cACP,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,YAAA;AAAA"}
package/dist/driver_impl.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { columnLabel, streamPTableRows } from "./csv_writer.js";
2
2
  import { logPFrames } from "./logging.js";
3
3
  import { PFramePool } from "./pframe_pool.js";
4
+ import { embedInlineColumnTypeSpecs } from "./ptable_shared.js";
4
5
  import { PTableDefPool } from "./ptable_def_pool.js";
5
6
  import { PTablePool } from "./ptable_pool.js";
6
7
  import { PTableCachePerFrame, PTableCachePerFrameOpsDefaults } from "./ptable_cache_per_frame.js";
@@ -109,7 +110,7 @@ var AbstractPFrameDriver = class {
109
110
  const pFrameGuard = _usingCtx3.u(new PoolEntryGuard(this.createPFrame(columns)));
110
111
  const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;
111
112
  const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));
112
- const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);
113
+ const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(embedInlineColumnTypeSpecs(sortedQuery));
113
114
  const pTableGuard = _usingCtx3.u(new PoolEntryGuard(this.pTableDefs.acquire({
114
115
  pFrameHandle: pFrameGuard.key,
115
116
  tableSpec,
package/dist/driver_impl.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"driver_impl.js","names":["fs"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n PFrameDriverError,\n XLSX_MAX_ROWS_PER_SHEET,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { expandAxes, findTableColumn, rewriteLegacyFilters } from \"@milaboratories/pframes-rs-wasm\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { columnLabel, streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n ExportPTableOptions,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n // `combinedSignal` is passed twice on purpose: into the native calls for cooperative\n // cancellation, and into `run` so the concurrency slot is released on abort even if a\n // native call fails to unwind. Dropping the `run` signal reintroduces the queue wedge.\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { result, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { result: { path: options.path, rowsWritten, bytesWritten }, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return result;\n }\n\n public async exportPTable(\n handle: PTableHandle,\n options: ExportPTableOptions,\n signal?: AbortSignal,\n ): Promise<void> {\n const { path, columnIndices } = options;\n\n this.logger(\"info\", `[ExportPTable] ENTER (handle = ${handle}, path = ${path})`);\n const startTime = performance.now();\n\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n const headers: Record<number, string> = {};\n for (const index of columnIndices) {\n headers[index] = columnLabel(def.tableSpec[index]);\n }\n\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const overallSize = await this.tableConcurrencyLimiter.run(async () => {\n // Cap data rows per xlsx sheet below Excel's hard limit of 1,048,576.\n if (path.toLowerCase().endsWith(\".xlsx\")) {\n const shape = await pTable.getShape({ signal: combinedSignal });\n if (shape.rows > XLSX_MAX_ROWS_PER_SHEET) {\n const error = new PFrameDriverError(`exportPTable failed`);\n error.cause = new Error(\n `xlsx export rejected: ${shape.rows} rows exceed the per-sheet ` +\n `limit of ${XLSX_MAX_ROWS_PER_SHEET} rows`,\n );\n throw error;\n }\n }\n\n await pTable.export(path, headers, { signal: combinedSignal });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[ExportPTable] complete (handle = ${handle}, path = ${path}), ` +\n `duration = ${durationMs}ms`,\n );\n }\n\n return overallSize;\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { columnSpecs } = this.pFrames.getByKey(handle);\n return columnSpecs[columnId] ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { data, overallSize } = await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 200k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, columnSpecs, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const columnSpec = columnSpecs[request.columnId];\n if (!columnSpec) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);\n throw error;\n }\n\n const axisIds = expandAxes(columnSpec.axesSpec);\n const tableSpec: PTableColumnSpec[] = [\n ...columnSpec.axesSpec.map(\n (axisSpec, i): PTableColumnSpec => ({\n type: \"axis\",\n id: axisIds[i],\n spec: axisSpec,\n }),\n ),\n { type: \"column\", id: request.columnId, spec: columnSpec },\n ];\n\n let axisIndex: number | undefined;\n if (request.axis !== undefined) {\n const index = findTableColumn(tableSpec, { type: \"axis\", id: request.axis });\n if (index < 0) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(\n `axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,\n );\n throw error;\n }\n axisIndex = index;\n }\n\n const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {\n columnId: request.columnId,\n axisIndex,\n filters: rewriteLegacyFilters({\n tableSpec,\n filters: migrateFilters(request.filters, this.logger),\n }),\n limit: request.limit,\n };\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });\n }, combinedSignal);\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { shape, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA4FA,MAAa,kCAA2D;CACtE,GAAG;CACH,GAAG;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAM,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,YAAY,QAAQ,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAI,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAI,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAI,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAI,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAI,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAO,UAAU,CAAC;EAKpD,MAAM,UADgB,SAHG,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,MAAM,yBAAyB,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,MACjC,eAAe,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,UAAU,SAAS,kBAAkB,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,YAAY,cAChB,mBACE,aAAa,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAI,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,UAAU,SAAS,wBAAwB,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAc,cAAc,oBAAoB,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAAc,YAAY;GAEtE,MAAM,cAAA,WAAA,EAAc,IAAI,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,CAAC,MAAM,EAAE,CAAC,CAC3E;GAQD,MAAM,EAAE,QAAQ,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACjF,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAW,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,UAAM,qBAAqB,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcA,KAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAAS,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,OAAM,SAAS,QADF,KAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,OAAM,SAAS,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,WAAW,SAAS;IAI9C,MAAM,cAAc,aAAa;AAEjC,QAAI,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,QAAQ;MAAE,MAAM,QAAQ;MAAM;MAAa;MAAc;KAAE;KAAa;MAChF,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,aACX,QACA,SACA,QACe;;;GACf,MAAM,EAAE,MAAM,kBAAkB;AAEhC,QAAK,OAAO,QAAQ,kCAAkC,OAAO,WAAW,KAAK,GAAG;GAChF,MAAM,YAAY,YAAY,KAAK;GAEnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAwB,CAAC,MAAM,EAAE,CAAC,CACnE;GAED,MAAM,UAAkC,EAAE;AAC1C,QAAK,MAAM,SAAS,cAClB,SAAQ,SAAS,YAAY,IAAI,UAAU,OAAO;GAMpD,MAAM,cAAc,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAErE,QAAI,KAAK,aAAa,CAAC,SAAS,QAAQ,EAAE;KACxC,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;AAC/D,SAAI,MAAM,OAAO,yBAAyB;MACxC,MAAM,QAAQ,IAAI,kBAAkB,sBAAsB;AAC1D,YAAM,wBAAQ,IAAI,MAChB,yBAAyB,MAAM,KAAK,sCACtB,wBAAwB,OACvC;AACD,YAAM;;;AAIV,UAAM,OAAO,OAAO,MAAM,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAE9D,MAAM,cAAc,MAAM,WAAW,SAAS;AAE9C,QAAI,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,qCAAqC,OAAO,WAAW,KAAK,gBAC5C,WAAW,IAC5B;;AAGH,WAAO;MACN,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;;;;;;;CAO/E,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,CAAC,iBAAiB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,gBAAgB,KAAK,QAAQ,SAAS,OAAO;AACrD,SAAO,YAAY,aAAa;;CAGlC,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAI,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,YAAY,cAAc,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAIlF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MANI,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;MACvD;MACA,QAAQ;MACT,CAAC;KAGa,aADK,MAAM,WAAW,SAAS;KAClB;MAC3B,eAAe;AAElB,QAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAC9D,UAAO,UAAU,KAAK,MAAM,OAAO;IAC3B;IACN,MAAM,KAAK;IACZ,EAAE;;;;;;;CAGL,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAI,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,aAAa,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EACvF,MAAM,aAAa,MAAM;EAEzB,MAAM,aAAa,YAAY,QAAQ;AACvC,MAAI,CAAC,YAAY;GACf,MAAM,QAAQ,IAAI,kBAAkB,yBAAyB;AAC7D,SAAM,wBAAQ,IAAI,MAAM,UAAU,QAAQ,SAAS,kCAAkC;AACrF,SAAM;;EAGR,MAAM,UAAU,WAAW,WAAW,SAAS;EAC/C,MAAM,YAAgC,CACpC,GAAG,WAAW,SAAS,KACpB,UAAU,OAAyB;GAClC,MAAM;GACN,IAAI,QAAQ;GACZ,MAAM;GACP,EACF,EACD;GAAE,MAAM;GAAU,IAAI,QAAQ;GAAU,MAAM;GAAY,CAC3D;EAED,IAAI;AACJ,MAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,MAAM,QAAQ,gBAAgB,WAAW;IAAE,MAAM;IAAQ,IAAI,QAAQ;IAAM,CAAC;AAC5E,OAAI,QAAQ,GAAG;IACb,MAAM,QAAQ,IAAI,kBAAkB,yBAAyB;AAC7D,UAAM,wBAAQ,IAAI,MAChB,QAAQ,KAAK,UAAU,QAAQ,KAAK,CAAC,kCAAkC,QAAQ,WAChF;AACD,UAAM;;AAER,eAAY;;EAGd,MAAM,kBAAwD;GAC5D,UAAU,QAAQ;GAClB;GACA,SAAS,qBAAqB;IAC5B;IACA,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,CAAC;GACF,OAAO,QAAQ;GAChB;EAED,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBAAgB,iBAAiB,EAAE,QAAQ,gBAAgB,CAAC;KACnF,eAAe;;CAOpB,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAOhF,WAAO;KAAE,OANK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAIc,aAFI,MAAM,WAAW,SAAS;KAEjB;MAC5B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAQ/E,WAAO;KAAE,MAPI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAIa,aAFK,MAAM,WAAW,SAAS;KAElB;MAC3B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,KAAI,MAAM,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
1
+ {"version":3,"file":"driver_impl.js","names":["fs"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n PFrameDriverError,\n XLSX_MAX_ROWS_PER_SHEET,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { expandAxes, findTableColumn, rewriteLegacyFilters } from \"@milaboratories/pframes-rs-wasm\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { columnLabel, streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n ExportPTableOptions,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport { embedInlineColumnTypeSpecs } from \"./ptable_shared\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(\n embedInlineColumnTypeSpecs(sortedQuery),\n );\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n // `combinedSignal` is passed twice on purpose: into the native calls for cooperative\n // cancellation, and into `run` so the concurrency slot is released on abort even if a\n // native call fails to unwind. Dropping the `run` signal reintroduces the queue wedge.\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { result, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { result: { path: options.path, rowsWritten, bytesWritten }, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return result;\n }\n\n public async exportPTable(\n handle: PTableHandle,\n options: ExportPTableOptions,\n signal?: AbortSignal,\n ): Promise<void> {\n const { path, columnIndices } = options;\n\n this.logger(\"info\", `[ExportPTable] ENTER (handle = ${handle}, path = ${path})`);\n const startTime = performance.now();\n\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n const headers: Record<number, string> = {};\n for (const index of columnIndices) {\n headers[index] = columnLabel(def.tableSpec[index]);\n }\n\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const overallSize = await this.tableConcurrencyLimiter.run(async () => {\n // Cap data rows per xlsx sheet below Excel's hard limit of 1,048,576.\n if (path.toLowerCase().endsWith(\".xlsx\")) {\n const shape = await pTable.getShape({ signal: combinedSignal });\n if (shape.rows > XLSX_MAX_ROWS_PER_SHEET) {\n const error = new PFrameDriverError(`exportPTable failed`);\n error.cause = new Error(\n `xlsx export rejected: ${shape.rows} rows exceed the per-sheet ` +\n `limit of ${XLSX_MAX_ROWS_PER_SHEET} rows`,\n );\n throw error;\n }\n }\n\n await pTable.export(path, headers, { signal: combinedSignal });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[ExportPTable] complete (handle = ${handle}, path = ${path}), ` +\n `duration = ${durationMs}ms`,\n );\n }\n\n return overallSize;\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { columnSpecs } = this.pFrames.getByKey(handle);\n return columnSpecs[columnId] ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n // keep()/cache live outside the limiter task: on abort `run` rejects and they are\n // skipped, so an abandoned (detached) operation never caches its result nor touches\n // `tableGuard` after the `using` block has already unref'd it.\n const { data, overallSize } = await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 200k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, columnSpecs, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const columnSpec = columnSpecs[request.columnId];\n if (!columnSpec) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);\n throw error;\n }\n\n const axisIds = expandAxes(columnSpec.axesSpec);\n const tableSpec: PTableColumnSpec[] = [\n ...columnSpec.axesSpec.map(\n (axisSpec, i): PTableColumnSpec => ({\n type: \"axis\",\n id: axisIds[i],\n spec: axisSpec,\n }),\n ),\n { type: \"column\", id: request.columnId, spec: columnSpec },\n ];\n\n let axisIndex: number | undefined;\n if (request.axis !== undefined) {\n const index = findTableColumn(tableSpec, { type: \"axis\", id: request.axis });\n if (index < 0) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(\n `axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,\n );\n throw error;\n }\n axisIndex = index;\n }\n\n const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {\n columnId: request.columnId,\n axisIndex,\n filters: rewriteLegacyFilters({\n tableSpec,\n filters: migrateFilters(request.filters, this.logger),\n }),\n limit: request.limit,\n };\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });\n }, combinedSignal);\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { shape, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await tableGuard.resource.cacheSize;\n\n return { data, overallSize };\n }, combinedSignal);\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA6FA,MAAa,kCAA2D;CACtE,GAAG;CACH,GAAG;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAM,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,YAAY,QAAQ,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAI,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAI,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAI,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAI,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAI,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAO,UAAU,CAAC;EAKpD,MAAM,UADgB,SAHG,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,MAAM,yBAAyB,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,MACjC,eAAe,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,UAAU,SAAS,kBAAkB,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,YAAY,cAChB,mBACE,aAAa,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAI,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,UAAU,SAAS,wBAAwB,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAc,cAAc,oBAAoB,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAC1C,2BAA2B,YAAY,CACxC;GAED,MAAM,cAAA,WAAA,EAAc,IAAI,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,CAAC,MAAM,EAAE,CAAC,CAC3E;GAQD,MAAM,EAAE,QAAQ,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACjF,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAW,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,UAAM,qBAAqB,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcA,KAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAAS,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,OAAM,SAAS,QADF,KAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,OAAM,SAAS,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,WAAW,SAAS;IAI9C,MAAM,cAAc,aAAa;AAEjC,QAAI,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,QAAQ;MAAE,MAAM,QAAQ;MAAM;MAAa;MAAc;KAAE;KAAa;MAChF,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,aACX,QACA,SACA,QACe;;;GACf,MAAM,EAAE,MAAM,kBAAkB;AAEhC,QAAK,OAAO,QAAQ,kCAAkC,OAAO,WAAW,KAAK,GAAG;GAChF,MAAM,YAAY,YAAY,KAAK;GAEnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAwB,CAAC,MAAM,EAAE,CAAC,CACnE;GAED,MAAM,UAAkC,EAAE;AAC1C,QAAK,MAAM,SAAS,cAClB,SAAQ,SAAS,YAAY,IAAI,UAAU,OAAO;GAMpD,MAAM,cAAc,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAErE,QAAI,KAAK,aAAa,CAAC,SAAS,QAAQ,EAAE;KACxC,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;AAC/D,SAAI,MAAM,OAAO,yBAAyB;MACxC,MAAM,QAAQ,IAAI,kBAAkB,sBAAsB;AAC1D,YAAM,wBAAQ,IAAI,MAChB,yBAAyB,MAAM,KAAK,sCACtB,wBAAwB,OACvC;AACD,YAAM;;;AAIV,UAAM,OAAO,OAAO,MAAM,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAE9D,MAAM,cAAc,MAAM,WAAW,SAAS;AAE9C,QAAI,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,qCAAqC,OAAO,WAAW,KAAK,gBAC5C,WAAW,IAC5B;;AAGH,WAAO;MACN,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;;;;;;;CAO/E,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,CAAC,iBAAiB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,gBAAgB,KAAK,QAAQ,SAAS,OAAO;AACrD,SAAO,YAAY,aAAa;;CAGlC,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAI,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,YAAY,cAAc,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAIlF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MANI,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;MACvD;MACA,QAAQ;MACT,CAAC;KAGa,aADK,MAAM,WAAW,SAAS;KAClB;MAC3B,eAAe;AAElB,QAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAC9D,UAAO,UAAU,KAAK,MAAM,OAAO;IAC3B;IACN,MAAM,KAAK;IACZ,EAAE;;;;;;;CAGL,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAI,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,aAAa,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EACvF,MAAM,aAAa,MAAM;EAEzB,MAAM,aAAa,YAAY,QAAQ;AACvC,MAAI,CAAC,YAAY;GACf,MAAM,QAAQ,IAAI,kBAAkB,yBAAyB;AAC7D,SAAM,wBAAQ,IAAI,MAAM,UAAU,QAAQ,SAAS,kCAAkC;AACrF,SAAM;;EAGR,MAAM,UAAU,WAAW,WAAW,SAAS;EAC/C,MAAM,YAAgC,CACpC,GAAG,WAAW,SAAS,KACpB,UAAU,OAAyB;GAClC,MAAM;GACN,IAAI,QAAQ;GACZ,MAAM;GACP,EACF,EACD;GAAE,MAAM;GAAU,IAAI,QAAQ;GAAU,MAAM;GAAY,CAC3D;EAED,IAAI;AACJ,MAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,MAAM,QAAQ,gBAAgB,WAAW;IAAE,MAAM;IAAQ,IAAI,QAAQ;IAAM,CAAC;AAC5E,OAAI,QAAQ,GAAG;IACb,MAAM,QAAQ,IAAI,kBAAkB,yBAAyB;AAC7D,UAAM,wBAAQ,IAAI,MAChB,QAAQ,KAAK,UAAU,QAAQ,KAAK,CAAC,kCAAkC,QAAQ,WAChF;AACD,UAAM;;AAER,eAAY;;EAGd,MAAM,kBAAwD;GAC5D,UAAU,QAAQ;GAClB;GACA,SAAS,qBAAqB;IAC5B;IACA,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,CAAC;GACF,OAAO,QAAQ;GAChB;EAED,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBAAgB,iBAAiB,EAAE,QAAQ,gBAAgB,CAAC;KACnF,eAAe;;CAOpB,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAOhF,WAAO;KAAE,OANK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAIc,aAFI,MAAM,WAAW,SAAS;KAEjB;MAC5B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAQ/E,WAAO;KAAE,MAPI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAIa,aAFK,MAAM,WAAW,SAAS;KAElB;MAC3B,eAAe;AAElB,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,KAAI,MAAM,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
package/dist/ptable_shared.cjs CHANGED
@@ -1,6 +1,32 @@
1
1
  require("./_virtual/_rolldown/runtime.cjs");
2
+ let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
2
3
  let _milaboratories_pl_model_middle_layer = require("@milaboratories/pl-model-middle-layer");
3
4
  //#region src/ptable_shared.ts
5
+ /**
6
+ * Make every inline column in a `SpecQuery` self-contained by embedding its
7
+ * column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the
8
+ * column's `PColumnSpec`.
9
+ */
10
+ function embedInlineColumnTypeSpecs(query) {
11
+ return (0, _milaboratories_pl_model_common.traverseQuerySpec)(query, {
12
+ column: (c) => c,
13
+ node: (node) => {
14
+ if (node.type !== "inlineColumn") return node;
15
+ const { axesSpec, valueType } = node.spec.spec;
16
+ const dataInfo = {
17
+ ...node.dataInfo,
18
+ typeSpec: {
19
+ axes: axesSpec.map((a) => a.type),
20
+ column: valueType
21
+ }
22
+ };
23
+ return {
24
+ ...node,
25
+ dataInfo
26
+ };
27
+ }
28
+ });
29
+ }
4
30
  function stableKeyFromFullPTableDef(data) {
5
31
  return (0, _milaboratories_pl_model_middle_layer.hashJson)(data);
6
32
  }
@@ -16,7 +42,7 @@ function lowerLegacyPTableDef(pFrameSpec, legacyDef) {
16
42
  filters: [...legacyDef.partitionFilters, ...legacyDef.filters],
17
43
  sorting: legacyDef.sorting
18
44
  };
19
- return pFrameSpec.evaluateQuery(pFrameSpec.rewriteLegacyQuery(legacy));
45
+ return pFrameSpec.evaluateQuery(embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)));
20
46
  }
21
47
  /**
22
48
  * Build a `FullPTableDef` from a legacy `PTableDef` plus the
@@ -31,6 +57,7 @@ function buildFullPTableDefFromLegacy(opts) {
31
57
  }
32
58
  //#endregion
33
59
  exports.buildFullPTableDefFromLegacy = buildFullPTableDefFromLegacy;
60
+ exports.embedInlineColumnTypeSpecs = embedInlineColumnTypeSpecs;
34
61
  exports.stableKeyFromFullPTableDef = stableKeyFromFullPTableDef;
35
62
 
36
63
  //# sourceMappingURL=ptable_shared.cjs.map
package/dist/ptable_shared.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"ptable_shared.cjs","names":[],"sources":["../src/ptable_shared.ts"],"sourcesContent":["import type {\n DataQuery,\n PFrameHandle,\n PObjectId,\n PTableColumnSpec,\n PTableDef,\n PTableHandle,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\n/**\n * PTable definition stored in the def / table pools. Always\n * carries a pre-lowered data query plus the corresponding output\n * `tableSpec`. Legacy `PTableDef<PObjectId>` inputs are lowered to\n * this shape via WASM-spec at acquire time.\n */\nexport type FullPTableDef = {\n pFrameHandle: PFrameHandle;\n tableSpec: PTableColumnSpec[];\n dataQuery: DataQuery;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n return hashJson(data) as string as PTableHandle;\n}\n\n/**\n * Lower a legacy `PTableDef<PObjectId>` to the data layer. Returns\n * the output `tableSpec` and the lowered `dataQuery`. Routes through\n * `rewriteLegacyQuery` + `evaluateQuery` on the caller-supplied\n * WASM-spec frame.\n */\nexport function lowerLegacyPTableDef(\n pFrameSpec: PFrameInternal.PFrameWasmV3,\n legacyDef: PTableDef<PObjectId>,\n): { tableSpec: PTableColumnSpec[]; dataQuery: DataQuery } {\n const legacy: PFrameInternal.LegacyQuery = {\n src: legacyDef.src,\n filters: [...legacyDef.partitionFilters, ...legacyDef.filters],\n sorting: legacyDef.sorting,\n };\n return pFrameSpec.evaluateQuery(pFrameSpec.rewriteLegacyQuery(legacy));\n}\n\n/**\n * Build a `FullPTableDef` from a legacy `PTableDef` plus the\n * PFrame's WASM-spec frame. Used by pool `acquireFromLegacy` methods\n * to keep lowering off the call site.\n */\nexport function buildFullPTableDefFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n}): FullPTableDef {\n return {\n pFrameHandle: opts.pFrameHandle,\n ...lowerLegacyPTableDef(opts.pFrameSpec, opts.def),\n };\n}\n"],"mappings":";;;AAsBA,SAAgB,2BAA2B,MAAmC;AAC5E,SAAA,GAAA,sCAAA,UAAgB,KAAK;;;;;;;;AASvB,SAAgB,qBACd,YACA,WACyD;CACzD,MAAM,SAAqC;EACzC,KAAK,UAAU;EACf,SAAS,CAAC,GAAG,UAAU,kBAAkB,GAAG,UAAU,QAAQ;EAC9D,SAAS,UAAU;EACpB;AACD,QAAO,WAAW,cAAc,WAAW,mBAAmB,OAAO,CAAC;;;;;;;AAQxE,SAAgB,6BAA6B,MAI3B;AAChB,QAAO;EACL,cAAc,KAAK;EACnB,GAAG,qBAAqB,KAAK,YAAY,KAAK,IAAI;EACnD"}
1
+ {"version":3,"file":"ptable_shared.cjs","names":[],"sources":["../src/ptable_shared.ts"],"sourcesContent":["import type {\n DataQuery,\n JsonDataInfo,\n PFrameHandle,\n PObjectId,\n PTableColumnSpec,\n PTableDef,\n PTableHandle,\n SpecQuery,\n} from \"@milaboratories/pl-model-common\";\nimport { traverseQuerySpec } from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\n/**\n * Make every inline column in a `SpecQuery` self-contained by embedding its\n * column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the\n * column's `PColumnSpec`.\n */\nexport function embedInlineColumnTypeSpecs(query: SpecQuery): SpecQuery {\n return traverseQuerySpec(query, {\n column: (c) => c,\n node: (node) => {\n if (node.type !== \"inlineColumn\") return node;\n const { axesSpec, valueType } = node.spec.spec;\n const dataInfo: JsonDataInfo & { typeSpec: PFrameInternal.PColumnValueTypeSpec } = {\n ...node.dataInfo,\n typeSpec: { axes: axesSpec.map((a) => a.type), column: valueType },\n };\n return { ...node, dataInfo };\n },\n });\n}\n\n/**\n * PTable definition stored in the def / table pools. Always\n * carries a pre-lowered data query plus the corresponding output\n * `tableSpec`. Legacy `PTableDef<PObjectId>` inputs are lowered to\n * this shape via WASM-spec at acquire time.\n */\nexport type FullPTableDef = {\n pFrameHandle: PFrameHandle;\n tableSpec: PTableColumnSpec[];\n dataQuery: DataQuery;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n return hashJson(data) as string as PTableHandle;\n}\n\n/**\n * Lower a legacy `PTableDef<PObjectId>` to the data layer. Returns\n * the output `tableSpec` and the lowered `dataQuery`. Routes through\n * `rewriteLegacyQuery` + `evaluateQuery` on the caller-supplied\n * WASM-spec frame.\n */\nexport function lowerLegacyPTableDef(\n pFrameSpec: PFrameInternal.PFrameWasmV3,\n legacyDef: PTableDef<PObjectId>,\n): { tableSpec: PTableColumnSpec[]; dataQuery: DataQuery } {\n const legacy: PFrameInternal.LegacyQuery = {\n src: legacyDef.src,\n filters: [...legacyDef.partitionFilters, ...legacyDef.filters],\n sorting: legacyDef.sorting,\n };\n return pFrameSpec.evaluateQuery(\n embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)),\n );\n}\n\n/**\n * Build a `FullPTableDef` from a legacy `PTableDef` plus the\n * PFrame's WASM-spec frame. Used by pool `acquireFromLegacy` methods\n * to keep lowering off the call site.\n */\nexport function buildFullPTableDefFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n}): FullPTableDef {\n return {\n pFrameHandle: opts.pFrameHandle,\n ...lowerLegacyPTableDef(opts.pFrameSpec, opts.def),\n };\n}\n"],"mappings":";;;;;;;;;AAkBA,SAAgB,2BAA2B,OAA6B;AACtE,SAAA,GAAA,gCAAA,mBAAyB,OAAO;EAC9B,SAAS,MAAM;EACf,OAAO,SAAS;AACd,OAAI,KAAK,SAAS,eAAgB,QAAO;GACzC,MAAM,EAAE,UAAU,cAAc,KAAK,KAAK;GAC1C,MAAM,WAA6E;IACjF,GAAG,KAAK;IACR,UAAU;KAAE,MAAM,SAAS,KAAK,MAAM,EAAE,KAAK;KAAE,QAAQ;KAAW;IACnE;AACD,UAAO;IAAE,GAAG;IAAM;IAAU;;EAE/B,CAAC;;AAeJ,SAAgB,2BAA2B,MAAmC;AAC5E,SAAA,GAAA,sCAAA,UAAgB,KAAK;;;;;;;;AASvB,SAAgB,qBACd,YACA,WACyD;CACzD,MAAM,SAAqC;EACzC,KAAK,UAAU;EACf,SAAS,CAAC,GAAG,UAAU,kBAAkB,GAAG,UAAU,QAAQ;EAC9D,SAAS,UAAU;EACpB;AACD,QAAO,WAAW,cAChB,2BAA2B,WAAW,mBAAmB,OAAO,CAAC,CAClE;;;;;;;AAQH,SAAgB,6BAA6B,MAI3B;AAChB,QAAO;EACL,cAAc,KAAK;EACnB,GAAG,qBAAqB,KAAK,YAAY,KAAK,IAAI;EACnD"}
package/dist/ptable_shared.js CHANGED
@@ -1,5 +1,31 @@
1
+ import { traverseQuerySpec } from "@milaboratories/pl-model-common";
1
2
  import { hashJson } from "@milaboratories/pl-model-middle-layer";
2
3
  //#region src/ptable_shared.ts
4
+ /**
5
+ * Make every inline column in a `SpecQuery` self-contained by embedding its
6
+ * column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the
7
+ * column's `PColumnSpec`.
8
+ */
9
+ function embedInlineColumnTypeSpecs(query) {
10
+ return traverseQuerySpec(query, {
11
+ column: (c) => c,
12
+ node: (node) => {
13
+ if (node.type !== "inlineColumn") return node;
14
+ const { axesSpec, valueType } = node.spec.spec;
15
+ const dataInfo = {
16
+ ...node.dataInfo,
17
+ typeSpec: {
18
+ axes: axesSpec.map((a) => a.type),
19
+ column: valueType
20
+ }
21
+ };
22
+ return {
23
+ ...node,
24
+ dataInfo
25
+ };
26
+ }
27
+ });
28
+ }
3
29
  function stableKeyFromFullPTableDef(data) {
4
30
  return hashJson(data);
5
31
  }
@@ -15,7 +41,7 @@ function lowerLegacyPTableDef(pFrameSpec, legacyDef) {
15
41
  filters: [...legacyDef.partitionFilters, ...legacyDef.filters],
16
42
  sorting: legacyDef.sorting
17
43
  };
18
- return pFrameSpec.evaluateQuery(pFrameSpec.rewriteLegacyQuery(legacy));
44
+ return pFrameSpec.evaluateQuery(embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)));
19
45
  }
20
46
  /**
21
47
  * Build a `FullPTableDef` from a legacy `PTableDef` plus the
@@ -29,6 +55,6 @@ function buildFullPTableDefFromLegacy(opts) {
29
55
  };
30
56
  }
31
57
  //#endregion
32
- export { buildFullPTableDefFromLegacy, stableKeyFromFullPTableDef };
58
+ export { buildFullPTableDefFromLegacy, embedInlineColumnTypeSpecs, stableKeyFromFullPTableDef };
33
59
 
34
60
  //# sourceMappingURL=ptable_shared.js.map
package/dist/ptable_shared.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"ptable_shared.js","names":[],"sources":["../src/ptable_shared.ts"],"sourcesContent":["import type {\n DataQuery,\n PFrameHandle,\n PObjectId,\n PTableColumnSpec,\n PTableDef,\n PTableHandle,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\n/**\n * PTable definition stored in the def / table pools. Always\n * carries a pre-lowered data query plus the corresponding output\n * `tableSpec`. Legacy `PTableDef<PObjectId>` inputs are lowered to\n * this shape via WASM-spec at acquire time.\n */\nexport type FullPTableDef = {\n pFrameHandle: PFrameHandle;\n tableSpec: PTableColumnSpec[];\n dataQuery: DataQuery;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n return hashJson(data) as string as PTableHandle;\n}\n\n/**\n * Lower a legacy `PTableDef<PObjectId>` to the data layer. Returns\n * the output `tableSpec` and the lowered `dataQuery`. Routes through\n * `rewriteLegacyQuery` + `evaluateQuery` on the caller-supplied\n * WASM-spec frame.\n */\nexport function lowerLegacyPTableDef(\n pFrameSpec: PFrameInternal.PFrameWasmV3,\n legacyDef: PTableDef<PObjectId>,\n): { tableSpec: PTableColumnSpec[]; dataQuery: DataQuery } {\n const legacy: PFrameInternal.LegacyQuery = {\n src: legacyDef.src,\n filters: [...legacyDef.partitionFilters, ...legacyDef.filters],\n sorting: legacyDef.sorting,\n };\n return pFrameSpec.evaluateQuery(pFrameSpec.rewriteLegacyQuery(legacy));\n}\n\n/**\n * Build a `FullPTableDef` from a legacy `PTableDef` plus the\n * PFrame's WASM-spec frame. Used by pool `acquireFromLegacy` methods\n * to keep lowering off the call site.\n */\nexport function buildFullPTableDefFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n}): FullPTableDef {\n return {\n pFrameHandle: opts.pFrameHandle,\n ...lowerLegacyPTableDef(opts.pFrameSpec, opts.def),\n };\n}\n"],"mappings":";;AAsBA,SAAgB,2BAA2B,MAAmC;AAC5E,QAAO,SAAS,KAAK;;;;;;;;AASvB,SAAgB,qBACd,YACA,WACyD;CACzD,MAAM,SAAqC;EACzC,KAAK,UAAU;EACf,SAAS,CAAC,GAAG,UAAU,kBAAkB,GAAG,UAAU,QAAQ;EAC9D,SAAS,UAAU;EACpB;AACD,QAAO,WAAW,cAAc,WAAW,mBAAmB,OAAO,CAAC;;;;;;;AAQxE,SAAgB,6BAA6B,MAI3B;AAChB,QAAO;EACL,cAAc,KAAK;EACnB,GAAG,qBAAqB,KAAK,YAAY,KAAK,IAAI;EACnD"}
1
+ {"version":3,"file":"ptable_shared.js","names":[],"sources":["../src/ptable_shared.ts"],"sourcesContent":["import type {\n DataQuery,\n JsonDataInfo,\n PFrameHandle,\n PObjectId,\n PTableColumnSpec,\n PTableDef,\n PTableHandle,\n SpecQuery,\n} from \"@milaboratories/pl-model-common\";\nimport { traverseQuerySpec } from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\n/**\n * Make every inline column in a `SpecQuery` self-contained by embedding its\n * column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the\n * column's `PColumnSpec`.\n */\nexport function embedInlineColumnTypeSpecs(query: SpecQuery): SpecQuery {\n return traverseQuerySpec(query, {\n column: (c) => c,\n node: (node) => {\n if (node.type !== \"inlineColumn\") return node;\n const { axesSpec, valueType } = node.spec.spec;\n const dataInfo: JsonDataInfo & { typeSpec: PFrameInternal.PColumnValueTypeSpec } = {\n ...node.dataInfo,\n typeSpec: { axes: axesSpec.map((a) => a.type), column: valueType },\n };\n return { ...node, dataInfo };\n },\n });\n}\n\n/**\n * PTable definition stored in the def / table pools. Always\n * carries a pre-lowered data query plus the corresponding output\n * `tableSpec`. Legacy `PTableDef<PObjectId>` inputs are lowered to\n * this shape via WASM-spec at acquire time.\n */\nexport type FullPTableDef = {\n pFrameHandle: PFrameHandle;\n tableSpec: PTableColumnSpec[];\n dataQuery: DataQuery;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n return hashJson(data) as string as PTableHandle;\n}\n\n/**\n * Lower a legacy `PTableDef<PObjectId>` to the data layer. Returns\n * the output `tableSpec` and the lowered `dataQuery`. Routes through\n * `rewriteLegacyQuery` + `evaluateQuery` on the caller-supplied\n * WASM-spec frame.\n */\nexport function lowerLegacyPTableDef(\n pFrameSpec: PFrameInternal.PFrameWasmV3,\n legacyDef: PTableDef<PObjectId>,\n): { tableSpec: PTableColumnSpec[]; dataQuery: DataQuery } {\n const legacy: PFrameInternal.LegacyQuery = {\n src: legacyDef.src,\n filters: [...legacyDef.partitionFilters, ...legacyDef.filters],\n sorting: legacyDef.sorting,\n };\n return pFrameSpec.evaluateQuery(\n embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)),\n );\n}\n\n/**\n * Build a `FullPTableDef` from a legacy `PTableDef` plus the\n * PFrame's WASM-spec frame. Used by pool `acquireFromLegacy` methods\n * to keep lowering off the call site.\n */\nexport function buildFullPTableDefFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n}): FullPTableDef {\n return {\n pFrameHandle: opts.pFrameHandle,\n ...lowerLegacyPTableDef(opts.pFrameSpec, opts.def),\n };\n}\n"],"mappings":";;;;;;;;AAkBA,SAAgB,2BAA2B,OAA6B;AACtE,QAAO,kBAAkB,OAAO;EAC9B,SAAS,MAAM;EACf,OAAO,SAAS;AACd,OAAI,KAAK,SAAS,eAAgB,QAAO;GACzC,MAAM,EAAE,UAAU,cAAc,KAAK,KAAK;GAC1C,MAAM,WAA6E;IACjF,GAAG,KAAK;IACR,UAAU;KAAE,MAAM,SAAS,KAAK,MAAM,EAAE,KAAK;KAAE,QAAQ;KAAW;IACnE;AACD,UAAO;IAAE,GAAG;IAAM;IAAU;;EAE/B,CAAC;;AAeJ,SAAgB,2BAA2B,MAAmC;AAC5E,QAAO,SAAS,KAAK;;;;;;;;AASvB,SAAgB,qBACd,YACA,WACyD;CACzD,MAAM,SAAqC;EACzC,KAAK,UAAU;EACf,SAAS,CAAC,GAAG,UAAU,kBAAkB,GAAG,UAAU,QAAQ;EAC9D,SAAS,UAAU;EACpB;AACD,QAAO,WAAW,cAChB,2BAA2B,WAAW,mBAAmB,OAAO,CAAC,CAClE;;;;;;;AAQH,SAAgB,6BAA6B,MAI3B;AAChB,QAAO;EACL,cAAc,KAAK;EACnB,GAAG,qBAAqB,KAAK,YAAY,KAAK,IAAI;EACnD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pf-driver",
3
- "version": "1.7.7",
3
+ "version": "1.7.8",
4
4
  "description": "PFrameDriver implementation abstracted from Middle Layer",
5
5
  "keywords": [],
6
6
  "license": "UNLICENSED",
@@ -24,10 +24,10 @@
24
24
  "@milaboratories/pframes-rs-wasm": "1.1.44",
25
25
  "es-toolkit": "^1.39.10",
26
26
  "lru-cache": "^11.2.2",
27
+ "@milaboratories/pl-model-common": "1.46.1",
28
+ "@milaboratories/pl-model-middle-layer": "1.30.4",
27
29
  "@milaboratories/helpers": "1.14.2",
28
- "@milaboratories/ts-helpers": "1.8.3",
29
- "@milaboratories/pl-model-middle-layer": "1.30.3",
30
- "@milaboratories/pl-model-common": "1.46.0"
30
+ "@milaboratories/ts-helpers": "1.8.3"
31
31
  },
32
32
  "devDependencies": {
33
33
  "@types/node": "~24.5.2",
@@ -35,9 +35,9 @@
35
35
  "fast-csv": "5.0.5",
36
36
  "typescript": "~5.9.3",
37
37
  "vitest": "^4.1.3",
38
- "@milaboratories/build-configs": "2.0.0",
38
+ "@milaboratories/ts-builder": "1.5.0",
39
39
  "@milaboratories/ts-configs": "1.2.3",
40
- "@milaboratories/ts-builder": "1.5.0"
40
+ "@milaboratories/build-configs": "2.0.0"
41
41
  },
42
42
  "engines": {
43
43
  "node": ">=22.19.0"
package/src/driver_impl.ts CHANGED
@@ -65,6 +65,7 @@ import {
65
65
  } from "./pframe_pool";
66
66
  import { PTableDefPool } from "./ptable_def_pool";
67
67
  import { PTablePool } from "./ptable_pool";
68
+ import { embedInlineColumnTypeSpecs } from "./ptable_shared";
68
69
  import {
69
70
  PTableCachePerFrame,
70
71
  PTableCachePerFrameOpsDefaults,
@@ -239,7 +240,9 @@ export class AbstractPFrameDriver<
239
240
  using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));
240
241
  const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;
241
242
  const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));
242
- const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);
243
+ const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(
244
+ embedInlineColumnTypeSpecs(sortedQuery),
245
+ );
243
246
 
244
247
  using pTableGuard = new PoolEntryGuard(
245
248
  this.pTableDefs.acquire({
package/src/ptable_shared.test.ts ADDED
@@ -0,0 +1,105 @@
1
+ import type {
2
+ JsonDataInfo,
3
+ PColumnSpec,
4
+ PObjectId,
5
+ SpecQuery,
6
+ } from "@milaboratories/pl-model-common";
7
+ import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
8
+ import { test } from "vitest";
9
+ import { embedInlineColumnTypeSpecs } from "./ptable_shared";
10
+
11
+ const COLUMN_SPEC: PColumnSpec = {
12
+ kind: "PColumn",
13
+ name: "inline1",
14
+ valueType: "Int",
15
+ axesSpec: [
16
+ { name: "a1", type: "String" },
17
+ { name: "a2", type: "Long" },
18
+ ],
19
+ };
20
+
21
+ function inlineColumnNode(): SpecQuery {
22
+ return {
23
+ type: "inlineColumn",
24
+ spec: { columnId: "col1" as PObjectId, spec: COLUMN_SPEC },
25
+ dataInfo: { type: "Json", keyLength: 2, data: { '["a",1]': 10 } },
26
+ };
27
+ }
28
+
29
+ /** Narrow to an inline column node and expose the (injected) typeSpec. */
30
+ function inlineDataInfo(query: SpecQuery): JsonDataInfo & {
31
+ typeSpec?: PFrameInternal.PColumnValueTypeSpec;
32
+ } {
33
+ if (query.type !== "inlineColumn") throw new Error(`expected inlineColumn, got ${query.type}`);
34
+ return query.dataInfo;
35
+ }
36
+
37
+ test("embedInlineColumnTypeSpecs injects typeSpec derived from the column spec", ({ expect }) => {
38
+ const result = embedInlineColumnTypeSpecs(inlineColumnNode());
39
+ const dataInfo = inlineDataInfo(result);
40
+
41
+ // typeSpec is the self-contained { axes, column } shape, derived from the
42
+ // inline column's PColumnSpec (axis types in order + the value type).
43
+ expect(dataInfo.typeSpec).toEqual({
44
+ axes: ["String", "Long"],
45
+ column: "Int",
46
+ });
47
+ });
48
+
49
+ test("embedInlineColumnTypeSpecs preserves existing dataInfo fields", ({ expect }) => {
50
+ const result = embedInlineColumnTypeSpecs(inlineColumnNode());
51
+ const dataInfo = inlineDataInfo(result);
52
+
53
+ expect(dataInfo.type).toBe("Json");
54
+ expect(dataInfo.keyLength).toBe(2);
55
+ expect(dataInfo.data).toEqual({ '["a",1]': 10 });
56
+ });
57
+
58
+ test("embedInlineColumnTypeSpecs leaves the inline column spec untouched", ({ expect }) => {
59
+ const result = embedInlineColumnTypeSpecs(inlineColumnNode());
60
+ if (result.type !== "inlineColumn") throw new Error("expected inlineColumn");
61
+ expect(result.spec).toEqual({ columnId: "col1" as PObjectId, spec: COLUMN_SPEC });
62
+ });
63
+
64
+ test("embedInlineColumnTypeSpecs traverses into nested join entries", ({ expect }) => {
65
+ const query: SpecQuery = {
66
+ type: "fullJoin",
67
+ entries: [{ entry: inlineColumnNode(), qualifications: [] }],
68
+ };
69
+
70
+ const result = embedInlineColumnTypeSpecs(query);
71
+ if (result.type !== "fullJoin") throw new Error("expected fullJoin");
72
+ const dataInfo = inlineDataInfo(result.entries[0].entry);
73
+
74
+ expect(dataInfo.typeSpec).toEqual({ axes: ["String", "Long"], column: "Int" });
75
+ });
76
+
77
+ test("embedInlineColumnTypeSpecs traverses outerJoin primary and secondary entries", ({
78
+ expect,
79
+ }) => {
80
+ const query: SpecQuery = {
81
+ type: "outerJoin",
82
+ primary: { entry: inlineColumnNode(), qualifications: [] },
83
+ secondary: [{ entry: inlineColumnNode(), qualifications: [] }],
84
+ };
85
+
86
+ const result = embedInlineColumnTypeSpecs(query);
87
+ if (result.type !== "outerJoin") throw new Error("expected outerJoin");
88
+
89
+ // Both traversal paths (primary via traverseEntry, secondary via map) must
90
+ // reach the inline column and embed its typeSpec.
91
+ expect(inlineDataInfo(result.primary.entry).typeSpec).toEqual({
92
+ axes: ["String", "Long"],
93
+ column: "Int",
94
+ });
95
+ expect(inlineDataInfo(result.secondary[0].entry).typeSpec).toEqual({
96
+ axes: ["String", "Long"],
97
+ column: "Int",
98
+ });
99
+ });
100
+
101
+ test("embedInlineColumnTypeSpecs leaves non-inline nodes unchanged", ({ expect }) => {
102
+ const query: SpecQuery = { type: "column", column: "col1" as PObjectId };
103
+ const result = embedInlineColumnTypeSpecs(query);
104
+ expect(result).toEqual(query);
105
+ });
package/src/ptable_shared.ts CHANGED
@@ -1,13 +1,36 @@
1
1
  import type {
2
2
  DataQuery,
3
+ JsonDataInfo,
3
4
  PFrameHandle,
4
5
  PObjectId,
5
6
  PTableColumnSpec,
6
7
  PTableDef,
7
8
  PTableHandle,
9
+ SpecQuery,
8
10
  } from "@milaboratories/pl-model-common";
11
+ import { traverseQuerySpec } from "@milaboratories/pl-model-common";
9
12
  import { hashJson, PFrameInternal } from "@milaboratories/pl-model-middle-layer";
10
13
 
14
+ /**
15
+ * Make every inline column in a `SpecQuery` self-contained by embedding its
16
+ * column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the
17
+ * column's `PColumnSpec`.
18
+ */
19
+ export function embedInlineColumnTypeSpecs(query: SpecQuery): SpecQuery {
20
+ return traverseQuerySpec(query, {
21
+ column: (c) => c,
22
+ node: (node) => {
23
+ if (node.type !== "inlineColumn") return node;
24
+ const { axesSpec, valueType } = node.spec.spec;
25
+ const dataInfo: JsonDataInfo & { typeSpec: PFrameInternal.PColumnValueTypeSpec } = {
26
+ ...node.dataInfo,
27
+ typeSpec: { axes: axesSpec.map((a) => a.type), column: valueType },
28
+ };
29
+ return { ...node, dataInfo };
30
+ },
31
+ });
32
+ }
33
+
11
34
  /**
12
35
  * PTable definition stored in the def / table pools. Always
13
36
  * carries a pre-lowered data query plus the corresponding output
@@ -39,7 +62,9 @@ export function lowerLegacyPTableDef(
39
62
  filters: [...legacyDef.partitionFilters, ...legacyDef.filters],
40
63
  sorting: legacyDef.sorting,
41
64
  };
42
- return pFrameSpec.evaluateQuery(pFrameSpec.rewriteLegacyQuery(legacy));
65
+ return pFrameSpec.evaluateQuery(
66
+ embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)),
67
+ );
43
68
  }
44
69
 
45
70
  /**