@milaboratories/pf-driver 1.4.17 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csv_writer.cjs.map +1 -1
- package/dist/csv_writer.js.map +1 -1
- package/dist/driver_impl.cjs +41 -5
- package/dist/driver_impl.cjs.map +1 -1
- package/dist/driver_impl.d.ts.map +1 -1
- package/dist/driver_impl.js +42 -6
- package/dist/driver_impl.js.map +1 -1
- package/dist/pframe_pool.cjs +9 -8
- package/dist/pframe_pool.cjs.map +1 -1
- package/dist/pframe_pool.js +9 -8
- package/dist/pframe_pool.js.map +1 -1
- package/dist/ptable_pool.cjs +2 -5
- package/dist/ptable_pool.cjs.map +1 -1
- package/dist/ptable_pool.js +2 -5
- package/dist/ptable_pool.js.map +1 -1
- package/package.json +7 -7
- package/src/csv_writer.ts +1 -1
- package/src/driver_impl.ts +46 -10
- package/src/pframe_pool.ts +16 -17
- package/src/ptable_pool.ts +3 -5
package/dist/csv_writer.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv_writer.cjs","names":["Annotation","ValueType"],"sources":["../src/csv_writer.ts"],"sourcesContent":["import {\n Annotation,\n isValueNA,\n readAnnotation,\n ValueType,\n type PTableColumnSpec,\n type PTableVector,\n type TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isNil } from \"@milaboratories/helpers\";\n\n/** Minimal subset of
|
|
1
|
+
{"version":3,"file":"csv_writer.cjs","names":["Annotation","ValueType"],"sources":["../src/csv_writer.ts"],"sourcesContent":["import {\n Annotation,\n isValueNA,\n readAnnotation,\n ValueType,\n type PTableColumnSpec,\n type PTableVector,\n type TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isNil } from \"@milaboratories/helpers\";\n\n/** Minimal subset of PTableV9 required by streamPTableRows. */\nexport interface PTableDataSource {\n getData(\n columnIndices: number[],\n options?: { range?: TableRange; signal?: AbortSignal },\n ): Promise<PTableVector[]>;\n}\n\n// ── Public API (high-level → low-level) ──────────────────────────────\n\n/** Format a CSV/TSV header row from column specs. Line ending is CRLF. */\nexport function formatHeader(specs: PTableColumnSpec[], separator: string): string {\n return specs.map((spec) => escapeField(columnLabel(spec), separator)).join(separator) + \"\\r\\n\";\n}\n\n/** Format a single data row from parallel vectors. Line ending is CRLF. */\nexport function formatRow(vectors: PTableVector[], rowIndex: number, separator: string): string {\n return (\n vectors\n .map((vector) => escapeField(serializeValue(vector, rowIndex), separator))\n .join(separator) + \"\\r\\n\"\n );\n}\n\n/**\n * Async generator that streams CSV/TSV content chunk by chunk.\n *\n * The caller is responsible for providing a concrete `range` (already clipped\n * to the table shape). When `range` is undefined the generator does nothing\n * beyond emitting an optional BOM and header.\n */\nexport interface StreamPTableRowsOptions {\n pTable: PTableDataSource;\n specs: PTableColumnSpec[];\n columnIndices: number[];\n range?: TableRange;\n chunkSize: number;\n separator: string;\n includeHeader: boolean;\n bom: boolean;\n signal?: AbortSignal;\n}\nexport async function* streamPTableRows(options: StreamPTableRowsOptions): AsyncIterable<string> {\n const { pTable, columnIndices, range, chunkSize, separator, signal, specs, includeHeader, bom } =\n options;\n\n if (bom) {\n yield \"\\uFEFF\";\n }\n\n if (includeHeader) {\n const selectedSpecs = columnIndices.map((index) => specs[index]);\n yield formatHeader(selectedSpecs, separator);\n }\n\n if (isNil(range)) {\n return;\n }\n\n const end = range.offset + range.length;\n\n for (let from = range.offset; from < end; from += chunkSize) {\n signal?.throwIfAborted();\n\n const length = Math.min(chunkSize, end - from);\n const subRange: TableRange = { offset: from, length };\n\n const vectors = await pTable.getData(columnIndices, { range: subRange, signal });\n\n const rows: string[] = [];\n for (let rowIndex = 0; rowIndex < length; rowIndex++) {\n rows.push(formatRow(vectors, rowIndex, separator));\n }\n yield rows.join(\"\");\n }\n}\n\n// ── Helpers (low-level) ──────────────────────────────────────────────\n\n/** Extract a human-readable label from a PTableColumnSpec. */\nfunction columnLabel(spec: PTableColumnSpec): string {\n const annotation = readAnnotation(spec.spec, Annotation.Label);\n return isNil(annotation) ? spec.spec.name : annotation.trim();\n}\n\n/**\n * RFC 4180 field escaping.\n * Quote if the field contains the separator, a double-quote, CR, or LF.\n * Embedded `\"` are doubled.\n */\nfunction escapeField(value: string, separator: string): string {\n return needsQuoting(value, separator) ? '\"' + value.replace(/\"/g, '\"\"') + '\"' : value;\n}\n\n/** Returns true when the value must be wrapped in double-quotes. */\nfunction needsQuoting(value: string, separator: string): boolean {\n return (\n value.includes(separator) || value.includes('\"') || value.includes(\"\\r\") || value.includes(\"\\n\")\n );\n}\n\n/**\n * Serialize one cell value from a typed vector to its string representation.\n *\n * - `null` / `undefined` -> `\"\"`\n * - `bigint` -> `String(x)`\n * - `NaN` / `+Inf` / `-Inf` -> `\"\"`\n */\nfunction serializeValue(vector: PTableVector, rowIndex: number): string {\n const rawValue = vector.data[rowIndex];\n\n if (isNil(rawValue)) {\n return \"\";\n }\n\n if (isValueNA(vector, rowIndex)) {\n return \"\";\n }\n\n switch (vector.type) {\n case ValueType.Long: {\n // BigInt64Array element — may be stored as bigint\n return String(rawValue);\n }\n case ValueType.Float:\n case ValueType.Double: {\n const numeric = rawValue as number;\n return Number.isNaN(numeric) || !Number.isFinite(numeric) ? \"\" : String(numeric);\n }\n case ValueType.Int: {\n return String(rawValue);\n }\n case ValueType.String: {\n return rawValue as string;\n }\n case ValueType.Bytes: {\n return \"\";\n }\n default: {\n return String(rawValue);\n }\n }\n}\n"],"mappings":";;;;;AAsBA,SAAgB,aAAa,OAA2B,WAA2B;AACjF,QAAO,MAAM,KAAK,SAAS,YAAY,YAAY,KAAK,EAAE,UAAU,CAAC,CAAC,KAAK,UAAU,GAAG;;;AAI1F,SAAgB,UAAU,SAAyB,UAAkB,WAA2B;AAC9F,QACE,QACG,KAAK,WAAW,YAAY,eAAe,QAAQ,SAAS,EAAE,UAAU,CAAC,CACzE,KAAK,UAAU,GAAG;;AAsBzB,gBAAuB,iBAAiB,SAAyD;CAC/F,MAAM,EAAE,QAAQ,eAAe,OAAO,WAAW,WAAW,QAAQ,OAAO,eAAe,QACxF;AAEF,KAAI,IACF,OAAM;AAGR,KAAI,cAEF,OAAM,aADgB,cAAc,KAAK,UAAU,MAAM,OAAO,EAC9B,UAAU;AAG9C,MAAA,GAAA,wBAAA,OAAU,MAAM,CACd;CAGF,MAAM,MAAM,MAAM,SAAS,MAAM;AAEjC,MAAK,IAAI,OAAO,MAAM,QAAQ,OAAO,KAAK,QAAQ,WAAW;AAC3D,UAAQ,gBAAgB;EAExB,MAAM,SAAS,KAAK,IAAI,WAAW,MAAM,KAAK;EAC9C,MAAM,WAAuB;GAAE,QAAQ;GAAM;GAAQ;EAErD,MAAM,UAAU,MAAM,OAAO,QAAQ,eAAe;GAAE,OAAO;GAAU;GAAQ,CAAC;EAEhF,MAAM,OAAiB,EAAE;AACzB,OAAK,IAAI,WAAW,GAAG,WAAW,QAAQ,WACxC,MAAK,KAAK,UAAU,SAAS,UAAU,UAAU,CAAC;AAEpD,QAAM,KAAK,KAAK,GAAG;;;;AAOvB,SAAS,YAAY,MAAgC;CACnD,MAAM,cAAA,GAAA,gCAAA,gBAA4B,KAAK,MAAMA,gCAAAA,WAAW,MAAM;AAC9D,SAAA,GAAA,wBAAA,OAAa,WAAW,GAAG,KAAK,KAAK,OAAO,WAAW,MAAM;;;;;;;AAQ/D,SAAS,YAAY,OAAe,WAA2B;AAC7D,QAAO,aAAa,OAAO,UAAU,GAAG,OAAM,MAAM,QAAQ,MAAM,OAAK,GAAG,OAAM;;;AAIlF,SAAS,aAAa,OAAe,WAA4B;AAC/D,QACE,MAAM,SAAS,UAAU,IAAI,MAAM,SAAS,KAAI,IAAI,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK;;;;;;;;;AAWpG,SAAS,eAAe,QAAsB,UAA0B;CACtE,MAAM,WAAW,OAAO,KAAK;AAE7B,MAAA,GAAA,wBAAA,OAAU,SAAS,CACjB,QAAO;AAGT,MAAA,GAAA,gCAAA,WAAc,QAAQ,SAAS,CAC7B,QAAO;AAGT,SAAQ,OAAO,MAAf;EACE,KAAKC,gCAAAA,UAAU,KAEb,QAAO,OAAO,SAAS;EAEzB,KAAKA,gCAAAA,UAAU;EACf,KAAKA,gCAAAA,UAAU,QAAQ;GACrB,MAAM,UAAU;AAChB,UAAO,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,QAAQ,GAAG,KAAK,OAAO,QAAQ;;EAElF,KAAKA,gCAAAA,UAAU,IACb,QAAO,OAAO,SAAS;EAEzB,KAAKA,gCAAAA,UAAU,OACb,QAAO;EAET,KAAKA,gCAAAA,UAAU,MACb,QAAO;EAET,QACE,QAAO,OAAO,SAAS"}
|
package/dist/csv_writer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv_writer.js","names":[],"sources":["../src/csv_writer.ts"],"sourcesContent":["import {\n Annotation,\n isValueNA,\n readAnnotation,\n ValueType,\n type PTableColumnSpec,\n type PTableVector,\n type TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isNil } from \"@milaboratories/helpers\";\n\n/** Minimal subset of
|
|
1
|
+
{"version":3,"file":"csv_writer.js","names":[],"sources":["../src/csv_writer.ts"],"sourcesContent":["import {\n Annotation,\n isValueNA,\n readAnnotation,\n ValueType,\n type PTableColumnSpec,\n type PTableVector,\n type TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isNil } from \"@milaboratories/helpers\";\n\n/** Minimal subset of PTableV9 required by streamPTableRows. */\nexport interface PTableDataSource {\n getData(\n columnIndices: number[],\n options?: { range?: TableRange; signal?: AbortSignal },\n ): Promise<PTableVector[]>;\n}\n\n// ── Public API (high-level → low-level) ──────────────────────────────\n\n/** Format a CSV/TSV header row from column specs. Line ending is CRLF. */\nexport function formatHeader(specs: PTableColumnSpec[], separator: string): string {\n return specs.map((spec) => escapeField(columnLabel(spec), separator)).join(separator) + \"\\r\\n\";\n}\n\n/** Format a single data row from parallel vectors. Line ending is CRLF. */\nexport function formatRow(vectors: PTableVector[], rowIndex: number, separator: string): string {\n return (\n vectors\n .map((vector) => escapeField(serializeValue(vector, rowIndex), separator))\n .join(separator) + \"\\r\\n\"\n );\n}\n\n/**\n * Async generator that streams CSV/TSV content chunk by chunk.\n *\n * The caller is responsible for providing a concrete `range` (already clipped\n * to the table shape). When `range` is undefined the generator does nothing\n * beyond emitting an optional BOM and header.\n */\nexport interface StreamPTableRowsOptions {\n pTable: PTableDataSource;\n specs: PTableColumnSpec[];\n columnIndices: number[];\n range?: TableRange;\n chunkSize: number;\n separator: string;\n includeHeader: boolean;\n bom: boolean;\n signal?: AbortSignal;\n}\nexport async function* streamPTableRows(options: StreamPTableRowsOptions): AsyncIterable<string> {\n const { pTable, columnIndices, range, chunkSize, separator, signal, specs, includeHeader, bom } =\n options;\n\n if (bom) {\n yield \"\\uFEFF\";\n }\n\n if (includeHeader) {\n const selectedSpecs = columnIndices.map((index) => specs[index]);\n yield formatHeader(selectedSpecs, separator);\n }\n\n if (isNil(range)) {\n return;\n }\n\n const end = range.offset + range.length;\n\n for (let from = range.offset; from < end; from += chunkSize) {\n signal?.throwIfAborted();\n\n const length = Math.min(chunkSize, end - from);\n const subRange: TableRange = { offset: from, length };\n\n const vectors = await pTable.getData(columnIndices, { range: subRange, signal });\n\n const rows: string[] = [];\n for (let rowIndex = 0; rowIndex < length; rowIndex++) {\n rows.push(formatRow(vectors, rowIndex, separator));\n }\n yield rows.join(\"\");\n }\n}\n\n// ── Helpers (low-level) ──────────────────────────────────────────────\n\n/** Extract a human-readable label from a PTableColumnSpec. */\nfunction columnLabel(spec: PTableColumnSpec): string {\n const annotation = readAnnotation(spec.spec, Annotation.Label);\n return isNil(annotation) ? spec.spec.name : annotation.trim();\n}\n\n/**\n * RFC 4180 field escaping.\n * Quote if the field contains the separator, a double-quote, CR, or LF.\n * Embedded `\"` are doubled.\n */\nfunction escapeField(value: string, separator: string): string {\n return needsQuoting(value, separator) ? '\"' + value.replace(/\"/g, '\"\"') + '\"' : value;\n}\n\n/** Returns true when the value must be wrapped in double-quotes. */\nfunction needsQuoting(value: string, separator: string): boolean {\n return (\n value.includes(separator) || value.includes('\"') || value.includes(\"\\r\") || value.includes(\"\\n\")\n );\n}\n\n/**\n * Serialize one cell value from a typed vector to its string representation.\n *\n * - `null` / `undefined` -> `\"\"`\n * - `bigint` -> `String(x)`\n * - `NaN` / `+Inf` / `-Inf` -> `\"\"`\n */\nfunction serializeValue(vector: PTableVector, rowIndex: number): string {\n const rawValue = vector.data[rowIndex];\n\n if (isNil(rawValue)) {\n return \"\";\n }\n\n if (isValueNA(vector, rowIndex)) {\n return \"\";\n }\n\n switch (vector.type) {\n case ValueType.Long: {\n // BigInt64Array element — may be stored as bigint\n return String(rawValue);\n }\n case ValueType.Float:\n case ValueType.Double: {\n const numeric = rawValue as number;\n return Number.isNaN(numeric) || !Number.isFinite(numeric) ? \"\" : String(numeric);\n }\n case ValueType.Int: {\n return String(rawValue);\n }\n case ValueType.String: {\n return rawValue as string;\n }\n case ValueType.Bytes: {\n return \"\";\n }\n default: {\n return String(rawValue);\n }\n }\n}\n"],"mappings":";;;;AAsBA,SAAgB,aAAa,OAA2B,WAA2B;AACjF,QAAO,MAAM,KAAK,SAAS,YAAY,YAAY,KAAK,EAAE,UAAU,CAAC,CAAC,KAAK,UAAU,GAAG;;;AAI1F,SAAgB,UAAU,SAAyB,UAAkB,WAA2B;AAC9F,QACE,QACG,KAAK,WAAW,YAAY,eAAe,QAAQ,SAAS,EAAE,UAAU,CAAC,CACzE,KAAK,UAAU,GAAG;;AAsBzB,gBAAuB,iBAAiB,SAAyD;CAC/F,MAAM,EAAE,QAAQ,eAAe,OAAO,WAAW,WAAW,QAAQ,OAAO,eAAe,QACxF;AAEF,KAAI,IACF,OAAM;AAGR,KAAI,cAEF,OAAM,aADgB,cAAc,KAAK,UAAU,MAAM,OAAO,EAC9B,UAAU;AAG9C,KAAI,MAAM,MAAM,CACd;CAGF,MAAM,MAAM,MAAM,SAAS,MAAM;AAEjC,MAAK,IAAI,OAAO,MAAM,QAAQ,OAAO,KAAK,QAAQ,WAAW;AAC3D,UAAQ,gBAAgB;EAExB,MAAM,SAAS,KAAK,IAAI,WAAW,MAAM,KAAK;EAC9C,MAAM,WAAuB;GAAE,QAAQ;GAAM;GAAQ;EAErD,MAAM,UAAU,MAAM,OAAO,QAAQ,eAAe;GAAE,OAAO;GAAU;GAAQ,CAAC;EAEhF,MAAM,OAAiB,EAAE;AACzB,OAAK,IAAI,WAAW,GAAG,WAAW,QAAQ,WACxC,MAAK,KAAK,UAAU,SAAS,UAAU,UAAU,CAAC;AAEpD,QAAM,KAAK,KAAK,GAAG;;;;AAOvB,SAAS,YAAY,MAAgC;CACnD,MAAM,aAAa,eAAe,KAAK,MAAM,WAAW,MAAM;AAC9D,QAAO,MAAM,WAAW,GAAG,KAAK,KAAK,OAAO,WAAW,MAAM;;;;;;;AAQ/D,SAAS,YAAY,OAAe,WAA2B;AAC7D,QAAO,aAAa,OAAO,UAAU,GAAG,OAAM,MAAM,QAAQ,MAAM,OAAK,GAAG,OAAM;;;AAIlF,SAAS,aAAa,OAAe,WAA4B;AAC/D,QACE,MAAM,SAAS,UAAU,IAAI,MAAM,SAAS,KAAI,IAAI,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK;;;;;;;;;AAWpG,SAAS,eAAe,QAAsB,UAA0B;CACtE,MAAM,WAAW,OAAO,KAAK;AAE7B,KAAI,MAAM,SAAS,CACjB,QAAO;AAGT,KAAI,UAAU,QAAQ,SAAS,CAC7B,QAAO;AAGT,SAAQ,OAAO,MAAf;EACE,KAAK,UAAU,KAEb,QAAO,OAAO,SAAS;EAEzB,KAAK,UAAU;EACf,KAAK,UAAU,QAAQ;GACrB,MAAM,UAAU;AAChB,UAAO,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,QAAQ,GAAG,KAAK,OAAO,QAAQ;;EAElF,KAAK,UAAU,IACb,QAAO,OAAO,SAAS;EAEzB,KAAK,UAAU,OACb,QAAO;EAET,KAAK,UAAU,MACb,QAAO;EAET,QACE,QAAO,OAAO,SAAS"}
|
package/dist/driver_impl.cjs
CHANGED
|
@@ -11,6 +11,7 @@ let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common")
|
|
|
11
11
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
12
12
|
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
13
13
|
let _milaboratories_pframes_rs_node = require("@milaboratories/pframes-rs-node");
|
|
14
|
+
let _milaboratories_pframes_rs_wasm = require("@milaboratories/pframes-rs-wasm");
|
|
14
15
|
let node_os = require("node:os");
|
|
15
16
|
let node_fs = require("node:fs");
|
|
16
17
|
node_fs = require_runtime.__toESM(node_fs);
|
|
@@ -252,14 +253,49 @@ var AbstractPFrameDriver = class {
|
|
|
252
253
|
}
|
|
253
254
|
async getUniqueValues(handle, request, signal) {
|
|
254
255
|
if (require_logging.logPFrames()) this.logger("info", `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, _milaboratories_pl_model_common.bigintReplacer)}`);
|
|
255
|
-
const { pFrameDataPromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
256
|
+
const { pFrameDataPromise, pFrameSpec, disposeSignal } = this.pFrames.getByKey(handle);
|
|
256
257
|
const pFrameData = await pFrameDataPromise;
|
|
258
|
+
const column = pFrameSpec.listColumns().find((c) => c.columnId === request.columnId);
|
|
259
|
+
if (!column) {
|
|
260
|
+
const error = new _milaboratories_pl_model_common.PFrameDriverError(`getUniqueValues failed`);
|
|
261
|
+
error.cause = /* @__PURE__ */ new Error(`column ${request.columnId} is not registered in the PFrame`);
|
|
262
|
+
throw error;
|
|
263
|
+
}
|
|
264
|
+
const axisIds = (0, _milaboratories_pframes_rs_wasm.expandAxes)(column.spec.axesSpec);
|
|
265
|
+
const tableSpec = [...column.spec.axesSpec.map((axisSpec, i) => ({
|
|
266
|
+
type: "axis",
|
|
267
|
+
id: axisIds[i],
|
|
268
|
+
spec: axisSpec
|
|
269
|
+
})), {
|
|
270
|
+
type: "column",
|
|
271
|
+
id: request.columnId,
|
|
272
|
+
spec: column.spec
|
|
273
|
+
}];
|
|
274
|
+
let axisIndex;
|
|
275
|
+
if (request.axis !== void 0) {
|
|
276
|
+
const index = (0, _milaboratories_pframes_rs_wasm.findTableColumn)(tableSpec, {
|
|
277
|
+
type: "axis",
|
|
278
|
+
id: request.axis
|
|
279
|
+
});
|
|
280
|
+
if (index < 0) {
|
|
281
|
+
const error = new _milaboratories_pl_model_common.PFrameDriverError(`getUniqueValues failed`);
|
|
282
|
+
error.cause = /* @__PURE__ */ new Error(`axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`);
|
|
283
|
+
throw error;
|
|
284
|
+
}
|
|
285
|
+
axisIndex = index;
|
|
286
|
+
}
|
|
287
|
+
const resolvedRequest = {
|
|
288
|
+
columnId: request.columnId,
|
|
289
|
+
axisIndex,
|
|
290
|
+
filters: (0, _milaboratories_pframes_rs_wasm.rewriteLegacyFilters)({
|
|
291
|
+
tableSpec,
|
|
292
|
+
filters: migrateFilters(request.filters, this.logger)
|
|
293
|
+
}),
|
|
294
|
+
limit: request.limit
|
|
295
|
+
};
|
|
257
296
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
258
297
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
259
|
-
return await pFrameData.getUniqueValues({
|
|
260
|
-
...request,
|
|
261
|
-
filters: migrateFilters(request.filters, this.logger)
|
|
262
|
-
}, { signal: combinedSignal });
|
|
298
|
+
return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });
|
|
263
299
|
});
|
|
264
300
|
}
|
|
265
301
|
async getSpec(handle) {
|
package/dist/driver_impl.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_impl.cjs","names":["PTableCachePerFrameOpsDefaults","PTableCachePlainOpsDefaults","PFrameFactory","ConcurrencyLimitingExecutor","PFramePool","PTableDefPool","PTablePool","PTableCachePerFrame","PTableCachePlain","ValueType","PoolEntryGuard","logPFrames","streamPTableRows","fs","Readable","zlib","bigintReplacer"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n return await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await pTable.getFootprint({ signal: combinedSignal });\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { path: options.path, rowsWritten, bytesWritten };\n });\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const info = pFrameSpec.listColumns().find((c) => c.columnId === columnId);\n return info?.spec ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 150k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n });\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(\n {\n ...request,\n filters: migrateFilters(request.filters, this.logger),\n },\n {\n signal: combinedSignal,\n },\n );\n });\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { shape, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { data, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAwFA,MAAa,kCAA2D;CACtE,GAAGA,+BAAAA;CACH,GAAGC,2BAAAA;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAMC,gCAAAA,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,aAAA,GAAA,QAAA,SAAoB,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAIC,2BAAAA,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAIA,2BAAAA,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAIC,oBAAAA,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAIC,wBAAAA,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAIC,oBAAAA,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAIC,+BAAAA,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAIC,2BAAAA,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOC,gCAAAA,UAAU,CAAC;EAKpD,MAAM,WAAA,GAAA,gCAAA,UAJmB,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,OAAA,GAAA,gCAAA,0BAA+B,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,OAAA,GAAA,gCAAA,gBAClB,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,mBAAqC,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAIC,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAA,GAAA,gCAAA,eACJ,oBAAA,GAAA,gCAAA,cACe,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,yBAA2C,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAID,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,eAAA,GAAA,gCAAA,gBAAA,GAAA,gCAAA,qBAAgD,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAAc,YAAY;GAEtE,MAAM,cAAA,WAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAID,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,EAAA,GAAA,wBAAA,OAAO,EAAE,CAAC,CAC3E;AAED,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACxD,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAWE,mBAAAA,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,WAAA,GAAA,2BAAA,sBAA2B,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcC,QAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAASC,YAAAA,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,QAAA,GAAA,qBAAA,UAAe,QADFC,UAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,QAAA,GAAA,qBAAA,UAAe,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AACzE,SAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;IAI7E,MAAM,cAAc,aAAa;AAEjC,QAAIJ,gBAAAA,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,MAAM,QAAQ;KAAM;KAAa;KAAc;KACxD;;;;;;;CAOJ,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,EAAA,GAAA,gCAAA,kBAAkB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SADa,WAAW,aAAa,CAAC,MAAM,MAAM,EAAE,aAAa,SAAS,EAC7D,QAAQ;;CAGvB,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAIA,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAASK,gCAAAA,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,aAAA,GAAA,gCAAA,eAA0B,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAIN,wBAAAA,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IAIxD,MAAM,OAAO,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;KACvD;KACA,QAAQ;KACT,CAAC;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;AACF,SAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAE9D,WAAO,UAAU,KAAK,MAAM,OAAO;KAC3B;KACN,MAAM,KAAK;KACZ,EAAE;KACH;;;;;;;CAGJ,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAASK,gCAAAA,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EAC1E,MAAM,aAAa,MAAM;EAEzB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBACtB;IACE,GAAG;IACH,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,EACD,EACE,QAAQ,gBACT,CACF;IACD;;CAOJ,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIN,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAShF,WAAO;KAAE,OARK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAMc,aAJI,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE2B;KAC7B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIA,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MATI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAMa,aAJK,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE0B;KAC5B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,MAAA,GAAA,wBAAA,OAAU,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
|
|
1
|
+
{"version":3,"file":"driver_impl.cjs","names":["PTableCachePerFrameOpsDefaults","PTableCachePlainOpsDefaults","PFrameFactory","ConcurrencyLimitingExecutor","PFramePool","PTableDefPool","PTablePool","PTableCachePerFrame","PTableCachePlain","ValueType","PoolEntryGuard","logPFrames","streamPTableRows","fs","Readable","zlib","bigintReplacer","PFrameDriverError"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n PFrameDriverError,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { expandAxes, findTableColumn, rewriteLegacyFilters } from \"@milaboratories/pframes-rs-wasm\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n return await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await pTable.getFootprint({ signal: combinedSignal });\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { path: options.path, rowsWritten, bytesWritten };\n });\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const info = pFrameSpec.listColumns().find((c) => c.columnId === columnId);\n return info?.spec ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 150k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n });\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, pFrameSpec, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const column = pFrameSpec.listColumns().find((c) => c.columnId === request.columnId);\n if (!column) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);\n throw error;\n }\n\n const axisIds = expandAxes(column.spec.axesSpec);\n const tableSpec: PTableColumnSpec[] = [\n ...column.spec.axesSpec.map(\n (axisSpec, i): PTableColumnSpec => ({\n type: \"axis\",\n id: axisIds[i],\n spec: axisSpec,\n }),\n ),\n { type: \"column\", id: request.columnId, spec: column.spec },\n ];\n\n let axisIndex: number | undefined;\n if (request.axis !== undefined) {\n const index = findTableColumn(tableSpec, { type: \"axis\", id: request.axis });\n if (index < 0) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(\n `axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,\n );\n throw error;\n }\n axisIndex = index;\n }\n\n const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {\n columnId: request.columnId,\n axisIndex,\n filters: rewriteLegacyFilters({\n tableSpec,\n filters: migrateFilters(request.filters, this.logger),\n }),\n limit: request.limit,\n };\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });\n });\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { shape, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { data, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0FA,MAAa,kCAA2D;CACtE,GAAGA,+BAAAA;CACH,GAAGC,2BAAAA;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAMC,gCAAAA,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,aAAA,GAAA,QAAA,SAAoB,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAIC,2BAAAA,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAIA,2BAAAA,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAIC,oBAAAA,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAIC,wBAAAA,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAIC,oBAAAA,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAIC,+BAAAA,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAIC,2BAAAA,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOC,gCAAAA,UAAU,CAAC;EAKpD,MAAM,WAAA,GAAA,gCAAA,UAJmB,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,OAAA,GAAA,gCAAA,0BAA+B,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,OAAA,GAAA,gCAAA,gBAClB,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,mBAAqC,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAIC,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAA,GAAA,gCAAA,eACJ,oBAAA,GAAA,gCAAA,cACe,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,WAAA,GAAA,gCAAA,WAAA,GAAA,gCAAA,yBAA2C,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAID,wBAAAA,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,eAAA,GAAA,gCAAA,gBAAA,GAAA,gCAAA,qBAAgD,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAAc,YAAY;GAEtE,MAAM,cAAA,WAAA,EAAc,IAAIA,wBAAAA,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAID,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,EAAA,GAAA,wBAAA,OAAO,EAAE,CAAC,CAC3E;AAED,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACxD,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAWE,mBAAAA,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,WAAA,GAAA,2BAAA,sBAA2B,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcC,QAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAASC,YAAAA,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,QAAA,GAAA,qBAAA,UAAe,QADFC,UAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,QAAA,GAAA,qBAAA,UAAe,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AACzE,SAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;IAI7E,MAAM,cAAc,aAAa;AAEjC,QAAIJ,gBAAAA,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,MAAM,QAAQ;KAAM;KAAa;KAAc;KACxD;;;;;;;CAOJ,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,EAAA,GAAA,gCAAA,kBAAkB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SADa,WAAW,aAAa,CAAC,MAAM,MAAM,EAAE,aAAa,SAAS,EAC7D,QAAQ;;CAGvB,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAIA,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAASK,gCAAAA,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,aAAA,GAAA,gCAAA,eAA0B,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAIN,wBAAAA,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IAIxD,MAAM,OAAO,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;KACvD;KACA,QAAQ;KACT,CAAC;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;AACF,SAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAE9D,WAAO,UAAU,KAAK,MAAM,OAAO;KAC3B;KACN,MAAM,KAAK;KACZ,EAAE;KACH;;;;;;;CAGJ,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAASK,gCAAAA,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,YAAY,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EACtF,MAAM,aAAa,MAAM;EAEzB,MAAM,SAAS,WAAW,aAAa,CAAC,MAAM,MAAM,EAAE,aAAa,QAAQ,SAAS;AACpF,MAAI,CAAC,QAAQ;GACX,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,yBAAyB;AAC7D,SAAM,wBAAQ,IAAI,MAAM,UAAU,QAAQ,SAAS,kCAAkC;AACrF,SAAM;;EAGR,MAAM,WAAA,GAAA,gCAAA,YAAqB,OAAO,KAAK,SAAS;EAChD,MAAM,YAAgC,CACpC,GAAG,OAAO,KAAK,SAAS,KACrB,UAAU,OAAyB;GAClC,MAAM;GACN,IAAI,QAAQ;GACZ,MAAM;GACP,EACF,EACD;GAAE,MAAM;GAAU,IAAI,QAAQ;GAAU,MAAM,OAAO;GAAM,CAC5D;EAED,IAAI;AACJ,MAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,MAAM,SAAA,GAAA,gCAAA,iBAAwB,WAAW;IAAE,MAAM;IAAQ,IAAI,QAAQ;IAAM,CAAC;AAC5E,OAAI,QAAQ,GAAG;IACb,MAAM,QAAQ,IAAIA,gCAAAA,kBAAkB,yBAAyB;AAC7D,UAAM,wBAAQ,IAAI,MAChB,QAAQ,KAAK,UAAU,QAAQ,KAAK,CAAC,kCAAkC,QAAQ,WAChF;AACD,UAAM;;AAER,eAAY;;EAGd,MAAM,kBAAwD;GAC5D,UAAU,QAAQ;GAClB;GACA,UAAA,GAAA,gCAAA,sBAA8B;IAC5B;IACA,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,CAAC;GACF,OAAO,QAAQ;GAChB;EAED,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBAAgB,iBAAiB,EAAE,QAAQ,gBAAgB,CAAC;IACpF;;CAOJ,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIP,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAShF,WAAO;KAAE,OARK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAMc,aAJI,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE2B;KAC7B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAIA,wBAAAA,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MATI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAMa,aAJK,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE0B;KAC5B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,MAAA,GAAA,wBAAA,OAAU,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_impl.d.ts","names":[],"sources":["../src/driver_impl.ts"],"mappings":";;;;;;;;;
|
|
1
|
+
{"version":3,"file":"driver_impl.d.ts","names":[],"sources":["../src/driver_impl.ts"],"mappings":";;;;;;;;;UA4EiB,iBAAA,mBAAoC,gBAAA,UAC3C,mBAAA,CAAsB,SAAA,GAAY,eAAA;AAAA,UAE3B,kBAAA,mBAAqC,gBAAA,UAC5C,oBAAA,CAAuB,SAAA,GAAY,eAAA;AAAA,KAEjC,uBAAA,GAA0B,sBAAA,GACpC,mBAAA;EAPmD,iFASjD,iBAAA,UARM;EAUN,iBAAA;AAAA;AAAA,cAGS,+BAAA,EAAiC,uBAAA;AAAA,KAOlC,gBAAA,gCAAgD,gBAAA,KAC1D,IAAA,EAAM,WAAA,EACN,IAAA,EAAM,WAAA,KACH,cAAA,CAAe,QAAA,CAAS,SAAA;AAAA,cAEhB,oBAAA,gCAEO,gBAAA,aACP,4BAAA,CAA6B,WAAA;EAAA,iBACvB,MAAA;EAAA,iBAEA,iBAAA;EAAA,iBACA,kBAAA;EAAA,iBAEA,eAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,UAAA;EAAA,iBACA,OAAA;EAAA,iBAEA,mBAAA;EAAA,iBACA,gBAAA;EAAA,iBAEA,uBAAA;EAAA,iBACA,uBAAA;EAEJ,SAAA,CAAA,GAAa,OAAA,CAAQ,UAAA;;IAKhC,MAAA;IACA,iBAAA;IACA,kBAAA;IACA,SAAA;IACA,OAAA;IACA;EAAA;IAEA,MAAA,GAAS,cAAA,CAAe,MAAA;IACxB,iBAAA,EAAmB,iBAAA,CAAkB,SAAA;IACrC,kBAAA,EAAoB,kBAAA,CAAmB,SAAA;IACvC,SAAA;IACA,OAAA,GAAU,uBAAA;IACV,eAAA,EAAiB,gBAAA,CAAiB,WAAA,EAAa,SAAA;EAAA;EAyB3C,OAAA,CAAA,GAAW,OAAA;EAAA,CAUV,MAAA,CAAO,YAAA,KAAiB,OAAA;EAQxB,YAAA,CAAa,GAAA,EAAK,SAAA,CAAU,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAa9D,YAAA,CAAa,MAAA,EAAQ,SAAA,CAAU,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAsCjE,cAAA,CAAe,GAAA,EAAK,WAAA,CAAY,OAAA,CAAQ,WAAA,KAAgB,SAAA,CAAU,YAAA;EAmC5D,eAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,wBAAA,GACR,OAAA,CAAQ,uBAAA;EA2EE,WAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,kBAAA,GACR,OAAA,CAAQ,mBAAA;EAoCE,aAAA,CACX,MAAA,EAAQ,YAAA,EACR,QAAA,EAAU,SAAA,GACT,OAAA,CAAQ,WAAA;EAME,WAAA,CAAY,MAAA,EAAQ,YAAA,GAAe,OAAA,CAAQ,gBAAA;EAK3C,kBAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,yBAAA,CAA0B,SAAA,GACnC,KAAA,EAAO,UAAA,cACP,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,0BAAA;EA0CE,eAAA,CACX,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,mBAAA,EACT,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,oBAAA;EA+DE,OAAA,CAAQ,MAAA,EAAQ,YAAA,GAAe,OAAA,CAAQ,gBAAA;EAKvC,QAAA,CAAS,MAAA,EAAQ,YAAA,EAAc,MAAA,GAAS,WAAA,GAAc,OAAA,CAAQ,WAAA;EAwB9D,OAAA,CACX,MAAA,EAAQ,YAAA,EACR,aAAA,YACA,KAAA,EAAO,UAAA,cACP,MAAA,GAAS,WAAA,GACR,OAAA,CAAQ,YAAA;AAAA"}
|
package/dist/driver_impl.js
CHANGED
|
@@ -6,10 +6,11 @@ import { PTablePool } from "./ptable_pool.js";
|
|
|
6
6
|
import { PTableCachePerFrame, PTableCachePerFrameOpsDefaults } from "./ptable_cache_per_frame.js";
|
|
7
7
|
import { PTableCachePlain, PTableCachePlainOpsDefaults } from "./ptable_cache_plain.js";
|
|
8
8
|
import { _usingCtx } from "./_virtual/_@oxc-project_runtime@0.123.0/helpers/usingCtx.js";
|
|
9
|
-
import { ValueType, bigintReplacer, canonicalizeJson, collectSpecQueryColumns, extractAllColumns, mapPObjectData, mapPTableDef, mapSpecQueryColumns, resolveAnnotationParents, sortPTableDef, sortSpecQuery, uniqueBy } from "@milaboratories/pl-model-common";
|
|
9
|
+
import { PFrameDriverError, ValueType, bigintReplacer, canonicalizeJson, collectSpecQueryColumns, extractAllColumns, mapPObjectData, mapPTableDef, mapSpecQueryColumns, resolveAnnotationParents, sortPTableDef, sortSpecQuery, uniqueBy } from "@milaboratories/pl-model-common";
|
|
10
10
|
import { ConcurrencyLimitingExecutor, createPathAtomically } from "@milaboratories/ts-helpers";
|
|
11
11
|
import { PoolEntryGuard, isNil } from "@milaboratories/helpers";
|
|
12
12
|
import { PFrameFactory } from "@milaboratories/pframes-rs-node";
|
|
13
|
+
import { expandAxes, findTableColumn, rewriteLegacyFilters } from "@milaboratories/pframes-rs-wasm";
|
|
13
14
|
import { tmpdir } from "node:os";
|
|
14
15
|
import * as fs$1 from "node:fs";
|
|
15
16
|
import { Readable } from "node:stream";
|
|
@@ -249,14 +250,49 @@ var AbstractPFrameDriver = class {
|
|
|
249
250
|
}
|
|
250
251
|
async getUniqueValues(handle, request, signal) {
|
|
251
252
|
if (logPFrames()) this.logger("info", `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
|
|
252
|
-
const { pFrameDataPromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
253
|
+
const { pFrameDataPromise, pFrameSpec, disposeSignal } = this.pFrames.getByKey(handle);
|
|
253
254
|
const pFrameData = await pFrameDataPromise;
|
|
255
|
+
const column = pFrameSpec.listColumns().find((c) => c.columnId === request.columnId);
|
|
256
|
+
if (!column) {
|
|
257
|
+
const error = new PFrameDriverError(`getUniqueValues failed`);
|
|
258
|
+
error.cause = /* @__PURE__ */ new Error(`column ${request.columnId} is not registered in the PFrame`);
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
const axisIds = expandAxes(column.spec.axesSpec);
|
|
262
|
+
const tableSpec = [...column.spec.axesSpec.map((axisSpec, i) => ({
|
|
263
|
+
type: "axis",
|
|
264
|
+
id: axisIds[i],
|
|
265
|
+
spec: axisSpec
|
|
266
|
+
})), {
|
|
267
|
+
type: "column",
|
|
268
|
+
id: request.columnId,
|
|
269
|
+
spec: column.spec
|
|
270
|
+
}];
|
|
271
|
+
let axisIndex;
|
|
272
|
+
if (request.axis !== void 0) {
|
|
273
|
+
const index = findTableColumn(tableSpec, {
|
|
274
|
+
type: "axis",
|
|
275
|
+
id: request.axis
|
|
276
|
+
});
|
|
277
|
+
if (index < 0) {
|
|
278
|
+
const error = new PFrameDriverError(`getUniqueValues failed`);
|
|
279
|
+
error.cause = /* @__PURE__ */ new Error(`axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`);
|
|
280
|
+
throw error;
|
|
281
|
+
}
|
|
282
|
+
axisIndex = index;
|
|
283
|
+
}
|
|
284
|
+
const resolvedRequest = {
|
|
285
|
+
columnId: request.columnId,
|
|
286
|
+
axisIndex,
|
|
287
|
+
filters: rewriteLegacyFilters({
|
|
288
|
+
tableSpec,
|
|
289
|
+
filters: migrateFilters(request.filters, this.logger)
|
|
290
|
+
}),
|
|
291
|
+
limit: request.limit
|
|
292
|
+
};
|
|
254
293
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
255
294
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
256
|
-
return await pFrameData.getUniqueValues({
|
|
257
|
-
...request,
|
|
258
|
-
filters: migrateFilters(request.filters, this.logger)
|
|
259
|
-
}, { signal: combinedSignal });
|
|
295
|
+
return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });
|
|
260
296
|
});
|
|
261
297
|
}
|
|
262
298
|
async getSpec(handle) {
|
package/dist/driver_impl.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_impl.js","names":["fs"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n return await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await pTable.getFootprint({ signal: combinedSignal });\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { path: options.path, rowsWritten, bytesWritten };\n });\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const info = pFrameSpec.listColumns().find((c) => c.columnId === columnId);\n return info?.spec ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 150k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n });\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(\n {\n ...request,\n filters: migrateFilters(request.filters, this.logger),\n },\n {\n signal: combinedSignal,\n },\n );\n });\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { shape, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { data, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAwFA,MAAa,kCAA2D;CACtE,GAAG;CACH,GAAG;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAM,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,YAAY,QAAQ,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAI,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAI,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAI,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAI,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAI,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAO,UAAU,CAAC;EAKpD,MAAM,UADgB,SAHG,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,MAAM,yBAAyB,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,MACjC,eAAe,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,UAAU,SAAS,kBAAkB,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,YAAY,cAChB,mBACE,aAAa,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAI,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,UAAU,SAAS,wBAAwB,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAc,cAAc,oBAAoB,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAAc,YAAY;GAEtE,MAAM,cAAA,WAAA,EAAc,IAAI,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,CAAC,MAAM,EAAE,CAAC,CAC3E;AAED,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACxD,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAW,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,UAAM,qBAAqB,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcA,KAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAAS,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,OAAM,SAAS,QADF,KAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,OAAM,SAAS,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AACzE,SAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;IAI7E,MAAM,cAAc,aAAa;AAEjC,QAAI,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,MAAM,QAAQ;KAAM;KAAa;KAAc;KACxD;;;;;;;CAOJ,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,CAAC,iBAAiB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SADa,WAAW,aAAa,CAAC,MAAM,MAAM,EAAE,aAAa,SAAS,EAC7D,QAAQ;;CAGvB,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAI,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,YAAY,cAAc,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IAIxD,MAAM,OAAO,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;KACvD;KACA,QAAQ;KACT,CAAC;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;AACF,SAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAE9D,WAAO,UAAU,KAAK,MAAM,OAAO;KAC3B;KACN,MAAM,KAAK;KACZ,EAAE;KACH;;;;;;;CAGJ,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAI,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EAC1E,MAAM,aAAa,MAAM;EAEzB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBACtB;IACE,GAAG;IACH,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,EACD,EACE,QAAQ,gBACT,CACF;IACD;;CAOJ,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAShF,WAAO;KAAE,OARK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAMc,aAJI,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE2B;KAC7B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MATI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAMa,aAJK,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE0B;KAC5B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,KAAI,MAAM,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
|
|
1
|
+
{"version":3,"file":"driver_impl.js","names":["fs"],"sources":["../src/driver_impl.ts"],"sourcesContent":["import {\n mapPObjectData,\n mapPTableDef,\n extractAllColumns,\n uniqueBy,\n canonicalizeJson,\n bigintReplacer,\n ValueType,\n type CalculateTableDataRequest,\n type CalculateTableDataResponse,\n type FindColumnsRequest,\n type FindColumnsResponse,\n type PColumnIdAndSpec,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n type PTableColumnSpec,\n type PTableHandle,\n type PTableShape,\n type PTableVector,\n type TableRange,\n type UniqueValuesRequest,\n type UniqueValuesResponse,\n type PColumn,\n type PFrameDef,\n type PTableDef,\n type PTableRecordSingleValueFilterV2,\n type PTableRecordFilter,\n type JsonSerializable,\n type PTableDefV2,\n mapSpecQueryColumns,\n collectSpecQueryColumns,\n sortSpecQuery,\n sortPTableDef,\n resolveAnnotationParents,\n PFrameDriverError,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n ConcurrencyLimitingExecutor,\n createPathAtomically,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport { isNil, PoolEntryGuard, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { expandAxes, findTableColumn, rewriteLegacyFilters } from \"@milaboratories/pframes-rs-wasm\";\nimport { tmpdir } from \"node:os\";\nimport * as fs from \"node:fs\";\nimport { Readable } from \"node:stream\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\nimport { streamPTableRows } from \"./csv_writer\";\nimport type {\n AbstractInternalPFrameDriver,\n WritePTableToFsOptions,\n WritePTableToFsResult,\n} from \"./driver_decl\";\nimport { logPFrames } from \"./logging\";\nimport {\n PFramePool,\n type LocalBlobProvider as PoolLocalBlobProvider,\n type RemoteBlobProvider as PoolRemoteBlobProvider,\n} from \"./pframe_pool\";\nimport { PTableDefPool } from \"./ptable_def_pool\";\nimport { PTablePool } from \"./ptable_pool\";\nimport {\n PTableCachePerFrame,\n PTableCachePerFrameOpsDefaults,\n type PTableCachePerFrameOps,\n} from \"./ptable_cache_per_frame\";\nimport {\n PTableCachePlain,\n PTableCachePlainOpsDefaults,\n type PTableCachePlainOps,\n} from \"./ptable_cache_plain\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolLocalBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable>\n extends PoolRemoteBlobProvider<TreeEntry>, AsyncDisposable {}\n\nexport type AbstractPFrameDriverOps = PTableCachePerFrameOps &\n PTableCachePlainOps & {\n /** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */\n pFrameConcurrency: number;\n /** Concurrency limits for `getShape` and `getData` requests */\n pTableConcurrency: number;\n };\n\nexport const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps = {\n ...PTableCachePerFrameOpsDefaults,\n ...PTableCachePlainOpsDefaults,\n pFrameConcurrency: 1, // 1 join is executed in parallel and utilize all RAM and CPU cores\n pTableConcurrency: 1, // 1 joined table is read from disk at a time, which matches 1 table the user can view in the UI\n};\n\nexport type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (\n spec: PColumnSpec,\n data: PColumnData,\n) => PFrameInternal.DataInfo<TreeEntry>;\n\nexport class AbstractPFrameDriver<\n PColumnData,\n TreeEntry extends JsonSerializable,\n> implements AbstractInternalPFrameDriver<PColumnData> {\n private readonly logger: PFrameInternal.Logger;\n\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>;\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n\n private readonly resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n\n private readonly pFrames: PFramePool<TreeEntry>;\n private readonly pTableDefs: PTableDefPool;\n private readonly pTables: PTablePool<TreeEntry>;\n\n private readonly pTableCachePerFrame: PTableCachePerFrame;\n private readonly pTableCachePlain: PTableCachePlain;\n\n private readonly frameConcurrencyLimiter: ConcurrencyLimitingExecutor;\n private readonly tableConcurrencyLimiter: ConcurrencyLimitingExecutor;\n\n public async pprofDump(): Promise<Uint8Array> {\n return await PFrameFactory.pprofDump();\n }\n\n public constructor({\n logger = () => {},\n localBlobProvider,\n remoteBlobProvider,\n spillPath = tmpdir(),\n options = AbstractPFrameDriverOpsDefaults,\n resolveDataInfo,\n }: {\n logger?: PFrameInternal.Logger;\n localBlobProvider: LocalBlobProvider<TreeEntry>;\n remoteBlobProvider: RemoteBlobProvider<TreeEntry>;\n spillPath?: string;\n options?: AbstractPFrameDriverOps;\n resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;\n }) {\n this.logger = logger;\n\n this.localBlobProvider = localBlobProvider;\n this.remoteBlobProvider = remoteBlobProvider;\n\n this.resolveDataInfo = resolveDataInfo;\n\n this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);\n this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);\n\n this.pFrames = new PFramePool(\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n spillPath,\n );\n this.pTableDefs = new PTableDefPool(this.logger);\n this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);\n\n this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);\n this.pTableCachePlain = new PTableCachePlain(this.logger, options);\n }\n\n async dispose(): Promise<void> {\n void (await Promise.allSettled([\n this.pTables[Symbol.asyncDispose](),\n this.pTableDefs[Symbol.asyncDispose](),\n this.pFrames[Symbol.asyncDispose](),\n this.localBlobProvider[Symbol.asyncDispose](),\n this.remoteBlobProvider[Symbol.asyncDispose](),\n ]));\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n return await this.dispose();\n }\n\n //\n // Internal / Config API Methods\n //\n\n public createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle> {\n const ValueTypes = new Set(Object.values(ValueType));\n const supportedColumns = def\n .filter((column) => ValueTypes.has(column.spec.valueType))\n .map((c) => ({ ...c, spec: resolveAnnotationParents(c.spec) }));\n const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);\n const columns = uniqueColumns.map((c) =>\n mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)),\n );\n\n return this.pFrames.acquire(columns);\n }\n\n public createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(extractAllColumns(rawDef.src), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const sortedDef = sortPTableDef(\n migrateTableFilter(\n mapPTableDef(rawDef, (c) => c.id),\n this.logger,\n ),\n );\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquireFromLegacy({\n pFrameHandle: pFrameGuard.key,\n def: sortedDef,\n pFrameSpec,\n }).entry,\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle> {\n const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);\n using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns));\n const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec;\n const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));\n const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery);\n\n using pTableGuard = new PoolEntryGuard(\n this.pTableDefs.acquire({\n pFrameHandle: pFrameGuard.key,\n tableSpec,\n dataQuery,\n }),\n );\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Create PTable call (pFrameHandle = ${pFrameGuard.key}; pTableHandle = ${pTableGuard.key})`,\n );\n }\n\n const pFrameEntry = pFrameGuard.keep();\n const pTableEntry = pTableGuard.keep();\n const unref = () => {\n pTableEntry.unref();\n pFrameEntry.unref();\n };\n return {\n key: pTableEntry.key,\n resource: pTableEntry.resource,\n unref,\n [Symbol.dispose]: unref,\n };\n }\n\n public async writePTableToFs(\n handle: PTableHandle,\n options: WritePTableToFsOptions,\n ): Promise<WritePTableToFsResult> {\n this.logger(\n \"info\",\n `[WritePTableToFs] ENTER (handle = ${handle}, path = ${options.path}, format = ${options.format}, compression = ${options.compression ?? \"auto\"}, columns = ${options.columnIndices.length})`,\n );\n const startTime = performance.now();\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any(\n [options.signal, disposeSignal].filter((s): s is AbortSignal => !isNil(s)),\n );\n\n return await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({ signal: combinedSignal });\n const clippedRange = clipRange(options.range, shape);\n const specs = def.tableSpec;\n const separator = options.format === \"tsv\" ? \"\\t\" : \",\";\n\n const iterable = streamPTableRows({\n pTable,\n specs,\n columnIndices: options.columnIndices,\n range: clippedRange,\n chunkSize: options.chunkSize ?? 50_000,\n separator,\n includeHeader: options.includeHeader ?? true,\n bom: options.bom ?? true,\n signal: combinedSignal,\n });\n\n const miLogger: MiLogger = {\n info: (msg) => this.logger(\"info\", String(msg)),\n warn: (msg) => this.logger(\"warn\", String(msg)),\n error: (msg) => this.logger(\"error\", String(msg)),\n };\n\n let bytesWritten = 0;\n await createPathAtomically(miLogger, options.path, async (tempPath) => {\n const writeStream = fs.createWriteStream(tempPath, { flags: \"wx\" });\n const source = Readable.from(iterable, { objectMode: false });\n if (options.compression?.type === \"gzip\") {\n const gzip = zlib.createGzip({ level: options.compression.level ?? 6 });\n await pipeline(source, gzip, writeStream, { signal: combinedSignal });\n } else {\n await pipeline(source, writeStream, { signal: combinedSignal });\n }\n bytesWritten = writeStream.bytesWritten;\n });\n\n const overallSize = await pTable.getFootprint({ signal: combinedSignal });\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n\n // rowsWritten equals the clipped range length — the generator streams the\n // entire effective range without early termination, so this is accurate.\n const rowsWritten = clippedRange.length;\n\n if (logPFrames()) {\n const durationMs = Math.round(performance.now() - startTime);\n this.logger(\n \"info\",\n `[WritePTableToFs] complete (handle = ${handle}, columns = ${options.columnIndices.length}, rows = ${rowsWritten}, bytes = ${bytesWritten}, duration = ${durationMs}ms)`,\n );\n }\n\n return { path: options.path, rowsWritten, bytesWritten };\n });\n }\n\n //\n // PFrame instance methods\n //\n\n public async findColumns(\n handle: PFrameHandle,\n request: FindColumnsRequest,\n ): Promise<FindColumnsResponse> {\n const iRequest: PFrameInternal.FindColumnsRequest = {\n ...request,\n compatibleWith:\n request.compatibleWith.length !== 0\n ? [\n {\n axesSpec: [\n ...new Map(\n request.compatibleWith.map((item) => [canonicalizeJson(item), item] as const),\n ).values(),\n ],\n qualifications: [],\n },\n ]\n : [],\n };\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const response = pFrameSpec.findColumns(iRequest);\n return {\n hits: response.hits\n .filter(\n (h) =>\n // only exactly matching columns\n h.mappingVariants.length === 0 ||\n h.mappingVariants.some(\n (v) =>\n v.qualifications.forHit.length === 0 &&\n v.qualifications.forQueries.every((q) => q.length === 0),\n ),\n )\n .map((h) => h.hit),\n };\n }\n\n public async getColumnSpec(\n handle: PFrameHandle,\n columnId: PObjectId,\n ): Promise<PColumnSpec | null> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const info = pFrameSpec.listColumns().find((c) => c.columnId === columnId);\n return info?.spec ?? null;\n }\n\n public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n return pFrameSpec.listColumns().map(({ columnId, spec }) => ({ columnId, spec }));\n }\n\n public async calculateTableData(\n handle: PFrameHandle,\n request: CalculateTableDataRequest<PObjectId>,\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<CalculateTableDataResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameSpec } = this.pFrames.getByKey(handle);\n const sortedDef = sortPTableDef(migrateTableFilter(request, this.logger));\n const { def, entry } = this.pTables.acquireFromLegacy({\n pFrameHandle: handle,\n def: sortedDef,\n pFrameSpec,\n });\n using tableGuard = new PoolEntryGuard(entry);\n const tableSpec = def.tableSpec;\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n // TODO: throw error when more then 150k rows is requested\n // after pf-plots migration to stream API\n\n const data = await pTable.getData([...tableSpec.keys()], {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n this.pTableCachePerFrame.cache(tableGuard.keep(), overallSize);\n\n return tableSpec.map((spec, i) => ({\n spec: spec,\n data: data[i],\n }));\n });\n }\n\n public async getUniqueValues(\n handle: PFrameHandle,\n request: UniqueValuesRequest,\n signal?: AbortSignal,\n ): Promise<UniqueValuesResponse> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,\n );\n }\n\n const { pFrameDataPromise, pFrameSpec, disposeSignal } = this.pFrames.getByKey(handle);\n const pFrameData = await pFrameDataPromise;\n\n const column = pFrameSpec.listColumns().find((c) => c.columnId === request.columnId);\n if (!column) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);\n throw error;\n }\n\n const axisIds = expandAxes(column.spec.axesSpec);\n const tableSpec: PTableColumnSpec[] = [\n ...column.spec.axesSpec.map(\n (axisSpec, i): PTableColumnSpec => ({\n type: \"axis\",\n id: axisIds[i],\n spec: axisSpec,\n }),\n ),\n { type: \"column\", id: request.columnId, spec: column.spec },\n ];\n\n let axisIndex: number | undefined;\n if (request.axis !== undefined) {\n const index = findTableColumn(tableSpec, { type: \"axis\", id: request.axis });\n if (index < 0) {\n const error = new PFrameDriverError(`getUniqueValues failed`);\n error.cause = new Error(\n `axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,\n );\n throw error;\n }\n axisIndex = index;\n }\n\n const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {\n columnId: request.columnId,\n axisIndex,\n filters: rewriteLegacyFilters({\n tableSpec,\n filters: migrateFilters(request.filters, this.logger),\n }),\n limit: request.limit,\n };\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n return await this.frameConcurrencyLimiter.run(async () => {\n return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });\n });\n }\n\n //\n // PTable instance methods\n //\n\n public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {\n const { def } = this.pTableDefs.getByKey(handle);\n return def.tableSpec;\n }\n\n public async getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const shape = await pTable.getShape({\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { shape, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return shape;\n }\n\n public async getData(\n handle: PTableHandle,\n columnIndices: number[],\n range: TableRange | undefined,\n signal?: AbortSignal,\n ): Promise<PTableVector[]> {\n const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);\n using tableGuard = new PoolEntryGuard(this.pTables.acquire(def));\n\n const { pTablePromise, disposeSignal } = tableGuard.resource;\n const pTable = await pTablePromise;\n\n const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));\n const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {\n const data = await pTable.getData(columnIndices, {\n range,\n signal: combinedSignal,\n });\n\n const overallSize = await pTable.getFootprint({\n signal: combinedSignal,\n });\n\n return { data, overallSize };\n });\n\n this.pTableCachePlain.cache(tableGuard.keep(), overallSize, defDisposeSignal);\n return data;\n }\n}\n\n/** Clamp range to table shape. When range is undefined, returns full table range. */\nfunction clipRange(range: undefined | TableRange, shape: PTableShape): TableRange {\n if (isNil(range)) {\n return { offset: 0, length: shape.rows };\n }\n const clampedOffset = Math.min(range.offset, shape.rows);\n const clampedLength = Math.min(range.length, shape.rows - clampedOffset);\n return { offset: clampedOffset, length: clampedLength };\n}\n\nfunction migrateFilters(\n filters: PTableRecordFilter[],\n logger: PFrameInternal.Logger,\n): PTableRecordFilter[] {\n const filtersV1 = [];\n const filtersV2: PTableRecordSingleValueFilterV2[] = [];\n for (const filter of filters) {\n if ((filter.type as unknown) === \"bySingleColumn\") {\n filtersV1.push(filter);\n filtersV2.push({\n ...filter,\n type: \"bySingleColumnV2\",\n });\n } else {\n filtersV2.push(filter);\n }\n }\n if (filtersV1.length > 0) {\n const filtersV1Json = JSON.stringify(filtersV1);\n logger(\n \"warn\",\n `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`,\n );\n }\n return filtersV2;\n}\n\nfunction migrateTableFilter<T>(\n def: Omit<PTableDef<T>, \"partitionFilters\"> | PTableDef<T>,\n logger: PFrameInternal.Logger,\n): PTableDef<T> {\n if (!(\"partitionFilters\" in def)) {\n // For old blocks assume all axes filters to be partition filters\n return {\n ...def,\n partitionFilters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"axis\"),\n logger,\n ),\n filters: migrateFilters(\n def.filters.filter((f) => f.column.type === \"column\"),\n logger,\n ),\n };\n }\n return {\n ...def,\n partitionFilters: migrateFilters(def.partitionFilters, logger),\n filters: migrateFilters(def.filters, logger),\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA0FA,MAAa,kCAA2D;CACtE,GAAG;CACH,GAAG;CACH,mBAAmB;CACnB,mBAAmB;CACpB;AAOD,IAAa,uBAAb,MAGuD;CACrD;CAEA;CACA;CAEA;CAEA;CACA;CACA;CAEA;CACA;CAEA;CACA;CAEA,MAAa,YAAiC;AAC5C,SAAO,MAAM,cAAc,WAAW;;CAGxC,YAAmB,EACjB,eAAe,IACf,mBACA,oBACA,YAAY,QAAQ,EACpB,UAAU,iCACV,mBAQC;AACD,OAAK,SAAS;AAEd,OAAK,oBAAoB;AACzB,OAAK,qBAAqB;AAE1B,OAAK,kBAAkB;AAEvB,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AACzF,OAAK,0BAA0B,IAAI,4BAA4B,QAAQ,kBAAkB;AAEzF,OAAK,UAAU,IAAI,WACjB,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,UACD;AACD,OAAK,aAAa,IAAI,cAAc,KAAK,OAAO;AAChD,OAAK,UAAU,IAAI,WAAW,KAAK,SAAS,KAAK,YAAY,KAAK,OAAO;AAEzE,OAAK,sBAAsB,IAAI,oBAAoB,KAAK,QAAQ,QAAQ;AACxE,OAAK,mBAAmB,IAAI,iBAAiB,KAAK,QAAQ,QAAQ;;CAGpE,MAAM,UAAyB;AACvB,QAAM,QAAQ,WAAW;GAC7B,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,WAAW,OAAO,eAAe;GACtC,KAAK,QAAQ,OAAO,eAAe;GACnC,KAAK,kBAAkB,OAAO,eAAe;GAC7C,KAAK,mBAAmB,OAAO,eAAe;GAC/C,CAAC;;CAGJ,OAAO,OAAO,gBAA+B;AAC3C,SAAO,MAAM,KAAK,SAAS;;CAO7B,aAAoB,KAA+D;EACjF,MAAM,aAAa,IAAI,IAAI,OAAO,OAAO,UAAU,CAAC;EAKpD,MAAM,UADgB,SAHG,IACtB,QAAQ,WAAW,WAAW,IAAI,OAAO,KAAK,UAAU,CAAC,CACzD,KAAK,OAAO;GAAE,GAAG;GAAG,MAAM,yBAAyB,EAAE,KAAK;GAAE,EAAE,GACf,WAAW,OAAO,GAAG,CACzC,KAAK,MACjC,eAAe,IAAI,MAAM,KAAK,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAC1D;AAED,SAAO,KAAK,QAAQ,QAAQ,QAAQ;;CAGtC,aAAoB,QAAkE;;;GACpF,MAAM,UAAU,SAAS,kBAAkB,OAAO,IAAI,GAAG,MAAM,EAAE,GAAG;GACpE,MAAM,cAAA,YAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,YAAY,cAChB,mBACE,aAAa,SAAS,MAAM,EAAE,GAAG,EACjC,KAAK,OACN,CACF;GACD,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAA,YAAA,EAAc,IAAI,eACtB,KAAK,WAAW,kBAAkB;IAChC,cAAc,YAAY;IAC1B,KAAK;IACL;IACD,CAAC,CAAC,MACJ,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,eAAsB,KAAiE;;;GACrF,MAAM,UAAU,SAAS,wBAAwB,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG;GACzE,MAAM,cAAA,WAAA,EAAc,IAAI,eAAe,KAAK,aAAa,QAAQ,CAAC,CAAA;GAClE,MAAM,aAAa,KAAK,QAAQ,SAAS,YAAY,IAAI,CAAC;GAC1D,MAAM,cAAc,cAAc,oBAAoB,IAAI,QAAQ,MAAM,EAAE,GAAG,CAAC;GAC9E,MAAM,EAAE,WAAW,cAAc,WAAW,cAAc,YAAY;GAEtE,MAAM,cAAA,WAAA,EAAc,IAAI,eACtB,KAAK,WAAW,QAAQ;IACtB,cAAc,YAAY;IAC1B;IACA;IACD,CAAC,CACH,CAAA;AACD,OAAI,YAAY,CACd,MAAK,OACH,QACA,sCAAsC,YAAY,IAAI,mBAAmB,YAAY,IAAI,GAC1F;GAGH,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc,YAAY,MAAM;GACtC,MAAM,cAAc;AAClB,gBAAY,OAAO;AACnB,gBAAY,OAAO;;AAErB,UAAO;IACL,KAAK,YAAY;IACjB,UAAU,YAAY;IACtB;KACC,OAAO,UAAU;IACnB;;;;;;;CAGH,MAAa,gBACX,QACA,SACgC;;;AAChC,QAAK,OACH,QACA,qCAAqC,OAAO,WAAW,QAAQ,KAAK,aAAa,QAAQ,OAAO,kBAAkB,QAAQ,eAAe,OAAO,cAAc,QAAQ,cAAc,OAAO,GAC5L;GACD,MAAM,YAAY,YAAY,KAAK;GACnC,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAChE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IACjC,CAAC,QAAQ,QAAQ,cAAc,CAAC,QAAQ,MAAwB,CAAC,MAAM,EAAE,CAAC,CAC3E;AAED,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IACxD,MAAM,QAAQ,MAAM,OAAO,SAAS,EAAE,QAAQ,gBAAgB,CAAC;IAC/D,MAAM,eAAe,UAAU,QAAQ,OAAO,MAAM;IACpD,MAAM,QAAQ,IAAI;IAClB,MAAM,YAAY,QAAQ,WAAW,QAAQ,MAAO;IAEpD,MAAM,WAAW,iBAAiB;KAChC;KACA;KACA,eAAe,QAAQ;KACvB,OAAO;KACP,WAAW,QAAQ,aAAa;KAChC;KACA,eAAe,QAAQ,iBAAiB;KACxC,KAAK,QAAQ,OAAO;KACpB,QAAQ;KACT,CAAC;IAEF,MAAM,WAAqB;KACzB,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,IAAI,CAAC;KAC/C,QAAQ,QAAQ,KAAK,OAAO,SAAS,OAAO,IAAI,CAAC;KAClD;IAED,IAAI,eAAe;AACnB,UAAM,qBAAqB,UAAU,QAAQ,MAAM,OAAO,aAAa;KACrE,MAAM,cAAcA,KAAG,kBAAkB,UAAU,EAAE,OAAO,MAAM,CAAC;KACnE,MAAM,SAAS,SAAS,KAAK,UAAU,EAAE,YAAY,OAAO,CAAC;AAC7D,SAAI,QAAQ,aAAa,SAAS,OAEhC,OAAM,SAAS,QADF,KAAK,WAAW,EAAE,OAAO,QAAQ,YAAY,SAAS,GAAG,CAAC,EAC1C,aAAa,EAAE,QAAQ,gBAAgB,CAAC;SAErE,OAAM,SAAS,QAAQ,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AAEjE,oBAAe,YAAY;MAC3B;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAAE,QAAQ,gBAAgB,CAAC;AACzE,SAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;IAI7E,MAAM,cAAc,aAAa;AAEjC,QAAI,YAAY,EAAE;KAChB,MAAM,aAAa,KAAK,MAAM,YAAY,KAAK,GAAG,UAAU;AAC5D,UAAK,OACH,QACA,wCAAwC,OAAO,cAAc,QAAQ,cAAc,OAAO,WAAW,YAAY,YAAY,aAAa,eAAe,WAAW,KACrK;;AAGH,WAAO;KAAE,MAAM,QAAQ;KAAM;KAAa;KAAc;KACxD;;;;;;;CAOJ,MAAa,YACX,QACA,SAC8B;EAC9B,MAAM,WAA8C;GAClD,GAAG;GACH,gBACE,QAAQ,eAAe,WAAW,IAC9B,CACE;IACE,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,eAAe,KAAK,SAAS,CAAC,iBAAiB,KAAK,EAAE,KAAK,CAAU,CAC9E,CAAC,QAAQ,CACX;IACD,gBAAgB,EAAE;IACnB,CACF,GACD,EAAE;GACT;EAED,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SAAO,EACL,MAFe,WAAW,YAAY,SAAS,CAEhC,KACZ,QACE,MAEC,EAAE,gBAAgB,WAAW,KAC7B,EAAE,gBAAgB,MACf,MACC,EAAE,eAAe,OAAO,WAAW,KACnC,EAAE,eAAe,WAAW,OAAO,MAAM,EAAE,WAAW,EAAE,CAC3D,CACJ,CACA,KAAK,MAAM,EAAE,IAAI,EACrB;;CAGH,MAAa,cACX,QACA,UAC6B;EAC7B,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AAEpD,SADa,WAAW,aAAa,CAAC,MAAM,MAAM,EAAE,aAAa,SAAS,EAC7D,QAAQ;;CAGvB,MAAa,YAAY,QAAmD;EAC1E,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;AACpD,SAAO,WAAW,aAAa,CAAC,KAAK,EAAE,UAAU,YAAY;GAAE;GAAU;GAAM,EAAE;;CAGnF,MAAa,mBACX,QACA,SACA,OACA,QACqC;;;AACrC,OAAI,YAAY,CACd,MAAK,OACH,QACA,qCAAqC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAClG;GAGH,MAAM,EAAE,eAAe,KAAK,QAAQ,SAAS,OAAO;GACpD,MAAM,YAAY,cAAc,mBAAmB,SAAS,KAAK,OAAO,CAAC;GACzE,MAAM,EAAE,KAAK,UAAU,KAAK,QAAQ,kBAAkB;IACpD,cAAc;IACd,KAAK;IACL;IACD,CAAC;GACF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,MAAM,CAAA;GAC5C,MAAM,YAAY,IAAI;GACtB,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,UAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;IAIxD,MAAM,OAAO,MAAM,OAAO,QAAQ,CAAC,GAAG,UAAU,MAAM,CAAC,EAAE;KACvD;KACA,QAAQ;KACT,CAAC;IAEF,MAAM,cAAc,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;AACF,SAAK,oBAAoB,MAAM,WAAW,MAAM,EAAE,YAAY;AAE9D,WAAO,UAAU,KAAK,MAAM,OAAO;KAC3B;KACN,MAAM,KAAK;KACZ,EAAE;KACH;;;;;;;CAGJ,MAAa,gBACX,QACA,SACA,QAC+B;AAC/B,MAAI,YAAY,CACd,MAAK,OACH,QACA,kCAAkC,OAAO,cAAc,KAAK,UAAU,SAAS,eAAe,GAC/F;EAGH,MAAM,EAAE,mBAAmB,YAAY,kBAAkB,KAAK,QAAQ,SAAS,OAAO;EACtF,MAAM,aAAa,MAAM;EAEzB,MAAM,SAAS,WAAW,aAAa,CAAC,MAAM,MAAM,EAAE,aAAa,QAAQ,SAAS;AACpF,MAAI,CAAC,QAAQ;GACX,MAAM,QAAQ,IAAI,kBAAkB,yBAAyB;AAC7D,SAAM,wBAAQ,IAAI,MAAM,UAAU,QAAQ,SAAS,kCAAkC;AACrF,SAAM;;EAGR,MAAM,UAAU,WAAW,OAAO,KAAK,SAAS;EAChD,MAAM,YAAgC,CACpC,GAAG,OAAO,KAAK,SAAS,KACrB,UAAU,OAAyB;GAClC,MAAM;GACN,IAAI,QAAQ;GACZ,MAAM;GACP,EACF,EACD;GAAE,MAAM;GAAU,IAAI,QAAQ;GAAU,MAAM,OAAO;GAAM,CAC5D;EAED,IAAI;AACJ,MAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,MAAM,QAAQ,gBAAgB,WAAW;IAAE,MAAM;IAAQ,IAAI,QAAQ;IAAM,CAAC;AAC5E,OAAI,QAAQ,GAAG;IACb,MAAM,QAAQ,IAAI,kBAAkB,yBAAyB;AAC7D,UAAM,wBAAQ,IAAI,MAChB,QAAQ,KAAK,UAAU,QAAQ,KAAK,CAAC,kCAAkC,QAAQ,WAChF;AACD,UAAM;;AAER,eAAY;;EAGd,MAAM,kBAAwD;GAC5D,UAAU,QAAQ;GAClB;GACA,SAAS,qBAAqB;IAC5B;IACA,SAAS,eAAe,QAAQ,SAAS,KAAK,OAAO;IACtD,CAAC;GACF,OAAO,QAAQ;GAChB;EAED,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;AAClF,SAAO,MAAM,KAAK,wBAAwB,IAAI,YAAY;AACxD,UAAO,MAAM,WAAW,gBAAgB,iBAAiB,EAAE,QAAQ,gBAAgB,CAAC;IACpF;;CAOJ,MAAa,QAAQ,QAAmD;EACtE,MAAM,EAAE,QAAQ,KAAK,WAAW,SAAS,OAAO;AAChD,SAAO,IAAI;;CAGb,MAAa,SAAS,QAAsB,QAA4C;;;GACtF,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,OAAO,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAShF,WAAO;KAAE,OARK,MAAM,OAAO,SAAS,EAClC,QAAQ,gBACT,CAAC;KAMc,aAJI,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE2B;KAC7B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;CAGT,MAAa,QACX,QACA,eACA,OACA,QACyB;;;GACzB,MAAM,EAAE,KAAK,eAAe,qBAAqB,KAAK,WAAW,SAAS,OAAO;GACjF,MAAM,aAAA,WAAA,EAAa,IAAI,eAAe,KAAK,QAAQ,QAAQ,IAAI,CAAC,CAAA;GAEhE,MAAM,EAAE,eAAe,kBAAkB,WAAW;GACpD,MAAM,SAAS,MAAM;GAErB,MAAM,iBAAiB,YAAY,IAAI,CAAC,QAAQ,cAAc,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;GAClF,MAAM,EAAE,MAAM,gBAAgB,MAAM,KAAK,wBAAwB,IAAI,YAAY;AAU/E,WAAO;KAAE,MATI,MAAM,OAAO,QAAQ,eAAe;MAC/C;MACA,QAAQ;MACT,CAAC;KAMa,aAJK,MAAM,OAAO,aAAa,EAC5C,QAAQ,gBACT,CAAC;KAE0B;KAC5B;AAEF,QAAK,iBAAiB,MAAM,WAAW,MAAM,EAAE,aAAa,iBAAiB;AAC7E,UAAO;;;;;;;;;AAKX,SAAS,UAAU,OAA+B,OAAgC;AAChF,KAAI,MAAM,MAAM,CACd,QAAO;EAAE,QAAQ;EAAG,QAAQ,MAAM;EAAM;CAE1C,MAAM,gBAAgB,KAAK,IAAI,MAAM,QAAQ,MAAM,KAAK;AAExD,QAAO;EAAE,QAAQ;EAAe,QADV,KAAK,IAAI,MAAM,QAAQ,MAAM,OAAO,cAAc;EACjB;;AAGzD,SAAS,eACP,SACA,QACsB;CACtB,MAAM,YAAY,EAAE;CACpB,MAAM,YAA+C,EAAE;AACvD,MAAK,MAAM,UAAU,QACnB,KAAK,OAAO,SAAqB,kBAAkB;AACjD,YAAU,KAAK,OAAO;AACtB,YAAU,KAAK;GACb,GAAG;GACH,MAAM;GACP,CAAC;OAEF,WAAU,KAAK,OAAO;AAG1B,KAAI,UAAU,SAAS,EAErB,QACE,QACA,6EAHoB,KAAK,UAAU,UAAU,GAI9C;AAEH,QAAO;;AAGT,SAAS,mBACP,KACA,QACc;AACd,KAAI,EAAE,sBAAsB,KAE1B,QAAO;EACL,GAAG;EACH,kBAAkB,eAChB,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,OAAO,EACnD,OACD;EACD,SAAS,eACP,IAAI,QAAQ,QAAQ,MAAM,EAAE,OAAO,SAAS,SAAS,EACrD,OACD;EACF;AAEH,QAAO;EACL,GAAG;EACH,kBAAkB,eAAe,IAAI,kBAAkB,OAAO;EAC9D,SAAS,eAAe,IAAI,SAAS,OAAO;EAC7C"}
|
package/dist/pframe_pool.cjs
CHANGED
|
@@ -3,8 +3,8 @@ const require_logging = require("./logging.cjs");
|
|
|
3
3
|
let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
|
|
4
4
|
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
5
5
|
let _milaboratories_pframes_rs_node = require("@milaboratories/pframes-rs-node");
|
|
6
|
-
let _milaboratories_pl_model_middle_layer = require("@milaboratories/pl-model-middle-layer");
|
|
7
6
|
let _milaboratories_pframes_rs_wasm = require("@milaboratories/pframes-rs-wasm");
|
|
7
|
+
let _milaboratories_pl_model_middle_layer = require("@milaboratories/pl-model-middle-layer");
|
|
8
8
|
let es_toolkit = require("es-toolkit");
|
|
9
9
|
//#region src/pframe_pool.ts
|
|
10
10
|
var PFrameHolder = class {
|
|
@@ -70,17 +70,19 @@ var PFrameHolder = class {
|
|
|
70
70
|
spillPath: this.spillPath,
|
|
71
71
|
logger
|
|
72
72
|
});
|
|
73
|
-
const promises = [];
|
|
74
73
|
try {
|
|
75
74
|
pFrameData.setDataSource({
|
|
76
75
|
...this.localBlobProvider.makeDataSource(this.disposeSignal),
|
|
77
76
|
parquetServer: this.remoteBlobProvider.httpServerInfo()
|
|
78
77
|
});
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
78
|
+
this.pFrameDataPromise = pFrameData.addColumns(jsonifiedColumns.map((c) => ({
|
|
79
|
+
id: c.id,
|
|
80
|
+
typeSpec: {
|
|
81
|
+
axes: c.spec.axesSpec.map((a) => a.type),
|
|
82
|
+
columns: [c.spec.valueType]
|
|
83
|
+
},
|
|
84
|
+
data: c.data
|
|
85
|
+
})), { signal: this.disposeSignal }).then(() => pFrameData).catch((err) => {
|
|
84
86
|
this.dispose();
|
|
85
87
|
pFrameData.dispose();
|
|
86
88
|
const error = new _milaboratories_pl_model_common.PFrameDriverError("PFrame creation failed asynchronously");
|
|
@@ -88,7 +90,6 @@ var PFrameHolder = class {
|
|
|
88
90
|
throw error;
|
|
89
91
|
});
|
|
90
92
|
} catch (err) {
|
|
91
|
-
Promise.allSettled(promises);
|
|
92
93
|
pFrameData.dispose();
|
|
93
94
|
throw err;
|
|
94
95
|
}
|
package/dist/pframe_pool.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pframe_pool.cjs","names":["ValueType","PFrameInternal","PFrameFactory","PFrameDriverError","RefCountPoolBase","logPFrames","bigintReplacer"],"sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n assertNever,\n bigintReplacer,\n canonicalizeJson,\n ensureError,\n mapPObjectData,\n PFrameDriverError,\n ValueType,\n resolveAnnotationParents,\n type JsonSerializable,\n type PColumn,\n type PColumnSpec,\n type PFrameHandle,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { createPFrame as createPFrameSpec } from \"@milaboratories/pframes-rs-wasm\";\nimport { mapValues } from \"es-toolkit\";\nimport { logPFrames } from \"./logging\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\">;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n public readonly pFrameDataPromise: Promise<PFrameInternal.PFrameV13>;\n /**\n * WASM-spec frame built from this PFrame's columns. Source of truth\n * for spec-side operations: column discovery, selector resolution,\n * legacy-query lowering.\n */\n public readonly pFrameSpec: PFrameInternal.PFrameWasmV3;\n private readonly abortController = new AbortController();\n\n private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n constructor(\n frameId: PFrameInternal.PFrameId,\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ) {\n const ValueTypes = new Set(Object.values(ValueType));\n const specColumnsMap: Record<string, PColumnSpec> = {};\n for (const c of columns) {\n if (ValueTypes.has(c.spec.valueType)) {\n specColumnsMap[c.id] = resolveAnnotationParents(c.spec);\n }\n }\n this.pFrameSpec = createPFrameSpec(specColumnsMap);\n\n try {\n const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const localBlob = this.localBlobProvider.acquire(blob);\n this.localBlobs.push(localBlob);\n return localBlob.key;\n };\n\n const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const remoteBlob = this.remoteBlobProvider.acquire(blob);\n this.remoteBlobs.push(remoteBlob);\n return `${remoteBlob.key}${PFrameInternal.ParquetExtension}` as PFrameInternal.PFrameBlobId;\n };\n\n const mapColumnData = (\n data: PFrameInternal.DataInfo<TreeEntry>,\n ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n switch (data.type) {\n case \"Json\":\n return { ...data };\n case \"JsonPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, makeLocalBlobId),\n };\n case \"BinaryPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n index: makeLocalBlobId(v.index),\n values: makeLocalBlobId(v.values),\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n ...v,\n data: makeRemoteBlobId(v.data),\n })),\n };\n default:\n assertNever(data);\n }\n };\n\n const jsonifiedColumns = columns.map((column) => ({\n ...column,\n data: mapColumnData(column.data),\n }));\n\n const pFrameData = PFrameFactory.createPFrame({ frameId, spillPath: this.spillPath, logger });\n const promises: Promise<void>[] = [];\n try {\n pFrameData.setDataSource({\n ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n parquetServer: this.remoteBlobProvider.httpServerInfo(),\n });\n\n for (const column of jsonifiedColumns) {\n pFrameData.addColumnSpec(column.id, column.spec);\n promises.push(\n pFrameData.setColumnData(column.id, column.data, { signal: this.disposeSignal }),\n );\n }\n\n this.pFrameDataPromise = Promise.all(promises)\n .then(() => pFrameData)\n .catch((err) => {\n this.dispose();\n pFrameData.dispose();\n const error = new PFrameDriverError(\"PFrame creation failed asynchronously\");\n error.cause = new Error(\n `PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,\n { cause: ensureError(err) },\n );\n throw error;\n });\n } catch (err: unknown) {\n // setDataSource / addColumnSpec / setColumnData threw synchronously\n // before pFrameDataPromise was assigned — dispose the addon's\n // pFrameData explicitly (the dispose() path can't reach it yet).\n // `allSettled` attaches handlers to any in-flight setColumnData\n // promises so they don't trigger unhandled-rejection warnings\n // when the dispose below rejects them.\n void Promise.allSettled(promises);\n pFrameData.dispose();\n throw err;\n }\n } catch (err: unknown) {\n // Release everything allocated so far in this constructor:\n // acquired blobs, the WASM spec frame, and the abort signal.\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n const error = new PFrameDriverError(\"PFrame creation failed synchronously\");\n error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(columns)}`, {\n cause: ensureError(err),\n });\n throw error;\n }\n }\n\n public get disposeSignal(): AbortSignal {\n return this.abortController.signal;\n }\n\n private dispose(): void {\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n void this.pFrameDataPromise\n .then((pFrameData) => pFrameData.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n\n [Symbol.dispose](): void {\n this.dispose();\n }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n PFrameHandle,\n PFrameHolder<TreeEntry>\n> {\n constructor(\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n private readonly logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n ) {\n super();\n }\n\n protected calculateParamsKey(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ): PFrameHandle {\n return stableKeyFromPFrameData(params);\n }\n\n protected createNewResource(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n key: PFrameHandle,\n ): PFrameHolder<TreeEntry> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PFrame creation (pFrameHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n return new PFrameHolder(\n key,\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n this.spillPath,\n params,\n );\n }\n\n public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PFrame handle`);\n error.cause = new Error(`PFrame with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n const orderedData = [...data].map((column) =>\n mapPObjectData(column, (r) => {\n let result: {\n type: string;\n keyLength: number;\n payload: {\n key: string;\n value: null | number | string | [string, string];\n }[];\n };\n const type = r.type;\n switch (type) {\n case \"Json\":\n result = {\n type: r.type,\n keyLength: r.keyLength,\n payload: Object.entries(r.data).map(([part, value]) => ({\n key: part,\n value,\n })),\n };\n break;\n case \"JsonPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: canonicalizeJson(info),\n })),\n };\n break;\n case \"BinaryPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n })),\n };\n break;\n case \"ParquetPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value:\n info.dataDigest ||\n ([\n canonicalizeJson(info.data),\n JSON.stringify({ axes: info.axes, column: info.column }),\n ] as const),\n })),\n };\n break;\n default:\n throw new PFrameDriverError(\n `unsupported resource type: ${JSON.stringify(type satisfies never)}`,\n );\n }\n result.payload.sort((lhs, rhs) => (lhs.key < rhs.key ? -1 : 1));\n return result;\n }),\n );\n orderedData.sort((lhs, rhs) => (lhs.id < rhs.id ? -1 : 1));\n return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"mappings":";;;;;;;;;AA+BA,IAAa,eAAb,MAAoF;CAClF;;;;;;CAMA;CACA,kBAAmC,IAAI,iBAAiB;CAExD,aAAwE,EAAE;CAC1E,cAAyE,EAAE;CAE3E,YACE,SACA,mBACA,oBACA,QACA,WACA,SACA;AALiB,OAAA,oBAAA;AACA,OAAA,qBAAA;AAEA,OAAA,YAAA;EAGjB,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOA,gCAAAA,UAAU,CAAC;EACpD,MAAM,iBAA8C,EAAE;AACtD,OAAK,MAAM,KAAK,QACd,KAAI,WAAW,IAAI,EAAE,KAAK,UAAU,CAClC,gBAAe,EAAE,OAAA,GAAA,gCAAA,0BAA+B,EAAE,KAAK;AAG3D,OAAK,cAAA,GAAA,gCAAA,cAA8B,eAAe;AAElD,MAAI;GACF,MAAM,mBAAmB,SAAiD;IACxE,MAAM,YAAY,KAAK,kBAAkB,QAAQ,KAAK;AACtD,SAAK,WAAW,KAAK,UAAU;AAC/B,WAAO,UAAU;;GAGnB,MAAM,oBAAoB,SAAiD;IACzE,MAAM,aAAa,KAAK,mBAAmB,QAAQ,KAAK;AACxD,SAAK,YAAY,KAAK,WAAW;AACjC,WAAO,GAAG,WAAW,MAAMC,sCAAAA,eAAe;;GAG5C,MAAM,iBACJ,SACyD;AACzD,YAAQ,KAAK,MAAb;KACE,KAAK,OACH,QAAO,EAAE,GAAG,MAAM;KACpB,KAAK,kBACH,QAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,WAAiB,KAAK,OAAO,gBAAgB;MAC9C;KACH,KAAK,oBACH,QAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,WAAiB,KAAK,QAAQ,OAAO;OACnC,OAAO,gBAAgB,EAAE,MAAM;OAC/B,QAAQ,gBAAgB,EAAE,OAAO;OAClC,EAAE;MACJ;KACH,KAAK,qBACH,QAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,WAAiB,KAAK,QAAQ,OAAO;OACnC,GAAG;OACH,MAAM,iBAAiB,EAAE,KAAK;OAC/B,EAAE;MACJ;KACH,QACE,EAAA,GAAA,gCAAA,aAAY,KAAK;;;GAIvB,MAAM,mBAAmB,QAAQ,KAAK,YAAY;IAChD,GAAG;IACH,MAAM,cAAc,OAAO,KAAK;IACjC,EAAE;GAEH,MAAM,aAAaC,gCAAAA,cAAc,aAAa;IAAE;IAAS,WAAW,KAAK;IAAW;IAAQ,CAAC;GAC7F,MAAM,WAA4B,EAAE;AACpC,OAAI;AACF,eAAW,cAAc;KACvB,GAAG,KAAK,kBAAkB,eAAe,KAAK,cAAc;KAC5D,eAAe,KAAK,mBAAmB,gBAAgB;KACxD,CAAC;AAEF,SAAK,MAAM,UAAU,kBAAkB;AACrC,gBAAW,cAAc,OAAO,IAAI,OAAO,KAAK;AAChD,cAAS,KACP,WAAW,cAAc,OAAO,IAAI,OAAO,MAAM,EAAE,QAAQ,KAAK,eAAe,CAAC,CACjF;;AAGH,SAAK,oBAAoB,QAAQ,IAAI,SAAS,CAC3C,WAAW,WAAW,CACtB,OAAO,QAAQ;AACd,UAAK,SAAS;AACd,gBAAW,SAAS;KACpB,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,wCAAwC;AAC5E,WAAM,QAAQ,IAAI,MAChB,0CAA0C,KAAK,UAAU,iBAAiB,IAC1E,EAAE,QAAA,GAAA,gCAAA,aAAmB,IAAI,EAAE,CAC5B;AACD,WAAM;MACN;YACG,KAAc;AAOhB,YAAQ,WAAW,SAAS;AACjC,eAAW,SAAS;AACpB,UAAM;;WAED,KAAc;AAGrB,QAAK,gBAAgB,OAAO;AAC5B,QAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,QAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,QAAK,WAAW,OAAO,UAAU;GACjC,MAAM,QAAQ,IAAIA,gCAAAA,kBAAkB,uCAAuC;AAC3E,SAAM,QAAQ,IAAI,MAAM,0CAA0C,KAAK,UAAU,QAAQ,IAAI,EAC3F,QAAA,GAAA,gCAAA,aAAmB,IAAI,EACxB,CAAC;AACF,SAAM;;;CAIV,IAAW,gBAA6B;AACtC,SAAO,KAAK,gBAAgB;;CAG9B,UAAwB;AACtB,OAAK,gBAAgB,OAAO;AAC5B,OAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,OAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,OAAK,WAAW,OAAO,UAAU;AAC5B,OAAK,kBACP,MAAM,eAAe,WAAW,SAAS,CAAC,CAC1C,YAAY,GAEX;;CAGN,CAAC,OAAO,WAAiB;AACvB,OAAK,SAAS;;;AAIlB,IAAa,aAAb,cAAoEC,wBAAAA,iBAIlE;CACA,YACE,mBACA,oBACA,QACA,WACA;AACA,SAAO;AALU,OAAA,oBAAA;AACA,OAAA,qBAAA;AACA,OAAA,SAAA;AACA,OAAA,YAAA;;CAKnB,mBACE,QACc;AACd,SAAO,wBAAwB,OAAO;;CAGxC,kBACE,QACA,KACyB;AACzB,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQC,gCAAAA,eAAe,GACxF;AAEH,SAAO,IAAI,aACT,KACA,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,KAAK,WACL,OACD;;CAGH,SAAgB,KAA4C;EAC1D,MAAM,WAAW,MAAM,YAAY,IAAI;AACvC,MAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAIH,gCAAAA,kBAAkB,wBAAwB;AAC5D,SAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,YAAY;AAC9D,SAAM;;AAER,SAAO;;;AAIX,SAAS,wBACP,MACc;CACd,MAAM,cAAc,CAAC,GAAG,KAAK,CAAC,KAAK,YAAA,GAAA,gCAAA,gBAClB,SAAS,MAAM;EAC5B,IAAI;EAQJ,MAAM,OAAO,EAAE;AACf,UAAQ,MAAR;GACE,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,YAAY;MACtD,KAAK;MACL;MACD,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,QAAA,GAAA,gCAAA,kBAAwB,KAAK;MAC9B,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,EAAA,GAAA,gCAAA,kBAAkB,KAAK,MAAM,GAAA,GAAA,gCAAA,kBAAmB,KAAK,OAAO,CAAC;MACrE,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OACE,KAAK,cACJ,EAAA,GAAA,gCAAA,kBACkB,KAAK,KAAK,EAC3B,KAAK,UAAU;OAAE,MAAM,KAAK;OAAM,QAAQ,KAAK;OAAQ,CAAC,CACzD;MACJ,EAAE;KACJ;AACD;GACF,QACE,OAAM,IAAIA,gCAAAA,kBACR,8BAA8B,KAAK,UAAU,KAAqB,GACnE;;AAEL,SAAO,QAAQ,MAAM,KAAK,QAAS,IAAI,MAAM,IAAI,MAAM,KAAK,EAAG;AAC/D,SAAO;GACP,CACH;AACD,aAAY,MAAM,KAAK,QAAS,IAAI,KAAK,IAAI,KAAK,KAAK,EAAG;AAC1D,SAAA,GAAA,sCAAA,UAAgB,YAAY"}
|
|
1
|
+
{"version":3,"file":"pframe_pool.cjs","names":["ValueType","PFrameInternal","PFrameFactory","PFrameDriverError","RefCountPoolBase","logPFrames","bigintReplacer"],"sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n assertNever,\n bigintReplacer,\n canonicalizeJson,\n ensureError,\n mapPObjectData,\n PFrameDriverError,\n ValueType,\n resolveAnnotationParents,\n type JsonSerializable,\n type PColumn,\n type PColumnSpec,\n type PFrameHandle,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { createPFrame as createPFrameSpec } from \"@milaboratories/pframes-rs-wasm\";\nimport { mapValues } from \"es-toolkit\";\nimport { logPFrames } from \"./logging\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\">;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n public readonly pFrameDataPromise: Promise<PFrameInternal.PFrameV14>;\n /**\n * WASM-spec frame built from this PFrame's columns. Source of truth\n * for spec-side operations: column discovery, selector resolution,\n * legacy-query lowering.\n */\n public readonly pFrameSpec: PFrameInternal.PFrameWasmV3;\n private readonly abortController = new AbortController();\n\n private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n constructor(\n frameId: PFrameInternal.PFrameId,\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ) {\n const ValueTypes = new Set(Object.values(ValueType));\n const specColumnsMap: Record<string, PColumnSpec> = {};\n for (const c of columns) {\n if (ValueTypes.has(c.spec.valueType)) {\n specColumnsMap[c.id] = resolveAnnotationParents(c.spec);\n }\n }\n this.pFrameSpec = createPFrameSpec(specColumnsMap);\n\n try {\n const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const localBlob = this.localBlobProvider.acquire(blob);\n this.localBlobs.push(localBlob);\n return localBlob.key;\n };\n\n const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const remoteBlob = this.remoteBlobProvider.acquire(blob);\n this.remoteBlobs.push(remoteBlob);\n return `${remoteBlob.key}${PFrameInternal.ParquetExtension}` as PFrameInternal.PFrameBlobId;\n };\n\n const mapColumnData = (\n data: PFrameInternal.DataInfo<TreeEntry>,\n ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n switch (data.type) {\n case \"Json\":\n return { ...data };\n case \"JsonPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, makeLocalBlobId),\n };\n case \"BinaryPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n index: makeLocalBlobId(v.index),\n values: makeLocalBlobId(v.values),\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n ...v,\n data: makeRemoteBlobId(v.data),\n })),\n };\n default:\n assertNever(data);\n }\n };\n\n const jsonifiedColumns = columns.map((column) => ({\n ...column,\n data: mapColumnData(column.data),\n }));\n\n const pFrameData = PFrameFactory.createPFrame({ frameId, spillPath: this.spillPath, logger });\n try {\n pFrameData.setDataSource({\n ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n parquetServer: this.remoteBlobProvider.httpServerInfo(),\n });\n\n this.pFrameDataPromise = pFrameData\n .addColumns(\n jsonifiedColumns.map((c) => ({\n id: c.id,\n typeSpec: {\n axes: c.spec.axesSpec.map((a) => a.type),\n columns: [c.spec.valueType],\n },\n data: c.data,\n })),\n { signal: this.disposeSignal },\n )\n .then(() => pFrameData)\n .catch((err) => {\n this.dispose();\n pFrameData.dispose();\n const error = new PFrameDriverError(\"PFrame creation failed asynchronously\");\n error.cause = new Error(\n `PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,\n { cause: ensureError(err) },\n );\n throw error;\n });\n } catch (err: unknown) {\n // setDataSource / addColumns threw synchronously before\n // pFrameDataPromise was assigned — dispose the addon's pFrameData\n // explicitly (the dispose() path can't reach it yet).\n pFrameData.dispose();\n throw err;\n }\n } catch (err: unknown) {\n // Release everything allocated so far in this constructor:\n // acquired blobs, the WASM spec frame, and the abort signal.\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n const error = new PFrameDriverError(\"PFrame creation failed synchronously\");\n error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(columns)}`, {\n cause: ensureError(err),\n });\n throw error;\n }\n }\n\n public get disposeSignal(): AbortSignal {\n return this.abortController.signal;\n }\n\n private dispose(): void {\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n void this.pFrameDataPromise\n .then((pFrameData) => pFrameData.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n\n [Symbol.dispose](): void {\n this.dispose();\n }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n PFrameHandle,\n PFrameHolder<TreeEntry>\n> {\n constructor(\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n private readonly logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n ) {\n super();\n }\n\n protected calculateParamsKey(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ): PFrameHandle {\n return stableKeyFromPFrameData(params);\n }\n\n protected createNewResource(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n key: PFrameHandle,\n ): PFrameHolder<TreeEntry> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PFrame creation (pFrameHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n return new PFrameHolder(\n key,\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n this.spillPath,\n params,\n );\n }\n\n public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PFrame handle`);\n error.cause = new Error(`PFrame with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n const orderedData = [...data].map((column) =>\n mapPObjectData(column, (r) => {\n let result: {\n type: string;\n keyLength: number;\n payload: {\n key: string;\n value: null | number | string | [string, string];\n }[];\n };\n const type = r.type;\n switch (type) {\n case \"Json\":\n result = {\n type: r.type,\n keyLength: r.keyLength,\n payload: Object.entries(r.data).map(([part, value]) => ({\n key: part,\n value,\n })),\n };\n break;\n case \"JsonPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: canonicalizeJson(info),\n })),\n };\n break;\n case \"BinaryPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n })),\n };\n break;\n case \"ParquetPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value:\n info.dataDigest ||\n ([\n canonicalizeJson(info.data),\n JSON.stringify({ axes: info.axes, column: info.column }),\n ] as const),\n })),\n };\n break;\n default:\n throw new PFrameDriverError(\n `unsupported resource type: ${JSON.stringify(type satisfies never)}`,\n );\n }\n result.payload.sort((lhs, rhs) => (lhs.key < rhs.key ? -1 : 1));\n return result;\n }),\n );\n orderedData.sort((lhs, rhs) => (lhs.id < rhs.id ? -1 : 1));\n return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"mappings":";;;;;;;;;AA+BA,IAAa,eAAb,MAAoF;CAClF;;;;;;CAMA;CACA,kBAAmC,IAAI,iBAAiB;CAExD,aAAwE,EAAE;CAC1E,cAAyE,EAAE;CAE3E,YACE,SACA,mBACA,oBACA,QACA,WACA,SACA;AALiB,OAAA,oBAAA;AACA,OAAA,qBAAA;AAEA,OAAA,YAAA;EAGjB,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOA,gCAAAA,UAAU,CAAC;EACpD,MAAM,iBAA8C,EAAE;AACtD,OAAK,MAAM,KAAK,QACd,KAAI,WAAW,IAAI,EAAE,KAAK,UAAU,CAClC,gBAAe,EAAE,OAAA,GAAA,gCAAA,0BAA+B,EAAE,KAAK;AAG3D,OAAK,cAAA,GAAA,gCAAA,cAA8B,eAAe;AAElD,MAAI;GACF,MAAM,mBAAmB,SAAiD;IACxE,MAAM,YAAY,KAAK,kBAAkB,QAAQ,KAAK;AACtD,SAAK,WAAW,KAAK,UAAU;AAC/B,WAAO,UAAU;;GAGnB,MAAM,oBAAoB,SAAiD;IACzE,MAAM,aAAa,KAAK,mBAAmB,QAAQ,KAAK;AACxD,SAAK,YAAY,KAAK,WAAW;AACjC,WAAO,GAAG,WAAW,MAAMC,sCAAAA,eAAe;;GAG5C,MAAM,iBACJ,SACyD;AACzD,YAAQ,KAAK,MAAb;KACE,KAAK,OACH,QAAO,EAAE,GAAG,MAAM;KACpB,KAAK,kBACH,QAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,WAAiB,KAAK,OAAO,gBAAgB;MAC9C;KACH,KAAK,oBACH,QAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,WAAiB,KAAK,QAAQ,OAAO;OACnC,OAAO,gBAAgB,EAAE,MAAM;OAC/B,QAAQ,gBAAgB,EAAE,OAAO;OAClC,EAAE;MACJ;KACH,KAAK,qBACH,QAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,WAAiB,KAAK,QAAQ,OAAO;OACnC,GAAG;OACH,MAAM,iBAAiB,EAAE,KAAK;OAC/B,EAAE;MACJ;KACH,QACE,EAAA,GAAA,gCAAA,aAAY,KAAK;;;GAIvB,MAAM,mBAAmB,QAAQ,KAAK,YAAY;IAChD,GAAG;IACH,MAAM,cAAc,OAAO,KAAK;IACjC,EAAE;GAEH,MAAM,aAAaC,gCAAAA,cAAc,aAAa;IAAE;IAAS,WAAW,KAAK;IAAW;IAAQ,CAAC;AAC7F,OAAI;AACF,eAAW,cAAc;KACvB,GAAG,KAAK,kBAAkB,eAAe,KAAK,cAAc;KAC5D,eAAe,KAAK,mBAAmB,gBAAgB;KACxD,CAAC;AAEF,SAAK,oBAAoB,WACtB,WACC,iBAAiB,KAAK,OAAO;KAC3B,IAAI,EAAE;KACN,UAAU;MACR,MAAM,EAAE,KAAK,SAAS,KAAK,MAAM,EAAE,KAAK;MACxC,SAAS,CAAC,EAAE,KAAK,UAAU;MAC5B;KACD,MAAM,EAAE;KACT,EAAE,EACH,EAAE,QAAQ,KAAK,eAAe,CAC/B,CACA,WAAW,WAAW,CACtB,OAAO,QAAQ;AACd,UAAK,SAAS;AACd,gBAAW,SAAS;KACpB,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,wCAAwC;AAC5E,WAAM,QAAQ,IAAI,MAChB,0CAA0C,KAAK,UAAU,iBAAiB,IAC1E,EAAE,QAAA,GAAA,gCAAA,aAAmB,IAAI,EAAE,CAC5B;AACD,WAAM;MACN;YACG,KAAc;AAIrB,eAAW,SAAS;AACpB,UAAM;;WAED,KAAc;AAGrB,QAAK,gBAAgB,OAAO;AAC5B,QAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,QAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,QAAK,WAAW,OAAO,UAAU;GACjC,MAAM,QAAQ,IAAIA,gCAAAA,kBAAkB,uCAAuC;AAC3E,SAAM,QAAQ,IAAI,MAAM,0CAA0C,KAAK,UAAU,QAAQ,IAAI,EAC3F,QAAA,GAAA,gCAAA,aAAmB,IAAI,EACxB,CAAC;AACF,SAAM;;;CAIV,IAAW,gBAA6B;AACtC,SAAO,KAAK,gBAAgB;;CAG9B,UAAwB;AACtB,OAAK,gBAAgB,OAAO;AAC5B,OAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,OAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,OAAK,WAAW,OAAO,UAAU;AAC5B,OAAK,kBACP,MAAM,eAAe,WAAW,SAAS,CAAC,CAC1C,YAAY,GAEX;;CAGN,CAAC,OAAO,WAAiB;AACvB,OAAK,SAAS;;;AAIlB,IAAa,aAAb,cAAoEC,wBAAAA,iBAIlE;CACA,YACE,mBACA,oBACA,QACA,WACA;AACA,SAAO;AALU,OAAA,oBAAA;AACA,OAAA,qBAAA;AACA,OAAA,SAAA;AACA,OAAA,YAAA;;CAKnB,mBACE,QACc;AACd,SAAO,wBAAwB,OAAO;;CAGxC,kBACE,QACA,KACyB;AACzB,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQC,gCAAAA,eAAe,GACxF;AAEH,SAAO,IAAI,aACT,KACA,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,KAAK,WACL,OACD;;CAGH,SAAgB,KAA4C;EAC1D,MAAM,WAAW,MAAM,YAAY,IAAI;AACvC,MAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAIH,gCAAAA,kBAAkB,wBAAwB;AAC5D,SAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,YAAY;AAC9D,SAAM;;AAER,SAAO;;;AAIX,SAAS,wBACP,MACc;CACd,MAAM,cAAc,CAAC,GAAG,KAAK,CAAC,KAAK,YAAA,GAAA,gCAAA,gBAClB,SAAS,MAAM;EAC5B,IAAI;EAQJ,MAAM,OAAO,EAAE;AACf,UAAQ,MAAR;GACE,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,YAAY;MACtD,KAAK;MACL;MACD,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,QAAA,GAAA,gCAAA,kBAAwB,KAAK;MAC9B,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,EAAA,GAAA,gCAAA,kBAAkB,KAAK,MAAM,GAAA,GAAA,gCAAA,kBAAmB,KAAK,OAAO,CAAC;MACrE,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OACE,KAAK,cACJ,EAAA,GAAA,gCAAA,kBACkB,KAAK,KAAK,EAC3B,KAAK,UAAU;OAAE,MAAM,KAAK;OAAM,QAAQ,KAAK;OAAQ,CAAC,CACzD;MACJ,EAAE;KACJ;AACD;GACF,QACE,OAAM,IAAIA,gCAAAA,kBACR,8BAA8B,KAAK,UAAU,KAAqB,GACnE;;AAEL,SAAO,QAAQ,MAAM,KAAK,QAAS,IAAI,MAAM,IAAI,MAAM,KAAK,EAAG;AAC/D,SAAO;GACP,CACH;AACD,aAAY,MAAM,KAAK,QAAS,IAAI,KAAK,IAAI,KAAK,KAAK,EAAG;AAC1D,SAAA,GAAA,sCAAA,UAAgB,YAAY"}
|
package/dist/pframe_pool.js
CHANGED
|
@@ -2,8 +2,8 @@ import { logPFrames } from "./logging.js";
|
|
|
2
2
|
import { PFrameDriverError, ValueType, assertNever, bigintReplacer, canonicalizeJson, ensureError, mapPObjectData, resolveAnnotationParents } from "@milaboratories/pl-model-common";
|
|
3
3
|
import { RefCountPoolBase } from "@milaboratories/helpers";
|
|
4
4
|
import { PFrameFactory } from "@milaboratories/pframes-rs-node";
|
|
5
|
-
import { PFrameInternal, hashJson } from "@milaboratories/pl-model-middle-layer";
|
|
6
5
|
import { createPFrame } from "@milaboratories/pframes-rs-wasm";
|
|
6
|
+
import { PFrameInternal, hashJson } from "@milaboratories/pl-model-middle-layer";
|
|
7
7
|
import { mapValues } from "es-toolkit";
|
|
8
8
|
//#region src/pframe_pool.ts
|
|
9
9
|
var PFrameHolder = class {
|
|
@@ -69,17 +69,19 @@ var PFrameHolder = class {
|
|
|
69
69
|
spillPath: this.spillPath,
|
|
70
70
|
logger
|
|
71
71
|
});
|
|
72
|
-
const promises = [];
|
|
73
72
|
try {
|
|
74
73
|
pFrameData.setDataSource({
|
|
75
74
|
...this.localBlobProvider.makeDataSource(this.disposeSignal),
|
|
76
75
|
parquetServer: this.remoteBlobProvider.httpServerInfo()
|
|
77
76
|
});
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
77
|
+
this.pFrameDataPromise = pFrameData.addColumns(jsonifiedColumns.map((c) => ({
|
|
78
|
+
id: c.id,
|
|
79
|
+
typeSpec: {
|
|
80
|
+
axes: c.spec.axesSpec.map((a) => a.type),
|
|
81
|
+
columns: [c.spec.valueType]
|
|
82
|
+
},
|
|
83
|
+
data: c.data
|
|
84
|
+
})), { signal: this.disposeSignal }).then(() => pFrameData).catch((err) => {
|
|
83
85
|
this.dispose();
|
|
84
86
|
pFrameData.dispose();
|
|
85
87
|
const error = new PFrameDriverError("PFrame creation failed asynchronously");
|
|
@@ -87,7 +89,6 @@ var PFrameHolder = class {
|
|
|
87
89
|
throw error;
|
|
88
90
|
});
|
|
89
91
|
} catch (err) {
|
|
90
|
-
Promise.allSettled(promises);
|
|
91
92
|
pFrameData.dispose();
|
|
92
93
|
throw err;
|
|
93
94
|
}
|
package/dist/pframe_pool.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pframe_pool.js","names":["createPFrameSpec"],"sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n assertNever,\n bigintReplacer,\n canonicalizeJson,\n ensureError,\n mapPObjectData,\n PFrameDriverError,\n ValueType,\n resolveAnnotationParents,\n type JsonSerializable,\n type PColumn,\n type PColumnSpec,\n type PFrameHandle,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { createPFrame as createPFrameSpec } from \"@milaboratories/pframes-rs-wasm\";\nimport { mapValues } from \"es-toolkit\";\nimport { logPFrames } from \"./logging\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\">;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n public readonly pFrameDataPromise: Promise<PFrameInternal.PFrameV13>;\n /**\n * WASM-spec frame built from this PFrame's columns. Source of truth\n * for spec-side operations: column discovery, selector resolution,\n * legacy-query lowering.\n */\n public readonly pFrameSpec: PFrameInternal.PFrameWasmV3;\n private readonly abortController = new AbortController();\n\n private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n constructor(\n frameId: PFrameInternal.PFrameId,\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ) {\n const ValueTypes = new Set(Object.values(ValueType));\n const specColumnsMap: Record<string, PColumnSpec> = {};\n for (const c of columns) {\n if (ValueTypes.has(c.spec.valueType)) {\n specColumnsMap[c.id] = resolveAnnotationParents(c.spec);\n }\n }\n this.pFrameSpec = createPFrameSpec(specColumnsMap);\n\n try {\n const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const localBlob = this.localBlobProvider.acquire(blob);\n this.localBlobs.push(localBlob);\n return localBlob.key;\n };\n\n const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const remoteBlob = this.remoteBlobProvider.acquire(blob);\n this.remoteBlobs.push(remoteBlob);\n return `${remoteBlob.key}${PFrameInternal.ParquetExtension}` as PFrameInternal.PFrameBlobId;\n };\n\n const mapColumnData = (\n data: PFrameInternal.DataInfo<TreeEntry>,\n ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n switch (data.type) {\n case \"Json\":\n return { ...data };\n case \"JsonPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, makeLocalBlobId),\n };\n case \"BinaryPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n index: makeLocalBlobId(v.index),\n values: makeLocalBlobId(v.values),\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n ...v,\n data: makeRemoteBlobId(v.data),\n })),\n };\n default:\n assertNever(data);\n }\n };\n\n const jsonifiedColumns = columns.map((column) => ({\n ...column,\n data: mapColumnData(column.data),\n }));\n\n const pFrameData = PFrameFactory.createPFrame({ frameId, spillPath: this.spillPath, logger });\n const promises: Promise<void>[] = [];\n try {\n pFrameData.setDataSource({\n ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n parquetServer: this.remoteBlobProvider.httpServerInfo(),\n });\n\n for (const column of jsonifiedColumns) {\n pFrameData.addColumnSpec(column.id, column.spec);\n promises.push(\n pFrameData.setColumnData(column.id, column.data, { signal: this.disposeSignal }),\n );\n }\n\n this.pFrameDataPromise = Promise.all(promises)\n .then(() => pFrameData)\n .catch((err) => {\n this.dispose();\n pFrameData.dispose();\n const error = new PFrameDriverError(\"PFrame creation failed asynchronously\");\n error.cause = new Error(\n `PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,\n { cause: ensureError(err) },\n );\n throw error;\n });\n } catch (err: unknown) {\n // setDataSource / addColumnSpec / setColumnData threw synchronously\n // before pFrameDataPromise was assigned — dispose the addon's\n // pFrameData explicitly (the dispose() path can't reach it yet).\n // `allSettled` attaches handlers to any in-flight setColumnData\n // promises so they don't trigger unhandled-rejection warnings\n // when the dispose below rejects them.\n void Promise.allSettled(promises);\n pFrameData.dispose();\n throw err;\n }\n } catch (err: unknown) {\n // Release everything allocated so far in this constructor:\n // acquired blobs, the WASM spec frame, and the abort signal.\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n const error = new PFrameDriverError(\"PFrame creation failed synchronously\");\n error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(columns)}`, {\n cause: ensureError(err),\n });\n throw error;\n }\n }\n\n public get disposeSignal(): AbortSignal {\n return this.abortController.signal;\n }\n\n private dispose(): void {\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n void this.pFrameDataPromise\n .then((pFrameData) => pFrameData.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n\n [Symbol.dispose](): void {\n this.dispose();\n }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n PFrameHandle,\n PFrameHolder<TreeEntry>\n> {\n constructor(\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n private readonly logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n ) {\n super();\n }\n\n protected calculateParamsKey(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ): PFrameHandle {\n return stableKeyFromPFrameData(params);\n }\n\n protected createNewResource(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n key: PFrameHandle,\n ): PFrameHolder<TreeEntry> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PFrame creation (pFrameHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n return new PFrameHolder(\n key,\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n this.spillPath,\n params,\n );\n }\n\n public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PFrame handle`);\n error.cause = new Error(`PFrame with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n const orderedData = [...data].map((column) =>\n mapPObjectData(column, (r) => {\n let result: {\n type: string;\n keyLength: number;\n payload: {\n key: string;\n value: null | number | string | [string, string];\n }[];\n };\n const type = r.type;\n switch (type) {\n case \"Json\":\n result = {\n type: r.type,\n keyLength: r.keyLength,\n payload: Object.entries(r.data).map(([part, value]) => ({\n key: part,\n value,\n })),\n };\n break;\n case \"JsonPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: canonicalizeJson(info),\n })),\n };\n break;\n case \"BinaryPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n })),\n };\n break;\n case \"ParquetPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value:\n info.dataDigest ||\n ([\n canonicalizeJson(info.data),\n JSON.stringify({ axes: info.axes, column: info.column }),\n ] as const),\n })),\n };\n break;\n default:\n throw new PFrameDriverError(\n `unsupported resource type: ${JSON.stringify(type satisfies never)}`,\n );\n }\n result.payload.sort((lhs, rhs) => (lhs.key < rhs.key ? -1 : 1));\n return result;\n }),\n );\n orderedData.sort((lhs, rhs) => (lhs.id < rhs.id ? -1 : 1));\n return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"mappings":";;;;;;;;AA+BA,IAAa,eAAb,MAAoF;CAClF;;;;;;CAMA;CACA,kBAAmC,IAAI,iBAAiB;CAExD,aAAwE,EAAE;CAC1E,cAAyE,EAAE;CAE3E,YACE,SACA,mBACA,oBACA,QACA,WACA,SACA;AALiB,OAAA,oBAAA;AACA,OAAA,qBAAA;AAEA,OAAA,YAAA;EAGjB,MAAM,aAAa,IAAI,IAAI,OAAO,OAAO,UAAU,CAAC;EACpD,MAAM,iBAA8C,EAAE;AACtD,OAAK,MAAM,KAAK,QACd,KAAI,WAAW,IAAI,EAAE,KAAK,UAAU,CAClC,gBAAe,EAAE,MAAM,yBAAyB,EAAE,KAAK;AAG3D,OAAK,aAAaA,aAAiB,eAAe;AAElD,MAAI;GACF,MAAM,mBAAmB,SAAiD;IACxE,MAAM,YAAY,KAAK,kBAAkB,QAAQ,KAAK;AACtD,SAAK,WAAW,KAAK,UAAU;AAC/B,WAAO,UAAU;;GAGnB,MAAM,oBAAoB,SAAiD;IACzE,MAAM,aAAa,KAAK,mBAAmB,QAAQ,KAAK;AACxD,SAAK,YAAY,KAAK,WAAW;AACjC,WAAO,GAAG,WAAW,MAAM,eAAe;;GAG5C,MAAM,iBACJ,SACyD;AACzD,YAAQ,KAAK,MAAb;KACE,KAAK,OACH,QAAO,EAAE,GAAG,MAAM;KACpB,KAAK,kBACH,QAAO;MACL,GAAG;MACH,OAAO,UAAU,KAAK,OAAO,gBAAgB;MAC9C;KACH,KAAK,oBACH,QAAO;MACL,GAAG;MACH,OAAO,UAAU,KAAK,QAAQ,OAAO;OACnC,OAAO,gBAAgB,EAAE,MAAM;OAC/B,QAAQ,gBAAgB,EAAE,OAAO;OAClC,EAAE;MACJ;KACH,KAAK,qBACH,QAAO;MACL,GAAG;MACH,OAAO,UAAU,KAAK,QAAQ,OAAO;OACnC,GAAG;OACH,MAAM,iBAAiB,EAAE,KAAK;OAC/B,EAAE;MACJ;KACH,QACE,aAAY,KAAK;;;GAIvB,MAAM,mBAAmB,QAAQ,KAAK,YAAY;IAChD,GAAG;IACH,MAAM,cAAc,OAAO,KAAK;IACjC,EAAE;GAEH,MAAM,aAAa,cAAc,aAAa;IAAE;IAAS,WAAW,KAAK;IAAW;IAAQ,CAAC;GAC7F,MAAM,WAA4B,EAAE;AACpC,OAAI;AACF,eAAW,cAAc;KACvB,GAAG,KAAK,kBAAkB,eAAe,KAAK,cAAc;KAC5D,eAAe,KAAK,mBAAmB,gBAAgB;KACxD,CAAC;AAEF,SAAK,MAAM,UAAU,kBAAkB;AACrC,gBAAW,cAAc,OAAO,IAAI,OAAO,KAAK;AAChD,cAAS,KACP,WAAW,cAAc,OAAO,IAAI,OAAO,MAAM,EAAE,QAAQ,KAAK,eAAe,CAAC,CACjF;;AAGH,SAAK,oBAAoB,QAAQ,IAAI,SAAS,CAC3C,WAAW,WAAW,CACtB,OAAO,QAAQ;AACd,UAAK,SAAS;AACd,gBAAW,SAAS;KACpB,MAAM,QAAQ,IAAI,kBAAkB,wCAAwC;AAC5E,WAAM,QAAQ,IAAI,MAChB,0CAA0C,KAAK,UAAU,iBAAiB,IAC1E,EAAE,OAAO,YAAY,IAAI,EAAE,CAC5B;AACD,WAAM;MACN;YACG,KAAc;AAOhB,YAAQ,WAAW,SAAS;AACjC,eAAW,SAAS;AACpB,UAAM;;WAED,KAAc;AAGrB,QAAK,gBAAgB,OAAO;AAC5B,QAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,QAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,QAAK,WAAW,OAAO,UAAU;GACjC,MAAM,QAAQ,IAAI,kBAAkB,uCAAuC;AAC3E,SAAM,QAAQ,IAAI,MAAM,0CAA0C,KAAK,UAAU,QAAQ,IAAI,EAC3F,OAAO,YAAY,IAAI,EACxB,CAAC;AACF,SAAM;;;CAIV,IAAW,gBAA6B;AACtC,SAAO,KAAK,gBAAgB;;CAG9B,UAAwB;AACtB,OAAK,gBAAgB,OAAO;AAC5B,OAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,OAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,OAAK,WAAW,OAAO,UAAU;AAC5B,OAAK,kBACP,MAAM,eAAe,WAAW,SAAS,CAAC,CAC1C,YAAY,GAEX;;CAGN,CAAC,OAAO,WAAiB;AACvB,OAAK,SAAS;;;AAIlB,IAAa,aAAb,cAAoE,iBAIlE;CACA,YACE,mBACA,oBACA,QACA,WACA;AACA,SAAO;AALU,OAAA,oBAAA;AACA,OAAA,qBAAA;AACA,OAAA,SAAA;AACA,OAAA,YAAA;;CAKnB,mBACE,QACc;AACd,SAAO,wBAAwB,OAAO;;CAGxC,kBACE,QACA,KACyB;AACzB,MAAI,YAAY,CACd,MAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQ,eAAe,GACxF;AAEH,SAAO,IAAI,aACT,KACA,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,KAAK,WACL,OACD;;CAGH,SAAgB,KAA4C;EAC1D,MAAM,WAAW,MAAM,YAAY,IAAI;AACvC,MAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAI,kBAAkB,wBAAwB;AAC5D,SAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,YAAY;AAC9D,SAAM;;AAER,SAAO;;;AAIX,SAAS,wBACP,MACc;CACd,MAAM,cAAc,CAAC,GAAG,KAAK,CAAC,KAAK,WACjC,eAAe,SAAS,MAAM;EAC5B,IAAI;EAQJ,MAAM,OAAO,EAAE;AACf,UAAQ,MAAR;GACE,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,YAAY;MACtD,KAAK;MACL;MACD,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,iBAAiB,KAAK;MAC9B,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,CAAC,iBAAiB,KAAK,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC;MACrE,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OACE,KAAK,cACJ,CACC,iBAAiB,KAAK,KAAK,EAC3B,KAAK,UAAU;OAAE,MAAM,KAAK;OAAM,QAAQ,KAAK;OAAQ,CAAC,CACzD;MACJ,EAAE;KACJ;AACD;GACF,QACE,OAAM,IAAI,kBACR,8BAA8B,KAAK,UAAU,KAAqB,GACnE;;AAEL,SAAO,QAAQ,MAAM,KAAK,QAAS,IAAI,MAAM,IAAI,MAAM,KAAK,EAAG;AAC/D,SAAO;GACP,CACH;AACD,aAAY,MAAM,KAAK,QAAS,IAAI,KAAK,IAAI,KAAK,KAAK,EAAG;AAC1D,QAAO,SAAS,YAAY"}
|
|
1
|
+
{"version":3,"file":"pframe_pool.js","names":["createPFrameSpec"],"sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n assertNever,\n bigintReplacer,\n canonicalizeJson,\n ensureError,\n mapPObjectData,\n PFrameDriverError,\n ValueType,\n resolveAnnotationParents,\n type JsonSerializable,\n type PColumn,\n type PColumnSpec,\n type PFrameHandle,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { createPFrame as createPFrameSpec } from \"@milaboratories/pframes-rs-wasm\";\nimport { mapValues } from \"es-toolkit\";\nimport { logPFrames } from \"./logging\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\">;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n public readonly pFrameDataPromise: Promise<PFrameInternal.PFrameV14>;\n /**\n * WASM-spec frame built from this PFrame's columns. Source of truth\n * for spec-side operations: column discovery, selector resolution,\n * legacy-query lowering.\n */\n public readonly pFrameSpec: PFrameInternal.PFrameWasmV3;\n private readonly abortController = new AbortController();\n\n private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n constructor(\n frameId: PFrameInternal.PFrameId,\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ) {\n const ValueTypes = new Set(Object.values(ValueType));\n const specColumnsMap: Record<string, PColumnSpec> = {};\n for (const c of columns) {\n if (ValueTypes.has(c.spec.valueType)) {\n specColumnsMap[c.id] = resolveAnnotationParents(c.spec);\n }\n }\n this.pFrameSpec = createPFrameSpec(specColumnsMap);\n\n try {\n const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const localBlob = this.localBlobProvider.acquire(blob);\n this.localBlobs.push(localBlob);\n return localBlob.key;\n };\n\n const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const remoteBlob = this.remoteBlobProvider.acquire(blob);\n this.remoteBlobs.push(remoteBlob);\n return `${remoteBlob.key}${PFrameInternal.ParquetExtension}` as PFrameInternal.PFrameBlobId;\n };\n\n const mapColumnData = (\n data: PFrameInternal.DataInfo<TreeEntry>,\n ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n switch (data.type) {\n case \"Json\":\n return { ...data };\n case \"JsonPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, makeLocalBlobId),\n };\n case \"BinaryPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n index: makeLocalBlobId(v.index),\n values: makeLocalBlobId(v.values),\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n ...v,\n data: makeRemoteBlobId(v.data),\n })),\n };\n default:\n assertNever(data);\n }\n };\n\n const jsonifiedColumns = columns.map((column) => ({\n ...column,\n data: mapColumnData(column.data),\n }));\n\n const pFrameData = PFrameFactory.createPFrame({ frameId, spillPath: this.spillPath, logger });\n try {\n pFrameData.setDataSource({\n ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n parquetServer: this.remoteBlobProvider.httpServerInfo(),\n });\n\n this.pFrameDataPromise = pFrameData\n .addColumns(\n jsonifiedColumns.map((c) => ({\n id: c.id,\n typeSpec: {\n axes: c.spec.axesSpec.map((a) => a.type),\n columns: [c.spec.valueType],\n },\n data: c.data,\n })),\n { signal: this.disposeSignal },\n )\n .then(() => pFrameData)\n .catch((err) => {\n this.dispose();\n pFrameData.dispose();\n const error = new PFrameDriverError(\"PFrame creation failed asynchronously\");\n error.cause = new Error(\n `PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,\n { cause: ensureError(err) },\n );\n throw error;\n });\n } catch (err: unknown) {\n // setDataSource / addColumns threw synchronously before\n // pFrameDataPromise was assigned — dispose the addon's pFrameData\n // explicitly (the dispose() path can't reach it yet).\n pFrameData.dispose();\n throw err;\n }\n } catch (err: unknown) {\n // Release everything allocated so far in this constructor:\n // acquired blobs, the WASM spec frame, and the abort signal.\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n const error = new PFrameDriverError(\"PFrame creation failed synchronously\");\n error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(columns)}`, {\n cause: ensureError(err),\n });\n throw error;\n }\n }\n\n public get disposeSignal(): AbortSignal {\n return this.abortController.signal;\n }\n\n private dispose(): void {\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n void this.pFrameDataPromise\n .then((pFrameData) => pFrameData.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n\n [Symbol.dispose](): void {\n this.dispose();\n }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n PFrameHandle,\n PFrameHolder<TreeEntry>\n> {\n constructor(\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n private readonly logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n ) {\n super();\n }\n\n protected calculateParamsKey(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ): PFrameHandle {\n return stableKeyFromPFrameData(params);\n }\n\n protected createNewResource(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n key: PFrameHandle,\n ): PFrameHolder<TreeEntry> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PFrame creation (pFrameHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n return new PFrameHolder(\n key,\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n this.spillPath,\n params,\n );\n }\n\n public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PFrame handle`);\n error.cause = new Error(`PFrame with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n const orderedData = [...data].map((column) =>\n mapPObjectData(column, (r) => {\n let result: {\n type: string;\n keyLength: number;\n payload: {\n key: string;\n value: null | number | string | [string, string];\n }[];\n };\n const type = r.type;\n switch (type) {\n case \"Json\":\n result = {\n type: r.type,\n keyLength: r.keyLength,\n payload: Object.entries(r.data).map(([part, value]) => ({\n key: part,\n value,\n })),\n };\n break;\n case \"JsonPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: canonicalizeJson(info),\n })),\n };\n break;\n case \"BinaryPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n })),\n };\n break;\n case \"ParquetPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value:\n info.dataDigest ||\n ([\n canonicalizeJson(info.data),\n JSON.stringify({ axes: info.axes, column: info.column }),\n ] as const),\n })),\n };\n break;\n default:\n throw new PFrameDriverError(\n `unsupported resource type: ${JSON.stringify(type satisfies never)}`,\n );\n }\n result.payload.sort((lhs, rhs) => (lhs.key < rhs.key ? -1 : 1));\n return result;\n }),\n );\n orderedData.sort((lhs, rhs) => (lhs.id < rhs.id ? -1 : 1));\n return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"mappings":";;;;;;;;AA+BA,IAAa,eAAb,MAAoF;CAClF;;;;;;CAMA;CACA,kBAAmC,IAAI,iBAAiB;CAExD,aAAwE,EAAE;CAC1E,cAAyE,EAAE;CAE3E,YACE,SACA,mBACA,oBACA,QACA,WACA,SACA;AALiB,OAAA,oBAAA;AACA,OAAA,qBAAA;AAEA,OAAA,YAAA;EAGjB,MAAM,aAAa,IAAI,IAAI,OAAO,OAAO,UAAU,CAAC;EACpD,MAAM,iBAA8C,EAAE;AACtD,OAAK,MAAM,KAAK,QACd,KAAI,WAAW,IAAI,EAAE,KAAK,UAAU,CAClC,gBAAe,EAAE,MAAM,yBAAyB,EAAE,KAAK;AAG3D,OAAK,aAAaA,aAAiB,eAAe;AAElD,MAAI;GACF,MAAM,mBAAmB,SAAiD;IACxE,MAAM,YAAY,KAAK,kBAAkB,QAAQ,KAAK;AACtD,SAAK,WAAW,KAAK,UAAU;AAC/B,WAAO,UAAU;;GAGnB,MAAM,oBAAoB,SAAiD;IACzE,MAAM,aAAa,KAAK,mBAAmB,QAAQ,KAAK;AACxD,SAAK,YAAY,KAAK,WAAW;AACjC,WAAO,GAAG,WAAW,MAAM,eAAe;;GAG5C,MAAM,iBACJ,SACyD;AACzD,YAAQ,KAAK,MAAb;KACE,KAAK,OACH,QAAO,EAAE,GAAG,MAAM;KACpB,KAAK,kBACH,QAAO;MACL,GAAG;MACH,OAAO,UAAU,KAAK,OAAO,gBAAgB;MAC9C;KACH,KAAK,oBACH,QAAO;MACL,GAAG;MACH,OAAO,UAAU,KAAK,QAAQ,OAAO;OACnC,OAAO,gBAAgB,EAAE,MAAM;OAC/B,QAAQ,gBAAgB,EAAE,OAAO;OAClC,EAAE;MACJ;KACH,KAAK,qBACH,QAAO;MACL,GAAG;MACH,OAAO,UAAU,KAAK,QAAQ,OAAO;OACnC,GAAG;OACH,MAAM,iBAAiB,EAAE,KAAK;OAC/B,EAAE;MACJ;KACH,QACE,aAAY,KAAK;;;GAIvB,MAAM,mBAAmB,QAAQ,KAAK,YAAY;IAChD,GAAG;IACH,MAAM,cAAc,OAAO,KAAK;IACjC,EAAE;GAEH,MAAM,aAAa,cAAc,aAAa;IAAE;IAAS,WAAW,KAAK;IAAW;IAAQ,CAAC;AAC7F,OAAI;AACF,eAAW,cAAc;KACvB,GAAG,KAAK,kBAAkB,eAAe,KAAK,cAAc;KAC5D,eAAe,KAAK,mBAAmB,gBAAgB;KACxD,CAAC;AAEF,SAAK,oBAAoB,WACtB,WACC,iBAAiB,KAAK,OAAO;KAC3B,IAAI,EAAE;KACN,UAAU;MACR,MAAM,EAAE,KAAK,SAAS,KAAK,MAAM,EAAE,KAAK;MACxC,SAAS,CAAC,EAAE,KAAK,UAAU;MAC5B;KACD,MAAM,EAAE;KACT,EAAE,EACH,EAAE,QAAQ,KAAK,eAAe,CAC/B,CACA,WAAW,WAAW,CACtB,OAAO,QAAQ;AACd,UAAK,SAAS;AACd,gBAAW,SAAS;KACpB,MAAM,QAAQ,IAAI,kBAAkB,wCAAwC;AAC5E,WAAM,QAAQ,IAAI,MAChB,0CAA0C,KAAK,UAAU,iBAAiB,IAC1E,EAAE,OAAO,YAAY,IAAI,EAAE,CAC5B;AACD,WAAM;MACN;YACG,KAAc;AAIrB,eAAW,SAAS;AACpB,UAAM;;WAED,KAAc;AAGrB,QAAK,gBAAgB,OAAO;AAC5B,QAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,QAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,QAAK,WAAW,OAAO,UAAU;GACjC,MAAM,QAAQ,IAAI,kBAAkB,uCAAuC;AAC3E,SAAM,QAAQ,IAAI,MAAM,0CAA0C,KAAK,UAAU,QAAQ,IAAI,EAC3F,OAAO,YAAY,IAAI,EACxB,CAAC;AACF,SAAM;;;CAIV,IAAW,gBAA6B;AACtC,SAAO,KAAK,gBAAgB;;CAG9B,UAAwB;AACtB,OAAK,gBAAgB,OAAO;AAC5B,OAAK,WAAW,SAAS,UAAU,MAAM,OAAO,CAAC;AACjD,OAAK,YAAY,SAAS,UAAU,MAAM,OAAO,CAAC;AAClD,OAAK,WAAW,OAAO,UAAU;AAC5B,OAAK,kBACP,MAAM,eAAe,WAAW,SAAS,CAAC,CAC1C,YAAY,GAEX;;CAGN,CAAC,OAAO,WAAiB;AACvB,OAAK,SAAS;;;AAIlB,IAAa,aAAb,cAAoE,iBAIlE;CACA,YACE,mBACA,oBACA,QACA,WACA;AACA,SAAO;AALU,OAAA,oBAAA;AACA,OAAA,qBAAA;AACA,OAAA,SAAA;AACA,OAAA,YAAA;;CAKnB,mBACE,QACc;AACd,SAAO,wBAAwB,OAAO;;CAGxC,kBACE,QACA,KACyB;AACzB,MAAI,YAAY,CACd,MAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQ,eAAe,GACxF;AAEH,SAAO,IAAI,aACT,KACA,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,KAAK,WACL,OACD;;CAGH,SAAgB,KAA4C;EAC1D,MAAM,WAAW,MAAM,YAAY,IAAI;AACvC,MAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAI,kBAAkB,wBAAwB;AAC5D,SAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,YAAY;AAC9D,SAAM;;AAER,SAAO;;;AAIX,SAAS,wBACP,MACc;CACd,MAAM,cAAc,CAAC,GAAG,KAAK,CAAC,KAAK,WACjC,eAAe,SAAS,MAAM;EAC5B,IAAI;EAQJ,MAAM,OAAO,EAAE;AACf,UAAQ,MAAR;GACE,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,YAAY;MACtD,KAAK;MACL;MACD,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,iBAAiB,KAAK;MAC9B,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,CAAC,iBAAiB,KAAK,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC;MACrE,EAAE;KACJ;AACD;GACF,KAAK;AACH,aAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OACE,KAAK,cACJ,CACC,iBAAiB,KAAK,KAAK,EAC3B,KAAK,UAAU;OAAE,MAAM,KAAK;OAAM,QAAQ,KAAK;OAAQ,CAAC,CACzD;MACJ,EAAE;KACJ;AACD;GACF,QACE,OAAM,IAAI,kBACR,8BAA8B,KAAK,UAAU,KAAqB,GACnE;;AAEL,SAAO,QAAQ,MAAM,KAAK,QAAS,IAAI,MAAM,IAAI,MAAM,KAAK,EAAG;AAC/D,SAAO;GACP,CACH;AACD,aAAY,MAAM,KAAK,QAAS,IAAI,KAAK,IAAI,KAAK,KAAK,EAAG;AAC1D,QAAO,SAAS,YAAY"}
|
package/dist/ptable_pool.cjs
CHANGED
|
@@ -38,11 +38,8 @@ var PTablePool = class extends _milaboratories_helpers.RefCountPoolBase {
|
|
|
38
38
|
const { pFrameDataPromise: pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);
|
|
39
39
|
const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
|
|
40
40
|
const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
|
|
41
|
-
const {
|
|
42
|
-
return new PTableHolder(pFrameHandle, combinedSignal, pFramePromise.then((pFrame) => pFrame.
|
|
43
|
-
tableSpec,
|
|
44
|
-
dataQuery
|
|
45
|
-
})));
|
|
41
|
+
const { dataQuery } = params;
|
|
42
|
+
return new PTableHolder(pFrameHandle, combinedSignal, pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery)));
|
|
46
43
|
}
|
|
47
44
|
getByKey(key) {
|
|
48
45
|
const resource = super.tryGetByKey(key);
|
package/dist/ptable_pool.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_pool.cjs","names":["RefCountPoolBase","stableKeyFromFullPTableDef","logPFrames","bigintReplacer","PFrameDriverError","buildFullPTableDefFromLegacy"],"sources":["../src/ptable_pool.ts"],"sourcesContent":["import {\n bigintReplacer,\n PFrameDriverError,\n type PFrameHandle,\n type PTableDef,\n type PTableHandle,\n type JsonSerializable,\n type PObjectId,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { logPFrames } from \"./logging\";\nimport type { PFramePool } from \"./pframe_pool\";\nimport {\n buildFullPTableDefFromLegacy,\n stableKeyFromFullPTableDef,\n type FullPTableDef,\n} from \"./ptable_shared\";\nimport type { PTableDefPool } from \"./ptable_def_pool\";\n\nexport class PTableHolder implements Disposable {\n private readonly abortController = new AbortController();\n private readonly combinedDisposeSignal: AbortSignal;\n\n constructor(\n public readonly pFrame: PFrameHandle,\n pFrameDisposeSignal: AbortSignal,\n public readonly pTablePromise: Promise<PFrameInternal.
|
|
1
|
+
{"version":3,"file":"ptable_pool.cjs","names":["RefCountPoolBase","stableKeyFromFullPTableDef","logPFrames","bigintReplacer","PFrameDriverError","buildFullPTableDefFromLegacy"],"sources":["../src/ptable_pool.ts"],"sourcesContent":["import {\n bigintReplacer,\n PFrameDriverError,\n type PFrameHandle,\n type PTableDef,\n type PTableHandle,\n type JsonSerializable,\n type PObjectId,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { logPFrames } from \"./logging\";\nimport type { PFramePool } from \"./pframe_pool\";\nimport {\n buildFullPTableDefFromLegacy,\n stableKeyFromFullPTableDef,\n type FullPTableDef,\n} from \"./ptable_shared\";\nimport type { PTableDefPool } from \"./ptable_def_pool\";\n\nexport class PTableHolder implements Disposable {\n private readonly abortController = new AbortController();\n private readonly combinedDisposeSignal: AbortSignal;\n\n constructor(\n public readonly pFrame: PFrameHandle,\n pFrameDisposeSignal: AbortSignal,\n public readonly pTablePromise: Promise<PFrameInternal.PTableV9>,\n private readonly predecessor?: PoolEntry<PTableHandle, PTableHolder>,\n ) {\n this.combinedDisposeSignal = AbortSignal.any([\n pFrameDisposeSignal,\n this.abortController.signal,\n ]);\n }\n\n public get disposeSignal(): AbortSignal {\n return this.combinedDisposeSignal;\n }\n\n [Symbol.dispose](): void {\n this.abortController.abort();\n this.predecessor?.unref();\n void this.pTablePromise\n .then((pTable) => pTable.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n}\n\nexport class PTablePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n FullPTableDef,\n PTableHandle,\n PTableHolder\n> {\n constructor(\n private readonly pFrames: PFramePool<TreeEntry>,\n private readonly pTableDefs: PTableDefPool,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: FullPTableDef): PTableHandle {\n return stableKeyFromFullPTableDef(params);\n }\n\n protected createNewResource(params: FullPTableDef, key: PTableHandle): PTableHolder {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PTable creation (pTableHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n\n const { pFrameHandle } = params;\n const { pFrameDataPromise: pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);\n\n const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;\n const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));\n\n const { dataQuery } = params;\n const table = pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery));\n return new PTableHolder(pFrameHandle, combinedSignal, table);\n }\n\n public getByKey(key: PTableHandle): PTableHolder {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PTable handle`);\n error.cause = new Error(`PTable with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n\n /**\n * Acquire a PTable from a legacy `PTableDef` + column specs map.\n * Lowers the input via WASM-spec and stores the resulting\n * `{ tableSpec, dataQuery }` shape. Returns the lowered def\n * alongside the pool entry.\n */\n public acquireFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n }): { def: FullPTableDef; entry: PoolEntry<PTableHandle, PTableHolder> } {\n const def = buildFullPTableDefFromLegacy(opts);\n return { def, entry: this.acquire(def) };\n }\n}\n"],"mappings":";;;;;;AAoBA,IAAa,eAAb,MAAgD;CAC9C,kBAAmC,IAAI,iBAAiB;CACxD;CAEA,YACE,QACA,qBACA,eACA,aACA;AAJgB,OAAA,SAAA;AAEA,OAAA,gBAAA;AACC,OAAA,cAAA;AAEjB,OAAK,wBAAwB,YAAY,IAAI,CAC3C,qBACA,KAAK,gBAAgB,OACtB,CAAC;;CAGJ,IAAW,gBAA6B;AACtC,SAAO,KAAK;;CAGd,CAAC,OAAO,WAAiB;AACvB,OAAK,gBAAgB,OAAO;AAC5B,OAAK,aAAa,OAAO;AACpB,OAAK,cACP,MAAM,WAAW,OAAO,SAAS,CAAC,CAClC,YAAY,GAEX;;;AAIR,IAAa,aAAb,cAAoEA,wBAAAA,iBAIlE;CACA,YACE,SACA,YACA,QACA;AACA,SAAO;AAJU,OAAA,UAAA;AACA,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAqC;AAChE,SAAOC,sBAAAA,2BAA2B,OAAO;;CAG3C,kBAA4B,QAAuB,KAAiC;AAClF,MAAIC,gBAAAA,YAAY,CACd,MAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQC,gCAAAA,eAAe,GACxF;EAGH,MAAM,EAAE,iBAAiB;EACzB,MAAM,EAAE,mBAAmB,eAAe,kBAAkB,KAAK,QAAQ,SAAS,aAAa;EAE/F,MAAM,mBAAmB,KAAK,WAAW,YAAY,IAAI,EAAE;EAC3D,MAAM,iBAAiB,YAAY,IAAI,CAAC,eAAe,iBAAiB,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;EAE5F,MAAM,EAAE,cAAc;AAEtB,SAAO,IAAI,aAAa,cAAc,gBADxB,cAAc,MAAM,WAAW,OAAO,YAAY,KAAK,UAAU,CAAC,CACpB;;CAG9D,SAAgB,KAAiC;EAC/C,MAAM,WAAW,MAAM,YAAY,IAAI;AACvC,MAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,wBAAwB;AAC5D,SAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,YAAY;AAC9D,SAAM;;AAER,SAAO;;;;;;;;CAST,kBAAyB,MAIgD;EACvE,MAAM,MAAMC,sBAAAA,6BAA6B,KAAK;AAC9C,SAAO;GAAE;GAAK,OAAO,KAAK,QAAQ,IAAI;GAAE"}
|
package/dist/ptable_pool.js
CHANGED
|
@@ -37,11 +37,8 @@ var PTablePool = class extends RefCountPoolBase {
|
|
|
37
37
|
const { pFrameDataPromise: pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);
|
|
38
38
|
const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
|
|
39
39
|
const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
|
|
40
|
-
const {
|
|
41
|
-
return new PTableHolder(pFrameHandle, combinedSignal, pFramePromise.then((pFrame) => pFrame.
|
|
42
|
-
tableSpec,
|
|
43
|
-
dataQuery
|
|
44
|
-
})));
|
|
40
|
+
const { dataQuery } = params;
|
|
41
|
+
return new PTableHolder(pFrameHandle, combinedSignal, pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery)));
|
|
45
42
|
}
|
|
46
43
|
getByKey(key) {
|
|
47
44
|
const resource = super.tryGetByKey(key);
|
package/dist/ptable_pool.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_pool.js","names":[],"sources":["../src/ptable_pool.ts"],"sourcesContent":["import {\n bigintReplacer,\n PFrameDriverError,\n type PFrameHandle,\n type PTableDef,\n type PTableHandle,\n type JsonSerializable,\n type PObjectId,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { logPFrames } from \"./logging\";\nimport type { PFramePool } from \"./pframe_pool\";\nimport {\n buildFullPTableDefFromLegacy,\n stableKeyFromFullPTableDef,\n type FullPTableDef,\n} from \"./ptable_shared\";\nimport type { PTableDefPool } from \"./ptable_def_pool\";\n\nexport class PTableHolder implements Disposable {\n private readonly abortController = new AbortController();\n private readonly combinedDisposeSignal: AbortSignal;\n\n constructor(\n public readonly pFrame: PFrameHandle,\n pFrameDisposeSignal: AbortSignal,\n public readonly pTablePromise: Promise<PFrameInternal.
|
|
1
|
+
{"version":3,"file":"ptable_pool.js","names":[],"sources":["../src/ptable_pool.ts"],"sourcesContent":["import {\n bigintReplacer,\n PFrameDriverError,\n type PFrameHandle,\n type PTableDef,\n type PTableHandle,\n type JsonSerializable,\n type PObjectId,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { logPFrames } from \"./logging\";\nimport type { PFramePool } from \"./pframe_pool\";\nimport {\n buildFullPTableDefFromLegacy,\n stableKeyFromFullPTableDef,\n type FullPTableDef,\n} from \"./ptable_shared\";\nimport type { PTableDefPool } from \"./ptable_def_pool\";\n\nexport class PTableHolder implements Disposable {\n private readonly abortController = new AbortController();\n private readonly combinedDisposeSignal: AbortSignal;\n\n constructor(\n public readonly pFrame: PFrameHandle,\n pFrameDisposeSignal: AbortSignal,\n public readonly pTablePromise: Promise<PFrameInternal.PTableV9>,\n private readonly predecessor?: PoolEntry<PTableHandle, PTableHolder>,\n ) {\n this.combinedDisposeSignal = AbortSignal.any([\n pFrameDisposeSignal,\n this.abortController.signal,\n ]);\n }\n\n public get disposeSignal(): AbortSignal {\n return this.combinedDisposeSignal;\n }\n\n [Symbol.dispose](): void {\n this.abortController.abort();\n this.predecessor?.unref();\n void this.pTablePromise\n .then((pTable) => pTable.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n}\n\nexport class PTablePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n FullPTableDef,\n PTableHandle,\n PTableHolder\n> {\n constructor(\n private readonly pFrames: PFramePool<TreeEntry>,\n private readonly pTableDefs: PTableDefPool,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: FullPTableDef): PTableHandle {\n return stableKeyFromFullPTableDef(params);\n }\n\n protected createNewResource(params: FullPTableDef, key: PTableHandle): PTableHolder {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PTable creation (pTableHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n\n const { pFrameHandle } = params;\n const { pFrameDataPromise: pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);\n\n const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;\n const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));\n\n const { dataQuery } = params;\n const table = pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery));\n return new PTableHolder(pFrameHandle, combinedSignal, table);\n }\n\n public getByKey(key: PTableHandle): PTableHolder {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PTable handle`);\n error.cause = new Error(`PTable with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n\n /**\n * Acquire a PTable from a legacy `PTableDef` + column specs map.\n * Lowers the input via WASM-spec and stores the resulting\n * `{ tableSpec, dataQuery }` shape. Returns the lowered def\n * alongside the pool entry.\n */\n public acquireFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n }): { def: FullPTableDef; entry: PoolEntry<PTableHandle, PTableHolder> } {\n const def = buildFullPTableDefFromLegacy(opts);\n return { def, entry: this.acquire(def) };\n }\n}\n"],"mappings":";;;;;AAoBA,IAAa,eAAb,MAAgD;CAC9C,kBAAmC,IAAI,iBAAiB;CACxD;CAEA,YACE,QACA,qBACA,eACA,aACA;AAJgB,OAAA,SAAA;AAEA,OAAA,gBAAA;AACC,OAAA,cAAA;AAEjB,OAAK,wBAAwB,YAAY,IAAI,CAC3C,qBACA,KAAK,gBAAgB,OACtB,CAAC;;CAGJ,IAAW,gBAA6B;AACtC,SAAO,KAAK;;CAGd,CAAC,OAAO,WAAiB;AACvB,OAAK,gBAAgB,OAAO;AAC5B,OAAK,aAAa,OAAO;AACpB,OAAK,cACP,MAAM,WAAW,OAAO,SAAS,CAAC,CAClC,YAAY,GAEX;;;AAIR,IAAa,aAAb,cAAoE,iBAIlE;CACA,YACE,SACA,YACA,QACA;AACA,SAAO;AAJU,OAAA,UAAA;AACA,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAqC;AAChE,SAAO,2BAA2B,OAAO;;CAG3C,kBAA4B,QAAuB,KAAiC;AAClF,MAAI,YAAY,CACd,MAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQ,eAAe,GACxF;EAGH,MAAM,EAAE,iBAAiB;EACzB,MAAM,EAAE,mBAAmB,eAAe,kBAAkB,KAAK,QAAQ,SAAS,aAAa;EAE/F,MAAM,mBAAmB,KAAK,WAAW,YAAY,IAAI,EAAE;EAC3D,MAAM,iBAAiB,YAAY,IAAI,CAAC,eAAe,iBAAiB,CAAC,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC;EAE5F,MAAM,EAAE,cAAc;AAEtB,SAAO,IAAI,aAAa,cAAc,gBADxB,cAAc,MAAM,WAAW,OAAO,YAAY,KAAK,UAAU,CAAC,CACpB;;CAG9D,SAAgB,KAAiC;EAC/C,MAAM,WAAW,MAAM,YAAY,IAAI;AACvC,MAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAI,kBAAkB,wBAAwB;AAC5D,SAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,YAAY;AAC9D,SAAM;;AAER,SAAO;;;;;;;;CAST,kBAAyB,MAIgD;EACvE,MAAM,MAAM,6BAA6B,KAAK;AAC9C,SAAO;GAAE;GAAK,OAAO,KAAK,QAAQ,IAAI;GAAE"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pf-driver",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.1",
|
|
4
4
|
"description": "PFrameDriver implementation abstracted from Middle Layer",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -20,13 +20,13 @@
|
|
|
20
20
|
}
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@milaboratories/pframes-rs-node": "1.1.
|
|
24
|
-
"@milaboratories/pframes-rs-wasm": "1.1.
|
|
23
|
+
"@milaboratories/pframes-rs-node": "1.1.38",
|
|
24
|
+
"@milaboratories/pframes-rs-wasm": "1.1.38",
|
|
25
25
|
"es-toolkit": "^1.39.10",
|
|
26
26
|
"lru-cache": "^11.2.2",
|
|
27
27
|
"@milaboratories/helpers": "1.14.2",
|
|
28
|
-
"@milaboratories/pl-model-
|
|
29
|
-
"@milaboratories/pl-model-
|
|
28
|
+
"@milaboratories/pl-model-middle-layer": "1.27.0",
|
|
29
|
+
"@milaboratories/pl-model-common": "1.43.0",
|
|
30
30
|
"@milaboratories/ts-helpers": "1.8.2"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
@@ -35,9 +35,9 @@
|
|
|
35
35
|
"fast-csv": "5.0.5",
|
|
36
36
|
"typescript": "~5.9.3",
|
|
37
37
|
"vitest": "^4.1.3",
|
|
38
|
+
"@milaboratories/build-configs": "2.0.0",
|
|
38
39
|
"@milaboratories/ts-builder": "1.5.0",
|
|
39
|
-
"@milaboratories/ts-configs": "1.2.3"
|
|
40
|
-
"@milaboratories/build-configs": "2.0.0"
|
|
40
|
+
"@milaboratories/ts-configs": "1.2.3"
|
|
41
41
|
},
|
|
42
42
|
"engines": {
|
|
43
43
|
"node": ">=22.19.0"
|
package/src/csv_writer.ts
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
} from "@milaboratories/pl-model-common";
|
|
10
10
|
import { isNil } from "@milaboratories/helpers";
|
|
11
11
|
|
|
12
|
-
/** Minimal subset of
|
|
12
|
+
/** Minimal subset of PTableV9 required by streamPTableRows. */
|
|
13
13
|
export interface PTableDataSource {
|
|
14
14
|
getData(
|
|
15
15
|
columnIndices: number[],
|
package/src/driver_impl.ts
CHANGED
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
sortSpecQuery,
|
|
34
34
|
sortPTableDef,
|
|
35
35
|
resolveAnnotationParents,
|
|
36
|
+
PFrameDriverError,
|
|
36
37
|
} from "@milaboratories/pl-model-common";
|
|
37
38
|
import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
38
39
|
import {
|
|
@@ -42,6 +43,7 @@ import {
|
|
|
42
43
|
} from "@milaboratories/ts-helpers";
|
|
43
44
|
import { isNil, PoolEntryGuard, type PoolEntry } from "@milaboratories/helpers";
|
|
44
45
|
import { PFrameFactory } from "@milaboratories/pframes-rs-node";
|
|
46
|
+
import { expandAxes, findTableColumn, rewriteLegacyFilters } from "@milaboratories/pframes-rs-wasm";
|
|
45
47
|
import { tmpdir } from "node:os";
|
|
46
48
|
import * as fs from "node:fs";
|
|
47
49
|
import { Readable } from "node:stream";
|
|
@@ -455,20 +457,54 @@ export class AbstractPFrameDriver<
|
|
|
455
457
|
);
|
|
456
458
|
}
|
|
457
459
|
|
|
458
|
-
const { pFrameDataPromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
460
|
+
const { pFrameDataPromise, pFrameSpec, disposeSignal } = this.pFrames.getByKey(handle);
|
|
459
461
|
const pFrameData = await pFrameDataPromise;
|
|
460
462
|
|
|
463
|
+
const column = pFrameSpec.listColumns().find((c) => c.columnId === request.columnId);
|
|
464
|
+
if (!column) {
|
|
465
|
+
const error = new PFrameDriverError(`getUniqueValues failed`);
|
|
466
|
+
error.cause = new Error(`column ${request.columnId} is not registered in the PFrame`);
|
|
467
|
+
throw error;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const axisIds = expandAxes(column.spec.axesSpec);
|
|
471
|
+
const tableSpec: PTableColumnSpec[] = [
|
|
472
|
+
...column.spec.axesSpec.map(
|
|
473
|
+
(axisSpec, i): PTableColumnSpec => ({
|
|
474
|
+
type: "axis",
|
|
475
|
+
id: axisIds[i],
|
|
476
|
+
spec: axisSpec,
|
|
477
|
+
}),
|
|
478
|
+
),
|
|
479
|
+
{ type: "column", id: request.columnId, spec: column.spec },
|
|
480
|
+
];
|
|
481
|
+
|
|
482
|
+
let axisIndex: number | undefined;
|
|
483
|
+
if (request.axis !== undefined) {
|
|
484
|
+
const index = findTableColumn(tableSpec, { type: "axis", id: request.axis });
|
|
485
|
+
if (index < 0) {
|
|
486
|
+
const error = new PFrameDriverError(`getUniqueValues failed`);
|
|
487
|
+
error.cause = new Error(
|
|
488
|
+
`axis ${JSON.stringify(request.axis)} not found among axes of column ${request.columnId}`,
|
|
489
|
+
);
|
|
490
|
+
throw error;
|
|
491
|
+
}
|
|
492
|
+
axisIndex = index;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
const resolvedRequest: PFrameInternal.UniqueValuesRequestV2 = {
|
|
496
|
+
columnId: request.columnId,
|
|
497
|
+
axisIndex,
|
|
498
|
+
filters: rewriteLegacyFilters({
|
|
499
|
+
tableSpec,
|
|
500
|
+
filters: migrateFilters(request.filters, this.logger),
|
|
501
|
+
}),
|
|
502
|
+
limit: request.limit,
|
|
503
|
+
};
|
|
504
|
+
|
|
461
505
|
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
462
506
|
return await this.frameConcurrencyLimiter.run(async () => {
|
|
463
|
-
return await pFrameData.getUniqueValues(
|
|
464
|
-
{
|
|
465
|
-
...request,
|
|
466
|
-
filters: migrateFilters(request.filters, this.logger),
|
|
467
|
-
},
|
|
468
|
-
{
|
|
469
|
-
signal: combinedSignal,
|
|
470
|
-
},
|
|
471
|
-
);
|
|
507
|
+
return await pFrameData.getUniqueValues(resolvedRequest, { signal: combinedSignal });
|
|
472
508
|
});
|
|
473
509
|
}
|
|
474
510
|
|
package/src/pframe_pool.ts
CHANGED
|
@@ -30,7 +30,7 @@ export interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {
|
|
33
|
-
public readonly pFrameDataPromise: Promise<PFrameInternal.
|
|
33
|
+
public readonly pFrameDataPromise: Promise<PFrameInternal.PFrameV14>;
|
|
34
34
|
/**
|
|
35
35
|
* WASM-spec frame built from this PFrame's columns. Source of truth
|
|
36
36
|
* for spec-side operations: column discovery, selector resolution,
|
|
@@ -110,21 +110,24 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
110
110
|
}));
|
|
111
111
|
|
|
112
112
|
const pFrameData = PFrameFactory.createPFrame({ frameId, spillPath: this.spillPath, logger });
|
|
113
|
-
const promises: Promise<void>[] = [];
|
|
114
113
|
try {
|
|
115
114
|
pFrameData.setDataSource({
|
|
116
115
|
...this.localBlobProvider.makeDataSource(this.disposeSignal),
|
|
117
116
|
parquetServer: this.remoteBlobProvider.httpServerInfo(),
|
|
118
117
|
});
|
|
119
118
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
119
|
+
this.pFrameDataPromise = pFrameData
|
|
120
|
+
.addColumns(
|
|
121
|
+
jsonifiedColumns.map((c) => ({
|
|
122
|
+
id: c.id,
|
|
123
|
+
typeSpec: {
|
|
124
|
+
axes: c.spec.axesSpec.map((a) => a.type),
|
|
125
|
+
columns: [c.spec.valueType],
|
|
126
|
+
},
|
|
127
|
+
data: c.data,
|
|
128
|
+
})),
|
|
129
|
+
{ signal: this.disposeSignal },
|
|
130
|
+
)
|
|
128
131
|
.then(() => pFrameData)
|
|
129
132
|
.catch((err) => {
|
|
130
133
|
this.dispose();
|
|
@@ -137,13 +140,9 @@ export class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposa
|
|
|
137
140
|
throw error;
|
|
138
141
|
});
|
|
139
142
|
} catch (err: unknown) {
|
|
140
|
-
// setDataSource /
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
// `allSettled` attaches handlers to any in-flight setColumnData
|
|
144
|
-
// promises so they don't trigger unhandled-rejection warnings
|
|
145
|
-
// when the dispose below rejects them.
|
|
146
|
-
void Promise.allSettled(promises);
|
|
143
|
+
// setDataSource / addColumns threw synchronously before
|
|
144
|
+
// pFrameDataPromise was assigned — dispose the addon's pFrameData
|
|
145
|
+
// explicitly (the dispose() path can't reach it yet).
|
|
147
146
|
pFrameData.dispose();
|
|
148
147
|
throw err;
|
|
149
148
|
}
|
package/src/ptable_pool.ts
CHANGED
|
@@ -25,7 +25,7 @@ export class PTableHolder implements Disposable {
|
|
|
25
25
|
constructor(
|
|
26
26
|
public readonly pFrame: PFrameHandle,
|
|
27
27
|
pFrameDisposeSignal: AbortSignal,
|
|
28
|
-
public readonly pTablePromise: Promise<PFrameInternal.
|
|
28
|
+
public readonly pTablePromise: Promise<PFrameInternal.PTableV9>,
|
|
29
29
|
private readonly predecessor?: PoolEntry<PTableHandle, PTableHolder>,
|
|
30
30
|
) {
|
|
31
31
|
this.combinedDisposeSignal = AbortSignal.any([
|
|
@@ -80,10 +80,8 @@ export class PTablePool<TreeEntry extends JsonSerializable> extends RefCountPool
|
|
|
80
80
|
const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
|
|
81
81
|
const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
|
|
82
82
|
|
|
83
|
-
const {
|
|
84
|
-
const table = pFramePromise.then((pFrame) =>
|
|
85
|
-
pFrame.createTableV2(key, { tableSpec, dataQuery }),
|
|
86
|
-
);
|
|
83
|
+
const { dataQuery } = params;
|
|
84
|
+
const table = pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery));
|
|
87
85
|
return new PTableHolder(pFrameHandle, combinedSignal, table);
|
|
88
86
|
}
|
|
89
87
|
|