@milaboratories/pf-driver 1.8.5 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/driver_impl.js +1 -1
- package/dist/driver_impl.js.map +1 -1
- package/dist/pframe_pool.js +1 -1
- package/dist/pframe_pool.js.map +1 -1
- package/dist/ptable_def_pool.js.map +1 -1
- package/dist/ptable_pool.js.map +1 -1
- package/dist/ptable_shared.d.ts +1 -2
- package/dist/ptable_shared.js.map +1 -1
- package/package.json +10 -11
- package/src/driver_impl.ts +1 -1
- package/src/pframe_pool.ts +2 -2
- package/src/ptable_def_pool.ts +2 -1
- package/src/ptable_pool.ts +2 -1
- package/src/ptable_shared.ts +4 -3
- package/dist/_virtual/_@oxc-project_runtime@0.137.0/helpers/esm/usingCtx.cjs +0 -57
- package/dist/_virtual/_rolldown/runtime.cjs +0 -23
- package/dist/csv_writer.cjs +0 -83
- package/dist/csv_writer.cjs.map +0 -1
- package/dist/data_info_helpers.cjs +0 -23
- package/dist/data_info_helpers.cjs.map +0 -1
- package/dist/driver_double.cjs +0 -114
- package/dist/driver_double.cjs.map +0 -1
- package/dist/driver_impl.cjs +0 -455
- package/dist/driver_impl.cjs.map +0 -1
- package/dist/index.cjs +0 -9
- package/dist/logging.cjs +0 -9
- package/dist/logging.cjs.map +0 -1
- package/dist/pframe_pool.cjs +0 -215
- package/dist/pframe_pool.cjs.map +0 -1
- package/dist/ptable_cache_per_frame.cjs +0 -58
- package/dist/ptable_cache_per_frame.cjs.map +0 -1
- package/dist/ptable_cache_plain.cjs +0 -45
- package/dist/ptable_cache_plain.cjs.map +0 -1
- package/dist/ptable_def_pool.cjs +0 -66
- package/dist/ptable_def_pool.cjs.map +0 -1
- package/dist/ptable_pool.cjs +0 -84
- package/dist/ptable_pool.cjs.map +0 -1
- package/dist/ptable_shared.cjs +0 -63
- package/dist/ptable_shared.cjs.map +0 -1
package/dist/pframe_pool.cjs
DELETED
|
@@ -1,215 +0,0 @@
|
|
|
1
|
-
const require_logging = require("./logging.cjs");
|
|
2
|
-
let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
|
|
3
|
-
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
4
|
-
let _milaboratories_pframes_rs_node = require("@milaboratories/pframes-rs-node");
|
|
5
|
-
let _milaboratories_pframes_rs_wasm = require("@milaboratories/pframes-rs-wasm");
|
|
6
|
-
let _milaboratories_pl_model_middle_layer = require("@milaboratories/pl-model-middle-layer");
|
|
7
|
-
let es_toolkit = require("es-toolkit");
|
|
8
|
-
//#region src/pframe_pool.ts
|
|
9
|
-
var PFrameHolder = class {
|
|
10
|
-
localBlobProvider;
|
|
11
|
-
remoteBlobProvider;
|
|
12
|
-
spillPath;
|
|
13
|
-
pFrameDataPromise;
|
|
14
|
-
/**
|
|
15
|
-
* WASM-spec frame built from this PFrame's columns. Source of truth
|
|
16
|
-
* for spec-side operations: column discovery, selector resolution,
|
|
17
|
-
* legacy-query lowering.
|
|
18
|
-
*/
|
|
19
|
-
pFrameSpec;
|
|
20
|
-
columnSpecs;
|
|
21
|
-
abortController = new AbortController();
|
|
22
|
-
localBlobs = [];
|
|
23
|
-
remoteBlobs = [];
|
|
24
|
-
constructor(frameId, localBlobProvider, remoteBlobProvider, logger, spillPath, columns) {
|
|
25
|
-
this.localBlobProvider = localBlobProvider;
|
|
26
|
-
this.remoteBlobProvider = remoteBlobProvider;
|
|
27
|
-
this.spillPath = spillPath;
|
|
28
|
-
const ValueTypes = new Set(Object.values(_milaboratories_pl_model_common.ValueType));
|
|
29
|
-
const specColumnsMap = Object.create(null);
|
|
30
|
-
for (const c of columns) if (ValueTypes.has(c.spec.valueType)) specColumnsMap[c.id] = (0, _milaboratories_pl_model_common.resolveAnnotationParents)(c.spec);
|
|
31
|
-
this.pFrameSpec = (0, _milaboratories_pframes_rs_wasm.createPFrame)(specColumnsMap);
|
|
32
|
-
this.columnSpecs = specColumnsMap;
|
|
33
|
-
try {
|
|
34
|
-
const makeLocalBlobId = (blob) => {
|
|
35
|
-
const localBlob = this.localBlobProvider.acquire(blob);
|
|
36
|
-
this.localBlobs.push(localBlob);
|
|
37
|
-
return localBlob.key;
|
|
38
|
-
};
|
|
39
|
-
const makeRemoteBlobId = (blob) => {
|
|
40
|
-
const remoteBlob = this.remoteBlobProvider.acquire(blob);
|
|
41
|
-
this.remoteBlobs.push(remoteBlob);
|
|
42
|
-
return `${remoteBlob.key}${_milaboratories_pl_model_middle_layer.PFrameInternal.ParquetExtension}`;
|
|
43
|
-
};
|
|
44
|
-
const mapColumnData = (data) => {
|
|
45
|
-
switch (data.type) {
|
|
46
|
-
case "Json": return { ...data };
|
|
47
|
-
case "JsonPartitioned": return {
|
|
48
|
-
...data,
|
|
49
|
-
parts: (0, es_toolkit.mapValues)(data.parts, makeLocalBlobId)
|
|
50
|
-
};
|
|
51
|
-
case "BinaryPartitioned": return {
|
|
52
|
-
...data,
|
|
53
|
-
parts: (0, es_toolkit.mapValues)(data.parts, (v) => ({
|
|
54
|
-
index: makeLocalBlobId(v.index),
|
|
55
|
-
values: makeLocalBlobId(v.values)
|
|
56
|
-
}))
|
|
57
|
-
};
|
|
58
|
-
case "ParquetPartitioned": return {
|
|
59
|
-
...data,
|
|
60
|
-
parts: (0, es_toolkit.mapValues)(data.parts, (v) => ({
|
|
61
|
-
...v,
|
|
62
|
-
data: makeRemoteBlobId(v.data)
|
|
63
|
-
}))
|
|
64
|
-
};
|
|
65
|
-
default: (0, _milaboratories_pl_model_common.assertNever)(data);
|
|
66
|
-
}
|
|
67
|
-
};
|
|
68
|
-
const jsonifiedColumns = columns.map((column) => ({
|
|
69
|
-
...column,
|
|
70
|
-
data: mapColumnData(column.data)
|
|
71
|
-
}));
|
|
72
|
-
const pFrameData = _milaboratories_pframes_rs_node.PFrameFactory.createPFrame({
|
|
73
|
-
frameId,
|
|
74
|
-
spillPath: this.spillPath,
|
|
75
|
-
logger
|
|
76
|
-
});
|
|
77
|
-
try {
|
|
78
|
-
pFrameData.setDataSource({
|
|
79
|
-
...this.localBlobProvider.makeDataSource(this.disposeSignal),
|
|
80
|
-
parquetServer: this.remoteBlobProvider.httpServerInfo()
|
|
81
|
-
});
|
|
82
|
-
this.pFrameDataPromise = pFrameData.addColumns(jsonifiedColumns.map((c) => ({
|
|
83
|
-
id: c.id,
|
|
84
|
-
data: {
|
|
85
|
-
...c.data,
|
|
86
|
-
typeSpec: {
|
|
87
|
-
axes: c.spec.axesSpec.map((a) => a.type),
|
|
88
|
-
column: c.spec.valueType
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
})), { signal: this.disposeSignal }).then(() => pFrameData).catch((err) => {
|
|
92
|
-
this.dispose();
|
|
93
|
-
pFrameData.dispose();
|
|
94
|
-
const error = new _milaboratories_pl_model_common.PFrameDriverError("PFrame creation failed asynchronously");
|
|
95
|
-
error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`, { cause: (0, _milaboratories_pl_model_common.ensureError)(err) });
|
|
96
|
-
throw error;
|
|
97
|
-
});
|
|
98
|
-
} catch (err) {
|
|
99
|
-
pFrameData.dispose();
|
|
100
|
-
throw err;
|
|
101
|
-
}
|
|
102
|
-
} catch (err) {
|
|
103
|
-
this.abortController.abort();
|
|
104
|
-
this.localBlobs.forEach((entry) => entry.unref());
|
|
105
|
-
this.remoteBlobs.forEach((entry) => entry.unref());
|
|
106
|
-
this.pFrameSpec[Symbol.dispose]();
|
|
107
|
-
const error = new _milaboratories_pl_model_common.PFrameDriverError("PFrame creation failed synchronously");
|
|
108
|
-
error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(columns)}`, { cause: (0, _milaboratories_pl_model_common.ensureError)(err) });
|
|
109
|
-
throw error;
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
get disposeSignal() {
|
|
113
|
-
return this.abortController.signal;
|
|
114
|
-
}
|
|
115
|
-
dispose() {
|
|
116
|
-
this.abortController.abort();
|
|
117
|
-
this.localBlobs.forEach((entry) => entry.unref());
|
|
118
|
-
this.remoteBlobs.forEach((entry) => entry.unref());
|
|
119
|
-
this.pFrameSpec[Symbol.dispose]();
|
|
120
|
-
this.pFrameDataPromise.then((pFrameData) => pFrameData.dispose()).catch(() => {});
|
|
121
|
-
}
|
|
122
|
-
[Symbol.dispose]() {
|
|
123
|
-
this.dispose();
|
|
124
|
-
}
|
|
125
|
-
};
|
|
126
|
-
var PFramePool = class extends _milaboratories_helpers.RefCountPoolBase {
|
|
127
|
-
localBlobProvider;
|
|
128
|
-
remoteBlobProvider;
|
|
129
|
-
logger;
|
|
130
|
-
spillPath;
|
|
131
|
-
constructor(localBlobProvider, remoteBlobProvider, logger, spillPath) {
|
|
132
|
-
super();
|
|
133
|
-
this.localBlobProvider = localBlobProvider;
|
|
134
|
-
this.remoteBlobProvider = remoteBlobProvider;
|
|
135
|
-
this.logger = logger;
|
|
136
|
-
this.spillPath = spillPath;
|
|
137
|
-
}
|
|
138
|
-
calculateParamsKey(params) {
|
|
139
|
-
return stableKeyFromPFrameData(params);
|
|
140
|
-
}
|
|
141
|
-
createNewResource(params, key) {
|
|
142
|
-
if (require_logging.logPFrames()) this.logger("info", `PFrame creation (pFrameHandle = ${key}): ${JSON.stringify(params, _milaboratories_pl_model_common.bigintReplacer)}`);
|
|
143
|
-
return new PFrameHolder(key, this.localBlobProvider, this.remoteBlobProvider, this.logger, this.spillPath, params);
|
|
144
|
-
}
|
|
145
|
-
getByKey(key) {
|
|
146
|
-
const resource = super.tryGetByKey(key);
|
|
147
|
-
if (!resource) {
|
|
148
|
-
const error = new _milaboratories_pl_model_common.PFrameDriverError(`Invalid PFrame handle`);
|
|
149
|
-
error.cause = /* @__PURE__ */ new Error(`PFrame with handle ${key} not found`);
|
|
150
|
-
throw error;
|
|
151
|
-
}
|
|
152
|
-
return resource;
|
|
153
|
-
}
|
|
154
|
-
};
|
|
155
|
-
function stableKeyFromPFrameData(data) {
|
|
156
|
-
const orderedData = [...data].map((column) => (0, _milaboratories_pl_model_common.mapPObjectData)(column, (r) => {
|
|
157
|
-
let result;
|
|
158
|
-
const type = r.type;
|
|
159
|
-
switch (type) {
|
|
160
|
-
case "Json":
|
|
161
|
-
result = {
|
|
162
|
-
type: r.type,
|
|
163
|
-
keyLength: r.keyLength,
|
|
164
|
-
payload: Object.entries(r.data).map(([part, value]) => ({
|
|
165
|
-
key: part,
|
|
166
|
-
value
|
|
167
|
-
}))
|
|
168
|
-
};
|
|
169
|
-
break;
|
|
170
|
-
case "JsonPartitioned":
|
|
171
|
-
result = {
|
|
172
|
-
type: r.type,
|
|
173
|
-
keyLength: r.partitionKeyLength,
|
|
174
|
-
payload: Object.entries(r.parts).map(([part, info]) => ({
|
|
175
|
-
key: part,
|
|
176
|
-
value: (0, _milaboratories_pl_model_common.canonicalizeJson)(info)
|
|
177
|
-
}))
|
|
178
|
-
};
|
|
179
|
-
break;
|
|
180
|
-
case "BinaryPartitioned":
|
|
181
|
-
result = {
|
|
182
|
-
type: r.type,
|
|
183
|
-
keyLength: r.partitionKeyLength,
|
|
184
|
-
payload: Object.entries(r.parts).map(([part, info]) => ({
|
|
185
|
-
key: part,
|
|
186
|
-
value: [(0, _milaboratories_pl_model_common.canonicalizeJson)(info.index), (0, _milaboratories_pl_model_common.canonicalizeJson)(info.values)]
|
|
187
|
-
}))
|
|
188
|
-
};
|
|
189
|
-
break;
|
|
190
|
-
case "ParquetPartitioned":
|
|
191
|
-
result = {
|
|
192
|
-
type: r.type,
|
|
193
|
-
keyLength: r.partitionKeyLength,
|
|
194
|
-
payload: Object.entries(r.parts).map(([part, info]) => ({
|
|
195
|
-
key: part,
|
|
196
|
-
value: info.dataDigest || [(0, _milaboratories_pl_model_common.canonicalizeJson)(info.data), JSON.stringify({
|
|
197
|
-
axes: info.axes,
|
|
198
|
-
column: info.column
|
|
199
|
-
})]
|
|
200
|
-
}))
|
|
201
|
-
};
|
|
202
|
-
break;
|
|
203
|
-
default: throw new _milaboratories_pl_model_common.PFrameDriverError(`unsupported resource type: ${JSON.stringify(type)}`);
|
|
204
|
-
}
|
|
205
|
-
result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);
|
|
206
|
-
return result;
|
|
207
|
-
}));
|
|
208
|
-
orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);
|
|
209
|
-
return (0, _milaboratories_pl_model_middle_layer.hashJson)(orderedData);
|
|
210
|
-
}
|
|
211
|
-
//#endregion
|
|
212
|
-
exports.PFrameHolder = PFrameHolder;
|
|
213
|
-
exports.PFramePool = PFramePool;
|
|
214
|
-
|
|
215
|
-
//# sourceMappingURL=pframe_pool.cjs.map
|
package/dist/pframe_pool.cjs.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"pframe_pool.cjs","names":["ValueType","PFrameInternal","PFrameFactory","PFrameDriverError","RefCountPoolBase","logPFrames","bigintReplacer"],"sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n assertNever,\n bigintReplacer,\n canonicalizeJson,\n ensureError,\n mapPObjectData,\n PFrameDriverError,\n ValueType,\n resolveAnnotationParents,\n type JsonSerializable,\n type PColumn,\n type PColumnSpec,\n type PFrameHandle,\n type PObjectId,\n} from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { PFrameFactory } from \"@milaboratories/pframes-rs-node\";\nimport { createPFrame as createPFrameSpec } from \"@milaboratories/pframes-rs-wasm\";\nimport { mapValues } from \"es-toolkit\";\nimport { logPFrames } from \"./logging\";\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\">;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n public readonly pFrameDataPromise: Promise<PFrameInternal.PFrameV18>;\n /**\n * WASM-spec frame built from this PFrame's columns. Source of truth\n * for spec-side operations: column discovery, selector resolution,\n * legacy-query lowering.\n */\n public readonly pFrameSpec: PFrameInternal.PFrameWasmV3;\n public readonly columnSpecs: Readonly<Record<PObjectId, PColumnSpec>>;\n private readonly abortController = new AbortController();\n\n private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n constructor(\n frameId: PFrameInternal.PFrameId,\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ) {\n const ValueTypes = new Set(Object.values(ValueType));\n const specColumnsMap: Record<PObjectId, PColumnSpec> = Object.create(null);\n for (const c of columns) {\n if (ValueTypes.has(c.spec.valueType)) {\n specColumnsMap[c.id] = resolveAnnotationParents(c.spec);\n }\n }\n this.pFrameSpec = createPFrameSpec(specColumnsMap);\n this.columnSpecs = specColumnsMap;\n\n try {\n const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const localBlob = this.localBlobProvider.acquire(blob);\n this.localBlobs.push(localBlob);\n return localBlob.key;\n };\n\n const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n const remoteBlob = this.remoteBlobProvider.acquire(blob);\n this.remoteBlobs.push(remoteBlob);\n return `${remoteBlob.key}${PFrameInternal.ParquetExtension}` as PFrameInternal.PFrameBlobId;\n };\n\n const mapColumnData = (\n data: PFrameInternal.DataInfo<TreeEntry>,\n ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n switch (data.type) {\n case \"Json\":\n return { ...data };\n case \"JsonPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, makeLocalBlobId),\n };\n case \"BinaryPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n index: makeLocalBlobId(v.index),\n values: makeLocalBlobId(v.values),\n })),\n };\n case \"ParquetPartitioned\":\n return {\n ...data,\n parts: mapValues(data.parts, (v) => ({\n ...v,\n data: makeRemoteBlobId(v.data),\n })),\n };\n default:\n assertNever(data);\n }\n };\n\n const jsonifiedColumns = columns.map((column) => ({\n ...column,\n data: mapColumnData(column.data),\n }));\n\n const pFrameData = PFrameFactory.createPFrame({ frameId, spillPath: this.spillPath, logger });\n try {\n pFrameData.setDataSource({\n ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n parquetServer: this.remoteBlobProvider.httpServerInfo(),\n });\n\n this.pFrameDataPromise = pFrameData\n .addColumns(\n jsonifiedColumns.map((c) => ({\n id: c.id,\n data: {\n ...c.data,\n typeSpec: {\n axes: c.spec.axesSpec.map((a) => a.type),\n column: c.spec.valueType,\n },\n },\n })),\n { signal: this.disposeSignal },\n )\n .then(() => pFrameData)\n .catch((err: unknown) => {\n this.dispose();\n pFrameData.dispose();\n const error = new PFrameDriverError(\"PFrame creation failed asynchronously\");\n error.cause = new Error(\n `PFrame cannot be created from columns: ${JSON.stringify(jsonifiedColumns)}`,\n { cause: ensureError(err) },\n );\n throw error;\n });\n } catch (err: unknown) {\n // setDataSource / addColumns threw synchronously before\n // pFrameDataPromise was assigned — dispose the addon's pFrameData\n // explicitly (the dispose() path can't reach it yet).\n pFrameData.dispose();\n throw err;\n }\n } catch (err: unknown) {\n // Release everything allocated so far in this constructor:\n // acquired blobs, the WASM spec frame, and the abort signal.\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n const error = new PFrameDriverError(\"PFrame creation failed synchronously\");\n error.cause = new Error(`PFrame cannot be created from columns: ${JSON.stringify(columns)}`, {\n cause: ensureError(err),\n });\n throw error;\n }\n }\n\n public get disposeSignal(): AbortSignal {\n return this.abortController.signal;\n }\n\n private dispose(): void {\n this.abortController.abort();\n this.localBlobs.forEach((entry) => entry.unref());\n this.remoteBlobs.forEach((entry) => entry.unref());\n this.pFrameSpec[Symbol.dispose]();\n void this.pFrameDataPromise\n .then((pFrameData) => pFrameData.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n\n [Symbol.dispose](): void {\n this.dispose();\n }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n PFrameHandle,\n PFrameHolder<TreeEntry>\n> {\n constructor(\n private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n private readonly logger: PFrameInternal.Logger,\n private readonly spillPath: string,\n ) {\n super();\n }\n\n protected calculateParamsKey(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n ): PFrameHandle {\n return stableKeyFromPFrameData(params);\n }\n\n protected createNewResource(\n params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n key: PFrameHandle,\n ): PFrameHolder<TreeEntry> {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PFrame creation (pFrameHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n return new PFrameHolder(\n key,\n this.localBlobProvider,\n this.remoteBlobProvider,\n this.logger,\n this.spillPath,\n params,\n );\n }\n\n public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PFrame handle`);\n error.cause = new Error(`PFrame with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n const orderedData = [...data].map((column) =>\n mapPObjectData(column, (r) => {\n let result: {\n type: string;\n keyLength: number;\n payload: {\n key: string;\n value: null | number | string | [string, string];\n }[];\n };\n const type = r.type;\n switch (type) {\n case \"Json\":\n result = {\n type: r.type,\n keyLength: r.keyLength,\n payload: Object.entries(r.data).map(([part, value]) => ({\n key: part,\n value,\n })),\n };\n break;\n case \"JsonPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: canonicalizeJson(info),\n })),\n };\n break;\n case \"BinaryPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n })),\n };\n break;\n case \"ParquetPartitioned\":\n result = {\n type: r.type,\n keyLength: r.partitionKeyLength,\n payload: Object.entries(r.parts).map(([part, info]) => ({\n key: part,\n value:\n info.dataDigest ||\n ([\n canonicalizeJson(info.data),\n JSON.stringify({ axes: info.axes, column: info.column }),\n ] as const),\n })),\n };\n break;\n default:\n throw new PFrameDriverError(\n `unsupported resource type: ${JSON.stringify(type satisfies never)}`,\n );\n }\n result.payload.sort((lhs, rhs) => (lhs.key < rhs.key ? -1 : 1));\n return result;\n }),\n );\n orderedData.sort((lhs, rhs) => (lhs.id < rhs.id ? -1 : 1));\n return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"mappings":";;;;;;;;AAgCA,IAAa,eAAb,MAAoF;CAgB/D;CACA;CAEA;CAlBnB;;;;;;CAMA;CACA;CACA,kBAAmC,IAAI,gBAAgB;CAEvD,aAAwE,CAAC;CACzE,cAAyE,CAAC;CAE1E,YACE,SACA,mBACA,oBACA,QACA,WACA,SACA;EALiB,KAAA,oBAAA;EACA,KAAA,qBAAA;EAEA,KAAA,YAAA;EAGjB,MAAM,aAAa,IAAI,IAAI,OAAO,OAAOA,gCAAAA,SAAS,CAAC;EACnD,MAAM,iBAAiD,OAAO,OAAO,IAAI;EACzE,KAAK,MAAM,KAAK,SACd,IAAI,WAAW,IAAI,EAAE,KAAK,SAAS,GACjC,eAAe,EAAE,OAAA,GAAA,gCAAA,yBAAA,CAA+B,EAAE,IAAI;EAG1D,KAAK,cAAA,GAAA,gCAAA,aAAA,CAA8B,cAAc;EACjD,KAAK,cAAc;EAEnB,IAAI;GACF,MAAM,mBAAmB,SAAiD;IACxE,MAAM,YAAY,KAAK,kBAAkB,QAAQ,IAAI;IACrD,KAAK,WAAW,KAAK,SAAS;IAC9B,OAAO,UAAU;GACnB;GAEA,MAAM,oBAAoB,SAAiD;IACzE,MAAM,aAAa,KAAK,mBAAmB,QAAQ,IAAI;IACvD,KAAK,YAAY,KAAK,UAAU;IAChC,OAAO,GAAG,WAAW,MAAMC,sCAAAA,eAAe;GAC5C;GAEA,MAAM,iBACJ,SACyD;IACzD,QAAQ,KAAK,MAAb;KACE,KAAK,QACH,OAAO,EAAE,GAAG,KAAK;KACnB,KAAK,mBACH,OAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,UAAA,CAAiB,KAAK,OAAO,eAAe;KAC9C;KACF,KAAK,qBACH,OAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,UAAA,CAAiB,KAAK,QAAQ,OAAO;OACnC,OAAO,gBAAgB,EAAE,KAAK;OAC9B,QAAQ,gBAAgB,EAAE,MAAM;MAClC,EAAE;KACJ;KACF,KAAK,sBACH,OAAO;MACL,GAAG;MACH,QAAA,GAAA,WAAA,UAAA,CAAiB,KAAK,QAAQ,OAAO;OACnC,GAAG;OACH,MAAM,iBAAiB,EAAE,IAAI;MAC/B,EAAE;KACJ;KACF,SACE,CAAA,GAAA,gCAAA,YAAA,CAAY,IAAI;IACpB;GACF;GAEA,MAAM,mBAAmB,QAAQ,KAAK,YAAY;IAChD,GAAG;IACH,MAAM,cAAc,OAAO,IAAI;GACjC,EAAE;GAEF,MAAM,aAAaC,gCAAAA,cAAc,aAAa;IAAE;IAAS,WAAW,KAAK;IAAW;GAAO,CAAC;GAC5F,IAAI;IACF,WAAW,cAAc;KACvB,GAAG,KAAK,kBAAkB,eAAe,KAAK,aAAa;KAC3D,eAAe,KAAK,mBAAmB,eAAe;IACxD,CAAC;IAED,KAAK,oBAAoB,WACtB,WACC,iBAAiB,KAAK,OAAO;KAC3B,IAAI,EAAE;KACN,MAAM;MACJ,GAAG,EAAE;MACL,UAAU;OACR,MAAM,EAAE,KAAK,SAAS,KAAK,MAAM,EAAE,IAAI;OACvC,QAAQ,EAAE,KAAK;MACjB;KACF;IACF,EAAE,GACF,EAAE,QAAQ,KAAK,cAAc,CAC/B,CAAC,CACA,WAAW,UAAU,CAAC,CACtB,OAAO,QAAiB;KACvB,KAAK,QAAQ;KACb,WAAW,QAAQ;KACnB,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,uCAAuC;KAC3E,MAAM,QAAQ,IAAI,MAChB,0CAA0C,KAAK,UAAU,gBAAgB,KACzE,EAAE,QAAA,GAAA,gCAAA,YAAA,CAAmB,GAAG,EAAE,CAC5B;KACA,MAAM;IACR,CAAC;GACL,SAAS,KAAc;IAIrB,WAAW,QAAQ;IACnB,MAAM;GACR;EACF,SAAS,KAAc;GAGrB,KAAK,gBAAgB,MAAM;GAC3B,KAAK,WAAW,SAAS,UAAU,MAAM,MAAM,CAAC;GAChD,KAAK,YAAY,SAAS,UAAU,MAAM,MAAM,CAAC;GACjD,KAAK,WAAW,OAAO,QAAQ,CAAC;GAChC,MAAM,QAAQ,IAAIA,gCAAAA,kBAAkB,sCAAsC;GAC1E,MAAM,QAAQ,IAAI,MAAM,0CAA0C,KAAK,UAAU,OAAO,KAAK,EAC3F,QAAA,GAAA,gCAAA,YAAA,CAAmB,GAAG,EACxB,CAAC;GACD,MAAM;EACR;CACF;CAEA,IAAW,gBAA6B;EACtC,OAAO,KAAK,gBAAgB;CAC9B;CAEA,UAAwB;EACtB,KAAK,gBAAgB,MAAM;EAC3B,KAAK,WAAW,SAAS,UAAU,MAAM,MAAM,CAAC;EAChD,KAAK,YAAY,SAAS,UAAU,MAAM,MAAM,CAAC;EACjD,KAAK,WAAW,OAAO,QAAQ,CAAC;EAChC,KAAU,kBACP,MAAM,eAAe,WAAW,QAAQ,CAAC,CAAC,CAC1C,YAAY,CAEb,CAAC;CACL;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,QAAQ;CACf;AACF;AAEA,IAAa,aAAb,cAAoEC,wBAAAA,iBAIlE;CAEmB;CACA;CACA;CACA;CAJnB,YACE,mBACA,oBACA,QACA,WACA;EACA,MAAM;EALW,KAAA,oBAAA;EACA,KAAA,qBAAA;EACA,KAAA,SAAA;EACA,KAAA,YAAA;CAGnB;CAEA,mBACE,QACc;EACd,OAAO,wBAAwB,MAAM;CACvC;CAEA,kBACE,QACA,KACyB;EACzB,IAAIC,gBAAAA,WAAW,GACb,KAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQC,gCAAAA,cAAc,GACxF;EAEF,OAAO,IAAI,aACT,KACA,KAAK,mBACL,KAAK,oBACL,KAAK,QACL,KAAK,WACL,MACF;CACF;CAEA,SAAgB,KAA4C;EAC1D,MAAM,WAAW,MAAM,YAAY,GAAG;EACtC,IAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAIH,gCAAAA,kBAAkB,uBAAuB;GAC3D,MAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,WAAW;GAC7D,MAAM;EACR;EACA,OAAO;CACT;AACF;AAEA,SAAS,wBACP,MACc;CACd,MAAM,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,YAAA,GAAA,gCAAA,eAAA,CAClB,SAAS,MAAM;EAC5B,IAAI;EAQJ,MAAM,OAAO,EAAE;EACf,QAAQ,MAAR;GACE,KAAK;IACH,SAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,YAAY;MACtD,KAAK;MACL;KACF,EAAE;IACJ;IACA;GACF,KAAK;IACH,SAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,QAAA,GAAA,gCAAA,iBAAA,CAAwB,IAAI;KAC9B,EAAE;IACJ;IACA;GACF,KAAK;IACH,SAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OAAO,EAAA,GAAA,gCAAA,iBAAA,CAAkB,KAAK,KAAK,IAAA,GAAA,gCAAA,iBAAA,CAAoB,KAAK,MAAM,CAAC;KACrE,EAAE;IACJ;IACA;GACF,KAAK;IACH,SAAS;KACP,MAAM,EAAE;KACR,WAAW,EAAE;KACb,SAAS,OAAO,QAAQ,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW;MACtD,KAAK;MACL,OACE,KAAK,cACJ,EAAA,GAAA,gCAAA,iBAAA,CACkB,KAAK,IAAI,GAC1B,KAAK,UAAU;OAAE,MAAM,KAAK;OAAM,QAAQ,KAAK;MAAO,CAAC,CACzD;KACJ,EAAE;IACJ;IACA;GACF,SACE,MAAM,IAAIA,gCAAAA,kBACR,8BAA8B,KAAK,UAAU,IAAoB,GACnE;EACJ;EACA,OAAO,QAAQ,MAAM,KAAK,QAAS,IAAI,MAAM,IAAI,MAAM,KAAK,CAAE;EAC9D,OAAO;CACT,CAAC,CACH;CACA,YAAY,MAAM,KAAK,QAAS,IAAI,KAAK,IAAI,KAAK,KAAK,CAAE;CACzD,QAAA,GAAA,sCAAA,SAAA,CAAgB,WAAW;AAC7B"}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
const require_logging = require("./logging.cjs");
|
|
2
|
-
let lru_cache = require("lru-cache");
|
|
3
|
-
//#region src/ptable_cache_per_frame.ts
|
|
4
|
-
const PTableCachePerFrameOpsDefaults = {
|
|
5
|
-
pFrameCacheMaxCount: 18,
|
|
6
|
-
pFramesCacheMaxSize: 8 * 1024 * 1024 * 1024
|
|
7
|
-
};
|
|
8
|
-
var PTableCachePerFrame = class {
|
|
9
|
-
logger;
|
|
10
|
-
ops;
|
|
11
|
-
perFrame = /* @__PURE__ */ new Map();
|
|
12
|
-
global;
|
|
13
|
-
disposeListeners = /* @__PURE__ */ new Set();
|
|
14
|
-
constructor(logger, ops) {
|
|
15
|
-
this.logger = logger;
|
|
16
|
-
this.ops = ops;
|
|
17
|
-
this.global = new lru_cache.LRUCache({
|
|
18
|
-
maxSize: this.ops.pFramesCacheMaxSize,
|
|
19
|
-
dispose: (resource, key, reason) => {
|
|
20
|
-
if (reason === "evict") this.perFrame.get(resource.resource.pFrame)?.delete(key);
|
|
21
|
-
if (this.perFrame.get(resource.resource.pFrame)?.size === 0) this.perFrame.delete(resource.resource.pFrame);
|
|
22
|
-
resource.unref();
|
|
23
|
-
if (require_logging.logPFrames()) logger("info", `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
|
|
24
|
-
}
|
|
25
|
-
});
|
|
26
|
-
}
|
|
27
|
-
cache(resource, size) {
|
|
28
|
-
const key = resource.key;
|
|
29
|
-
if (require_logging.logPFrames()) this.logger("info", `calculateTableData cache - added PTable ${key} with size ${size}`);
|
|
30
|
-
this.global.set(key, resource, { size: Math.max(size, 1) });
|
|
31
|
-
let perFrame = this.perFrame.get(resource.resource.pFrame);
|
|
32
|
-
if (!perFrame) {
|
|
33
|
-
perFrame = new lru_cache.LRUCache({
|
|
34
|
-
max: this.ops.pFrameCacheMaxCount,
|
|
35
|
-
dispose: (_resource, key, reason) => {
|
|
36
|
-
if (reason === "evict") this.global.delete(key);
|
|
37
|
-
}
|
|
38
|
-
});
|
|
39
|
-
this.perFrame.set(resource.resource.pFrame, perFrame);
|
|
40
|
-
}
|
|
41
|
-
perFrame.set(key, resource);
|
|
42
|
-
if (!this.disposeListeners.has(key)) {
|
|
43
|
-
const disposeListener = () => {
|
|
44
|
-
this.perFrame.get(resource.resource.pFrame)?.delete(key);
|
|
45
|
-
this.global.delete(key);
|
|
46
|
-
this.disposeListeners.delete(key);
|
|
47
|
-
resource.resource.disposeSignal.removeEventListener("abort", disposeListener);
|
|
48
|
-
};
|
|
49
|
-
this.disposeListeners.add(key);
|
|
50
|
-
resource.resource.disposeSignal.addEventListener("abort", disposeListener);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
//#endregion
|
|
55
|
-
exports.PTableCachePerFrame = PTableCachePerFrame;
|
|
56
|
-
exports.PTableCachePerFrameOpsDefaults = PTableCachePerFrameOpsDefaults;
|
|
57
|
-
|
|
58
|
-
//# sourceMappingURL=ptable_cache_per_frame.cjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_cache_per_frame.cjs","names":["LRUCache","logPFrames"],"sources":["../src/ptable_cache_per_frame.ts"],"sourcesContent":["import type { PFrameHandle, PTableHandle } from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport type { PoolEntry } from \"@milaboratories/helpers\";\nimport { LRUCache } from \"lru-cache\";\nimport { logPFrames } from \"./logging\";\nimport type { PTableHolder } from \"./ptable_pool\";\n\nexport type PTableCachePerFrameOps = {\n /** Maximum number of `calculateTableData` results cached for each PFrame */\n pFrameCacheMaxCount: number;\n /**\n * Maximum size of `calculateTableData` results cached for PFrames overall.\n * The limit is soft, as the same table could be materialized with other requests and will not be deleted in such case.\n * Also each table has predecessors, overlapping predecessors will be counted twice, so the effective limit is smaller.\n */\n pFramesCacheMaxSize: number;\n};\n\nexport const PTableCachePerFrameOpsDefaults: PTableCachePerFrameOps = {\n pFrameCacheMaxCount: 18, // SHM trees create 3 PTables per graphic, we want to cache 6 graphics per PFrame\n pFramesCacheMaxSize: 8 * 1024 * 1024 * 1024, // 8 GB, same as blob driver cache (must be at least 2GB)\n};\n\nexport class PTableCachePerFrame {\n private readonly perFrame = new Map<\n PFrameHandle,\n LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>\n >();\n private readonly global: LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>;\n private readonly disposeListeners = new Set<PTableHandle>();\n\n constructor(\n private readonly logger: PFrameInternal.Logger,\n private readonly ops: PTableCachePerFrameOps,\n ) {\n this.global = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({\n maxSize: this.ops.pFramesCacheMaxSize,\n dispose: (resource, key, reason) => {\n if (reason === \"evict\") {\n this.perFrame.get(resource.resource.pFrame)?.delete(key);\n }\n\n if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {\n this.perFrame.delete(resource.resource.pFrame);\n }\n\n resource.unref();\n if (logPFrames()) {\n logger(\"info\", `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);\n }\n },\n });\n }\n\n public cache(resource: PoolEntry<PTableHandle, PTableHolder>, size: number): void {\n const key = resource.key;\n if (logPFrames()) {\n this.logger(\"info\", `calculateTableData cache - added PTable ${key} with size ${size}`);\n }\n\n this.global.set(key, resource, { size: Math.max(size, 1) }); // 1 is minimum size to avoid cache evictions\n\n let perFrame = this.perFrame.get(resource.resource.pFrame);\n if (!perFrame) {\n perFrame = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({\n max: this.ops.pFrameCacheMaxCount,\n dispose: (_resource, key, reason) => {\n if (reason === \"evict\") {\n this.global.delete(key);\n }\n },\n });\n this.perFrame.set(resource.resource.pFrame, perFrame);\n }\n perFrame.set(key, resource);\n\n if (!this.disposeListeners.has(key)) {\n const disposeListener = () => {\n this.perFrame.get(resource.resource.pFrame)?.delete(key);\n this.global.delete(key);\n\n this.disposeListeners.delete(key);\n resource.resource.disposeSignal.removeEventListener(\"abort\", disposeListener);\n };\n this.disposeListeners.add(key);\n resource.resource.disposeSignal.addEventListener(\"abort\", disposeListener);\n }\n }\n}\n"],"mappings":";;;AAkBA,MAAa,iCAAyD;CACpE,qBAAqB;CACrB,qBAAqB,IAAI,OAAO,OAAO;AACzC;AAEA,IAAa,sBAAb,MAAiC;CASZ;CACA;CATnB,2BAA4B,IAAI,IAG9B;CACF;CACA,mCAAoC,IAAI,IAAkB;CAE1D,YACE,QACA,KACA;EAFiB,KAAA,SAAA;EACA,KAAA,MAAA;EAEjB,KAAK,SAAS,IAAIA,UAAAA,SAA8D;GAC9E,SAAS,KAAK,IAAI;GAClB,UAAU,UAAU,KAAK,WAAW;IAClC,IAAI,WAAW,SACb,KAAK,SAAS,IAAI,SAAS,SAAS,MAAM,CAAC,EAAE,OAAO,GAAG;IAGzD,IAAI,KAAK,SAAS,IAAI,SAAS,SAAS,MAAM,CAAC,EAAE,SAAS,GACxD,KAAK,SAAS,OAAO,SAAS,SAAS,MAAM;IAG/C,SAAS,MAAM;IACf,IAAIC,gBAAAA,WAAW,GACb,OAAO,QAAQ,6CAA6C,IAAI,YAAY,OAAO,EAAE;GAEzF;EACF,CAAC;CACH;CAEA,MAAa,UAAiD,MAAoB;EAChF,MAAM,MAAM,SAAS;EACrB,IAAIA,gBAAAA,WAAW,GACb,KAAK,OAAO,QAAQ,2CAA2C,IAAI,aAAa,MAAM;EAGxF,KAAK,OAAO,IAAI,KAAK,UAAU,EAAE,MAAM,KAAK,IAAI,MAAM,CAAC,EAAE,CAAC;EAE1D,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,SAAS,MAAM;EACzD,IAAI,CAAC,UAAU;GACb,WAAW,IAAID,UAAAA,SAA8D;IAC3E,KAAK,KAAK,IAAI;IACd,UAAU,WAAW,KAAK,WAAW;KACnC,IAAI,WAAW,SACb,KAAK,OAAO,OAAO,GAAG;IAE1B;GACF,CAAC;GACD,KAAK,SAAS,IAAI,SAAS,SAAS,QAAQ,QAAQ;EACtD;EACA,SAAS,IAAI,KAAK,QAAQ;EAE1B,IAAI,CAAC,KAAK,iBAAiB,IAAI,GAAG,GAAG;GACnC,MAAM,wBAAwB;IAC5B,KAAK,SAAS,IAAI,SAAS,SAAS,MAAM,CAAC,EAAE,OAAO,GAAG;IACvD,KAAK,OAAO,OAAO,GAAG;IAEtB,KAAK,iBAAiB,OAAO,GAAG;IAChC,SAAS,SAAS,cAAc,oBAAoB,SAAS,eAAe;GAC9E;GACA,KAAK,iBAAiB,IAAI,GAAG;GAC7B,SAAS,SAAS,cAAc,iBAAiB,SAAS,eAAe;EAC3E;CACF;AACF"}
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
const require_logging = require("./logging.cjs");
|
|
2
|
-
let lru_cache = require("lru-cache");
|
|
3
|
-
//#region src/ptable_cache_plain.ts
|
|
4
|
-
const PTableCachePlainOpsDefaults = { pTablesCacheMaxSize: 32 * 1024 * 1024 * 1024 };
|
|
5
|
-
var PTableCachePlain = class {
|
|
6
|
-
logger;
|
|
7
|
-
global;
|
|
8
|
-
disposeListeners = /* @__PURE__ */ new Set();
|
|
9
|
-
constructor(logger, ops) {
|
|
10
|
-
this.logger = logger;
|
|
11
|
-
this.global = new lru_cache.LRUCache({
|
|
12
|
-
maxSize: ops.pTablesCacheMaxSize,
|
|
13
|
-
dispose: (resource, key, reason) => {
|
|
14
|
-
resource.unref();
|
|
15
|
-
if (require_logging.logPFrames()) logger("info", `createPTable cache - removed PTable ${key} (reason: ${reason})`);
|
|
16
|
-
}
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
cache(resource, size, defDisposeSignal) {
|
|
20
|
-
const key = resource.key;
|
|
21
|
-
if (require_logging.logPFrames()) this.logger("info", `createPTable cache - added PTable ${key} with size ${size}`);
|
|
22
|
-
const status = {};
|
|
23
|
-
this.global.set(key, resource, {
|
|
24
|
-
size: Math.max(size, 1),
|
|
25
|
-
status
|
|
26
|
-
});
|
|
27
|
-
if (status.maxEntrySizeExceeded) {
|
|
28
|
-
resource.unref();
|
|
29
|
-
if (require_logging.logPFrames()) this.logger("info", `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);
|
|
30
|
-
} else if (!this.disposeListeners.has(key)) {
|
|
31
|
-
const disposeListener = () => {
|
|
32
|
-
this.global.delete(key);
|
|
33
|
-
this.disposeListeners.delete(key);
|
|
34
|
-
defDisposeSignal.removeEventListener("abort", disposeListener);
|
|
35
|
-
};
|
|
36
|
-
this.disposeListeners.add(key);
|
|
37
|
-
defDisposeSignal.addEventListener("abort", disposeListener);
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
|
-
//#endregion
|
|
42
|
-
exports.PTableCachePlain = PTableCachePlain;
|
|
43
|
-
exports.PTableCachePlainOpsDefaults = PTableCachePlainOpsDefaults;
|
|
44
|
-
|
|
45
|
-
//# sourceMappingURL=ptable_cache_plain.cjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_cache_plain.cjs","names":["LRUCache","logPFrames"],"sources":["../src/ptable_cache_plain.ts"],"sourcesContent":["import type { PTableHandle } from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport type { PoolEntry } from \"@milaboratories/helpers\";\nimport { LRUCache } from \"lru-cache\";\nimport { logPFrames } from \"./logging\";\nimport type { PTableHolder } from \"./ptable_pool\";\n\nexport type PTableCachePlainOps = {\n /**\n * Maximum size of `createPTable` results cached on disk.\n * The limit is soft, as the same table could be materialized with other requests and will not be deleted in such case.\n * Also each table has predecessors, overlapping predecessors will be counted twice, so the effective limit is smaller.\n */\n pTablesCacheMaxSize: number;\n};\n\nexport const PTableCachePlainOpsDefaults: PTableCachePlainOps = {\n pTablesCacheMaxSize: 32 * 1024 * 1024 * 1024, // 32 GB (must be at least 8GB)\n};\n\nexport class PTableCachePlain {\n private readonly global: LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>;\n private readonly disposeListeners = new Set<PTableHandle>();\n\n constructor(\n private readonly logger: PFrameInternal.Logger,\n ops: PTableCachePlainOps,\n ) {\n this.global = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({\n maxSize: ops.pTablesCacheMaxSize,\n dispose: (resource, key, reason) => {\n resource.unref();\n if (logPFrames()) {\n logger(\"info\", `createPTable cache - removed PTable ${key} (reason: ${reason})`);\n }\n },\n });\n }\n\n public cache(\n resource: PoolEntry<PTableHandle, PTableHolder>,\n size: number,\n defDisposeSignal: AbortSignal,\n ): void {\n const key = resource.key;\n if (logPFrames()) {\n this.logger(\"info\", `createPTable cache - added PTable ${key} with size ${size}`);\n }\n\n const status: LRUCache.Status<PoolEntry<PTableHandle, PTableHolder>> = {};\n this.global.set(key, resource, { size: Math.max(size, 1), status }); // 1 is minimum size to avoid cache evictions\n\n if (status.maxEntrySizeExceeded) {\n resource.unref();\n if (logPFrames()) {\n this.logger(\"info\", `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);\n }\n } else {\n if (!this.disposeListeners.has(key)) {\n const disposeListener = () => {\n this.global.delete(key);\n\n this.disposeListeners.delete(key);\n defDisposeSignal.removeEventListener(\"abort\", disposeListener);\n };\n this.disposeListeners.add(key);\n defDisposeSignal.addEventListener(\"abort\", disposeListener);\n }\n }\n }\n}\n"],"mappings":";;;AAgBA,MAAa,8BAAmD,EAC9D,qBAAqB,KAAK,OAAO,OAAO,KAC1C;AAEA,IAAa,mBAAb,MAA8B;CAKT;CAJnB;CACA,mCAAoC,IAAI,IAAkB;CAE1D,YACE,QACA,KACA;EAFiB,KAAA,SAAA;EAGjB,KAAK,SAAS,IAAIA,UAAAA,SAA8D;GAC9E,SAAS,IAAI;GACb,UAAU,UAAU,KAAK,WAAW;IAClC,SAAS,MAAM;IACf,IAAIC,gBAAAA,WAAW,GACb,OAAO,QAAQ,uCAAuC,IAAI,YAAY,OAAO,EAAE;GAEnF;EACF,CAAC;CACH;CAEA,MACE,UACA,MACA,kBACM;EACN,MAAM,MAAM,SAAS;EACrB,IAAIA,gBAAAA,WAAW,GACb,KAAK,OAAO,QAAQ,qCAAqC,IAAI,aAAa,MAAM;EAGlF,MAAM,SAAiE,CAAC;EACxE,KAAK,OAAO,IAAI,KAAK,UAAU;GAAE,MAAM,KAAK,IAAI,MAAM,CAAC;GAAG;EAAO,CAAC;EAElE,IAAI,OAAO,sBAAsB;GAC/B,SAAS,MAAM;GACf,IAAIA,gBAAAA,WAAW,GACb,KAAK,OAAO,QAAQ,uCAAuC,IAAI,wBAAwB;EAE3F,OACE,IAAI,CAAC,KAAK,iBAAiB,IAAI,GAAG,GAAG;GACnC,MAAM,wBAAwB;IAC5B,KAAK,OAAO,OAAO,GAAG;IAEtB,KAAK,iBAAiB,OAAO,GAAG;IAChC,iBAAiB,oBAAoB,SAAS,eAAe;GAC/D;GACA,KAAK,iBAAiB,IAAI,GAAG;GAC7B,iBAAiB,iBAAiB,SAAS,eAAe;EAC5D;CAEJ;AACF"}
|
package/dist/ptable_def_pool.cjs
DELETED
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
const require_logging = require("./logging.cjs");
|
|
2
|
-
const require_ptable_shared = require("./ptable_shared.cjs");
|
|
3
|
-
let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
|
|
4
|
-
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
5
|
-
//#region src/ptable_def_pool.ts
|
|
6
|
-
var PTableDefHolder = class {
|
|
7
|
-
def;
|
|
8
|
-
pTableHandle;
|
|
9
|
-
logger;
|
|
10
|
-
abortController = new AbortController();
|
|
11
|
-
constructor(def, pTableHandle, logger) {
|
|
12
|
-
this.def = def;
|
|
13
|
-
this.pTableHandle = pTableHandle;
|
|
14
|
-
this.logger = logger;
|
|
15
|
-
if (require_logging.logPFrames()) this.logger("info", `PTable definition saved (pTableHandle = ${this.pTableHandle})`);
|
|
16
|
-
}
|
|
17
|
-
get disposeSignal() {
|
|
18
|
-
return this.abortController.signal;
|
|
19
|
-
}
|
|
20
|
-
[Symbol.dispose]() {
|
|
21
|
-
this.abortController.abort();
|
|
22
|
-
if (require_logging.logPFrames()) this.logger("info", `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);
|
|
23
|
-
}
|
|
24
|
-
};
|
|
25
|
-
var PTableDefPool = class extends _milaboratories_helpers.RefCountPoolBase {
|
|
26
|
-
logger;
|
|
27
|
-
constructor(logger) {
|
|
28
|
-
super();
|
|
29
|
-
this.logger = logger;
|
|
30
|
-
}
|
|
31
|
-
calculateParamsKey(params) {
|
|
32
|
-
return require_ptable_shared.stableKeyFromFullPTableDef(params);
|
|
33
|
-
}
|
|
34
|
-
createNewResource(params, key) {
|
|
35
|
-
return new PTableDefHolder(params, key, this.logger);
|
|
36
|
-
}
|
|
37
|
-
getByKey(key) {
|
|
38
|
-
const resource = super.tryGetByKey(key);
|
|
39
|
-
if (!resource) {
|
|
40
|
-
const error = new _milaboratories_pl_model_common.PFrameDriverError(`Invalid PTable handle`);
|
|
41
|
-
error.cause = /* @__PURE__ */ new Error(`PTable definition for handle ${key} not found`);
|
|
42
|
-
throw error;
|
|
43
|
-
}
|
|
44
|
-
return resource;
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Acquire a def from a legacy `PTableDef` + column specs map.
|
|
48
|
-
* Lowers the input via WASM-spec and stores the resulting
|
|
49
|
-
* `{ tableSpec, dataQuery }` shape. Returns the lowered def
|
|
50
|
-
* alongside the pool entry — callers that need `tableSpec`
|
|
51
|
-
* (e.g. `calculateTableData`) read it from `def.tableSpec`
|
|
52
|
-
* without re-deriving.
|
|
53
|
-
*/
|
|
54
|
-
acquireFromLegacy(opts) {
|
|
55
|
-
const def = require_ptable_shared.buildFullPTableDefFromLegacy(opts);
|
|
56
|
-
return {
|
|
57
|
-
def,
|
|
58
|
-
entry: this.acquire(def)
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
//#endregion
|
|
63
|
-
exports.PTableDefHolder = PTableDefHolder;
|
|
64
|
-
exports.PTableDefPool = PTableDefPool;
|
|
65
|
-
|
|
66
|
-
//# sourceMappingURL=ptable_def_pool.cjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_def_pool.cjs","names":["logPFrames","RefCountPoolBase","stableKeyFromFullPTableDef","PFrameDriverError","buildFullPTableDefFromLegacy"],"sources":["../src/ptable_def_pool.ts"],"sourcesContent":["import {\n PFrameDriverError,\n type PFrameHandle,\n type PObjectId,\n type PTableDef,\n type PTableHandle,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { logPFrames } from \"./logging\";\nimport type { FullPTableDef } from \"./ptable_shared\";\nimport { buildFullPTableDefFromLegacy, stableKeyFromFullPTableDef } from \"./ptable_shared\";\n\nexport class PTableDefHolder implements Disposable {\n private readonly abortController = new AbortController();\n\n constructor(\n public readonly def: FullPTableDef,\n private readonly pTableHandle: PTableHandle,\n private readonly logger: PFrameInternal.Logger,\n ) {\n if (logPFrames()) {\n this.logger(\"info\", `PTable definition saved (pTableHandle = ${this.pTableHandle})`);\n }\n }\n\n public get disposeSignal(): AbortSignal {\n return this.abortController.signal;\n }\n\n [Symbol.dispose](): void {\n this.abortController.abort();\n if (logPFrames()) {\n this.logger(\"info\", `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);\n }\n }\n}\n\nexport class PTableDefPool extends RefCountPoolBase<FullPTableDef, PTableHandle, PTableDefHolder> {\n constructor(private readonly logger: PFrameInternal.Logger) {\n super();\n }\n\n protected calculateParamsKey(params: FullPTableDef): PTableHandle {\n return stableKeyFromFullPTableDef(params);\n }\n\n protected createNewResource(params: FullPTableDef, key: PTableHandle): PTableDefHolder {\n return new PTableDefHolder(params, key, this.logger);\n }\n\n public getByKey(key: PTableHandle): PTableDefHolder {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PTable handle`);\n error.cause = new Error(`PTable definition for handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n\n /**\n * Acquire a def from a legacy `PTableDef` + column specs map.\n * Lowers the input via WASM-spec and stores the resulting\n * `{ tableSpec, dataQuery }` shape. Returns the lowered def\n * alongside the pool entry — callers that need `tableSpec`\n * (e.g. `calculateTableData`) read it from `def.tableSpec`\n * without re-deriving.\n */\n public acquireFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n }): { def: FullPTableDef; entry: PoolEntry<PTableHandle> } {\n const def = buildFullPTableDefFromLegacy(opts);\n return { def, entry: this.acquire(def) };\n }\n}\n"],"mappings":";;;;;AAaA,IAAa,kBAAb,MAAmD;CAI/B;CACC;CACA;CALnB,kBAAmC,IAAI,gBAAgB;CAEvD,YACE,KACA,cACA,QACA;EAHgB,KAAA,MAAA;EACC,KAAA,eAAA;EACA,KAAA,SAAA;EAEjB,IAAIA,gBAAAA,WAAW,GACb,KAAK,OAAO,QAAQ,2CAA2C,KAAK,aAAa,EAAE;CAEvF;CAEA,IAAW,gBAA6B;EACtC,OAAO,KAAK,gBAAgB;CAC9B;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,gBAAgB,MAAM;EAC3B,IAAIA,gBAAAA,WAAW,GACb,KAAK,OAAO,QAAQ,8CAA8C,KAAK,aAAa,EAAE;CAE1F;AACF;AAEA,IAAa,gBAAb,cAAmCC,wBAAAA,iBAA+D;CACnE;CAA7B,YAAY,QAAgD;EAC1D,MAAM;EADqB,KAAA,SAAA;CAE7B;CAEA,mBAA6B,QAAqC;EAChE,OAAOC,sBAAAA,2BAA2B,MAAM;CAC1C;CAEA,kBAA4B,QAAuB,KAAoC;EACrF,OAAO,IAAI,gBAAgB,QAAQ,KAAK,KAAK,MAAM;CACrD;CAEA,SAAgB,KAAoC;EAClD,MAAM,WAAW,MAAM,YAAY,GAAG;EACtC,IAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,uBAAuB;GAC3D,MAAM,wBAAQ,IAAI,MAAM,gCAAgC,IAAI,WAAW;GACvE,MAAM;EACR;EACA,OAAO;CACT;;;;;;;;;CAUA,kBAAyB,MAIkC;EACzD,MAAM,MAAMC,sBAAAA,6BAA6B,IAAI;EAC7C,OAAO;GAAE;GAAK,OAAO,KAAK,QAAQ,GAAG;EAAE;CACzC;AACF"}
|
package/dist/ptable_pool.cjs
DELETED
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
const require_logging = require("./logging.cjs");
|
|
2
|
-
const require_ptable_shared = require("./ptable_shared.cjs");
|
|
3
|
-
let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
|
|
4
|
-
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
5
|
-
//#region src/ptable_pool.ts
|
|
6
|
-
var PTableHolder = class {
|
|
7
|
-
pFrame;
|
|
8
|
-
pTablePromise;
|
|
9
|
-
predecessor;
|
|
10
|
-
abortController = new AbortController();
|
|
11
|
-
combinedDisposeSignal;
|
|
12
|
-
cacheSizePromise;
|
|
13
|
-
constructor(pFrame, pFrameDisposeSignal, pTablePromise, predecessor) {
|
|
14
|
-
this.pFrame = pFrame;
|
|
15
|
-
this.pTablePromise = pTablePromise;
|
|
16
|
-
this.predecessor = predecessor;
|
|
17
|
-
this.combinedDisposeSignal = AbortSignal.any([pFrameDisposeSignal, this.abortController.signal]);
|
|
18
|
-
}
|
|
19
|
-
get disposeSignal() {
|
|
20
|
-
return this.combinedDisposeSignal;
|
|
21
|
-
}
|
|
22
|
-
get cacheSize() {
|
|
23
|
-
if (this.cacheSizePromise === void 0) this.cacheSizePromise = this.pTablePromise.then((pTable) => pTable.getFootprint({
|
|
24
|
-
withPredecessors: false,
|
|
25
|
-
signal: this.abortController.signal
|
|
26
|
-
}));
|
|
27
|
-
return this.cacheSizePromise;
|
|
28
|
-
}
|
|
29
|
-
[Symbol.dispose]() {
|
|
30
|
-
this.abortController.abort();
|
|
31
|
-
this.predecessor?.unref();
|
|
32
|
-
this.pTablePromise.then((pTable) => pTable.dispose()).catch(() => {});
|
|
33
|
-
}
|
|
34
|
-
};
|
|
35
|
-
var PTablePool = class extends _milaboratories_helpers.RefCountPoolBase {
|
|
36
|
-
pFrames;
|
|
37
|
-
pTableDefs;
|
|
38
|
-
logger;
|
|
39
|
-
constructor(pFrames, pTableDefs, logger) {
|
|
40
|
-
super();
|
|
41
|
-
this.pFrames = pFrames;
|
|
42
|
-
this.pTableDefs = pTableDefs;
|
|
43
|
-
this.logger = logger;
|
|
44
|
-
}
|
|
45
|
-
calculateParamsKey(params) {
|
|
46
|
-
return require_ptable_shared.stableKeyFromFullPTableDef(params);
|
|
47
|
-
}
|
|
48
|
-
createNewResource(params, key) {
|
|
49
|
-
if (require_logging.logPFrames()) this.logger("info", `PTable creation (pTableHandle = ${key}): ${JSON.stringify(params, _milaboratories_pl_model_common.bigintReplacer)}`);
|
|
50
|
-
const { pFrameHandle } = params;
|
|
51
|
-
const { pFrameDataPromise: pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);
|
|
52
|
-
const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
|
|
53
|
-
const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
|
|
54
|
-
const { dataQuery } = params;
|
|
55
|
-
return new PTableHolder(pFrameHandle, combinedSignal, pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery)));
|
|
56
|
-
}
|
|
57
|
-
getByKey(key) {
|
|
58
|
-
const resource = super.tryGetByKey(key);
|
|
59
|
-
if (!resource) {
|
|
60
|
-
const error = new _milaboratories_pl_model_common.PFrameDriverError(`Invalid PTable handle`);
|
|
61
|
-
error.cause = /* @__PURE__ */ new Error(`PTable with handle ${key} not found`);
|
|
62
|
-
throw error;
|
|
63
|
-
}
|
|
64
|
-
return resource;
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Acquire a PTable from a legacy `PTableDef` + column specs map.
|
|
68
|
-
* Lowers the input via WASM-spec and stores the resulting
|
|
69
|
-
* `{ tableSpec, dataQuery }` shape. Returns the lowered def
|
|
70
|
-
* alongside the pool entry.
|
|
71
|
-
*/
|
|
72
|
-
acquireFromLegacy(opts) {
|
|
73
|
-
const def = require_ptable_shared.buildFullPTableDefFromLegacy(opts);
|
|
74
|
-
return {
|
|
75
|
-
def,
|
|
76
|
-
entry: this.acquire(def)
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
};
|
|
80
|
-
//#endregion
|
|
81
|
-
exports.PTableHolder = PTableHolder;
|
|
82
|
-
exports.PTablePool = PTablePool;
|
|
83
|
-
|
|
84
|
-
//# sourceMappingURL=ptable_pool.cjs.map
|
package/dist/ptable_pool.cjs.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_pool.cjs","names":["RefCountPoolBase","stableKeyFromFullPTableDef","logPFrames","bigintReplacer","PFrameDriverError","buildFullPTableDefFromLegacy"],"sources":["../src/ptable_pool.ts"],"sourcesContent":["import {\n bigintReplacer,\n PFrameDriverError,\n type PFrameHandle,\n type PTableDef,\n type PTableHandle,\n type JsonSerializable,\n type PObjectId,\n} from \"@milaboratories/pl-model-common\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { RefCountPoolBase, type PoolEntry } from \"@milaboratories/helpers\";\nimport { logPFrames } from \"./logging\";\nimport type { PFramePool } from \"./pframe_pool\";\nimport {\n buildFullPTableDefFromLegacy,\n stableKeyFromFullPTableDef,\n type FullPTableDef,\n} from \"./ptable_shared\";\nimport type { PTableDefPool } from \"./ptable_def_pool\";\n\nexport class PTableHolder implements Disposable {\n private readonly abortController = new AbortController();\n private readonly combinedDisposeSignal: AbortSignal;\n private cacheSizePromise?: Promise<number>;\n\n constructor(\n public readonly pFrame: PFrameHandle,\n pFrameDisposeSignal: AbortSignal,\n public readonly pTablePromise: Promise<PFrameInternal.PTableV12>,\n private readonly predecessor?: PoolEntry<PTableHandle, PTableHolder>,\n ) {\n this.combinedDisposeSignal = AbortSignal.any([\n pFrameDisposeSignal,\n this.abortController.signal,\n ]);\n }\n\n public get disposeSignal(): AbortSignal {\n return this.combinedDisposeSignal;\n }\n\n public get cacheSize(): Promise<number> {\n if (this.cacheSizePromise === undefined) {\n this.cacheSizePromise = this.pTablePromise.then((pTable) =>\n pTable.getFootprint({\n withPredecessors: false,\n signal: this.abortController.signal,\n }),\n );\n }\n return this.cacheSizePromise;\n }\n\n [Symbol.dispose](): void {\n this.abortController.abort();\n this.predecessor?.unref();\n void this.pTablePromise\n .then((pTable) => pTable.dispose())\n .catch(() => {\n /* mute error */\n });\n }\n}\n\nexport class PTablePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<\n FullPTableDef,\n PTableHandle,\n PTableHolder\n> {\n constructor(\n private readonly pFrames: PFramePool<TreeEntry>,\n private readonly pTableDefs: PTableDefPool,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: FullPTableDef): PTableHandle {\n return stableKeyFromFullPTableDef(params);\n }\n\n protected createNewResource(params: FullPTableDef, key: PTableHandle): PTableHolder {\n if (logPFrames()) {\n this.logger(\n \"info\",\n `PTable creation (pTableHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,\n );\n }\n\n const { pFrameHandle } = params;\n const { pFrameDataPromise: pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);\n\n const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;\n const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));\n\n const { dataQuery } = params;\n const table = pFramePromise.then((pFrame) => pFrame.createTable(key, dataQuery));\n return new PTableHolder(pFrameHandle, combinedSignal, table);\n }\n\n public getByKey(key: PTableHandle): PTableHolder {\n const resource = super.tryGetByKey(key);\n if (!resource) {\n const error = new PFrameDriverError(`Invalid PTable handle`);\n error.cause = new Error(`PTable with handle ${key} not found`);\n throw error;\n }\n return resource;\n }\n\n /**\n * Acquire a PTable from a legacy `PTableDef` + column specs map.\n * Lowers the input via WASM-spec and stores the resulting\n * `{ tableSpec, dataQuery }` shape. Returns the lowered def\n * alongside the pool entry.\n */\n public acquireFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n }): { def: FullPTableDef; entry: PoolEntry<PTableHandle, PTableHolder> } {\n const def = buildFullPTableDefFromLegacy(opts);\n return { def, entry: this.acquire(def) };\n }\n}\n"],"mappings":";;;;;AAoBA,IAAa,eAAb,MAAgD;CAM5B;CAEA;CACC;CARnB,kBAAmC,IAAI,gBAAgB;CACvD;CACA;CAEA,YACE,QACA,qBACA,eACA,aACA;EAJgB,KAAA,SAAA;EAEA,KAAA,gBAAA;EACC,KAAA,cAAA;EAEjB,KAAK,wBAAwB,YAAY,IAAI,CAC3C,qBACA,KAAK,gBAAgB,MACvB,CAAC;CACH;CAEA,IAAW,gBAA6B;EACtC,OAAO,KAAK;CACd;CAEA,IAAW,YAA6B;EACtC,IAAI,KAAK,qBAAqB,KAAA,GAC5B,KAAK,mBAAmB,KAAK,cAAc,MAAM,WAC/C,OAAO,aAAa;GAClB,kBAAkB;GAClB,QAAQ,KAAK,gBAAgB;EAC/B,CAAC,CACH;EAEF,OAAO,KAAK;CACd;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,gBAAgB,MAAM;EAC3B,KAAK,aAAa,MAAM;EACxB,KAAU,cACP,MAAM,WAAW,OAAO,QAAQ,CAAC,CAAC,CAClC,YAAY,CAEb,CAAC;CACL;AACF;AAEA,IAAa,aAAb,cAAoEA,wBAAAA,iBAIlE;CAEmB;CACA;CACA;CAHnB,YACE,SACA,YACA,QACA;EACA,MAAM;EAJW,KAAA,UAAA;EACA,KAAA,aAAA;EACA,KAAA,SAAA;CAGnB;CAEA,mBAA6B,QAAqC;EAChE,OAAOC,sBAAAA,2BAA2B,MAAM;CAC1C;CAEA,kBAA4B,QAAuB,KAAiC;EAClF,IAAIC,gBAAAA,WAAW,GACb,KAAK,OACH,QACA,mCAAmC,IAAI,KAAU,KAAK,UAAU,QAAQC,gCAAAA,cAAc,GACxF;EAGF,MAAM,EAAE,iBAAiB;EACzB,MAAM,EAAE,mBAAmB,eAAe,kBAAkB,KAAK,QAAQ,SAAS,YAAY;EAE9F,MAAM,mBAAmB,KAAK,WAAW,YAAY,GAAG,CAAC,EAAE;EAC3D,MAAM,iBAAiB,YAAY,IAAI,CAAC,eAAe,gBAAgB,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,CAAC,CAAC;EAE3F,MAAM,EAAE,cAAc;EAEtB,OAAO,IAAI,aAAa,cAAc,gBADxB,cAAc,MAAM,WAAW,OAAO,YAAY,KAAK,SAAS,CACpB,CAAC;CAC7D;CAEA,SAAgB,KAAiC;EAC/C,MAAM,WAAW,MAAM,YAAY,GAAG;EACtC,IAAI,CAAC,UAAU;GACb,MAAM,QAAQ,IAAIC,gCAAAA,kBAAkB,uBAAuB;GAC3D,MAAM,wBAAQ,IAAI,MAAM,sBAAsB,IAAI,WAAW;GAC7D,MAAM;EACR;EACA,OAAO;CACT;;;;;;;CAQA,kBAAyB,MAIgD;EACvE,MAAM,MAAMC,sBAAAA,6BAA6B,IAAI;EAC7C,OAAO;GAAE;GAAK,OAAO,KAAK,QAAQ,GAAG;EAAE;CACzC;AACF"}
|
package/dist/ptable_shared.cjs
DELETED
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
|
|
2
|
-
let _milaboratories_pl_model_middle_layer = require("@milaboratories/pl-model-middle-layer");
|
|
3
|
-
//#region src/ptable_shared.ts
|
|
4
|
-
/**
|
|
5
|
-
* Make every inline column in a `SpecQuery` self-contained by embedding its
|
|
6
|
-
* column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the
|
|
7
|
-
* column's `PColumnSpec`.
|
|
8
|
-
*/
|
|
9
|
-
function embedInlineColumnTypeSpecs(query) {
|
|
10
|
-
return (0, _milaboratories_pl_model_common.traverseQuerySpec)(query, {
|
|
11
|
-
column: (c) => c,
|
|
12
|
-
node: (node) => {
|
|
13
|
-
if (node.type !== "inlineColumn") return node;
|
|
14
|
-
const { axesSpec, valueType } = node.spec.spec;
|
|
15
|
-
const dataInfo = {
|
|
16
|
-
...node.dataInfo,
|
|
17
|
-
typeSpec: {
|
|
18
|
-
axes: axesSpec.map((a) => a.type),
|
|
19
|
-
column: valueType
|
|
20
|
-
}
|
|
21
|
-
};
|
|
22
|
-
return {
|
|
23
|
-
...node,
|
|
24
|
-
dataInfo
|
|
25
|
-
};
|
|
26
|
-
}
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
function stableKeyFromFullPTableDef(data) {
|
|
30
|
-
return (0, _milaboratories_pl_model_middle_layer.hashJson)(data);
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Lower a legacy `PTableDef<PObjectId>` to the data layer. Returns
|
|
34
|
-
* the output `tableSpec` and the lowered `dataQuery`. Routes through
|
|
35
|
-
* `rewriteLegacyQuery` + `evaluateQuery` on the caller-supplied
|
|
36
|
-
* WASM-spec frame.
|
|
37
|
-
*/
|
|
38
|
-
function lowerLegacyPTableDef(pFrameSpec, legacyDef) {
|
|
39
|
-
const legacy = {
|
|
40
|
-
src: legacyDef.src,
|
|
41
|
-
filters: [...legacyDef.partitionFilters, ...legacyDef.filters],
|
|
42
|
-
sorting: legacyDef.sorting
|
|
43
|
-
};
|
|
44
|
-
return pFrameSpec.evaluateQuery(embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)));
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Build a `FullPTableDef` from a legacy `PTableDef` plus the
|
|
48
|
-
* PFrame's WASM-spec frame. Used by pool `acquireFromLegacy` methods
|
|
49
|
-
* to keep lowering off the call site.
|
|
50
|
-
*/
|
|
51
|
-
function buildFullPTableDefFromLegacy(opts) {
|
|
52
|
-
return {
|
|
53
|
-
pFrameHandle: opts.pFrameHandle,
|
|
54
|
-
...lowerLegacyPTableDef(opts.pFrameSpec, opts.def)
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
//#endregion
|
|
58
|
-
exports.buildFullPTableDefFromLegacy = buildFullPTableDefFromLegacy;
|
|
59
|
-
exports.embedInlineColumnTypeSpecs = embedInlineColumnTypeSpecs;
|
|
60
|
-
exports.lowerLegacyPTableDef = lowerLegacyPTableDef;
|
|
61
|
-
exports.stableKeyFromFullPTableDef = stableKeyFromFullPTableDef;
|
|
62
|
-
|
|
63
|
-
//# sourceMappingURL=ptable_shared.cjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ptable_shared.cjs","names":[],"sources":["../src/ptable_shared.ts"],"sourcesContent":["import type {\n DataQuery,\n JsonDataInfo,\n PFrameHandle,\n PObjectId,\n PTableColumnSpec,\n PTableDef,\n PTableHandle,\n SpecQuery,\n} from \"@milaboratories/pl-model-common\";\nimport { traverseQuerySpec } from \"@milaboratories/pl-model-common\";\nimport { hashJson, PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\n/**\n * Make every inline column in a `SpecQuery` self-contained by embedding its\n * column `typeSpec` (`{ axes, column }`) into `dataInfo`, derived from the\n * column's `PColumnSpec`.\n */\nexport function embedInlineColumnTypeSpecs(query: SpecQuery): SpecQuery {\n return traverseQuerySpec(query, {\n column: (c) => c,\n node: (node) => {\n if (node.type !== \"inlineColumn\") return node;\n const { axesSpec, valueType } = node.spec.spec;\n const dataInfo: JsonDataInfo & { typeSpec: PFrameInternal.PColumnValueTypeSpec } = {\n ...node.dataInfo,\n typeSpec: { axes: axesSpec.map((a) => a.type), column: valueType },\n };\n return { ...node, dataInfo };\n },\n });\n}\n\n/**\n * PTable definition stored in the def / table pools. Always\n * carries a pre-lowered data query plus the corresponding output\n * `tableSpec`. Legacy `PTableDef<PObjectId>` inputs are lowered to\n * this shape via WASM-spec at acquire time.\n */\nexport type FullPTableDef = {\n pFrameHandle: PFrameHandle;\n tableSpec: PTableColumnSpec[];\n dataQuery: DataQuery;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n return hashJson(data) as string as PTableHandle;\n}\n\n/**\n * Lower a legacy `PTableDef<PObjectId>` to the data layer. Returns\n * the output `tableSpec` and the lowered `dataQuery`. Routes through\n * `rewriteLegacyQuery` + `evaluateQuery` on the caller-supplied\n * WASM-spec frame.\n */\nexport function lowerLegacyPTableDef(\n pFrameSpec: PFrameInternal.PFrameWasmV3,\n legacyDef: PTableDef<PObjectId>,\n): { tableSpec: PTableColumnSpec[]; dataQuery: DataQuery } {\n const legacy: PFrameInternal.LegacyQuery = {\n src: legacyDef.src,\n filters: [...legacyDef.partitionFilters, ...legacyDef.filters],\n sorting: legacyDef.sorting,\n };\n return pFrameSpec.evaluateQuery(\n embedInlineColumnTypeSpecs(pFrameSpec.rewriteLegacyQuery(legacy)),\n );\n}\n\n/**\n * Build a `FullPTableDef` from a legacy `PTableDef` plus the\n * PFrame's WASM-spec frame. Used by pool `acquireFromLegacy` methods\n * to keep lowering off the call site.\n */\nexport function buildFullPTableDefFromLegacy(opts: {\n pFrameHandle: PFrameHandle;\n def: PTableDef<PObjectId>;\n pFrameSpec: PFrameInternal.PFrameWasmV3;\n}): FullPTableDef {\n return {\n pFrameHandle: opts.pFrameHandle,\n ...lowerLegacyPTableDef(opts.pFrameSpec, opts.def),\n };\n}\n"],"mappings":";;;;;;;;AAkBA,SAAgB,2BAA2B,OAA6B;CACtE,QAAA,GAAA,gCAAA,kBAAA,CAAyB,OAAO;EAC9B,SAAS,MAAM;EACf,OAAO,SAAS;GACd,IAAI,KAAK,SAAS,gBAAgB,OAAO;GACzC,MAAM,EAAE,UAAU,cAAc,KAAK,KAAK;GAC1C,MAAM,WAA6E;IACjF,GAAG,KAAK;IACR,UAAU;KAAE,MAAM,SAAS,KAAK,MAAM,EAAE,IAAI;KAAG,QAAQ;IAAU;GACnE;GACA,OAAO;IAAE,GAAG;IAAM;GAAS;EAC7B;CACF,CAAC;AACH;AAcA,SAAgB,2BAA2B,MAAmC;CAC5E,QAAA,GAAA,sCAAA,SAAA,CAAgB,IAAI;AACtB;;;;;;;AAQA,SAAgB,qBACd,YACA,WACyD;CACzD,MAAM,SAAqC;EACzC,KAAK,UAAU;EACf,SAAS,CAAC,GAAG,UAAU,kBAAkB,GAAG,UAAU,OAAO;EAC7D,SAAS,UAAU;CACrB;CACA,OAAO,WAAW,cAChB,2BAA2B,WAAW,mBAAmB,MAAM,CAAC,CAClE;AACF;;;;;;AAOA,SAAgB,6BAA6B,MAI3B;CAChB,OAAO;EACL,cAAc,KAAK;EACnB,GAAG,qBAAqB,KAAK,YAAY,KAAK,GAAG;CACnD;AACF"}
|