@milaboratories/pf-driver 1.0.61 → 1.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_virtual/_@oxc-project_runtime@0.114.0/helpers/usingCtx.cjs +59 -0
- package/dist/_virtual/_@oxc-project_runtime@0.114.0/helpers/usingCtx.js +58 -0
- package/dist/_virtual/_rolldown/runtime.cjs +29 -0
- package/dist/data_info_helpers.cjs +20 -19
- package/dist/data_info_helpers.cjs.map +1 -1
- package/dist/data_info_helpers.d.ts +6 -2
- package/dist/data_info_helpers.js +19 -17
- package/dist/data_info_helpers.js.map +1 -1
- package/dist/driver_decl.d.ts +38 -34
- package/dist/driver_double.cjs +98 -97
- package/dist/driver_double.cjs.map +1 -1
- package/dist/driver_double.d.ts +17 -10
- package/dist/driver_double.js +95 -95
- package/dist/driver_double.js.map +1 -1
- package/dist/driver_impl.cjs +233 -296
- package/dist/driver_impl.cjs.map +1 -1
- package/dist/driver_impl.d.ts +59 -52
- package/dist/driver_impl.js +232 -294
- package/dist/driver_impl.js.map +1 -1
- package/dist/index.cjs +10 -14
- package/dist/index.d.ts +5 -5
- package/dist/index.js +5 -4
- package/dist/logging.cjs +4 -3
- package/dist/logging.cjs.map +1 -1
- package/dist/logging.js +4 -2
- package/dist/logging.js.map +1 -1
- package/dist/pframe_pool.cjs +177 -196
- package/dist/pframe_pool.cjs.map +1 -1
- package/dist/pframe_pool.d.ts +12 -31
- package/dist/pframe_pool.js +177 -194
- package/dist/pframe_pool.js.map +1 -1
- package/dist/ptable_cache_per_frame.cjs +52 -63
- package/dist/ptable_cache_per_frame.cjs.map +1 -1
- package/dist/ptable_cache_per_frame.d.ts +16 -23
- package/dist/ptable_cache_per_frame.js +51 -61
- package/dist/ptable_cache_per_frame.js.map +1 -1
- package/dist/ptable_cache_plain.cjs +41 -49
- package/dist/ptable_cache_plain.cjs.map +1 -1
- package/dist/ptable_cache_plain.d.ts +15 -19
- package/dist/ptable_cache_plain.js +40 -47
- package/dist/ptable_cache_plain.js.map +1 -1
- package/dist/ptable_def_pool.cjs +45 -53
- package/dist/ptable_def_pool.cjs.map +1 -1
- package/dist/ptable_def_pool.d.ts +3 -20
- package/dist/ptable_def_pool.js +45 -51
- package/dist/ptable_def_pool.js.map +1 -1
- package/dist/ptable_pool.cjs +145 -185
- package/dist/ptable_pool.cjs.map +1 -1
- package/dist/ptable_pool.d.ts +6 -28
- package/dist/ptable_pool.js +145 -183
- package/dist/ptable_pool.js.map +1 -1
- package/dist/ptable_shared.cjs +6 -5
- package/dist/ptable_shared.cjs.map +1 -1
- package/dist/ptable_shared.d.ts +1 -17
- package/dist/ptable_shared.js +5 -3
- package/dist/ptable_shared.js.map +1 -1
- package/package.json +7 -7
- package/dist/__external/.pnpm/@rollup_plugin-typescript@12.3.0_rollup@4.52.4_tslib@2.8.1_typescript@5.6.3/__external/tslib/tslib.es6.cjs +0 -77
- package/dist/__external/.pnpm/@rollup_plugin-typescript@12.3.0_rollup@4.52.4_tslib@2.8.1_typescript@5.6.3/__external/tslib/tslib.es6.cjs.map +0 -1
- package/dist/__external/.pnpm/@rollup_plugin-typescript@12.3.0_rollup@4.52.4_tslib@2.8.1_typescript@5.6.3/__external/tslib/tslib.es6.js +0 -74
- package/dist/__external/.pnpm/@rollup_plugin-typescript@12.3.0_rollup@4.52.4_tslib@2.8.1_typescript@5.6.3/__external/tslib/tslib.es6.js.map +0 -1
- package/dist/data_info_helpers.d.ts.map +0 -1
- package/dist/driver_decl.d.ts.map +0 -1
- package/dist/driver_double.d.ts.map +0 -1
- package/dist/driver_double.test.d.ts +0 -2
- package/dist/driver_double.test.d.ts.map +0 -1
- package/dist/driver_impl.d.ts.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/logging.d.ts +0 -2
- package/dist/logging.d.ts.map +0 -1
- package/dist/pframe_pool.d.ts.map +0 -1
- package/dist/ptable_cache_per_frame.d.ts.map +0 -1
- package/dist/ptable_cache_plain.d.ts.map +0 -1
- package/dist/ptable_def_pool.d.ts.map +0 -1
- package/dist/ptable_pool.d.ts.map +0 -1
- package/dist/ptable_shared.d.ts.map +0 -1
package/dist/driver_impl.d.ts
CHANGED
|
@@ -1,55 +1,62 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
/** Concurrency limits for `getShape` and `getData` requests */
|
|
16
|
-
pTableConcurrency: number;
|
|
1
|
+
import { AbstractInternalPFrameDriver } from "./driver_decl.js";
|
|
2
|
+
import { LocalBlobProvider as LocalBlobProvider$1, RemoteBlobProvider as RemoteBlobProvider$1 } from "./pframe_pool.js";
|
|
3
|
+
import { PTableCachePerFrameOps } from "./ptable_cache_per_frame.js";
|
|
4
|
+
import { PTableCachePlainOps } from "./ptable_cache_plain.js";
|
|
5
|
+
import { CalculateTableDataRequest, CalculateTableDataResponse, FindColumnsRequest, FindColumnsResponse, JsonSerializable, PColumn, PColumnIdAndSpec, PColumnSpec, PFrameDef, PFrameHandle, PObjectId, PTableColumnSpec, PTableDef, PTableDefV2, PTableHandle, PTableShape, PTableVector, TableRange, UniqueValuesRequest, UniqueValuesResponse } from "@platforma-sdk/model";
|
|
6
|
+
import { PoolEntry } from "@milaboratories/ts-helpers";
|
|
7
|
+
import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
8
|
+
|
|
9
|
+
//#region src/driver_impl.d.ts
|
|
10
|
+
interface LocalBlobProvider<TreeEntry extends JsonSerializable> extends LocalBlobProvider$1<TreeEntry> {}
|
|
11
|
+
interface RemoteBlobProvider<TreeEntry extends JsonSerializable> extends RemoteBlobProvider$1<TreeEntry>, AsyncDisposable {}
|
|
12
|
+
type AbstractPFrameDriverOps = PTableCachePerFrameOps & PTableCachePlainOps & {
|
|
13
|
+
/** Concurrency limits for `getUniqueValues` and `calculateTableData` requests */pFrameConcurrency: number; /** Concurrency limits for `getShape` and `getData` requests */
|
|
14
|
+
pTableConcurrency: number;
|
|
17
15
|
};
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
16
|
+
declare const AbstractPFrameDriverOpsDefaults: AbstractPFrameDriverOps;
|
|
17
|
+
type DataInfoResolver<PColumnData, TreeEntry extends JsonSerializable> = (spec: PColumnSpec, data: PColumnData) => PFrameInternal.DataInfo<TreeEntry>;
|
|
18
|
+
declare class AbstractPFrameDriver<PColumnData, TreeEntry extends JsonSerializable> implements AbstractInternalPFrameDriver<PColumnData> {
|
|
19
|
+
private readonly logger;
|
|
20
|
+
private readonly localBlobProvider;
|
|
21
|
+
private readonly remoteBlobProvider;
|
|
22
|
+
private readonly resolveDataInfo;
|
|
23
|
+
private readonly pFrames;
|
|
24
|
+
private readonly pTableDefs;
|
|
25
|
+
private readonly pTables;
|
|
26
|
+
private readonly pTableCachePerFrame;
|
|
27
|
+
private readonly pTableCachePlain;
|
|
28
|
+
private readonly frameConcurrencyLimiter;
|
|
29
|
+
private readonly tableConcurrencyLimiter;
|
|
30
|
+
pprofDump(): Promise<Uint8Array>;
|
|
31
|
+
constructor({
|
|
32
|
+
logger,
|
|
33
|
+
localBlobProvider,
|
|
34
|
+
remoteBlobProvider,
|
|
35
|
+
spillPath,
|
|
36
|
+
options,
|
|
37
|
+
resolveDataInfo
|
|
38
|
+
}: {
|
|
39
|
+
logger?: PFrameInternal.Logger;
|
|
40
|
+
localBlobProvider: LocalBlobProvider<TreeEntry>;
|
|
41
|
+
remoteBlobProvider: RemoteBlobProvider<TreeEntry>;
|
|
42
|
+
spillPath?: string;
|
|
43
|
+
options?: AbstractPFrameDriverOps;
|
|
44
|
+
resolveDataInfo: DataInfoResolver<PColumnData, TreeEntry>;
|
|
45
|
+
});
|
|
46
|
+
dispose(): Promise<void>;
|
|
47
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
48
|
+
createPFrame(def: PFrameDef<PColumn<PColumnData>>): PoolEntry<PFrameHandle>;
|
|
49
|
+
createPTable(rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle>;
|
|
50
|
+
createPTableV2(def: PTableDefV2<PColumn<PColumnData>>): PoolEntry<PTableHandle>;
|
|
51
|
+
findColumns(handle: PFrameHandle, request: FindColumnsRequest): Promise<FindColumnsResponse>;
|
|
52
|
+
getColumnSpec(handle: PFrameHandle, columnId: PObjectId): Promise<PColumnSpec | null>;
|
|
53
|
+
listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]>;
|
|
54
|
+
calculateTableData(handle: PFrameHandle, request: CalculateTableDataRequest<PObjectId>, range: TableRange | undefined, signal?: AbortSignal): Promise<CalculateTableDataResponse>;
|
|
55
|
+
getUniqueValues(handle: PFrameHandle, request: UniqueValuesRequest, signal?: AbortSignal): Promise<UniqueValuesResponse>;
|
|
56
|
+
getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]>;
|
|
57
|
+
getShape(handle: PTableHandle, signal?: AbortSignal): Promise<PTableShape>;
|
|
58
|
+
getData(handle: PTableHandle, columnIndices: number[], range: TableRange | undefined, signal?: AbortSignal): Promise<PTableVector[]>;
|
|
54
59
|
}
|
|
60
|
+
//#endregion
|
|
61
|
+
export { AbstractPFrameDriver, AbstractPFrameDriverOps, AbstractPFrameDriverOpsDefaults, DataInfoResolver, LocalBlobProvider, RemoteBlobProvider };
|
|
55
62
|
//# sourceMappingURL=driver_impl.d.ts.map
|
package/dist/driver_impl.js
CHANGED
|
@@ -1,303 +1,241 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { createPFrame } from
|
|
1
|
+
import { logPFrames } from "./logging.js";
|
|
2
|
+
import { PFramePool } from "./pframe_pool.js";
|
|
3
|
+
import { PTableDefPool } from "./ptable_def_pool.js";
|
|
4
|
+
import { PTablePool } from "./ptable_pool.js";
|
|
5
|
+
import { PTableCachePerFrame, PTableCachePerFrameOpsDefaults } from "./ptable_cache_per_frame.js";
|
|
6
|
+
import { PTableCachePlain, PTableCachePlainOpsDefaults } from "./ptable_cache_plain.js";
|
|
7
|
+
import { _usingCtx } from "./_virtual/_@oxc-project_runtime@0.114.0/helpers/usingCtx.js";
|
|
8
|
+
import { ValueType, bigintReplacer, canonicalizeJson, collectSpecQueryColumns, extractAllColumns, mapPObjectData, mapPTableDef, mapSpecQueryColumns, sortPTableDef, sortSpecQuery, uniqueBy } from "@platforma-sdk/model";
|
|
9
|
+
import { ConcurrencyLimitingExecutor } from "@milaboratories/ts-helpers";
|
|
10
|
+
import { PFrameFactory } from "@milaboratories/pframes-rs-node";
|
|
11
|
+
import { tmpdir } from "node:os";
|
|
12
|
+
import { createPFrame } from "@milaboratories/pframes-rs-wasm";
|
|
13
13
|
|
|
14
|
+
//#region src/driver_impl.ts
|
|
14
15
|
const AbstractPFrameDriverOpsDefaults = {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
...PTableCachePerFrameOpsDefaults,
|
|
17
|
+
...PTableCachePlainOpsDefaults,
|
|
18
|
+
pFrameConcurrency: 1,
|
|
19
|
+
pTableConcurrency: 1
|
|
20
|
+
};
|
|
21
|
+
var AbstractPFrameDriver = class {
|
|
22
|
+
logger;
|
|
23
|
+
localBlobProvider;
|
|
24
|
+
remoteBlobProvider;
|
|
25
|
+
resolveDataInfo;
|
|
26
|
+
pFrames;
|
|
27
|
+
pTableDefs;
|
|
28
|
+
pTables;
|
|
29
|
+
pTableCachePerFrame;
|
|
30
|
+
pTableCachePlain;
|
|
31
|
+
frameConcurrencyLimiter;
|
|
32
|
+
tableConcurrencyLimiter;
|
|
33
|
+
async pprofDump() {
|
|
34
|
+
return await PFrameFactory.pprofDump();
|
|
35
|
+
}
|
|
36
|
+
constructor({ logger = () => {}, localBlobProvider, remoteBlobProvider, spillPath = tmpdir(), options = AbstractPFrameDriverOpsDefaults, resolveDataInfo }) {
|
|
37
|
+
this.logger = logger;
|
|
38
|
+
this.localBlobProvider = localBlobProvider;
|
|
39
|
+
this.remoteBlobProvider = remoteBlobProvider;
|
|
40
|
+
this.resolveDataInfo = resolveDataInfo;
|
|
41
|
+
this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);
|
|
42
|
+
this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);
|
|
43
|
+
this.pFrames = new PFramePool(this.localBlobProvider, this.remoteBlobProvider, this.logger, spillPath);
|
|
44
|
+
this.pTableDefs = new PTableDefPool(this.logger);
|
|
45
|
+
this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);
|
|
46
|
+
this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);
|
|
47
|
+
this.pTableCachePlain = new PTableCachePlain(this.logger, options);
|
|
48
|
+
}
|
|
49
|
+
async dispose() {
|
|
50
|
+
return await this.remoteBlobProvider[Symbol.asyncDispose]();
|
|
51
|
+
}
|
|
52
|
+
async [Symbol.asyncDispose]() {
|
|
53
|
+
return await this.dispose();
|
|
54
|
+
}
|
|
55
|
+
createPFrame(def) {
|
|
56
|
+
const ValueTypes = new Set(Object.values(ValueType));
|
|
57
|
+
const columns = uniqueBy(def.filter((column) => ValueTypes.has(column.spec.valueType)), (column) => column.id).map((c) => mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)));
|
|
58
|
+
return this.pFrames.acquire(columns);
|
|
59
|
+
}
|
|
60
|
+
createPTable(rawDef) {
|
|
61
|
+
const pFrameEntry = this.createPFrame(extractAllColumns(rawDef.src));
|
|
62
|
+
const sortedDef = sortPTableDef(migrateTableFilter(mapPTableDef(rawDef, (c) => c.id), this.logger));
|
|
63
|
+
const pTableEntry = this.pTableDefs.acquire({
|
|
64
|
+
type: "v1",
|
|
65
|
+
def: sortedDef,
|
|
66
|
+
pFrameHandle: pFrameEntry.key
|
|
67
|
+
});
|
|
68
|
+
if (logPFrames()) this.logger("info", `Create PTable call (pFrameHandle = ${pFrameEntry.key}; pTableHandle = ${pTableEntry.key})`);
|
|
69
|
+
const unref = () => {
|
|
70
|
+
pTableEntry.unref();
|
|
71
|
+
pFrameEntry.unref();
|
|
72
|
+
};
|
|
73
|
+
return {
|
|
74
|
+
key: pTableEntry.key,
|
|
75
|
+
resource: pTableEntry.resource,
|
|
76
|
+
unref,
|
|
77
|
+
[Symbol.dispose]: unref
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
createPTableV2(def) {
|
|
81
|
+
const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);
|
|
82
|
+
const columnsMap = columns.reduce((acc, col) => (acc[col.id] = col.spec, acc), {});
|
|
83
|
+
const pFrameEntry = this.createPFrame(columns);
|
|
84
|
+
const specFrame = createPFrame(columnsMap);
|
|
85
|
+
const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));
|
|
86
|
+
const { tableSpec, dataQuery } = specFrame.evaluateQuery(sortedQuery);
|
|
87
|
+
const pTableEntry = this.pTableDefs.acquire({
|
|
88
|
+
type: "v2",
|
|
89
|
+
pFrameHandle: pFrameEntry.key,
|
|
90
|
+
def: {
|
|
91
|
+
tableSpec,
|
|
92
|
+
dataQuery
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
if (logPFrames()) this.logger("info", `Create PTable call (pFrameHandle = ${pFrameEntry.key}; pTableHandle = ${pTableEntry.key})`);
|
|
96
|
+
const unref = () => {
|
|
97
|
+
pTableEntry.unref();
|
|
98
|
+
pFrameEntry.unref();
|
|
99
|
+
};
|
|
100
|
+
return {
|
|
101
|
+
key: pTableEntry.key,
|
|
102
|
+
resource: pTableEntry.resource,
|
|
103
|
+
unref,
|
|
104
|
+
[Symbol.dispose]: unref
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
async findColumns(handle, request) {
|
|
108
|
+
const iRequest = {
|
|
109
|
+
...request,
|
|
110
|
+
compatibleWith: request.compatibleWith.length !== 0 ? [{
|
|
111
|
+
axesSpec: [...new Map(request.compatibleWith.map((item) => [canonicalizeJson(item), item])).values()],
|
|
112
|
+
qualifications: []
|
|
113
|
+
}] : []
|
|
114
|
+
};
|
|
115
|
+
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
116
|
+
return { hits: (await (await pFramePromise).findColumns(iRequest)).hits.filter((h) => h.mappingVariants.length === 0 || h.mappingVariants.some((v) => v.qualifications.forHit.length === 0 && v.qualifications.forQueries.every((q) => q.length === 0))).map((h) => h.hit) };
|
|
117
|
+
}
|
|
118
|
+
async getColumnSpec(handle, columnId) {
|
|
119
|
+
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
120
|
+
return await (await pFramePromise).getColumnSpec(columnId);
|
|
121
|
+
}
|
|
122
|
+
async listColumns(handle) {
|
|
123
|
+
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
124
|
+
return await (await pFramePromise).listColumns();
|
|
125
|
+
}
|
|
126
|
+
async calculateTableData(handle, request, range, signal) {
|
|
127
|
+
if (logPFrames()) this.logger("info", `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
|
|
128
|
+
const table = this.pTables.acquire({
|
|
129
|
+
type: "v1",
|
|
130
|
+
pFrameHandle: handle,
|
|
131
|
+
def: sortPTableDef(migrateTableFilter(request, this.logger))
|
|
132
|
+
});
|
|
133
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
134
|
+
const pTable = await pTablePromise;
|
|
135
|
+
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
136
|
+
return await this.frameConcurrencyLimiter.run(async () => {
|
|
137
|
+
try {
|
|
138
|
+
const spec = pTable.getSpec();
|
|
139
|
+
const data = await pTable.getData([...spec.keys()], {
|
|
140
|
+
range,
|
|
141
|
+
signal: combinedSignal
|
|
142
|
+
});
|
|
143
|
+
const overallSize = await pTable.getFootprint({ signal: combinedSignal });
|
|
144
|
+
this.pTableCachePerFrame.cache(table, overallSize);
|
|
145
|
+
return spec.map((spec, i) => ({
|
|
146
|
+
spec,
|
|
147
|
+
data: data[i]
|
|
148
|
+
}));
|
|
149
|
+
} catch (err) {
|
|
150
|
+
table.unref();
|
|
151
|
+
throw err;
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
async getUniqueValues(handle, request, signal) {
|
|
156
|
+
if (logPFrames()) this.logger("info", `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
|
|
157
|
+
const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
158
|
+
const pFrame = await pFramePromise;
|
|
159
|
+
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
160
|
+
return await this.frameConcurrencyLimiter.run(async () => {
|
|
161
|
+
return await pFrame.getUniqueValues({
|
|
162
|
+
...request,
|
|
163
|
+
filters: migrateFilters(request.filters, this.logger)
|
|
164
|
+
}, { signal: combinedSignal });
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
async getSpec(handle) {
|
|
168
|
+
try {
|
|
169
|
+
var _usingCtx$1 = _usingCtx();
|
|
170
|
+
const { def } = this.pTableDefs.getByKey(handle);
|
|
171
|
+
const { pTablePromise } = _usingCtx$1.u(this.pTables.acquire(def)).resource;
|
|
172
|
+
return (await pTablePromise).getSpec();
|
|
173
|
+
} catch (_) {
|
|
174
|
+
_usingCtx$1.e = _;
|
|
175
|
+
} finally {
|
|
176
|
+
_usingCtx$1.d();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
async getShape(handle, signal) {
|
|
180
|
+
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
181
|
+
const table = this.pTables.acquire(def);
|
|
182
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
183
|
+
const pTable = await pTablePromise;
|
|
184
|
+
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
185
|
+
const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
186
|
+
return {
|
|
187
|
+
shape: await pTable.getShape({ signal: combinedSignal }),
|
|
188
|
+
overallSize: await pTable.getFootprint({ signal: combinedSignal })
|
|
189
|
+
};
|
|
190
|
+
});
|
|
191
|
+
this.pTableCachePlain.cache(table, overallSize, defDisposeSignal);
|
|
192
|
+
return shape;
|
|
193
|
+
}
|
|
194
|
+
async getData(handle, columnIndices, range, signal) {
|
|
195
|
+
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
196
|
+
const table = this.pTables.acquire(def);
|
|
197
|
+
const { pTablePromise, disposeSignal } = table.resource;
|
|
198
|
+
const pTable = await pTablePromise;
|
|
199
|
+
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
200
|
+
const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
201
|
+
return {
|
|
202
|
+
data: await pTable.getData(columnIndices, {
|
|
203
|
+
range,
|
|
204
|
+
signal: combinedSignal
|
|
205
|
+
}),
|
|
206
|
+
overallSize: await pTable.getFootprint({ signal: combinedSignal })
|
|
207
|
+
};
|
|
208
|
+
});
|
|
209
|
+
this.pTableCachePlain.cache(table, overallSize, defDisposeSignal);
|
|
210
|
+
return data;
|
|
211
|
+
}
|
|
19
212
|
};
|
|
20
|
-
class AbstractPFrameDriver {
|
|
21
|
-
logger;
|
|
22
|
-
localBlobProvider;
|
|
23
|
-
remoteBlobProvider;
|
|
24
|
-
resolveDataInfo;
|
|
25
|
-
pFrames;
|
|
26
|
-
pTableDefs;
|
|
27
|
-
pTables;
|
|
28
|
-
pTableCachePerFrame;
|
|
29
|
-
pTableCachePlain;
|
|
30
|
-
frameConcurrencyLimiter;
|
|
31
|
-
tableConcurrencyLimiter;
|
|
32
|
-
async pprofDump() {
|
|
33
|
-
return await PFrameFactory.pprofDump();
|
|
34
|
-
}
|
|
35
|
-
constructor({ logger = () => { }, localBlobProvider, remoteBlobProvider, spillPath = tmpdir(), options = AbstractPFrameDriverOpsDefaults, resolveDataInfo, }) {
|
|
36
|
-
this.logger = logger;
|
|
37
|
-
this.localBlobProvider = localBlobProvider;
|
|
38
|
-
this.remoteBlobProvider = remoteBlobProvider;
|
|
39
|
-
this.resolveDataInfo = resolveDataInfo;
|
|
40
|
-
this.frameConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pFrameConcurrency);
|
|
41
|
-
this.tableConcurrencyLimiter = new ConcurrencyLimitingExecutor(options.pTableConcurrency);
|
|
42
|
-
this.pFrames = new PFramePool(this.localBlobProvider, this.remoteBlobProvider, this.logger, spillPath);
|
|
43
|
-
this.pTableDefs = new PTableDefPool(this.logger);
|
|
44
|
-
this.pTables = new PTablePool(this.pFrames, this.pTableDefs, this.logger);
|
|
45
|
-
this.pTableCachePerFrame = new PTableCachePerFrame(this.logger, options);
|
|
46
|
-
this.pTableCachePlain = new PTableCachePlain(this.logger, options);
|
|
47
|
-
}
|
|
48
|
-
async dispose() {
|
|
49
|
-
return await this.remoteBlobProvider[Symbol.asyncDispose]();
|
|
50
|
-
}
|
|
51
|
-
async [Symbol.asyncDispose]() {
|
|
52
|
-
return await this.dispose();
|
|
53
|
-
}
|
|
54
|
-
//
|
|
55
|
-
// Internal / Config API Methods
|
|
56
|
-
//
|
|
57
|
-
createPFrame(def) {
|
|
58
|
-
const ValueTypes = new Set(Object.values(ValueType));
|
|
59
|
-
const supportedColumns = def.filter((column) => ValueTypes.has(column.spec.valueType));
|
|
60
|
-
const uniqueColumns = uniqueBy(supportedColumns, (column) => column.id);
|
|
61
|
-
const columns = uniqueColumns.map((c) => mapPObjectData(c, (d) => this.resolveDataInfo(c.spec, d)));
|
|
62
|
-
return this.pFrames.acquire(columns);
|
|
63
|
-
}
|
|
64
|
-
createPTable(rawDef) {
|
|
65
|
-
const pFrameEntry = this.createPFrame(extractAllColumns(rawDef.src));
|
|
66
|
-
const sortedDef = sortPTableDef(migrateTableFilter(mapPTableDef(rawDef, (c) => c.id), this.logger));
|
|
67
|
-
const pTableEntry = this.pTableDefs.acquire({
|
|
68
|
-
type: "v1",
|
|
69
|
-
def: sortedDef,
|
|
70
|
-
pFrameHandle: pFrameEntry.key,
|
|
71
|
-
});
|
|
72
|
-
if (logPFrames()) {
|
|
73
|
-
this.logger("info", `Create PTable call (pFrameHandle = ${pFrameEntry.key}; pTableHandle = ${pTableEntry.key})`);
|
|
74
|
-
}
|
|
75
|
-
const unref = () => {
|
|
76
|
-
pTableEntry.unref();
|
|
77
|
-
pFrameEntry.unref();
|
|
78
|
-
};
|
|
79
|
-
return {
|
|
80
|
-
key: pTableEntry.key,
|
|
81
|
-
resource: pTableEntry.resource,
|
|
82
|
-
unref,
|
|
83
|
-
[Symbol.dispose]: unref,
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
createPTableV2(def) {
|
|
87
|
-
const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id);
|
|
88
|
-
const columnsMap = columns.reduce((acc, col) => ((acc[col.id] = col.spec), acc), {});
|
|
89
|
-
const pFrameEntry = this.createPFrame(columns);
|
|
90
|
-
const specFrame = createPFrame(columnsMap);
|
|
91
|
-
const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id));
|
|
92
|
-
const { tableSpec, dataQuery } = specFrame.evaluateQuery(sortedQuery);
|
|
93
|
-
const pTableEntry = this.pTableDefs.acquire({
|
|
94
|
-
type: "v2",
|
|
95
|
-
pFrameHandle: pFrameEntry.key,
|
|
96
|
-
def: {
|
|
97
|
-
tableSpec,
|
|
98
|
-
dataQuery,
|
|
99
|
-
},
|
|
100
|
-
});
|
|
101
|
-
if (logPFrames()) {
|
|
102
|
-
this.logger("info", `Create PTable call (pFrameHandle = ${pFrameEntry.key}; pTableHandle = ${pTableEntry.key})`);
|
|
103
|
-
}
|
|
104
|
-
const unref = () => {
|
|
105
|
-
pTableEntry.unref();
|
|
106
|
-
pFrameEntry.unref();
|
|
107
|
-
};
|
|
108
|
-
return {
|
|
109
|
-
key: pTableEntry.key,
|
|
110
|
-
resource: pTableEntry.resource,
|
|
111
|
-
unref,
|
|
112
|
-
[Symbol.dispose]: unref,
|
|
113
|
-
};
|
|
114
|
-
}
|
|
115
|
-
//
|
|
116
|
-
// PFrame instance methods
|
|
117
|
-
//
|
|
118
|
-
async findColumns(handle, request) {
|
|
119
|
-
const iRequest = {
|
|
120
|
-
...request,
|
|
121
|
-
compatibleWith: request.compatibleWith.length !== 0
|
|
122
|
-
? [
|
|
123
|
-
{
|
|
124
|
-
axesSpec: [
|
|
125
|
-
...new Map(request.compatibleWith.map((item) => [canonicalizeJson(item), item])).values(),
|
|
126
|
-
],
|
|
127
|
-
qualifications: [],
|
|
128
|
-
},
|
|
129
|
-
]
|
|
130
|
-
: [],
|
|
131
|
-
};
|
|
132
|
-
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
133
|
-
const pFrame = await pFramePromise;
|
|
134
|
-
const response = await pFrame.findColumns(iRequest);
|
|
135
|
-
return {
|
|
136
|
-
hits: response.hits
|
|
137
|
-
.filter((h) =>
|
|
138
|
-
// only exactly matching columns
|
|
139
|
-
h.mappingVariants.length === 0 ||
|
|
140
|
-
h.mappingVariants.some((v) => v.qualifications.forHit.length === 0 &&
|
|
141
|
-
v.qualifications.forQueries.every((q) => q.length === 0)))
|
|
142
|
-
.map((h) => h.hit),
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
async getColumnSpec(handle, columnId) {
|
|
146
|
-
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
147
|
-
const pFrame = await pFramePromise;
|
|
148
|
-
return await pFrame.getColumnSpec(columnId);
|
|
149
|
-
}
|
|
150
|
-
async listColumns(handle) {
|
|
151
|
-
const { pFramePromise } = this.pFrames.getByKey(handle);
|
|
152
|
-
const pFrame = await pFramePromise;
|
|
153
|
-
return await pFrame.listColumns();
|
|
154
|
-
}
|
|
155
|
-
async calculateTableData(handle, request, range, signal) {
|
|
156
|
-
if (logPFrames()) {
|
|
157
|
-
this.logger("info", `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
|
|
158
|
-
}
|
|
159
|
-
const table = this.pTables.acquire({
|
|
160
|
-
type: "v1",
|
|
161
|
-
pFrameHandle: handle,
|
|
162
|
-
def: sortPTableDef(migrateTableFilter(request, this.logger)),
|
|
163
|
-
});
|
|
164
|
-
const { pTablePromise, disposeSignal } = table.resource;
|
|
165
|
-
const pTable = await pTablePromise;
|
|
166
|
-
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
167
|
-
return await this.frameConcurrencyLimiter.run(async () => {
|
|
168
|
-
try {
|
|
169
|
-
// TODO: throw error when more then 150k rows is requested
|
|
170
|
-
// after pf-plots migration to stream API
|
|
171
|
-
const spec = pTable.getSpec();
|
|
172
|
-
const data = await pTable.getData([...spec.keys()], {
|
|
173
|
-
range,
|
|
174
|
-
signal: combinedSignal,
|
|
175
|
-
});
|
|
176
|
-
const overallSize = await pTable.getFootprint({
|
|
177
|
-
signal: combinedSignal,
|
|
178
|
-
});
|
|
179
|
-
this.pTableCachePerFrame.cache(table, overallSize);
|
|
180
|
-
return spec.map((spec, i) => ({
|
|
181
|
-
spec: spec,
|
|
182
|
-
data: data[i],
|
|
183
|
-
}));
|
|
184
|
-
}
|
|
185
|
-
catch (err) {
|
|
186
|
-
table.unref();
|
|
187
|
-
throw err;
|
|
188
|
-
}
|
|
189
|
-
});
|
|
190
|
-
}
|
|
191
|
-
async getUniqueValues(handle, request, signal) {
|
|
192
|
-
if (logPFrames()) {
|
|
193
|
-
this.logger("info", `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
|
|
194
|
-
}
|
|
195
|
-
const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
|
|
196
|
-
const pFrame = await pFramePromise;
|
|
197
|
-
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
198
|
-
return await this.frameConcurrencyLimiter.run(async () => {
|
|
199
|
-
return await pFrame.getUniqueValues({
|
|
200
|
-
...request,
|
|
201
|
-
filters: migrateFilters(request.filters, this.logger),
|
|
202
|
-
}, {
|
|
203
|
-
signal: combinedSignal,
|
|
204
|
-
});
|
|
205
|
-
});
|
|
206
|
-
}
|
|
207
|
-
//
|
|
208
|
-
// PTable instance methods
|
|
209
|
-
//
|
|
210
|
-
async getSpec(handle) {
|
|
211
|
-
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
212
|
-
try {
|
|
213
|
-
const { def } = this.pTableDefs.getByKey(handle);
|
|
214
|
-
const table = __addDisposableResource(env_1, this.pTables.acquire(def), false);
|
|
215
|
-
const { pTablePromise } = table.resource;
|
|
216
|
-
const pTable = await pTablePromise;
|
|
217
|
-
return pTable.getSpec();
|
|
218
|
-
}
|
|
219
|
-
catch (e_1) {
|
|
220
|
-
env_1.error = e_1;
|
|
221
|
-
env_1.hasError = true;
|
|
222
|
-
}
|
|
223
|
-
finally {
|
|
224
|
-
__disposeResources(env_1);
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
async getShape(handle, signal) {
|
|
228
|
-
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
229
|
-
const table = this.pTables.acquire(def);
|
|
230
|
-
const { pTablePromise, disposeSignal } = table.resource;
|
|
231
|
-
const pTable = await pTablePromise;
|
|
232
|
-
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
233
|
-
const { shape, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
234
|
-
const shape = await pTable.getShape({
|
|
235
|
-
signal: combinedSignal,
|
|
236
|
-
});
|
|
237
|
-
const overallSize = await pTable.getFootprint({
|
|
238
|
-
signal: combinedSignal,
|
|
239
|
-
});
|
|
240
|
-
return { shape, overallSize };
|
|
241
|
-
});
|
|
242
|
-
this.pTableCachePlain.cache(table, overallSize, defDisposeSignal);
|
|
243
|
-
return shape;
|
|
244
|
-
}
|
|
245
|
-
async getData(handle, columnIndices, range, signal) {
|
|
246
|
-
const { def, disposeSignal: defDisposeSignal } = this.pTableDefs.getByKey(handle);
|
|
247
|
-
const table = this.pTables.acquire(def);
|
|
248
|
-
const { pTablePromise, disposeSignal } = table.resource;
|
|
249
|
-
const pTable = await pTablePromise;
|
|
250
|
-
const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
|
|
251
|
-
const { data, overallSize } = await this.tableConcurrencyLimiter.run(async () => {
|
|
252
|
-
const data = await pTable.getData(columnIndices, {
|
|
253
|
-
range,
|
|
254
|
-
signal: combinedSignal,
|
|
255
|
-
});
|
|
256
|
-
const overallSize = await pTable.getFootprint({
|
|
257
|
-
signal: combinedSignal,
|
|
258
|
-
});
|
|
259
|
-
return { data, overallSize };
|
|
260
|
-
});
|
|
261
|
-
this.pTableCachePlain.cache(table, overallSize, defDisposeSignal);
|
|
262
|
-
return data;
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
213
|
function migrateFilters(filters, logger) {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
filtersV2.push(filter);
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
if (filtersV1.length > 0) {
|
|
281
|
-
const filtersV1Json = JSON.stringify(filtersV1);
|
|
282
|
-
logger("warn", `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
|
|
283
|
-
}
|
|
284
|
-
return filtersV2;
|
|
214
|
+
const filtersV1 = [];
|
|
215
|
+
const filtersV2 = [];
|
|
216
|
+
for (const filter of filters) if (filter.type === "bySingleColumn") {
|
|
217
|
+
filtersV1.push(filter);
|
|
218
|
+
filtersV2.push({
|
|
219
|
+
...filter,
|
|
220
|
+
type: "bySingleColumnV2"
|
|
221
|
+
});
|
|
222
|
+
} else filtersV2.push(filter);
|
|
223
|
+
if (filtersV1.length > 0) logger("warn", `type overwritten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${JSON.stringify(filtersV1)}`);
|
|
224
|
+
return filtersV2;
|
|
285
225
|
}
|
|
286
226
|
function migrateTableFilter(def, logger) {
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
partitionFilters: migrateFilters(def.partitionFilters, logger),
|
|
298
|
-
filters: migrateFilters(def.filters, logger),
|
|
299
|
-
};
|
|
227
|
+
if (!("partitionFilters" in def)) return {
|
|
228
|
+
...def,
|
|
229
|
+
partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === "axis"), logger),
|
|
230
|
+
filters: migrateFilters(def.filters.filter((f) => f.column.type === "column"), logger)
|
|
231
|
+
};
|
|
232
|
+
return {
|
|
233
|
+
...def,
|
|
234
|
+
partitionFilters: migrateFilters(def.partitionFilters, logger),
|
|
235
|
+
filters: migrateFilters(def.filters, logger)
|
|
236
|
+
};
|
|
300
237
|
}
|
|
301
238
|
|
|
239
|
+
//#endregion
|
|
302
240
|
export { AbstractPFrameDriver, AbstractPFrameDriverOpsDefaults };
|
|
303
|
-
//# sourceMappingURL=driver_impl.js.map
|
|
241
|
+
//# sourceMappingURL=driver_impl.js.map
|