@milaboratories/pf-driver 1.0.39 → 1.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/data_info_helpers.cjs +1 -1
- package/dist/data_info_helpers.cjs.map +1 -1
- package/dist/data_info_helpers.d.ts +1 -1
- package/dist/data_info_helpers.d.ts.map +1 -1
- package/dist/data_info_helpers.js +1 -1
- package/dist/data_info_helpers.js.map +1 -1
- package/dist/driver_decl.d.ts +2 -2
- package/dist/driver_decl.d.ts.map +1 -1
- package/dist/driver_double.cjs +1 -1
- package/dist/driver_double.cjs.map +1 -1
- package/dist/driver_double.d.ts +6 -6
- package/dist/driver_double.d.ts.map +1 -1
- package/dist/driver_double.js +1 -1
- package/dist/driver_double.js.map +1 -1
- package/dist/driver_impl.cjs +35 -32
- package/dist/driver_impl.cjs.map +1 -1
- package/dist/driver_impl.d.ts +7 -7
- package/dist/driver_impl.d.ts.map +1 -1
- package/dist/driver_impl.js +35 -32
- package/dist/driver_impl.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/pframe_pool.cjs +21 -19
- package/dist/pframe_pool.cjs.map +1 -1
- package/dist/pframe_pool.d.ts +4 -4
- package/dist/pframe_pool.d.ts.map +1 -1
- package/dist/pframe_pool.js +21 -19
- package/dist/pframe_pool.js.map +1 -1
- package/dist/ptable_cache_per_frame.cjs +6 -6
- package/dist/ptable_cache_per_frame.cjs.map +1 -1
- package/dist/ptable_cache_per_frame.d.ts +4 -4
- package/dist/ptable_cache_per_frame.d.ts.map +1 -1
- package/dist/ptable_cache_per_frame.js +6 -6
- package/dist/ptable_cache_per_frame.js.map +1 -1
- package/dist/ptable_cache_plain.cjs +5 -5
- package/dist/ptable_cache_plain.cjs.map +1 -1
- package/dist/ptable_cache_plain.d.ts +4 -4
- package/dist/ptable_cache_plain.d.ts.map +1 -1
- package/dist/ptable_cache_plain.js +5 -5
- package/dist/ptable_cache_plain.js.map +1 -1
- package/dist/ptable_def_pool.cjs +2 -2
- package/dist/ptable_def_pool.cjs.map +1 -1
- package/dist/ptable_def_pool.d.ts +4 -4
- package/dist/ptable_def_pool.js +2 -2
- package/dist/ptable_def_pool.js.map +1 -1
- package/dist/ptable_pool.cjs +30 -26
- package/dist/ptable_pool.cjs.map +1 -1
- package/dist/ptable_pool.d.ts +6 -6
- package/dist/ptable_pool.d.ts.map +1 -1
- package/dist/ptable_pool.js +30 -26
- package/dist/ptable_pool.js.map +1 -1
- package/dist/ptable_shared.cjs.map +1 -1
- package/dist/ptable_shared.d.ts +1 -1
- package/dist/ptable_shared.js.map +1 -1
- package/package.json +29 -29
- package/src/data_info_helpers.ts +6 -7
- package/src/driver_decl.ts +8 -16
- package/src/driver_double.test.ts +89 -83
- package/src/driver_double.ts +23 -19
- package/src/driver_impl.ts +115 -80
- package/src/index.ts +4 -4
- package/src/pframe_pool.ts +42 -37
- package/src/ptable_cache_per_frame.ts +16 -13
- package/src/ptable_cache_plain.ts +16 -12
- package/src/ptable_def_pool.ts +7 -7
- package/src/ptable_pool.ts +62 -45
- package/src/ptable_shared.ts +2 -2
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type { PTableHandle } from
|
|
2
|
-
import type { PFrameInternal } from
|
|
3
|
-
import type { PoolEntry } from
|
|
4
|
-
import { LRUCache } from
|
|
5
|
-
import { logPFrames } from
|
|
6
|
-
import type { PTableHolder } from
|
|
1
|
+
import type { PTableHandle } from "@platforma-sdk/model";
|
|
2
|
+
import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
3
|
+
import type { PoolEntry } from "@milaboratories/ts-helpers";
|
|
4
|
+
import { LRUCache } from "lru-cache";
|
|
5
|
+
import { logPFrames } from "./logging";
|
|
6
|
+
import type { PTableHolder } from "./ptable_pool";
|
|
7
7
|
|
|
8
8
|
export type PTableCachePlainOps = {
|
|
9
9
|
/**
|
|
@@ -31,16 +31,20 @@ export class PTableCachePlain {
|
|
|
31
31
|
dispose: (resource, key, reason) => {
|
|
32
32
|
resource.unref();
|
|
33
33
|
if (logPFrames()) {
|
|
34
|
-
logger(
|
|
34
|
+
logger("info", `createPTable cache - removed PTable ${key} (reason: ${reason})`);
|
|
35
35
|
}
|
|
36
36
|
},
|
|
37
37
|
});
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
public cache(
|
|
40
|
+
public cache(
|
|
41
|
+
resource: PoolEntry<PTableHandle, PTableHolder>,
|
|
42
|
+
size: number,
|
|
43
|
+
defDisposeSignal: AbortSignal,
|
|
44
|
+
): void {
|
|
41
45
|
const key = resource.key;
|
|
42
46
|
if (logPFrames()) {
|
|
43
|
-
this.logger(
|
|
47
|
+
this.logger("info", `createPTable cache - added PTable ${key} with size ${size}`);
|
|
44
48
|
}
|
|
45
49
|
|
|
46
50
|
const status: LRUCache.Status<PoolEntry<PTableHandle, PTableHolder>> = {};
|
|
@@ -49,7 +53,7 @@ export class PTableCachePlain {
|
|
|
49
53
|
if (status.maxEntrySizeExceeded) {
|
|
50
54
|
resource.unref();
|
|
51
55
|
if (logPFrames()) {
|
|
52
|
-
this.logger(
|
|
56
|
+
this.logger("info", `createPTable cache - removed PTable ${key} (maxEntrySizeExceeded)`);
|
|
53
57
|
}
|
|
54
58
|
} else {
|
|
55
59
|
if (!this.disposeListeners.has(key)) {
|
|
@@ -57,10 +61,10 @@ export class PTableCachePlain {
|
|
|
57
61
|
this.global.delete(key);
|
|
58
62
|
|
|
59
63
|
this.disposeListeners.delete(key);
|
|
60
|
-
defDisposeSignal.removeEventListener(
|
|
64
|
+
defDisposeSignal.removeEventListener("abort", disposeListener);
|
|
61
65
|
};
|
|
62
66
|
this.disposeListeners.add(key);
|
|
63
|
-
defDisposeSignal.addEventListener(
|
|
67
|
+
defDisposeSignal.addEventListener("abort", disposeListener);
|
|
64
68
|
}
|
|
65
69
|
}
|
|
66
70
|
}
|
package/src/ptable_def_pool.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { PFrameDriverError, type PTableHandle } from
|
|
2
|
-
import type { PFrameInternal } from
|
|
3
|
-
import { RefCountPoolBase } from
|
|
4
|
-
import { logPFrames } from
|
|
5
|
-
import { stableKeyFromFullPTableDef, type FullPTableDef } from
|
|
1
|
+
import { PFrameDriverError, type PTableHandle } from "@platforma-sdk/model";
|
|
2
|
+
import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
3
|
+
import { RefCountPoolBase } from "@milaboratories/ts-helpers";
|
|
4
|
+
import { logPFrames } from "./logging";
|
|
5
|
+
import { stableKeyFromFullPTableDef, type FullPTableDef } from "./ptable_shared";
|
|
6
6
|
|
|
7
7
|
export class PTableDefHolder implements Disposable {
|
|
8
8
|
private readonly abortController = new AbortController();
|
|
@@ -13,7 +13,7 @@ export class PTableDefHolder implements Disposable {
|
|
|
13
13
|
private readonly logger: PFrameInternal.Logger,
|
|
14
14
|
) {
|
|
15
15
|
if (logPFrames()) {
|
|
16
|
-
this.logger(
|
|
16
|
+
this.logger("info", `PTable definition saved (pTableHandle = ${this.pTableHandle})`);
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -24,7 +24,7 @@ export class PTableDefHolder implements Disposable {
|
|
|
24
24
|
[Symbol.dispose](): void {
|
|
25
25
|
this.abortController.abort();
|
|
26
26
|
if (logPFrames()) {
|
|
27
|
-
this.logger(
|
|
27
|
+
this.logger("info", `PTable definition disposed (pTableHandle = ${this.pTableHandle})`);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
}
|
package/src/ptable_pool.ts
CHANGED
|
@@ -8,13 +8,13 @@ import {
|
|
|
8
8
|
type JsonSerializable,
|
|
9
9
|
type PColumnValue,
|
|
10
10
|
type PObjectId,
|
|
11
|
-
} from
|
|
12
|
-
import type { PFrameInternal } from
|
|
13
|
-
import { RefCountPoolBase, type PoolEntry } from
|
|
14
|
-
import { logPFrames } from
|
|
15
|
-
import type { PFramePool } from
|
|
16
|
-
import { stableKeyFromFullPTableDef, type FullPTableDef } from
|
|
17
|
-
import type { PTableDefPool } from
|
|
11
|
+
} from "@platforma-sdk/model";
|
|
12
|
+
import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
13
|
+
import { RefCountPoolBase, type PoolEntry } from "@milaboratories/ts-helpers";
|
|
14
|
+
import { logPFrames } from "./logging";
|
|
15
|
+
import type { PFramePool } from "./pframe_pool";
|
|
16
|
+
import { stableKeyFromFullPTableDef, type FullPTableDef } from "./ptable_shared";
|
|
17
|
+
import type { PTableDefPool } from "./ptable_def_pool";
|
|
18
18
|
|
|
19
19
|
export class PTableHolder implements Disposable {
|
|
20
20
|
private readonly abortController = new AbortController();
|
|
@@ -26,7 +26,10 @@ export class PTableHolder implements Disposable {
|
|
|
26
26
|
public readonly pTablePromise: Promise<PFrameInternal.PTableV8>,
|
|
27
27
|
private readonly predecessor?: PoolEntry<PTableHandle, PTableHolder>,
|
|
28
28
|
) {
|
|
29
|
-
this.combinedDisposeSignal = AbortSignal.any([
|
|
29
|
+
this.combinedDisposeSignal = AbortSignal.any([
|
|
30
|
+
pFrameDisposeSignal,
|
|
31
|
+
this.abortController.signal,
|
|
32
|
+
]);
|
|
30
33
|
}
|
|
31
34
|
|
|
32
35
|
public get disposeSignal(): AbortSignal {
|
|
@@ -38,12 +41,17 @@ export class PTableHolder implements Disposable {
|
|
|
38
41
|
this.predecessor?.unref();
|
|
39
42
|
void this.pTablePromise
|
|
40
43
|
.then((pTable) => pTable.dispose())
|
|
41
|
-
.catch(() => {
|
|
44
|
+
.catch(() => {
|
|
45
|
+
/* mute error */
|
|
46
|
+
});
|
|
42
47
|
}
|
|
43
48
|
}
|
|
44
49
|
|
|
45
|
-
export class PTablePool<TreeEntry extends JsonSerializable>
|
|
46
|
-
|
|
50
|
+
export class PTablePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<
|
|
51
|
+
FullPTableDef,
|
|
52
|
+
PTableHandle,
|
|
53
|
+
PTableHolder
|
|
54
|
+
> {
|
|
47
55
|
constructor(
|
|
48
56
|
private readonly pFrames: PFramePool<TreeEntry>,
|
|
49
57
|
private readonly pTableDefs: PTableDefPool,
|
|
@@ -58,9 +66,9 @@ export class PTablePool<TreeEntry extends JsonSerializable>
|
|
|
58
66
|
|
|
59
67
|
protected createNewResource(params: FullPTableDef, key: PTableHandle): PTableHolder {
|
|
60
68
|
if (logPFrames()) {
|
|
61
|
-
this.logger(
|
|
62
|
-
|
|
63
|
-
+ `${JSON.stringify(params, bigintReplacer)}`,
|
|
69
|
+
this.logger(
|
|
70
|
+
"info",
|
|
71
|
+
`PTable creation (pTableHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,
|
|
64
72
|
);
|
|
65
73
|
}
|
|
66
74
|
|
|
@@ -79,7 +87,9 @@ export class PTablePool<TreeEntry extends JsonSerializable>
|
|
|
79
87
|
sorting: [],
|
|
80
88
|
},
|
|
81
89
|
});
|
|
82
|
-
const {
|
|
90
|
+
const {
|
|
91
|
+
resource: { pTablePromise },
|
|
92
|
+
} = predecessor;
|
|
83
93
|
const sortedTable = pTablePromise.then((pTable) => pTable.sort(key, params.def.sorting));
|
|
84
94
|
return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
|
|
85
95
|
}
|
|
@@ -93,17 +103,21 @@ export class PTablePool<TreeEntry extends JsonSerializable>
|
|
|
93
103
|
filters: [],
|
|
94
104
|
},
|
|
95
105
|
});
|
|
96
|
-
const {
|
|
106
|
+
const {
|
|
107
|
+
resource: { pTablePromise },
|
|
108
|
+
} = predecessor;
|
|
97
109
|
const filteredTable = pTablePromise.then((pTable) => pTable.filter(key, params.def.filters));
|
|
98
110
|
return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
|
|
99
111
|
}
|
|
100
112
|
|
|
101
113
|
// 1. Join
|
|
102
|
-
const table = pFramePromise.then((pFrame) =>
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
114
|
+
const table = pFramePromise.then((pFrame) =>
|
|
115
|
+
pFrame.createTable(key, {
|
|
116
|
+
src: joinEntryToInternal(params.def.src),
|
|
117
|
+
// `params.def.filters` would be non-empty only when join has artificial columns
|
|
118
|
+
filters: [...params.def.partitionFilters, ...params.def.filters],
|
|
119
|
+
}),
|
|
120
|
+
);
|
|
107
121
|
return new PTableHolder(handle, combinedSignal, table);
|
|
108
122
|
}
|
|
109
123
|
|
|
@@ -120,16 +134,16 @@ export class PTablePool<TreeEntry extends JsonSerializable>
|
|
|
120
134
|
|
|
121
135
|
function hasArtificialColumns<T>(entry: JoinEntry<T>): boolean {
|
|
122
136
|
switch (entry.type) {
|
|
123
|
-
case
|
|
124
|
-
case
|
|
125
|
-
case
|
|
137
|
+
case "column":
|
|
138
|
+
case "slicedColumn":
|
|
139
|
+
case "inlineColumn":
|
|
126
140
|
return false;
|
|
127
|
-
case
|
|
141
|
+
case "artificialColumn":
|
|
128
142
|
return true;
|
|
129
|
-
case
|
|
130
|
-
case
|
|
143
|
+
case "full":
|
|
144
|
+
case "inner":
|
|
131
145
|
return entry.entries.some(hasArtificialColumns);
|
|
132
|
-
case
|
|
146
|
+
case "outer":
|
|
133
147
|
return hasArtificialColumns(entry.primary) || entry.secondary.some(hasArtificialColumns);
|
|
134
148
|
default:
|
|
135
149
|
assertNever(entry);
|
|
@@ -139,48 +153,51 @@ function hasArtificialColumns<T>(entry: JoinEntry<T>): boolean {
|
|
|
139
153
|
function joinEntryToInternal(entry: JoinEntry<PObjectId>): PFrameInternal.JoinEntryV4 {
|
|
140
154
|
const type = entry.type;
|
|
141
155
|
switch (type) {
|
|
142
|
-
case
|
|
156
|
+
case "column":
|
|
143
157
|
return {
|
|
144
|
-
type:
|
|
158
|
+
type: "column",
|
|
145
159
|
columnId: entry.column,
|
|
146
160
|
};
|
|
147
|
-
case
|
|
161
|
+
case "slicedColumn":
|
|
148
162
|
return {
|
|
149
|
-
type:
|
|
163
|
+
type: "slicedColumn",
|
|
150
164
|
columnId: entry.column,
|
|
151
165
|
newId: entry.newId,
|
|
152
166
|
axisFilters: entry.axisFilters,
|
|
153
167
|
};
|
|
154
|
-
case
|
|
168
|
+
case "artificialColumn":
|
|
155
169
|
return {
|
|
156
|
-
type:
|
|
170
|
+
type: "artificialColumn",
|
|
157
171
|
columnId: entry.column,
|
|
158
172
|
newId: entry.newId,
|
|
159
173
|
axesIndices: entry.axesIndices,
|
|
160
174
|
};
|
|
161
|
-
case
|
|
175
|
+
case "inlineColumn":
|
|
162
176
|
return {
|
|
163
|
-
type:
|
|
177
|
+
type: "inlineColumn",
|
|
164
178
|
newId: entry.column.id,
|
|
165
179
|
spec: entry.column.spec,
|
|
166
180
|
dataInfo: {
|
|
167
|
-
type:
|
|
181
|
+
type: "Json",
|
|
168
182
|
keyLength: entry.column.spec.axesSpec.length,
|
|
169
|
-
data: entry.column.data.reduce(
|
|
170
|
-
acc
|
|
171
|
-
|
|
172
|
-
|
|
183
|
+
data: entry.column.data.reduce(
|
|
184
|
+
(acc, row) => {
|
|
185
|
+
acc[JSON.stringify(row.key)] = row.val;
|
|
186
|
+
return acc;
|
|
187
|
+
},
|
|
188
|
+
{} as Record<string, PColumnValue>,
|
|
189
|
+
),
|
|
173
190
|
},
|
|
174
191
|
};
|
|
175
|
-
case
|
|
176
|
-
case
|
|
192
|
+
case "inner":
|
|
193
|
+
case "full":
|
|
177
194
|
return {
|
|
178
195
|
type: entry.type,
|
|
179
196
|
entries: entry.entries.map((col) => joinEntryToInternal(col)),
|
|
180
197
|
};
|
|
181
|
-
case
|
|
198
|
+
case "outer":
|
|
182
199
|
return {
|
|
183
|
-
type:
|
|
200
|
+
type: "outer",
|
|
184
201
|
primary: joinEntryToInternal(entry.primary),
|
|
185
202
|
secondary: entry.secondary.map((col) => joinEntryToInternal(col)),
|
|
186
203
|
};
|
package/src/ptable_shared.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { PObjectId, PFrameHandle, PTableDef, PTableHandle } from
|
|
2
|
-
import { hashJson } from
|
|
1
|
+
import type { PObjectId, PFrameHandle, PTableDef, PTableHandle } from "@platforma-sdk/model";
|
|
2
|
+
import { hashJson } from "@milaboratories/pl-model-middle-layer";
|
|
3
3
|
|
|
4
4
|
export type FullPTableDef = {
|
|
5
5
|
pFrameHandle: PFrameHandle;
|