@synnaxlabs/client 0.57.1 → 0.57.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{client-DDACDVZ-.js → client-CX6CMvCc.js} +3686 -3548
- package/dist/client-CX6CMvCc.js.map +1 -0
- package/dist/client.js +1 -1
- package/dist/src/arc/client.d.ts.map +1 -1
- package/dist/src/channel/client.d.ts +1 -0
- package/dist/src/channel/client.d.ts.map +1 -1
- package/dist/src/device/client.d.ts.map +1 -1
- package/dist/src/framer/streamer.d.ts +34 -18
- package/dist/src/framer/streamer.d.ts.map +1 -1
- package/dist/src/label/store.d.ts +1 -1
- package/dist/src/label/store.d.ts.map +1 -1
- package/dist/src/ontology/client.d.ts.map +1 -1
- package/dist/src/ontology/store.d.ts +16 -1
- package/dist/src/ontology/store.d.ts.map +1 -1
- package/dist/src/query/external.d.ts +1 -0
- package/dist/src/query/external.d.ts.map +1 -1
- package/dist/src/query/indexes.d.ts +38 -0
- package/dist/src/query/indexes.d.ts.map +1 -0
- package/dist/src/query/table.d.ts +20 -4
- package/dist/src/query/table.d.ts.map +1 -1
- package/dist/src/rack/client.d.ts.map +1 -1
- package/dist/src/ranger/client.d.ts +11 -1
- package/dist/src/ranger/client.d.ts.map +1 -1
- package/dist/src/status/client.d.ts.map +1 -1
- package/dist/src/status/status.d.ts +2 -0
- package/dist/src/status/status.d.ts.map +1 -1
- package/dist/src/task/client.d.ts +19 -1
- package/dist/src/task/client.d.ts.map +1 -1
- package/dist/src/testutil/client.d.ts +2 -0
- package/dist/src/testutil/client.d.ts.map +1 -1
- package/dist/src/testutil/proxy.d.ts +8 -0
- package/dist/src/testutil/proxy.d.ts.map +1 -1
- package/dist/testutil.js +80 -60
- package/dist/testutil.js.map +1 -1
- package/package.json +5 -5
- package/src/arc/client.ts +2 -20
- package/src/channel/client.ts +11 -6
- package/src/device/client.ts +4 -3
- package/src/framer/streamer.ts +71 -6
- package/src/label/store.ts +7 -2
- package/src/ontology/client.ts +15 -8
- package/src/ontology/store.ts +27 -5
- package/src/query/external.ts +1 -0
- package/src/query/indexes.ts +94 -0
- package/src/query/table.ts +74 -12
- package/src/rack/client.ts +5 -4
- package/src/ranger/client.ts +65 -24
- package/src/status/client.ts +15 -11
- package/src/status/status.ts +14 -0
- package/src/task/client.ts +54 -19
- package/src/testutil/client.ts +5 -1
- package/src/testutil/proxy.ts +59 -6
- package/dist/client-DDACDVZ-.js.map +0 -1
package/src/channel/client.ts
CHANGED
|
@@ -389,12 +389,14 @@ export class Client extends query.Retriever<
|
|
|
389
389
|
private readonly cfg: ClientConfig;
|
|
390
390
|
readonly writer: Writer;
|
|
391
391
|
readonly store: query.Table<Key, Channel>;
|
|
392
|
+
private readonly byName: query.LookupIndex<Key, Channel>;
|
|
392
393
|
|
|
393
394
|
constructor(cfg: ClientConfig) {
|
|
394
395
|
const { writer, statuses, ranges, cache } = cfg;
|
|
395
396
|
const statusStore = statuses.store;
|
|
396
397
|
const aliasStore = ranges.aliases;
|
|
397
398
|
const sugar = (payload: Payload): Channel => this.sugar(payload);
|
|
399
|
+
const byName = new query.LookupIndex<Key, Channel>((ch) => ch.name);
|
|
398
400
|
const store = cache.createTable<Key, Channel>({
|
|
399
401
|
name: "channels",
|
|
400
402
|
equal: (a, b) => deep.equal(a.payload, b.payload),
|
|
@@ -406,6 +408,7 @@ export class Client extends query.Retriever<
|
|
|
406
408
|
}),
|
|
407
409
|
query.createDeleteListener(DELETE_CHANNEL_NAME, keyZ),
|
|
408
410
|
],
|
|
411
|
+
indexes: [byName],
|
|
409
412
|
});
|
|
410
413
|
const composed = cache.derive<Key, Channel, Channel>({
|
|
411
414
|
name: "channel.composed",
|
|
@@ -461,6 +464,7 @@ export class Client extends query.Retriever<
|
|
|
461
464
|
this.cfg = cfg;
|
|
462
465
|
this.writer = writer;
|
|
463
466
|
this.store = store;
|
|
467
|
+
this.byName = byName;
|
|
464
468
|
}
|
|
465
469
|
|
|
466
470
|
/**
|
|
@@ -640,7 +644,7 @@ export class Client extends query.Retriever<
|
|
|
640
644
|
}
|
|
641
645
|
const names = normalized;
|
|
642
646
|
await this.writer.delete({ names });
|
|
643
|
-
const cached = this.
|
|
647
|
+
const cached = this.byName.get(names);
|
|
644
648
|
if (cached.length > 0) this.store.delete(cached.map((ch) => ch.key));
|
|
645
649
|
}
|
|
646
650
|
|
|
@@ -765,10 +769,11 @@ export class Client extends query.Retriever<
|
|
|
765
769
|
const { key, rangeKey } = query;
|
|
766
770
|
let ch = this.store.get(key);
|
|
767
771
|
if (ch == null) {
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
+
// Through the table, so concurrent misses coalesce into one request. A plot
|
|
773
|
+
// panel resolves one channel per telemetry source, all in the same tick.
|
|
774
|
+
const fetched = await this.store.retrieve([key]);
|
|
775
|
+
checkForMultipleOrNoResults("channel", key, fetched, true);
|
|
776
|
+
[ch] = fetched;
|
|
772
777
|
}
|
|
773
778
|
// A cached calculated channel without a cached status is ambiguous: the
|
|
774
779
|
// status may not exist, or may simply never have been fetched.
|
|
@@ -794,7 +799,7 @@ export class Client extends query.Retriever<
|
|
|
794
799
|
const resolved: Channel[] = [];
|
|
795
800
|
const missing: string[] = [];
|
|
796
801
|
for (const name of new Set(names)) {
|
|
797
|
-
const matches = this.
|
|
802
|
+
const matches = this.byName.get(name);
|
|
798
803
|
if (matches.length > 1) return null;
|
|
799
804
|
if (matches.length === 0) missing.push(name);
|
|
800
805
|
else resolved.push(matches[0]);
|
package/src/device/client.ts
CHANGED
|
@@ -424,10 +424,11 @@ export class Client extends query.Retriever<
|
|
|
424
424
|
}
|
|
425
425
|
|
|
426
426
|
private async fetchSingle(q: SingleQuery): Promise<Device> {
|
|
427
|
-
//
|
|
427
|
+
// Status-bearing queries bypass the table, which never holds status.
|
|
428
428
|
if (q.includeStatus !== true) {
|
|
429
|
-
const
|
|
430
|
-
|
|
429
|
+
const devices = await this.store.retrieve([q.key]);
|
|
430
|
+
checkForMultipleOrNoResults("Device", q, devices, true);
|
|
431
|
+
return devices[0];
|
|
431
432
|
}
|
|
432
433
|
const devices = await this.execRetrieve(q);
|
|
433
434
|
checkForMultipleOrNoResults("Device", q, devices, true);
|
package/src/framer/streamer.ts
CHANGED
|
@@ -7,8 +7,13 @@
|
|
|
7
7
|
// License, use of this software will be governed by the Apache License, Version 2.0,
|
|
8
8
|
// included in the file licenses/APL.txt.
|
|
9
9
|
|
|
10
|
-
import {
|
|
11
|
-
|
|
10
|
+
import {
|
|
11
|
+
EOF,
|
|
12
|
+
type Stream,
|
|
13
|
+
Unreachable,
|
|
14
|
+
type WebSocketClient,
|
|
15
|
+
} from "@synnaxlabs/freighter";
|
|
16
|
+
import { errors, Rate, TimeSpan, zod } from "@synnaxlabs/x";
|
|
12
17
|
import { z } from "zod";
|
|
13
18
|
|
|
14
19
|
import { type channel } from "@/channel";
|
|
@@ -23,6 +28,7 @@ const reqZ = z.object({
|
|
|
23
28
|
downsampleFactor: z.int(),
|
|
24
29
|
throttleRate: Rate.z.optional(),
|
|
25
30
|
excludeGroups: z.uint32().array().optional(),
|
|
31
|
+
keepAlive: TimeSpan.z.optional(),
|
|
26
32
|
});
|
|
27
33
|
|
|
28
34
|
/**
|
|
@@ -31,7 +37,11 @@ const reqZ = z.object({
|
|
|
31
37
|
*/
|
|
32
38
|
export interface StreamerRequest extends z.infer<typeof reqZ> {}
|
|
33
39
|
|
|
34
|
-
const resZ = z.object({
|
|
40
|
+
const resZ = z.object({
|
|
41
|
+
frame: frameZ,
|
|
42
|
+
/** Marks an empty response the Core emits so a dead connection is detectable. */
|
|
43
|
+
keepAlive: z.boolean().optional(),
|
|
44
|
+
});
|
|
35
45
|
|
|
36
46
|
/**
|
|
37
47
|
* Response interface for streaming frames from a Synnax cluster.
|
|
@@ -49,6 +59,10 @@ const intermediateStreamerConfigZ = z.object({
|
|
|
49
59
|
/** excludeGroups sets writer group IDs whose frames should be filtered out by the
|
|
50
60
|
Core. Used for telemetry bypass deduplication. */
|
|
51
61
|
excludeGroups: z.uint32().array().default([]),
|
|
62
|
+
/** Interval at which the Core emits keep-alive responses so a silently dead
|
|
63
|
+
connection fails reads instead of hanging forever. TimeSpan.ZERO disables
|
|
64
|
+
detection. Defaults to 5 seconds. */
|
|
65
|
+
keepAlive: TimeSpan.z.default(TimeSpan.seconds(5)),
|
|
52
66
|
});
|
|
53
67
|
|
|
54
68
|
/** Zod schema for {@link StreamerConfig}. A bare channel list parses as a config. */
|
|
@@ -73,7 +87,11 @@ export interface Streamer extends AsyncIterator<Frame>, AsyncIterable<Frame> {
|
|
|
73
87
|
update: (channels: channel.Params) => Promise<void>;
|
|
74
88
|
/** Close the streamer and free all associated resources. */
|
|
75
89
|
close: () => void;
|
|
76
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* Read the next frame of telemetry.
|
|
92
|
+
* @throws {Unreachable} if keep-alives were flowing and the stream then stays silent
|
|
93
|
+
* past the keep-alive deadline: the connection is presumed dead.
|
|
94
|
+
*/
|
|
77
95
|
read: () => Promise<Frame>;
|
|
78
96
|
}
|
|
79
97
|
|
|
@@ -101,14 +119,19 @@ export const createStreamOpener =
|
|
|
101
119
|
cfg.downsampleFactor,
|
|
102
120
|
cfg.throttleRate,
|
|
103
121
|
cfg.excludeGroups,
|
|
122
|
+
cfg.keepAlive,
|
|
104
123
|
);
|
|
105
124
|
stream.send({
|
|
106
125
|
keys: Array.from(adapter.keys),
|
|
107
126
|
downsampleFactor: cfg.downsampleFactor,
|
|
108
127
|
throttleRate: cfg.throttleRate,
|
|
109
128
|
excludeGroups: cfg.excludeGroups,
|
|
129
|
+
keepAlive: cfg.keepAlive,
|
|
110
130
|
});
|
|
111
|
-
|
|
131
|
+
// A keep-alive can beat the open ack onto the wire, so the ack is the first
|
|
132
|
+
// non-keep-alive response.
|
|
133
|
+
let res = await stream.receive();
|
|
134
|
+
while (res.keepAlive === true) res = await stream.receive();
|
|
112
135
|
return streamer;
|
|
113
136
|
};
|
|
114
137
|
|
|
@@ -124,12 +147,20 @@ export const openStreamer = async (
|
|
|
124
147
|
config: StreamerConfig,
|
|
125
148
|
): Promise<Streamer> => await createStreamOpener(retrieveChannels, client)(config);
|
|
126
149
|
|
|
150
|
+
// Missing this many keep-alive intervals in a row fails the pending read: one is normal
|
|
151
|
+
// jitter, three is a dead connection.
|
|
152
|
+
const KEEP_ALIVE_DEADLINE_FACTOR = 3;
|
|
153
|
+
|
|
127
154
|
class BaseStreamer implements Streamer {
|
|
128
155
|
private readonly stream: StreamProxy<typeof reqZ, typeof resZ>;
|
|
129
156
|
private readonly adapter: ReadAdapter;
|
|
130
157
|
private readonly downsampleFactor: number;
|
|
131
158
|
private readonly throttleRate: Rate;
|
|
132
159
|
private readonly excludeGroups: number[];
|
|
160
|
+
private readonly deadline: TimeSpan;
|
|
161
|
+
// Set once the Core proves keep-alive support by sending one, so the deadline never
|
|
162
|
+
// arms against a Core that will not send them.
|
|
163
|
+
private armed = false;
|
|
133
164
|
|
|
134
165
|
constructor(
|
|
135
166
|
stream: Stream<typeof reqZ, typeof resZ>,
|
|
@@ -137,12 +168,16 @@ class BaseStreamer implements Streamer {
|
|
|
137
168
|
downsampleFactor: number = 1,
|
|
138
169
|
throttleRate: Rate = new Rate(0),
|
|
139
170
|
excludeGroups: number[] = [],
|
|
171
|
+
keepAlive: TimeSpan = TimeSpan.ZERO,
|
|
140
172
|
) {
|
|
141
173
|
this.stream = new StreamProxy("Streamer", stream);
|
|
142
174
|
this.adapter = adapter;
|
|
143
175
|
this.downsampleFactor = downsampleFactor;
|
|
144
176
|
this.throttleRate = throttleRate;
|
|
145
177
|
this.excludeGroups = excludeGroups;
|
|
178
|
+
this.deadline = TimeSpan.milliseconds(
|
|
179
|
+
keepAlive.milliseconds * KEEP_ALIVE_DEADLINE_FACTOR,
|
|
180
|
+
);
|
|
146
181
|
}
|
|
147
182
|
|
|
148
183
|
get keys(): channel.Key[] {
|
|
@@ -160,7 +195,37 @@ class BaseStreamer implements Streamer {
|
|
|
160
195
|
}
|
|
161
196
|
|
|
162
197
|
async read(): Promise<Frame> {
|
|
163
|
-
|
|
198
|
+
while (true) {
|
|
199
|
+
const res = await this.receiveWithDeadline();
|
|
200
|
+
if (res.keepAlive === true) {
|
|
201
|
+
if (!this.deadline.isZero) this.armed = true;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
return this.adapter.adapt(new Frame(res.frame));
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
private async receiveWithDeadline(): Promise<z.infer<typeof resZ>> {
|
|
209
|
+
const received = this.stream.receive();
|
|
210
|
+
if (!this.armed) return await received;
|
|
211
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
212
|
+
const deadline = new Promise<never>((_, reject) => {
|
|
213
|
+
timer = setTimeout(() => {
|
|
214
|
+
const silence = this.deadline.toString();
|
|
215
|
+
const message = `streamer received no response for ${silence}`;
|
|
216
|
+
reject(new Unreachable({ message }));
|
|
217
|
+
}, this.deadline.milliseconds);
|
|
218
|
+
});
|
|
219
|
+
try {
|
|
220
|
+
return await Promise.race([received, deadline]);
|
|
221
|
+
} catch (err) {
|
|
222
|
+
// The read already failed for its caller; a late settle of the losing receive
|
|
223
|
+
// must not surface as an unhandled rejection.
|
|
224
|
+
received.catch(() => {});
|
|
225
|
+
throw errors.fromUnknown(err);
|
|
226
|
+
} finally {
|
|
227
|
+
clearTimeout(timer);
|
|
228
|
+
}
|
|
164
229
|
}
|
|
165
230
|
|
|
166
231
|
async update(channels: channel.Params): Promise<void> {
|
package/src/label/store.ts
CHANGED
|
@@ -21,8 +21,13 @@ export const matchLabeledBy = (rel: ontology.Relationship, id: ontology.ID): boo
|
|
|
21
21
|
|
|
22
22
|
/** Returns the cached labels attached to the given ontology ID. */
|
|
23
23
|
export const cachedLabelsOf = (
|
|
24
|
-
|
|
24
|
+
cache: ontology.Cache,
|
|
25
25
|
labels: query.Table<Key, Label>,
|
|
26
26
|
id: ontology.ID,
|
|
27
27
|
): Label[] =>
|
|
28
|
-
labels.get(
|
|
28
|
+
labels.get(
|
|
29
|
+
cache
|
|
30
|
+
.relationshipsFrom(id)
|
|
31
|
+
.filter((r) => r.type === LABELED_BY_ONTOLOGY_RELATIONSHIP_TYPE)
|
|
32
|
+
.map((r) => r.to.key),
|
|
33
|
+
);
|
package/src/ontology/client.ts
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
Cache,
|
|
36
36
|
RELATIONSHIP_DELETE_CHANNEL_NAME,
|
|
37
37
|
RELATIONSHIP_SET_CHANNEL_NAME,
|
|
38
|
+
type RelationshipIndexes,
|
|
38
39
|
RESOURCE_DELETE_CHANNEL_NAME,
|
|
39
40
|
RESOURCE_SET_CHANNEL_NAME,
|
|
40
41
|
} from "@/ontology/store";
|
|
@@ -194,10 +195,15 @@ export class Client extends query.Retriever<
|
|
|
194
195
|
|
|
195
196
|
constructor(cfg: ClientConfig) {
|
|
196
197
|
const { unary, cache } = cfg;
|
|
198
|
+
const relationshipIndexes: RelationshipIndexes = {
|
|
199
|
+
byTo: new query.LookupIndex<string, Relationship>((r) => idToString(r.to)),
|
|
200
|
+
byFrom: new query.LookupIndex<string, Relationship>((r) => idToString(r.from)),
|
|
201
|
+
};
|
|
197
202
|
const relationships = cache.createTable<string, Relationship>({
|
|
198
203
|
name: "relationships",
|
|
199
204
|
equal: (a, b) =>
|
|
200
205
|
idsEqual(a.from, b.from) && idsEqual(a.to, b.to) && a.type === b.type,
|
|
206
|
+
indexes: [relationshipIndexes.byTo, relationshipIndexes.byFrom],
|
|
201
207
|
listen: [
|
|
202
208
|
query.createSetListener(RELATIONSHIP_SET_CHANNEL_NAME, relationshipZ, {
|
|
203
209
|
key: (changed) => relationshipToString(changed),
|
|
@@ -244,7 +250,7 @@ export class Client extends query.Retriever<
|
|
|
244
250
|
});
|
|
245
251
|
this.cfg = cfg;
|
|
246
252
|
this.writer = new Writer(unary);
|
|
247
|
-
this.cache = new Cache(relationships, resources);
|
|
253
|
+
this.cache = new Cache(relationships, resources, relationshipIndexes);
|
|
248
254
|
this.children = this.dependentSurface(cache, "children", "to");
|
|
249
255
|
this.parents = this.dependentSurface(cache, "parents", "from");
|
|
250
256
|
}
|
|
@@ -466,13 +472,14 @@ export class Client extends query.Retriever<
|
|
|
466
472
|
): boolean {
|
|
467
473
|
if (q.types != null && !q.types.includes(resource.id.type)) return false;
|
|
468
474
|
const anchor = oppositeRelationshipDirection(direction);
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
(
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
475
|
+
const rels =
|
|
476
|
+
direction === "to"
|
|
477
|
+
? this.cache.relationshipsTo(resource.id)
|
|
478
|
+
: this.cache.relationshipsFrom(resource.id);
|
|
479
|
+
return rels.some(
|
|
480
|
+
(rel) =>
|
|
481
|
+
rel.type === PARENT_OF_RELATIONSHIP_TYPE &&
|
|
482
|
+
q.ids.includes(idToString(rel[anchor])),
|
|
476
483
|
);
|
|
477
484
|
}
|
|
478
485
|
|
package/src/ontology/store.ts
CHANGED
|
@@ -13,7 +13,6 @@ import {
|
|
|
13
13
|
type ID,
|
|
14
14
|
idsEqual,
|
|
15
15
|
idToString,
|
|
16
|
-
matchRelationship,
|
|
17
16
|
PARENT_OF_RELATIONSHIP_TYPE,
|
|
18
17
|
type Relationship,
|
|
19
18
|
type Resource,
|
|
@@ -26,26 +25,49 @@ export const RESOURCE_DELETE_CHANNEL_NAME = "sy_ontology_resource_delete";
|
|
|
26
25
|
export const RELATIONSHIP_SET_CHANNEL_NAME = "sy_ontology_relationship_set";
|
|
27
26
|
export const RELATIONSHIP_DELETE_CHANNEL_NAME = "sy_ontology_relationship_delete";
|
|
28
27
|
|
|
28
|
+
/**
|
|
29
|
+
* The secondary indexes registered on the relationships table, one per endpoint.
|
|
30
|
+
* Type filtering runs at the call site over the matched handful.
|
|
31
|
+
*/
|
|
32
|
+
export interface RelationshipIndexes {
|
|
33
|
+
/** Keyed by the stringified `to` end. */
|
|
34
|
+
byTo: query.LookupIndex<string, Relationship>;
|
|
35
|
+
/** Keyed by the stringified `from` end. */
|
|
36
|
+
byFrom: query.LookupIndex<string, Relationship>;
|
|
37
|
+
}
|
|
38
|
+
|
|
29
39
|
/** The ontology record tables: resources and the relationships between them. */
|
|
30
40
|
export class Cache {
|
|
31
41
|
readonly relationships: query.Table<string, Relationship>;
|
|
32
42
|
readonly resources: query.Table<string, Resource>;
|
|
43
|
+
private readonly indexes: RelationshipIndexes;
|
|
33
44
|
|
|
34
45
|
constructor(
|
|
35
46
|
relationships: query.Table<string, Relationship>,
|
|
36
47
|
resources: query.Table<string, Resource>,
|
|
48
|
+
indexes: RelationshipIndexes,
|
|
37
49
|
) {
|
|
38
50
|
this.relationships = relationships;
|
|
39
51
|
this.resources = resources;
|
|
52
|
+
this.indexes = indexes;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Returns the cached relationships pointing at the given ID. */
|
|
56
|
+
relationshipsTo(id: ID): Relationship[] {
|
|
57
|
+
return this.indexes.byTo.get(idToString(id));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Returns the cached relationships originating at the given ID. */
|
|
61
|
+
relationshipsFrom(id: ID): Relationship[] {
|
|
62
|
+
return this.indexes.byFrom.get(idToString(id));
|
|
40
63
|
}
|
|
41
64
|
|
|
42
65
|
/** Returns the cached parent ID of the given ontology ID, or null if unknown. */
|
|
43
66
|
parentID(id: ID): ID | null {
|
|
44
|
-
const
|
|
45
|
-
|
|
67
|
+
const rel = this.relationshipsTo(id).find(
|
|
68
|
+
(r) => r.type === PARENT_OF_RELATIONSHIP_TYPE,
|
|
46
69
|
);
|
|
47
|
-
|
|
48
|
-
return res[0].from;
|
|
70
|
+
return rel?.from ?? null;
|
|
49
71
|
}
|
|
50
72
|
|
|
51
73
|
/**
|
package/src/query/external.ts
CHANGED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Copyright 2026 Synnax Labs, Inc.
|
|
2
|
+
//
|
|
3
|
+
// Use of this software is governed by the Business Source License included in the file
|
|
4
|
+
// licenses/BSL.txt.
|
|
5
|
+
//
|
|
6
|
+
// As of the Change Date specified in that file, in accordance with the Business Source
|
|
7
|
+
// License, use of this software will be governed by the Apache License, Version 2.0,
|
|
8
|
+
// included in the file licenses/APL.txt.
|
|
9
|
+
|
|
10
|
+
import { type record, type state } from "@synnaxlabs/x";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* An exact-match secondary index over a table's live entries. `extract` reads one
|
|
14
|
+
* index value from each entry; a null leaves the entry out of the index. Register
|
|
15
|
+
* the index on its owning table's `indexes` param; the table keeps it current
|
|
16
|
+
* across every mutation, rollbacks and resets included. Tombstones are never
|
|
17
|
+
* indexed.
|
|
18
|
+
*/
|
|
19
|
+
export class LookupIndex<
|
|
20
|
+
Key extends record.Key = record.Key,
|
|
21
|
+
Value extends state.State = state.State,
|
|
22
|
+
IndexValue extends record.Key = record.Key,
|
|
23
|
+
> {
|
|
24
|
+
private readonly extract: (value: Value) => IndexValue | null;
|
|
25
|
+
private readonly buckets = new Map<IndexValue, Map<Key, Value>>();
|
|
26
|
+
private readonly indexed = new Map<Key, IndexValue>();
|
|
27
|
+
|
|
28
|
+
constructor(extract: (value: Value) => IndexValue | null) {
|
|
29
|
+
this.extract = extract;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Returns the entries whose extracted index value equals the given value. */
|
|
33
|
+
get(value: IndexValue): Value[];
|
|
34
|
+
/**
|
|
35
|
+
* Returns the entries matching any of the given values, in the order of the
|
|
36
|
+
* values and each entry once.
|
|
37
|
+
*/
|
|
38
|
+
get(values: IndexValue[]): Value[];
|
|
39
|
+
get(value: IndexValue | IndexValue[]): Value[] {
|
|
40
|
+
if (Array.isArray(value))
|
|
41
|
+
return Array.from(new Set(value)).flatMap((v) => this.get(v));
|
|
42
|
+
const bucket = this.buckets.get(value);
|
|
43
|
+
if (bucket == null) return [];
|
|
44
|
+
return Array.from(bucket.values());
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Records the entry, relocating it when its extracted value changed.
|
|
49
|
+
* Cache-internal surface: called by the owning table, not domain code.
|
|
50
|
+
*/
|
|
51
|
+
set(key: Key, value: Value): void {
|
|
52
|
+
const next = this.extract(value);
|
|
53
|
+
const prev = this.indexed.get(key);
|
|
54
|
+
if (prev !== undefined && prev !== next) this.unbucket(prev, key);
|
|
55
|
+
if (next == null) {
|
|
56
|
+
this.indexed.delete(key);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
let bucket = this.buckets.get(next);
|
|
60
|
+
if (bucket == null) {
|
|
61
|
+
bucket = new Map();
|
|
62
|
+
this.buckets.set(next, bucket);
|
|
63
|
+
}
|
|
64
|
+
bucket.set(key, value);
|
|
65
|
+
this.indexed.set(key, next);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Removes any mapping for the given entry key.
|
|
70
|
+
* Cache-internal surface: called by the owning table, not domain code.
|
|
71
|
+
*/
|
|
72
|
+
delete(key: Key): void {
|
|
73
|
+
const prev = this.indexed.get(key);
|
|
74
|
+
if (prev === undefined) return;
|
|
75
|
+
this.unbucket(prev, key);
|
|
76
|
+
this.indexed.delete(key);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Discards every mapping.
|
|
81
|
+
* Cache-internal surface: called by the owning table, not domain code.
|
|
82
|
+
*/
|
|
83
|
+
reset(): void {
|
|
84
|
+
this.buckets.clear();
|
|
85
|
+
this.indexed.clear();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
private unbucket(value: IndexValue, key: Key): void {
|
|
89
|
+
const bucket = this.buckets.get(value);
|
|
90
|
+
if (bucket == null) return;
|
|
91
|
+
bucket.delete(key);
|
|
92
|
+
if (bucket.size === 0) this.buckets.delete(value);
|
|
93
|
+
}
|
|
94
|
+
}
|
package/src/query/table.ts
CHANGED
|
@@ -23,6 +23,7 @@ import type z from "zod";
|
|
|
23
23
|
|
|
24
24
|
import { NotFoundError } from "@/errors";
|
|
25
25
|
import { Deleted } from "@/query/deleted";
|
|
26
|
+
import { type LookupIndex } from "@/query/indexes";
|
|
26
27
|
import { type Listener } from "@/query/streamer";
|
|
27
28
|
|
|
28
29
|
/**
|
|
@@ -98,6 +99,11 @@ export interface TableParams<
|
|
|
98
99
|
* @default TimeSpan.milliseconds(10)
|
|
99
100
|
*/
|
|
100
101
|
fetchDebounce?: CrudeTimeSpan;
|
|
102
|
+
/**
|
|
103
|
+
* Secondary indexes over the table's live entries. The table keeps each current
|
|
104
|
+
* across every mutation, rollbacks and resets included.
|
|
105
|
+
*/
|
|
106
|
+
indexes?: Array<LookupIndex<Key, Value>>;
|
|
101
107
|
}
|
|
102
108
|
|
|
103
109
|
/**
|
|
@@ -156,6 +162,7 @@ export class Table<
|
|
|
156
162
|
Key[],
|
|
157
163
|
Array<Keyed<Key, Value>>
|
|
158
164
|
> | null;
|
|
165
|
+
private readonly indexes: Array<LookupIndex<Key, Value>>;
|
|
159
166
|
private gen = 0;
|
|
160
167
|
|
|
161
168
|
constructor({
|
|
@@ -164,11 +171,13 @@ export class Table<
|
|
|
164
171
|
fetch,
|
|
165
172
|
hydrate = "set",
|
|
166
173
|
fetchDebounce = DEFAULT_FETCH_DEBOUNCE,
|
|
174
|
+
indexes = [],
|
|
167
175
|
}: TableParams<Key, Value>) {
|
|
168
176
|
this.onError = onError;
|
|
169
177
|
this.equal = equal;
|
|
170
178
|
this.fetchEntries = fetch;
|
|
171
179
|
this.hydrateMode = hydrate;
|
|
180
|
+
this.indexes = [...indexes];
|
|
172
181
|
this.fetchBatcher =
|
|
173
182
|
fetch == null
|
|
174
183
|
? null
|
|
@@ -177,7 +186,27 @@ export class Table<
|
|
|
177
186
|
exec: async (requests) => {
|
|
178
187
|
const keys = new Set<Key>();
|
|
179
188
|
requests.forEach(({ req }) => req.forEach((key) => keys.add(key)));
|
|
180
|
-
|
|
189
|
+
let fetched: Array<Keyed<Key, Value>>;
|
|
190
|
+
try {
|
|
191
|
+
fetched = await fetch(Array.from(keys));
|
|
192
|
+
} catch (exc) {
|
|
193
|
+
if (!NotFoundError.matches(exc) || requests.length === 1)
|
|
194
|
+
throw errors.fromUnknown(exc);
|
|
195
|
+
// A strict fetch rejects the whole batch when any caller's key
|
|
196
|
+
// has vanished. Refetch per caller so each settles exactly as
|
|
197
|
+
// its own request would have, keeping the batch transparent.
|
|
198
|
+
await Promise.all(
|
|
199
|
+
requests.map(async ({ req, resolve, reject }) => {
|
|
200
|
+
try {
|
|
201
|
+
const mine = new Set(req);
|
|
202
|
+
resolve((await fetch(req)).filter(({ key }) => mine.has(key)));
|
|
203
|
+
} catch (exc) {
|
|
204
|
+
reject(exc);
|
|
205
|
+
}
|
|
206
|
+
}),
|
|
207
|
+
);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
181
210
|
// The window's fetch carries other callers' keys too; each caller
|
|
182
211
|
// hydrates only the entries it asked for.
|
|
183
212
|
requests.forEach(({ req, resolve }) => {
|
|
@@ -188,6 +217,27 @@ export class Table<
|
|
|
188
217
|
});
|
|
189
218
|
}
|
|
190
219
|
|
|
220
|
+
private applySet(key: Key, value: Value): void {
|
|
221
|
+
this.entries.set(key, value);
|
|
222
|
+
for (const index of this.indexes) index.set(key, value);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
private applyDelete(key: Key): void {
|
|
226
|
+
this.entries.delete(key);
|
|
227
|
+
for (const index of this.indexes) index.delete(key);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Registers a secondary index and backfills it from the live entries, for a
|
|
232
|
+
* domain that owns an index's meaning but not the table it reads.
|
|
233
|
+
* @returns the index.
|
|
234
|
+
*/
|
|
235
|
+
index<I extends LookupIndex<Key, Value>>(index: I): I {
|
|
236
|
+
this.entries.forEach((value, key) => index.set(key, value));
|
|
237
|
+
this.indexes.push(index);
|
|
238
|
+
return index;
|
|
239
|
+
}
|
|
240
|
+
|
|
191
241
|
private setOne(
|
|
192
242
|
key: Key,
|
|
193
243
|
value: state.SetArg<Value | undefined>,
|
|
@@ -197,16 +247,16 @@ export class Table<
|
|
|
197
247
|
if (next == null || (prev != null && this.equal(next, prev, key))) return undefined;
|
|
198
248
|
const prevTombstone = this.tombstones.get(key);
|
|
199
249
|
this.tombstones.delete(key);
|
|
200
|
-
this.
|
|
250
|
+
this.applySet(key, next);
|
|
201
251
|
this.notify({ variant: "set", key, value: next });
|
|
202
252
|
|
|
203
253
|
return () => {
|
|
204
254
|
if (prev === undefined) {
|
|
205
|
-
this.
|
|
255
|
+
this.applyDelete(key);
|
|
206
256
|
if (prevTombstone != null) this.tombstones.set(key, prevTombstone);
|
|
207
257
|
this.notify({ variant: "delete", key });
|
|
208
258
|
} else {
|
|
209
|
-
this.
|
|
259
|
+
this.applySet(key, prev);
|
|
210
260
|
this.notify({ variant: "set", key, value: prev });
|
|
211
261
|
}
|
|
212
262
|
};
|
|
@@ -308,9 +358,10 @@ export class Table<
|
|
|
308
358
|
/**
|
|
309
359
|
* Resolves the given keys to records: serves cached entries and fetches the misses
|
|
310
360
|
* through the table's fetch, hydrating results under the declared mode. With refresh,
|
|
311
|
-
* every key is fetched regardless of presence
|
|
312
|
-
*
|
|
313
|
-
* omitted. Tables without a fetch
|
|
361
|
+
* every key is fetched regardless of presence and cached entries the fetch omits are
|
|
362
|
+
* tombstoned. Returns the table's entries for the found keys in input order,
|
|
363
|
+
* deduplicated; keys the cluster no longer has are omitted. Tables without a fetch
|
|
364
|
+
* serve cached entries only.
|
|
314
365
|
*/
|
|
315
366
|
async retrieve(keys: Key[], opts: { refresh?: boolean } = {}): Promise<Value[]> {
|
|
316
367
|
if (this.fetchBatcher != null) {
|
|
@@ -319,9 +370,19 @@ export class Table<
|
|
|
319
370
|
if (misses.length > 0) {
|
|
320
371
|
const gen = this.gen;
|
|
321
372
|
const fetched = await this.fetchBatcher.enqueue(misses);
|
|
322
|
-
if (gen === this.gen
|
|
323
|
-
if (opts.refresh === true)
|
|
324
|
-
|
|
373
|
+
if (gen === this.gen)
|
|
374
|
+
if (opts.refresh === true) {
|
|
375
|
+
// A refresh is authoritative for its keys: cached entries the
|
|
376
|
+
// fetch omitted vanished from the cluster and are tombstoned.
|
|
377
|
+
const present = new Set<Key>(fetched.map(({ key }) => key));
|
|
378
|
+
const vanished = misses.filter(
|
|
379
|
+
(key) => !present.has(key) && this.entries.has(key),
|
|
380
|
+
);
|
|
381
|
+
this.batch(() => {
|
|
382
|
+
if (vanished.length > 0) this.delete(vanished);
|
|
383
|
+
if (fetched.length > 0) this.set(fetched);
|
|
384
|
+
});
|
|
385
|
+
} else if (fetched.length > 0) this.ingest(fetched);
|
|
325
386
|
}
|
|
326
387
|
}
|
|
327
388
|
const seen = new Set<Key>();
|
|
@@ -375,7 +436,7 @@ export class Table<
|
|
|
375
436
|
|
|
376
437
|
this.batch(() =>
|
|
377
438
|
toDelete.forEach(({ key: k, value }) => {
|
|
378
|
-
this.
|
|
439
|
+
this.applyDelete(k);
|
|
379
440
|
if (tombstone && value != null)
|
|
380
441
|
this.tombstones.set(k, new Deleted(value, TimeStamp.now()));
|
|
381
442
|
this.notify({ variant: "delete", key: k });
|
|
@@ -387,7 +448,7 @@ export class Table<
|
|
|
387
448
|
toDelete.forEach(({ key: k, value }) => {
|
|
388
449
|
if (value == null) return;
|
|
389
450
|
this.tombstones.delete(k);
|
|
390
|
-
this.
|
|
451
|
+
this.applySet(k, value);
|
|
391
452
|
this.notify({ variant: "set", key: k, value });
|
|
392
453
|
}),
|
|
393
454
|
);
|
|
@@ -426,6 +487,7 @@ export class Table<
|
|
|
426
487
|
this.gen++;
|
|
427
488
|
this.entries.clear();
|
|
428
489
|
this.tombstones.clear();
|
|
490
|
+
for (const index of this.indexes) index.reset();
|
|
429
491
|
}
|
|
430
492
|
|
|
431
493
|
/**
|
package/src/rack/client.ts
CHANGED
|
@@ -307,11 +307,12 @@ export class Client extends query.Retriever<
|
|
|
307
307
|
}
|
|
308
308
|
|
|
309
309
|
private async fetchSingle(q: SingleQuery): Promise<Rack> {
|
|
310
|
-
// Names are not unique
|
|
311
|
-
//
|
|
310
|
+
// Names are not unique and the table never holds status, so only keyed status-free
|
|
311
|
+
// queries can be served from it.
|
|
312
312
|
if ("key" in q && q.includeStatus !== true) {
|
|
313
|
-
const
|
|
314
|
-
|
|
313
|
+
const racks = await this.store.retrieve([q.key]);
|
|
314
|
+
checkForMultipleOrNoResults("Rack", q, racks, true);
|
|
315
|
+
return this.sugar(racks[0]);
|
|
315
316
|
}
|
|
316
317
|
const racks = await this.execRetrieve(q);
|
|
317
318
|
checkForMultipleOrNoResults("Rack", q, racks, true);
|