@synnaxlabs/client 0.57.0 → 0.57.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{client-B08VDLSA.js → client-BsVW3qAW.js} +3909 -3818
- package/dist/client-BsVW3qAW.js.map +1 -0
- package/dist/client.js +477 -477
- package/dist/client.js.map +1 -1
- package/dist/src/framer/cache/dynamic.d.ts +21 -1
- package/dist/src/framer/cache/dynamic.d.ts.map +1 -1
- package/dist/src/framer/cache/reader.d.ts +16 -2
- package/dist/src/framer/cache/reader.d.ts.map +1 -1
- package/dist/src/framer/cache/unary.d.ts +6 -4
- package/dist/src/framer/cache/unary.d.ts.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/label/store.d.ts +1 -1
- package/dist/src/label/store.d.ts.map +1 -1
- package/dist/src/ontology/client.d.ts.map +1 -1
- package/dist/src/ontology/store.d.ts +16 -1
- package/dist/src/ontology/store.d.ts.map +1 -1
- package/dist/src/query/external.d.ts +1 -0
- package/dist/src/query/external.d.ts.map +1 -1
- package/dist/src/query/indexes.d.ts +32 -0
- package/dist/src/query/indexes.d.ts.map +1 -0
- package/dist/src/query/table.d.ts +10 -1
- package/dist/src/query/table.d.ts.map +1 -1
- package/dist/src/ranger/client.d.ts +2 -0
- package/dist/src/ranger/client.d.ts.map +1 -1
- package/dist/src/status/client.d.ts.map +1 -1
- package/dist/testutil.js +1 -1
- package/package.json +4 -4
- package/src/framer/cache/dynamic.ts +104 -43
- package/src/framer/cache/reader.ts +120 -49
- package/src/framer/cache/unary.ts +9 -5
- package/src/index.ts +1 -0
- package/src/label/store.ts +7 -2
- package/src/ontology/client.ts +15 -8
- package/src/ontology/store.ts +27 -5
- package/src/query/external.ts +1 -0
- package/src/query/indexes.ts +81 -0
- package/src/query/table.ts +25 -5
- package/src/ranger/client.ts +26 -16
- package/src/status/client.ts +14 -8
- package/dist/client-B08VDLSA.js.map +0 -1
|
@@ -13,6 +13,8 @@ import {
|
|
|
13
13
|
math,
|
|
14
14
|
MultiSeries,
|
|
15
15
|
Series,
|
|
16
|
+
Size,
|
|
17
|
+
TimeRange,
|
|
16
18
|
type TimeSpan,
|
|
17
19
|
TimeStamp,
|
|
18
20
|
} from "@synnaxlabs/x";
|
|
@@ -52,6 +54,34 @@ const MAX_DEF_WRITES = 100;
|
|
|
52
54
|
// Variable-rate types allocate bytes, not samples. Rough bytes-per-sample estimate.
|
|
53
55
|
const VARIABLE_DT_MULTIPLIER = 40;
|
|
54
56
|
|
|
57
|
+
/**
|
|
58
|
+
* @returns true when every byte of the series fits in the buffer's free space. Measured
|
|
59
|
+
* in bytes because a variable-length buffer counts its capacity that way.
|
|
60
|
+
*/
|
|
61
|
+
const fits = (buffer: Series, series: Series): boolean =>
|
|
62
|
+
series.byteLength.valueOf() <=
|
|
63
|
+
buffer.byteCapacity.valueOf() - buffer.byteLength.valueOf();
|
|
64
|
+
|
|
65
|
+
// Bounds on compacting a buffer that rotated early. Both have to trip: the fraction
|
|
66
|
+
// spares a nearly full buffer, the floor one whose waste is not worth a copy.
|
|
67
|
+
const COMPACT_MAX_FILL = 0.25;
|
|
68
|
+
const COMPACT_MIN_WASTE = Size.kilobytes(256);
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* @returns true when copying the buffer's samples into a right-sized series is worth
|
|
72
|
+
* the copy. A held buffer never qualifies: the holder keeps the original alive, so a
|
|
73
|
+
* copy would only add a second allocation, and it would break the object identity a
|
|
74
|
+
* consumer that both reads and streams deduplicates on.
|
|
75
|
+
*/
|
|
76
|
+
const shouldCompact = (buffer: Series): boolean => {
|
|
77
|
+
if (buffer.refCount > 0 || buffer.length === 0) return false;
|
|
78
|
+
const used = buffer.byteLength.valueOf();
|
|
79
|
+
const capacity = buffer.byteCapacity.valueOf();
|
|
80
|
+
return (
|
|
81
|
+
capacity - used > COMPACT_MIN_WASTE.valueOf() && used < capacity * COMPACT_MAX_FILL
|
|
82
|
+
);
|
|
83
|
+
};
|
|
84
|
+
|
|
55
85
|
/**
|
|
56
86
|
* A cache for channel data that maintains a single, rolling Series as a buffer
|
|
57
87
|
* for channel data. The buffer's data type is derived from the first written series,
|
|
@@ -96,6 +126,17 @@ export class Dynamic {
|
|
|
96
126
|
return this.curr;
|
|
97
127
|
}
|
|
98
128
|
|
|
129
|
+
/**
|
|
130
|
+
* @returns the time range the buffer's samples actually cover: its start to the last
|
|
131
|
+
* stamped write, or the wall clock when the buffer holds unstamped data. Unlike the
|
|
132
|
+
* buffer's provisional time range, the end never claims future time. Null when there
|
|
133
|
+
* is no buffer.
|
|
134
|
+
*/
|
|
135
|
+
get dataTimeRange(): TimeRange | null {
|
|
136
|
+
if (this.curr == null) return null;
|
|
137
|
+
return new TimeRange(this.curr.timeRange.start, this.currDataEnd ?? this.now());
|
|
138
|
+
}
|
|
139
|
+
|
|
99
140
|
/**
|
|
100
141
|
* @returns a list of buffers that were filled by the cache during the write. If the
|
|
101
142
|
* current buffer is able to fit all writes, no buffers will be returned.
|
|
@@ -115,7 +156,6 @@ export class Dynamic {
|
|
|
115
156
|
alignment: bigint,
|
|
116
157
|
start: TimeStamp,
|
|
117
158
|
source: Series,
|
|
118
|
-
sampleIndex: number,
|
|
119
159
|
): Series {
|
|
120
160
|
this.counter++;
|
|
121
161
|
const dt = this.props.transform.resolveDataType(source.dataType);
|
|
@@ -127,8 +167,7 @@ export class Dynamic {
|
|
|
127
167
|
let sampleOffset: math.Numeric = 0;
|
|
128
168
|
if (narrowing)
|
|
129
169
|
if (source.dataType.equals(DataType.TIMESTAMP)) sampleOffset = start.valueOf();
|
|
130
|
-
else if (
|
|
131
|
-
sampleOffset = BigInt(source.data[sampleIndex].valueOf());
|
|
170
|
+
else if (source.length > 0) sampleOffset = BigInt(source.data[0].valueOf());
|
|
132
171
|
return Series.alloc({
|
|
133
172
|
capacity: dt.isVariable ? capacity * VARIABLE_DT_MULTIPLIER : capacity,
|
|
134
173
|
dataType: dt,
|
|
@@ -140,6 +179,18 @@ export class Dynamic {
|
|
|
140
179
|
});
|
|
141
180
|
}
|
|
142
181
|
|
|
182
|
+
/**
|
|
183
|
+
* @returns the sample capacity for a buffer that must hold the given series whole:
|
|
184
|
+
* the configured size, or the series' own size when that is larger.
|
|
185
|
+
*/
|
|
186
|
+
private capacityFor(source: Series): number {
|
|
187
|
+
const dt = this.props.transform.resolveDataType(source.dataType);
|
|
188
|
+
const required = dt.isVariable
|
|
189
|
+
? Math.ceil(source.byteLength.valueOf() / VARIABLE_DT_MULTIPLIER)
|
|
190
|
+
: source.length;
|
|
191
|
+
return Math.max(this.nextBufferSize(), required);
|
|
192
|
+
}
|
|
193
|
+
|
|
143
194
|
// Unstamped series (virtual channels, writes without an in-frame index) fall back
|
|
144
195
|
// to the wall clock.
|
|
145
196
|
private allocStart(series: Series): TimeStamp {
|
|
@@ -148,28 +199,41 @@ export class Dynamic {
|
|
|
148
199
|
|
|
149
200
|
private allocCurr(
|
|
150
201
|
res: WriteResponse,
|
|
151
|
-
alignment: bigint,
|
|
152
|
-
start: TimeStamp,
|
|
153
202
|
source: Series,
|
|
154
|
-
|
|
203
|
+
start: TimeStamp | null,
|
|
155
204
|
): Series {
|
|
156
205
|
this.curr = this.allocate(
|
|
157
|
-
this.
|
|
158
|
-
alignment,
|
|
159
|
-
start,
|
|
206
|
+
this.capacityFor(source),
|
|
207
|
+
source.alignment,
|
|
208
|
+
start ?? this.allocStart(source),
|
|
160
209
|
source,
|
|
161
|
-
sampleIndex,
|
|
162
210
|
);
|
|
163
211
|
res.allocated.push(this.curr);
|
|
164
212
|
return this.curr;
|
|
165
213
|
}
|
|
166
214
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
215
|
+
/** @returns the timestamp the flushed buffer's end was stamped with, or null when
|
|
216
|
+
* there was no buffer. */
|
|
217
|
+
private flushCurr(res: WriteResponse): TimeStamp | null {
|
|
218
|
+
if (this.curr == null) return null;
|
|
219
|
+
const end = this.currDataEnd ?? this.now();
|
|
220
|
+
this.curr.timeRange.end = end;
|
|
170
221
|
this.currDataEnd = null;
|
|
171
|
-
res.flushed.push(this.curr);
|
|
222
|
+
res.flushed.push(this.compacted(this.curr, res));
|
|
172
223
|
this.curr = null;
|
|
224
|
+
return end;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* @returns a right-sized copy of a buffer that rotated with most of its space
|
|
229
|
+
* unused, or the buffer itself. A rotated buffer holds its whole allocation until
|
|
230
|
+
* it is collected, which is wasteful when an oversized series ended it early.
|
|
231
|
+
*/
|
|
232
|
+
private compacted(buffer: Series, res: WriteResponse): Series {
|
|
233
|
+
// A buffer allocated during this same write has not reached its subscribers yet,
|
|
234
|
+
// so its reference count cannot yet report who is about to hold it.
|
|
235
|
+
if (!shouldCompact(buffer) || res.allocated.series.includes(buffer)) return buffer;
|
|
236
|
+
return buffer.compact();
|
|
173
237
|
}
|
|
174
238
|
|
|
175
239
|
private _write(series: Series, res: WriteResponse): void {
|
|
@@ -194,9 +258,8 @@ export class Dynamic {
|
|
|
194
258
|
}
|
|
195
259
|
}
|
|
196
260
|
let curr = this.curr;
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
else {
|
|
261
|
+
let trimmed = false;
|
|
262
|
+
if (curr != null) {
|
|
200
263
|
// overlap > 0: the incoming series steps back into samples the current buffer
|
|
201
264
|
// already holds. overlap < 0: there is a gap between the buffer and the series.
|
|
202
265
|
const overlap = Number(curr.alignment + BigInt(curr.length) - series.alignment);
|
|
@@ -205,38 +268,36 @@ export class Dynamic {
|
|
|
205
268
|
// the cache with overlapping series. sub() is zero-copy.
|
|
206
269
|
if (overlap >= series.length) return;
|
|
207
270
|
series = series.sub(overlap);
|
|
271
|
+
trimmed = true;
|
|
208
272
|
} else if (overlap < 0) {
|
|
209
273
|
this.flushCurr(res);
|
|
210
|
-
curr =
|
|
211
|
-
res,
|
|
212
|
-
series.alignment,
|
|
213
|
-
this.allocStart(series),
|
|
214
|
-
series,
|
|
215
|
-
0,
|
|
216
|
-
);
|
|
274
|
+
curr = null;
|
|
217
275
|
}
|
|
218
276
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
277
|
+
const convert = (buffer: Series): Series =>
|
|
278
|
+
transform.convert(series, buffer.sampleOffset);
|
|
279
|
+
let rotationStart: TimeStamp | null = null;
|
|
280
|
+
if (curr != null) {
|
|
281
|
+
const converted = convert(curr);
|
|
282
|
+
// A series never splits across two buffers. The timestamp where a split falls is
|
|
283
|
+
// not in the data, so both sides would have to guess the boundary they share, and
|
|
284
|
+
// a guess lands outside the data whenever samples are irregular.
|
|
285
|
+
if (fits(curr, converted)) curr.write(converted);
|
|
286
|
+
else {
|
|
287
|
+
const end = this.flushCurr(res);
|
|
288
|
+
// A trimmed series keeps the whole frame's start, which is earlier than the
|
|
289
|
+
// samples it still holds. Those samples begin where the buffer they were
|
|
290
|
+
// trimmed against ends, so the next buffer starts there.
|
|
291
|
+
if (trimmed) rotationStart = end;
|
|
292
|
+
curr = null;
|
|
226
293
|
}
|
|
227
|
-
// The timestamp at the split point is unknowable from the data series, so
|
|
228
|
-
// both sides fall back to the wall clock.
|
|
229
|
-
this.currDataEnd = null;
|
|
230
|
-
this.flushCurr(res);
|
|
231
|
-
curr = this.allocCurr(
|
|
232
|
-
res,
|
|
233
|
-
series.alignment + BigInt(amountWritten),
|
|
234
|
-
this.now(),
|
|
235
|
-
series,
|
|
236
|
-
amountWritten,
|
|
237
|
-
);
|
|
238
|
-
series = series.slice(amountWritten);
|
|
239
294
|
}
|
|
295
|
+
if (curr == null) {
|
|
296
|
+
curr = this.allocCurr(res, series, rotationStart);
|
|
297
|
+
curr.write(convert(curr));
|
|
298
|
+
}
|
|
299
|
+
this.currDataEnd = series.timeRange.isZero ? null : series.timeRange.end;
|
|
300
|
+
this.updateAvgRate(series);
|
|
240
301
|
}
|
|
241
302
|
|
|
242
303
|
private updateAvgRate(series: Series): void {
|
|
@@ -7,11 +7,12 @@
|
|
|
7
7
|
// License, use of this software will be governed by the Apache License, Version 2.0,
|
|
8
8
|
// included in the file licenses/APL.txt.
|
|
9
9
|
|
|
10
|
+
import { Unreachable } from "@synnaxlabs/freighter";
|
|
10
11
|
import {
|
|
11
12
|
debounce,
|
|
13
|
+
errors,
|
|
12
14
|
MultiSeries,
|
|
13
15
|
type Series,
|
|
14
|
-
sync,
|
|
15
16
|
TimeRange,
|
|
16
17
|
TimeSpan,
|
|
17
18
|
} from "@synnaxlabs/x";
|
|
@@ -31,10 +32,18 @@ interface ReadRequest {
|
|
|
31
32
|
gaps: TimeRange[];
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
// Tracks one read's outstanding fetches, so the read settles when its last fetch lands.
|
|
36
|
+
// The first failure among them wins.
|
|
37
|
+
interface Tracker {
|
|
38
|
+
entry: debounce.Entry<ReadRequest>;
|
|
39
|
+
remaining: number;
|
|
40
|
+
failure?: unknown;
|
|
41
|
+
}
|
|
42
|
+
|
|
34
43
|
interface BatchFetch {
|
|
35
44
|
gap: TimeRange;
|
|
36
45
|
channels: Set<channel.Key>;
|
|
37
|
-
|
|
46
|
+
trackers: Set<Tracker>;
|
|
38
47
|
}
|
|
39
48
|
|
|
40
49
|
export interface ReaderProps {
|
|
@@ -53,17 +62,25 @@ export interface ReaderProps {
|
|
|
53
62
|
* @default TimeSpan.milliseconds(5)
|
|
54
63
|
*/
|
|
55
64
|
overlapThreshold?: TimeSpan;
|
|
65
|
+
/**
|
|
66
|
+
* Deadline for a single remote fetch. A fetch that does not settle within it rejects
|
|
67
|
+
* the reads it serves with an Unreachable error.
|
|
68
|
+
* @default TimeSpan.seconds(30)
|
|
69
|
+
*/
|
|
70
|
+
fetchTimeout?: TimeSpan;
|
|
56
71
|
}
|
|
57
72
|
|
|
58
73
|
/**
|
|
59
74
|
* Reads historical telemetry, serving from the cache and batch-filling gaps from the
|
|
60
|
-
* server. A
|
|
75
|
+
* server. A fetch failure rejects only the reads whose gaps were in the failed fetch.
|
|
61
76
|
*/
|
|
62
77
|
export class Reader {
|
|
63
78
|
private readonly props: Required<Omit<ReaderProps, "batchDebounce">>;
|
|
64
79
|
private readonly batcher: debounce.Batcher<ReadRequest>;
|
|
65
|
-
|
|
66
|
-
|
|
80
|
+
private closed = false;
|
|
81
|
+
// Reads whose fetches are in flight, so close can reject them without waiting on the
|
|
82
|
+
// fetches themselves.
|
|
83
|
+
private readonly inFlight = new Set<Tracker>();
|
|
67
84
|
|
|
68
85
|
constructor(props: ReaderProps) {
|
|
69
86
|
const {
|
|
@@ -71,90 +88,144 @@ export class Reader {
|
|
|
71
88
|
cache,
|
|
72
89
|
batchDebounce = TimeSpan.milliseconds(50),
|
|
73
90
|
overlapThreshold = TimeSpan.milliseconds(5),
|
|
91
|
+
fetchTimeout = TimeSpan.seconds(30),
|
|
74
92
|
} = props;
|
|
75
|
-
this.props = { readRemote, cache, overlapThreshold };
|
|
93
|
+
this.props = { readRemote, cache, overlapThreshold, fetchTimeout };
|
|
76
94
|
this.batcher = new debounce.Batcher({
|
|
77
95
|
interval: batchDebounce,
|
|
78
|
-
exec: async (entries) =>
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}),
|
|
96
|
+
exec: async (entries) => {
|
|
97
|
+
if (this.closed) throw new UnexpectedError("telemetry reader is closed");
|
|
98
|
+
await this.batchRead(entries);
|
|
99
|
+
},
|
|
83
100
|
});
|
|
84
101
|
}
|
|
85
102
|
|
|
86
103
|
/**
|
|
87
104
|
* Reads the given time range for the given channel.
|
|
88
105
|
* @throws {UnexpectedError} if the reader is closed while the read is pending.
|
|
106
|
+
* @throws {Unreachable} if a fetch serving the read exceeds the fetch timeout.
|
|
89
107
|
*/
|
|
90
108
|
async read(tr: TimeRange, channel: channel.Key): Promise<MultiSeries> {
|
|
91
109
|
const { cache } = this.props;
|
|
92
110
|
const unary = cache.get(channel);
|
|
93
111
|
const { series, gaps } = unary.read(tr);
|
|
94
112
|
if (gaps.length === 0) return series;
|
|
95
|
-
if (this.
|
|
113
|
+
if (this.closed) throw new UnexpectedError("telemetry reader is closed");
|
|
96
114
|
await this.batcher.enqueue({ channel, gaps });
|
|
97
115
|
return unary.read(tr).series;
|
|
98
116
|
}
|
|
99
117
|
|
|
100
118
|
private async batchRead(entries: Array<debounce.Entry<ReadRequest>>): Promise<void> {
|
|
101
|
-
const {
|
|
119
|
+
const { cache, overlapThreshold } = this.props;
|
|
102
120
|
const batched: BatchFetch[] = [];
|
|
121
|
+
const trackers = new Map<debounce.Entry<ReadRequest>, Tracker>();
|
|
103
122
|
entries.forEach((entry) => {
|
|
104
123
|
const unary = cache.get(entry.req.channel);
|
|
105
124
|
entry.req.gaps.forEach((rawGap) => {
|
|
106
|
-
//
|
|
107
|
-
//
|
|
125
|
+
// An earlier batch or a streaming write may have filled part of the gap while
|
|
126
|
+
// this entry sat in the debounce window.
|
|
108
127
|
unary.read(rawGap).gaps.forEach((gap) => {
|
|
128
|
+
let tracker = trackers.get(entry);
|
|
129
|
+
if (tracker == null) {
|
|
130
|
+
tracker = { entry, remaining: 0 };
|
|
131
|
+
trackers.set(entry, tracker);
|
|
132
|
+
this.inFlight.add(tracker);
|
|
133
|
+
}
|
|
109
134
|
const g = batched.find((r) => r.gap.equals(gap, overlapThreshold));
|
|
110
|
-
if (g == null)
|
|
135
|
+
if (g == null) {
|
|
111
136
|
batched.push({
|
|
112
137
|
gap,
|
|
113
138
|
channels: new Set([entry.req.channel]),
|
|
114
|
-
|
|
139
|
+
trackers: new Set([tracker]),
|
|
115
140
|
});
|
|
116
|
-
|
|
141
|
+
tracker.remaining++;
|
|
142
|
+
} else {
|
|
117
143
|
g.channels.add(entry.req.channel);
|
|
118
144
|
g.gap = TimeRange.max(g.gap, gap);
|
|
119
|
-
g.
|
|
145
|
+
if (!g.trackers.has(tracker)) {
|
|
146
|
+
g.trackers.add(tracker);
|
|
147
|
+
tracker.remaining++;
|
|
148
|
+
}
|
|
120
149
|
}
|
|
121
150
|
});
|
|
122
151
|
});
|
|
123
152
|
});
|
|
124
|
-
const failures = new Map<debounce.Entry<ReadRequest>, unknown>();
|
|
125
|
-
await Promise.all(
|
|
126
|
-
batched.map(async ({ gap, channels, entries: served }) => {
|
|
127
|
-
try {
|
|
128
|
-
const frame = await readRemote(gap, Array.from(channels));
|
|
129
|
-
// Group series by key in one pass; per-key get() scans the whole frame.
|
|
130
|
-
const grouped = new Map<channel.Key, Series[]>();
|
|
131
|
-
frame.forEach((k, s) => {
|
|
132
|
-
const key = k as channel.Key;
|
|
133
|
-
const existing = grouped.get(key);
|
|
134
|
-
if (existing == null) grouped.set(key, [s]);
|
|
135
|
-
else existing.push(s);
|
|
136
|
-
});
|
|
137
|
-
channels.forEach((key) =>
|
|
138
|
-
cache.get(key).writeStatic(new MultiSeries(grouped.get(key) ?? [])),
|
|
139
|
-
);
|
|
140
|
-
} catch (err) {
|
|
141
|
-
served.forEach((entry) => {
|
|
142
|
-
if (!failures.has(entry)) failures.set(entry, err);
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
}),
|
|
146
|
-
);
|
|
147
153
|
entries.forEach((entry) => {
|
|
148
|
-
|
|
149
|
-
if (err == null) entry.resolve();
|
|
150
|
-
else entry.reject(err);
|
|
154
|
+
if (!trackers.has(entry)) entry.resolve();
|
|
151
155
|
});
|
|
156
|
+
await Promise.all(batched.map(async (b) => await this.fetchGap(b)));
|
|
152
157
|
}
|
|
153
158
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
159
|
+
// Never throws: every outcome, including a deadline expiry and a post-close arrival,
|
|
160
|
+
// settles through the trackers.
|
|
161
|
+
private async fetchGap({ gap, channels, trackers }: BatchFetch): Promise<void> {
|
|
162
|
+
const { cache } = this.props;
|
|
163
|
+
let failure: unknown;
|
|
164
|
+
try {
|
|
165
|
+
const frame = await this.fetchWithDeadline(gap, Array.from(channels));
|
|
166
|
+
if (!this.closed) {
|
|
167
|
+
// Group series by key in one pass; per-key get() scans the whole frame.
|
|
168
|
+
const grouped = new Map<channel.Key, Series[]>();
|
|
169
|
+
frame.forEach((k, s) => {
|
|
170
|
+
const key = k as channel.Key;
|
|
171
|
+
const existing = grouped.get(key);
|
|
172
|
+
if (existing == null) grouped.set(key, [s]);
|
|
173
|
+
else existing.push(s);
|
|
174
|
+
});
|
|
175
|
+
channels.forEach((key) =>
|
|
176
|
+
cache.get(key).writeStatic(new MultiSeries(grouped.get(key) ?? [])),
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
} catch (err) {
|
|
180
|
+
failure = err;
|
|
181
|
+
}
|
|
182
|
+
trackers.forEach((tracker) => {
|
|
183
|
+
// close() already settled it.
|
|
184
|
+
if (!this.inFlight.has(tracker)) return;
|
|
185
|
+
if (failure != null) tracker.failure ??= failure;
|
|
186
|
+
tracker.remaining--;
|
|
187
|
+
if (tracker.remaining > 0) return;
|
|
188
|
+
this.inFlight.delete(tracker);
|
|
189
|
+
if (tracker.failure == null) tracker.entry.resolve();
|
|
190
|
+
else tracker.entry.reject(tracker.failure);
|
|
158
191
|
});
|
|
159
192
|
}
|
|
193
|
+
|
|
194
|
+
private async fetchWithDeadline(
|
|
195
|
+
gap: TimeRange,
|
|
196
|
+
channels: channel.Key[],
|
|
197
|
+
): Promise<Frame> {
|
|
198
|
+
const { readRemote, fetchTimeout } = this.props;
|
|
199
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
200
|
+
const deadline = new Promise<never>((_, reject) => {
|
|
201
|
+
timer = setTimeout(() => {
|
|
202
|
+
const timeout = fetchTimeout.toString();
|
|
203
|
+
const message = `gap read for ${gap.toString()} timed out after ${timeout}`;
|
|
204
|
+
reject(new Unreachable({ message }));
|
|
205
|
+
}, fetchTimeout.milliseconds);
|
|
206
|
+
});
|
|
207
|
+
const fetched = readRemote(gap, channels);
|
|
208
|
+
try {
|
|
209
|
+
return await Promise.race([fetched, deadline]);
|
|
210
|
+
} catch (err) {
|
|
211
|
+
// The read already failed for its callers; a late settle of the losing fetch must
|
|
212
|
+
// not surface as an unhandled rejection.
|
|
213
|
+
fetched.catch(() => {});
|
|
214
|
+
throw errors.fromUnknown(err);
|
|
215
|
+
} finally {
|
|
216
|
+
clearTimeout(timer);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Closes the reader, rejecting every pending read with an UnexpectedError. In-flight
|
|
222
|
+
* fetch results are discarded when they arrive.
|
|
223
|
+
*/
|
|
224
|
+
async close(): Promise<void> {
|
|
225
|
+
this.closed = true;
|
|
226
|
+
const err = new UnexpectedError("telemetry reader is closed");
|
|
227
|
+
this.batcher.close(err);
|
|
228
|
+
this.inFlight.forEach(({ entry }) => entry.reject(err));
|
|
229
|
+
this.inFlight.clear();
|
|
230
|
+
}
|
|
160
231
|
}
|
|
@@ -71,17 +71,21 @@ export class Unary {
|
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
* Reads cached data overlapping the given time range. The result includes the
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
74
|
+
* Reads cached data overlapping the given time range. The result includes the live
|
|
75
|
+
* leading buffer when the data it actually holds overlaps the range, judged by the
|
|
76
|
+
* last stamped write rather than the buffer's provisional MAX end, so the buffer
|
|
77
|
+
* never leaks into reads of spans it has no samples for. It never claims coverage
|
|
78
|
+
* either: its samples carry provisional leading alignments that cannot pair with
|
|
79
|
+
* fetched data on another channel, so its span is still reported as a gap to fetch. A
|
|
78
80
|
* span may therefore return in both its streamed and fetched representation.
|
|
79
81
|
*/
|
|
80
82
|
read(tr: TimeRange): DirtyReadResult {
|
|
81
83
|
this.checkOpen("read");
|
|
82
84
|
const res = this.static.dirtyRead(tr);
|
|
83
85
|
const buf = this.dynamic.leadingBuffer;
|
|
84
|
-
|
|
86
|
+
const dataTr = this.dynamic.dataTimeRange;
|
|
87
|
+
if (buf == null || dataTr == null || buf.length === 0 || !dataTr.overlapsWith(tr))
|
|
88
|
+
return res;
|
|
85
89
|
res.series.push(buf);
|
|
86
90
|
return res;
|
|
87
91
|
}
|
package/src/index.ts
CHANGED
package/src/label/store.ts
CHANGED
|
@@ -21,8 +21,13 @@ export const matchLabeledBy = (rel: ontology.Relationship, id: ontology.ID): boo
|
|
|
21
21
|
|
|
22
22
|
/** Returns the cached labels attached to the given ontology ID. */
|
|
23
23
|
export const cachedLabelsOf = (
|
|
24
|
-
|
|
24
|
+
cache: ontology.Cache,
|
|
25
25
|
labels: query.Table<Key, Label>,
|
|
26
26
|
id: ontology.ID,
|
|
27
27
|
): Label[] =>
|
|
28
|
-
labels.get(
|
|
28
|
+
labels.get(
|
|
29
|
+
cache
|
|
30
|
+
.relationshipsFrom(id)
|
|
31
|
+
.filter((r) => r.type === LABELED_BY_ONTOLOGY_RELATIONSHIP_TYPE)
|
|
32
|
+
.map((r) => r.to.key),
|
|
33
|
+
);
|
package/src/ontology/client.ts
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
Cache,
|
|
36
36
|
RELATIONSHIP_DELETE_CHANNEL_NAME,
|
|
37
37
|
RELATIONSHIP_SET_CHANNEL_NAME,
|
|
38
|
+
type RelationshipIndexes,
|
|
38
39
|
RESOURCE_DELETE_CHANNEL_NAME,
|
|
39
40
|
RESOURCE_SET_CHANNEL_NAME,
|
|
40
41
|
} from "@/ontology/store";
|
|
@@ -194,10 +195,15 @@ export class Client extends query.Retriever<
|
|
|
194
195
|
|
|
195
196
|
constructor(cfg: ClientConfig) {
|
|
196
197
|
const { unary, cache } = cfg;
|
|
198
|
+
const relationshipIndexes: RelationshipIndexes = {
|
|
199
|
+
byTo: new query.LookupIndex<string, Relationship>((r) => idToString(r.to)),
|
|
200
|
+
byFrom: new query.LookupIndex<string, Relationship>((r) => idToString(r.from)),
|
|
201
|
+
};
|
|
197
202
|
const relationships = cache.createTable<string, Relationship>({
|
|
198
203
|
name: "relationships",
|
|
199
204
|
equal: (a, b) =>
|
|
200
205
|
idsEqual(a.from, b.from) && idsEqual(a.to, b.to) && a.type === b.type,
|
|
206
|
+
indexes: [relationshipIndexes.byTo, relationshipIndexes.byFrom],
|
|
201
207
|
listen: [
|
|
202
208
|
query.createSetListener(RELATIONSHIP_SET_CHANNEL_NAME, relationshipZ, {
|
|
203
209
|
key: (changed) => relationshipToString(changed),
|
|
@@ -244,7 +250,7 @@ export class Client extends query.Retriever<
|
|
|
244
250
|
});
|
|
245
251
|
this.cfg = cfg;
|
|
246
252
|
this.writer = new Writer(unary);
|
|
247
|
-
this.cache = new Cache(relationships, resources);
|
|
253
|
+
this.cache = new Cache(relationships, resources, relationshipIndexes);
|
|
248
254
|
this.children = this.dependentSurface(cache, "children", "to");
|
|
249
255
|
this.parents = this.dependentSurface(cache, "parents", "from");
|
|
250
256
|
}
|
|
@@ -466,13 +472,14 @@ export class Client extends query.Retriever<
|
|
|
466
472
|
): boolean {
|
|
467
473
|
if (q.types != null && !q.types.includes(resource.id.type)) return false;
|
|
468
474
|
const anchor = oppositeRelationshipDirection(direction);
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
(
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
475
|
+
const rels =
|
|
476
|
+
direction === "to"
|
|
477
|
+
? this.cache.relationshipsTo(resource.id)
|
|
478
|
+
: this.cache.relationshipsFrom(resource.id);
|
|
479
|
+
return rels.some(
|
|
480
|
+
(rel) =>
|
|
481
|
+
rel.type === PARENT_OF_RELATIONSHIP_TYPE &&
|
|
482
|
+
q.ids.includes(idToString(rel[anchor])),
|
|
476
483
|
);
|
|
477
484
|
}
|
|
478
485
|
|
package/src/ontology/store.ts
CHANGED
|
@@ -13,7 +13,6 @@ import {
|
|
|
13
13
|
type ID,
|
|
14
14
|
idsEqual,
|
|
15
15
|
idToString,
|
|
16
|
-
matchRelationship,
|
|
17
16
|
PARENT_OF_RELATIONSHIP_TYPE,
|
|
18
17
|
type Relationship,
|
|
19
18
|
type Resource,
|
|
@@ -26,26 +25,49 @@ export const RESOURCE_DELETE_CHANNEL_NAME = "sy_ontology_resource_delete";
|
|
|
26
25
|
export const RELATIONSHIP_SET_CHANNEL_NAME = "sy_ontology_relationship_set";
|
|
27
26
|
export const RELATIONSHIP_DELETE_CHANNEL_NAME = "sy_ontology_relationship_delete";
|
|
28
27
|
|
|
28
|
+
/**
|
|
29
|
+
* The secondary indexes registered on the relationships table, one per endpoint.
|
|
30
|
+
* Type filtering runs at the call site over the matched handful.
|
|
31
|
+
*/
|
|
32
|
+
export interface RelationshipIndexes {
|
|
33
|
+
/** Keyed by the stringified `to` end. */
|
|
34
|
+
byTo: query.LookupIndex<string, Relationship>;
|
|
35
|
+
/** Keyed by the stringified `from` end. */
|
|
36
|
+
byFrom: query.LookupIndex<string, Relationship>;
|
|
37
|
+
}
|
|
38
|
+
|
|
29
39
|
/** The ontology record tables: resources and the relationships between them. */
|
|
30
40
|
export class Cache {
|
|
31
41
|
readonly relationships: query.Table<string, Relationship>;
|
|
32
42
|
readonly resources: query.Table<string, Resource>;
|
|
43
|
+
private readonly indexes: RelationshipIndexes;
|
|
33
44
|
|
|
34
45
|
constructor(
|
|
35
46
|
relationships: query.Table<string, Relationship>,
|
|
36
47
|
resources: query.Table<string, Resource>,
|
|
48
|
+
indexes: RelationshipIndexes,
|
|
37
49
|
) {
|
|
38
50
|
this.relationships = relationships;
|
|
39
51
|
this.resources = resources;
|
|
52
|
+
this.indexes = indexes;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Returns the cached relationships pointing at the given ID. */
|
|
56
|
+
relationshipsTo(id: ID): Relationship[] {
|
|
57
|
+
return this.indexes.byTo.get(idToString(id));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Returns the cached relationships originating at the given ID. */
|
|
61
|
+
relationshipsFrom(id: ID): Relationship[] {
|
|
62
|
+
return this.indexes.byFrom.get(idToString(id));
|
|
40
63
|
}
|
|
41
64
|
|
|
42
65
|
/** Returns the cached parent ID of the given ontology ID, or null if unknown. */
|
|
43
66
|
parentID(id: ID): ID | null {
|
|
44
|
-
const
|
|
45
|
-
|
|
67
|
+
const rel = this.relationshipsTo(id).find(
|
|
68
|
+
(r) => r.type === PARENT_OF_RELATIONSHIP_TYPE,
|
|
46
69
|
);
|
|
47
|
-
|
|
48
|
-
return res[0].from;
|
|
70
|
+
return rel?.from ?? null;
|
|
49
71
|
}
|
|
50
72
|
|
|
51
73
|
/**
|
package/src/query/external.ts
CHANGED