@cotal-ai/core 0.1.0 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts +8 -0
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +62 -2
- package/dist/agent-file.js.map +1 -1
- package/dist/channels.d.ts +68 -0
- package/dist/channels.d.ts.map +1 -0
- package/dist/channels.js +140 -0
- package/dist/channels.js.map +1 -0
- package/dist/endpoint.d.ts +127 -3
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +513 -30
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +36 -4
- package/dist/provision.js.map +1 -1
- package/dist/runtime.d.ts +66 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +2 -0
- package/dist/runtime.js.map +1 -0
- package/dist/spaces.d.ts +25 -0
- package/dist/spaces.d.ts.map +1 -0
- package/dist/spaces.js +96 -0
- package/dist/spaces.js.map +1 -0
- package/dist/streams.d.ts +16 -0
- package/dist/streams.d.ts.map +1 -1
- package/dist/streams.js +35 -5
- package/dist/streams.js.map +1 -1
- package/dist/subjects.d.ts +11 -0
- package/dist/subjects.d.ts.map +1 -1
- package/dist/subjects.js +13 -2
- package/dist/subjects.js.map +1 -1
- package/dist/types.d.ts +37 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +6 -1
package/dist/endpoint.js
CHANGED
|
@@ -2,11 +2,14 @@ import { EventEmitter } from "node:events";
|
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { connect, credsAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, } from "@nats-io/transport-node";
|
|
4
4
|
import { idFromCreds } from "./identity.js";
|
|
5
|
-
import { createSpaceStreams, dmDurableConfig, taskDurableConfig } from "./streams.js";
|
|
5
|
+
import { createSpaceStreams, dmDurableConfig, taskDurableConfig, MAX_MSGS_PER_SUBJECT } from "./streams.js";
|
|
6
6
|
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, } from "@nats-io/jetstream";
|
|
7
7
|
import { Kvm } from "@nats-io/kv";
|
|
8
|
-
import {
|
|
8
|
+
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, readChannelConfig, readChannelDefaults, } from "./channels.js";
|
|
9
|
+
import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatDurable, chatSubject, collapseFilterSubjects, controlServiceSubject, dmStream, dmDurable, isConcreteChannel, normalizeMentions, parseSubject, presenceBucket, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, } from "./subjects.js";
|
|
9
10
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
11
|
+
/** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
|
|
12
|
+
export const DEFAULT_SPACE = "main";
|
|
10
13
|
/**
|
|
11
14
|
* Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error).
|
|
12
15
|
*
|
|
@@ -36,6 +39,15 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
36
39
|
js;
|
|
37
40
|
jsm;
|
|
38
41
|
kv;
|
|
42
|
+
channelKv;
|
|
43
|
+
/** Live local cache of the channel registry (key = channel token), kept by a KV watch. */
|
|
44
|
+
channelConfigs = new Map();
|
|
45
|
+
channelDefaults = {};
|
|
46
|
+
/** Per-subscription join watermark: the stream frontier captured when a channel was joined.
|
|
47
|
+
* The tail ack-drops chat messages with `seq <= watermark` (suppresses pre-join history for
|
|
48
|
+
* a lagging joiner + dedups the backfill overlap). Keyed by the subscription pattern (may be
|
|
49
|
+
* wildcard), so the drop matches every concrete channel the pattern subsumes. */
|
|
50
|
+
joinSeq = new Map();
|
|
39
51
|
subs = [];
|
|
40
52
|
streamMsgs = [];
|
|
41
53
|
heartbeatTimer;
|
|
@@ -100,6 +112,14 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
100
112
|
await this.startPresenceWatch();
|
|
101
113
|
this.sweepTimer = setInterval(() => this.sweep(), Math.max(500, Math.floor(this.ttlMs / 3)));
|
|
102
114
|
}
|
|
115
|
+
// Open the channel registry bucket when we either watch it (live cache for the connector's
|
|
116
|
+
// pull/display) or consume (the join-time replay decision reads it fresh). Auth mode OPENs
|
|
117
|
+
// the bucket pre-created at `cotal up`; open mode lazily creates it.
|
|
118
|
+
if (this.doWatch || this.doConsume) {
|
|
119
|
+
this.channelKv = await openChannelRegistry(this.nc, this.space, { create: !this.creds });
|
|
120
|
+
if (this.doWatch)
|
|
121
|
+
await this.startChannelWatch();
|
|
122
|
+
}
|
|
103
123
|
if (this.doRegister) {
|
|
104
124
|
await this.publishPresence();
|
|
105
125
|
this.heartbeatTimer = setInterval(() => {
|
|
@@ -283,32 +303,151 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
283
303
|
await this.publishPresence();
|
|
284
304
|
}
|
|
285
305
|
// ---- channel discovery ---------------------------------------------------
|
|
286
|
-
/**
|
|
287
|
-
|
|
306
|
+
/** This channel's registry config from the live local cache (undefined if unset). */
|
|
307
|
+
getChannelConfig(channel) {
|
|
308
|
+
return this.channelConfigs.get(channel);
|
|
309
|
+
}
|
|
310
|
+
/** Effective replay-on-join policy for a channel: per-channel override ?? space default ??
|
|
311
|
+
* true. Reads the live cache, so it reflects runtime registry edits. */
|
|
312
|
+
channelReplay(channel) {
|
|
313
|
+
return effectiveReplay(this.channelConfigs.get(channel), this.channelDefaults);
|
|
314
|
+
}
|
|
315
|
+
// ---- dynamic subscription (join / leave mid-session) ---------------------
|
|
316
|
+
/** The channels this endpoint is currently subscribed to (live — reflects join/leave). */
|
|
317
|
+
joinedChannels() {
|
|
318
|
+
return [...this.channels];
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* Join a channel mid-session: add it to our chat durable's `filter_subjects` (same durable,
|
|
322
|
+
* same ack-floor, no teardown — `update` rides the self-scoped create grant), capture the
|
|
323
|
+
* stream frontier as this channel's join watermark, and backfill its history if replay is on.
|
|
324
|
+
* Idempotent: re-joining a channel already in our filter is a no-op (no re-backfill). Returns
|
|
325
|
+
* the number of historical messages backfilled (emitted as `historical` "message" events).
|
|
326
|
+
*/
|
|
327
|
+
async joinChannel(channel) {
|
|
328
|
+
if (!this.jsm)
|
|
329
|
+
throw new Error("endpoint not started");
|
|
330
|
+
if (this.channels.includes(channel))
|
|
331
|
+
return { joined: false, backfilled: 0 };
|
|
332
|
+
const next = collapseFilterSubjects([...this.channels, channel].map((ch) => chatSubject(this.space, "*", ch)));
|
|
333
|
+
// Arm the watermark BEFORE the filter flip (single-delivery: a tail message on the new
|
|
334
|
+
// channel is then either ≤ frontier → backfill-only or > frontier → tail-only, never both),
|
|
335
|
+
// and filter BEFORE backfill (gap-safe: backfill-first leaves a window in neither stream).
|
|
336
|
+
const armed = await this.armJoin([channel]);
|
|
337
|
+
await this.jsm.consumers.update(chatStream(this.space), chatDurable(this.card.id), {
|
|
338
|
+
filter_subjects: next,
|
|
339
|
+
});
|
|
340
|
+
this.channels.push(channel);
|
|
341
|
+
const backfilled = await this.backfillArmed(armed);
|
|
342
|
+
return { joined: true, backfilled };
|
|
343
|
+
}
|
|
344
|
+
/** Leave a channel mid-session: drop it from the durable's `filter_subjects`. Refuses to leave
|
|
345
|
+
* the *last* channel (an empty filter would match every chat subject — the opposite of
|
|
346
|
+
* leaving). Returns whether anything changed. */
|
|
347
|
+
async leaveChannel(channel) {
|
|
348
|
+
if (!this.jsm)
|
|
349
|
+
throw new Error("endpoint not started");
|
|
350
|
+
const i = this.channels.indexOf(channel);
|
|
351
|
+
if (i < 0)
|
|
352
|
+
return { left: false };
|
|
353
|
+
if (this.channels.length === 1)
|
|
354
|
+
throw new Error(`cannot leave "${channel}" — it is your only channel (an empty filter would subscribe to all)`);
|
|
355
|
+
const remaining = this.channels.filter((c) => c !== channel);
|
|
356
|
+
await this.jsm.consumers.update(chatStream(this.space), chatDurable(this.card.id), {
|
|
357
|
+
filter_subjects: collapseFilterSubjects(remaining.map((ch) => chatSubject(this.space, "*", ch))),
|
|
358
|
+
});
|
|
359
|
+
this.channels.splice(i, 1);
|
|
360
|
+
this.joinSeq.delete(channel);
|
|
361
|
+
return { left: true };
|
|
362
|
+
}
|
|
363
|
+
/** One coherent channel model for dashboards: every channel that has messages OR a registry
|
|
364
|
+
* entry (configured-but-empty), each tagged with its {@link ChannelConfig}. Works even on
|
|
365
|
+
* observer endpoints (no consumers needed). */
|
|
288
366
|
async listChannels() {
|
|
289
367
|
if (!this.nc)
|
|
290
368
|
throw new Error("endpoint not started");
|
|
291
369
|
const mgr = await jetstreamManager(this.nc);
|
|
292
|
-
|
|
370
|
+
// Subjects carry the sender (chat.<sender>.<channel>), so collapse across senders: sum
|
|
371
|
+
// each channel's counts regardless of who published.
|
|
372
|
+
const counts = new Map();
|
|
293
373
|
try {
|
|
294
|
-
info = await mgr.streams.info(chatStream(this.space), { subjects_filter: ">" });
|
|
374
|
+
const info = await mgr.streams.info(chatStream(this.space), { subjects_filter: ">" });
|
|
375
|
+
if (info.state.subjects) {
|
|
376
|
+
for (const [subject, count] of Object.entries(info.state.subjects)) {
|
|
377
|
+
const p = parseSubject(subject);
|
|
378
|
+
if (p?.kind === "chat")
|
|
379
|
+
counts.set(p.rest, (counts.get(p.rest) ?? 0) + count);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
295
382
|
}
|
|
296
383
|
catch {
|
|
297
|
-
|
|
384
|
+
/* stream missing — fall through to registry-only channels */
|
|
298
385
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
386
|
+
const channels = new Set([...counts.keys(), ...this.channelConfigs.keys()]);
|
|
387
|
+
return [...channels]
|
|
388
|
+
.map((channel) => ({
|
|
389
|
+
channel,
|
|
390
|
+
messages: counts.get(channel) ?? 0,
|
|
391
|
+
config: this.channelConfigs.get(channel),
|
|
392
|
+
}))
|
|
393
|
+
.sort((a, b) => a.channel.localeCompare(b.channel));
|
|
394
|
+
}
|
|
395
|
+
async channelMembers(channel) {
|
|
396
|
+
const mgr = await this.manager();
|
|
397
|
+
// Group channel patterns by each consumer's durable id-token (chat_<id> → token(id)).
|
|
398
|
+
// One peer has one chat consumer, so this is a straight per-peer collection; join/leave
|
|
399
|
+
// just mutates that consumer's filter_subjects, which the next call re-reads live.
|
|
400
|
+
const byTok = new Map();
|
|
401
|
+
for await (const ci of mgr.consumers.list(chatStream(this.space))) {
|
|
402
|
+
const tok = chatDurableToken(ci.config.durable_name ?? ci.name);
|
|
403
|
+
if (tok === null)
|
|
404
|
+
continue;
|
|
405
|
+
// The server may report a single filter as `filter_subject` or `filter_subjects` — both
|
|
406
|
+
// are the same datum; read whichever is present. Filters are already collapsed (the
|
|
407
|
+
// effective subscription), so parse the channel straight out of each.
|
|
408
|
+
const filters = ci.config.filter_subjects ?? (ci.config.filter_subject ? [ci.config.filter_subject] : []);
|
|
409
|
+
const set = byTok.get(tok) ?? new Set();
|
|
410
|
+
for (const f of filters) {
|
|
411
|
+
const p = parseSubject(f);
|
|
305
412
|
if (p?.kind === "chat")
|
|
306
|
-
|
|
413
|
+
set.add(p.rest);
|
|
307
414
|
}
|
|
415
|
+
byTok.set(tok, set);
|
|
308
416
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
417
|
+
// Join with presence for liveness. token() is lossy, so match forward: index the roster
|
|
418
|
+
// by token(id). A durable with no roster match is a ghost/foreign id — keep its token,
|
|
419
|
+
// never drop it.
|
|
420
|
+
const byToken = new Map();
|
|
421
|
+
for (const p of this.roster.values())
|
|
422
|
+
byToken.set(token(p.card.id), p);
|
|
423
|
+
const memberFor = (tok) => {
|
|
424
|
+
const p = byToken.get(tok);
|
|
425
|
+
return p
|
|
426
|
+
? { id: p.card.id, name: p.card.name, role: p.card.role, live: p.status !== "offline" }
|
|
427
|
+
: { id: tok, name: tok, live: false };
|
|
428
|
+
};
|
|
429
|
+
const byName = (a, b) => a.name.localeCompare(b.name);
|
|
430
|
+
if (channel !== undefined) {
|
|
431
|
+
const out = [];
|
|
432
|
+
for (const [tok, patterns] of byTok)
|
|
433
|
+
if ([...patterns].some((pat) => subjectMatches(pat, channel)))
|
|
434
|
+
out.push(memberFor(tok));
|
|
435
|
+
return out.sort(byName);
|
|
436
|
+
}
|
|
437
|
+
const map = new Map();
|
|
438
|
+
for (const [tok, patterns] of byTok) {
|
|
439
|
+
const m = memberFor(tok);
|
|
440
|
+
for (const pat of patterns) {
|
|
441
|
+
const arr = map.get(pat);
|
|
442
|
+
if (arr)
|
|
443
|
+
arr.push(m);
|
|
444
|
+
else
|
|
445
|
+
map.set(pat, [m]);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
for (const arr of map.values())
|
|
449
|
+
arr.sort(byName);
|
|
450
|
+
return map;
|
|
312
451
|
}
|
|
313
452
|
/** Fetch recent messages from a channel's JetStream backlog. */
|
|
314
453
|
async channelHistory(channel, opts) {
|
|
@@ -431,19 +570,47 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
431
570
|
}));
|
|
432
571
|
}
|
|
433
572
|
await this.pump(dmStream(this.space), dmDurable(id));
|
|
434
|
-
// Multicast:
|
|
573
|
+
// Multicast: a DeliverPolicy.New *tail* of our channels. History is NOT a durable replay —
|
|
574
|
+
// it's an explicit, per-channel backfill on join (replay-policy gated, below), the only
|
|
575
|
+
// shape that can honor per-channel policy given deliver_policy is consumer-wide.
|
|
435
576
|
if (this.channels.length) {
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
577
|
+
const durable = chatDurable(id);
|
|
578
|
+
const want = collapseFilterSubjects(this.channels.map((ch) => chatSubject(this.space, "*", ch)));
|
|
579
|
+
const info = await this.consumerInfo(chatStream(this.space), durable);
|
|
580
|
+
if (!info) {
|
|
581
|
+
// Fresh durable: a New tail (history is the explicit backfill below — the only shape
|
|
582
|
+
// that honors per-channel policy given deliver_policy is consumer-wide).
|
|
583
|
+
await this.jsm.consumers.add(chatStream(this.space), {
|
|
584
|
+
durable_name: durable,
|
|
585
|
+
filter_subjects: want,
|
|
586
|
+
ack_policy: AckPolicy.Explicit,
|
|
587
|
+
ack_wait,
|
|
588
|
+
deliver_policy: DeliverPolicy.New,
|
|
589
|
+
inactive_threshold,
|
|
590
|
+
});
|
|
591
|
+
// Arm the tail-drop watermarks BEFORE pump starts, so the tail can never deliver a
|
|
592
|
+
// just-created channel's message un-watermarked (which would double-emit: live + backfill).
|
|
593
|
+
const armed = await this.armJoin(this.channels);
|
|
594
|
+
await this.pump(chatStream(this.space), durable);
|
|
595
|
+
await this.backfillArmed(armed);
|
|
596
|
+
}
|
|
597
|
+
else {
|
|
598
|
+
// Rebind: reconcile the durable's filter to the CURRENT config (a config that changed
|
|
599
|
+
// between restarts is honored). Channels the config GAINED are backfilled like a fresh
|
|
600
|
+
// join; channels it LOST are dropped from the filter. An unchanged config = pure resume,
|
|
601
|
+
// empty diff, no re-replay.
|
|
602
|
+
await this.pump(chatStream(this.space), durable);
|
|
603
|
+
const haveFilters = info.config.filter_subjects ?? (info.config.filter_subject ? [info.config.filter_subject] : []);
|
|
604
|
+
// Channels the config gained = those not already covered by the durable's filters (a
|
|
605
|
+
// wildcard already covers its sub-channels). Backfill only those.
|
|
606
|
+
const gained = this.channels.filter((c) => !haveFilters.some((f) => subjectMatches(f, chatSubject(this.space, "*", c))));
|
|
607
|
+
// Arm watermarks for the gained channels BEFORE the filter reconcile flips them on.
|
|
608
|
+
const armed = gained.length ? await this.armJoin(gained) : undefined;
|
|
609
|
+
if (!sameSet(haveFilters, want))
|
|
610
|
+
await this.jsm.consumers.update(chatStream(this.space), durable, { filter_subjects: want });
|
|
611
|
+
if (armed)
|
|
612
|
+
await this.backfillArmed(armed);
|
|
613
|
+
}
|
|
447
614
|
}
|
|
448
615
|
// Anycast: a shared work-queue consumer for our role — one instance grabs each task.
|
|
449
616
|
// Open mode self-creates; auth mode BINDS the provisioner-pre-created svc_<role>
|
|
@@ -489,14 +656,264 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
489
656
|
m.ack(); // our own echo — advance past it
|
|
490
657
|
continue;
|
|
491
658
|
}
|
|
659
|
+
// No-replay + dedup (chat only): drop a message at/below this channel's join watermark
|
|
660
|
+
// — pre-join history the New tail still carries for a *lagging* joiner (cursor behind the
|
|
661
|
+
// frontier), and the overlap a replay backfill already delivered. Must ack, or JetStream
|
|
662
|
+
// redelivers it forever. The drop is here, before the message becomes model context.
|
|
663
|
+
if (parsed.kind === "chat") {
|
|
664
|
+
const wm = this.dropWatermark(parsed.rest);
|
|
665
|
+
if (wm !== undefined && m.seq <= wm) {
|
|
666
|
+
m.ack();
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
492
670
|
const delivery = { ack: () => m.ack(), nak: () => m.nak() };
|
|
493
|
-
this.emit("message", msg, delivery
|
|
671
|
+
this.emit("message", msg, delivery, {
|
|
672
|
+
historical: false,
|
|
673
|
+
kind: kindFromParsed(parsed.kind),
|
|
674
|
+
});
|
|
494
675
|
}
|
|
495
676
|
})().catch((e) => {
|
|
496
677
|
if (!this.stopped)
|
|
497
678
|
this.emit("error", e);
|
|
498
679
|
});
|
|
499
680
|
}
|
|
681
|
+
/** The highest join watermark among the joined subscriptions that cover `concreteChannel`
|
|
682
|
+
* (a wildcard sub like `team.>` covers `team.backend`), or undefined if none — the tail
|
|
683
|
+
* drops a chat message with `seq <= ` this. */
|
|
684
|
+
dropWatermark(concreteChannel) {
|
|
685
|
+
let wm;
|
|
686
|
+
for (const [pattern, seq] of this.joinSeq)
|
|
687
|
+
if (subjectMatches(pattern, concreteChannel) && (wm === undefined || seq > wm))
|
|
688
|
+
wm = seq;
|
|
689
|
+
return wm;
|
|
690
|
+
}
|
|
691
|
+
/** The durable's info (rebind) or null (fresh — 404). Gates create/backfill to the join event
|
|
692
|
+
* and exposes the current `filter_subjects` for restart reconciliation. */
|
|
693
|
+
async consumerInfo(stream, durable) {
|
|
694
|
+
if (!this.jsm)
|
|
695
|
+
throw new Error("endpoint not started");
|
|
696
|
+
try {
|
|
697
|
+
return await this.jsm.consumers.info(stream, durable);
|
|
698
|
+
}
|
|
699
|
+
catch {
|
|
700
|
+
return null; // 404 — fresh durable
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
/** Current frontier (last sequence) of the chat stream — a channel's join watermark, and the
|
|
704
|
+
* focus-watermark a connector captures on entering `focus` (recall reads ambient after it). */
|
|
705
|
+
async chatFrontier() {
|
|
706
|
+
if (!this.jsm)
|
|
707
|
+
throw new Error("endpoint not started");
|
|
708
|
+
return (await this.jsm.streams.info(chatStream(this.space))).state.last_seq;
|
|
709
|
+
}
|
|
710
|
+
/** Phase 1 of a join — arm each channel's tail-drop watermark at the current frontier. MUST run
|
|
711
|
+
* BEFORE the filter flip (consumers.update, or pump on a fresh create) so the tail can never
|
|
712
|
+
* carry a just-joined message un-watermarked — which would double-emit it (live + backfill).
|
|
713
|
+
* Returns the per-channel frontiers for {@link backfillArmed}. */
|
|
714
|
+
async armJoin(channels) {
|
|
715
|
+
const frontiers = new Map();
|
|
716
|
+
for (const ch of channels) {
|
|
717
|
+
const frontier = await this.chatFrontier();
|
|
718
|
+
this.joinSeq.set(ch, frontier);
|
|
719
|
+
frontiers.set(ch, frontier);
|
|
720
|
+
}
|
|
721
|
+
return frontiers;
|
|
722
|
+
}
|
|
723
|
+
/** Phase 2 of a join — backfill each armed channel's history up to its frontier (replay-gated),
|
|
724
|
+
* AFTER the filter flip. Returns the total backfilled. */
|
|
725
|
+
async backfillArmed(frontiers) {
|
|
726
|
+
let total = 0;
|
|
727
|
+
for (const [ch, frontier] of frontiers) {
|
|
728
|
+
const policy = await this.joinPolicyFresh(ch);
|
|
729
|
+
if (policy.replay)
|
|
730
|
+
total += await this.backfillChannel(ch, frontier, policy.windowMs);
|
|
731
|
+
}
|
|
732
|
+
return total;
|
|
733
|
+
}
|
|
734
|
+
/** Replay policy + backfill window read straight from the registry bucket (vs the watch cache)
|
|
735
|
+
* — the authoritative read for a join decision (a join is infrequent, and at startup the async
|
|
736
|
+
* cache may not have caught up). Falls to the built-in default only with no registry open. */
|
|
737
|
+
async joinPolicyFresh(channel) {
|
|
738
|
+
if (!this.channelKv)
|
|
739
|
+
return { replay: effectiveReplay(undefined, undefined) };
|
|
740
|
+
const [cfg, defaults] = await Promise.all([
|
|
741
|
+
readChannelConfig(this.channelKv, channel),
|
|
742
|
+
readChannelDefaults(this.channelKv),
|
|
743
|
+
]);
|
|
744
|
+
return { replay: effectiveReplay(cfg, defaults), windowMs: effectiveReplayWindowMs(cfg, defaults) };
|
|
745
|
+
}
|
|
746
|
+
/** Read a channel's retained history up to `upToSeq` via JetStream **Direct Get** (a read
|
|
747
|
+
* verb — no consumer create, so it rides a read-only grant) and emit each message as a
|
|
748
|
+
* `historical` "message" event. `sinceMs` bounds how far back via a native Direct-Get
|
|
749
|
+
* `start_time` (now − window); unset ⇒ the full retained window. New messages (`seq > upToSeq`)
|
|
750
|
+
* are skipped — the live tail owns them. Pages the batch API; the ack handle is a no-op. */
|
|
751
|
+
async backfillChannel(channel, upToSeq, sinceMs) {
|
|
752
|
+
if (!this.jsm)
|
|
753
|
+
throw new Error("endpoint not started");
|
|
754
|
+
const subject = chatSubject(this.space, "*", channel);
|
|
755
|
+
const collected = [];
|
|
756
|
+
// First page starts by time when a window is set (native), else from seq 1; after that we
|
|
757
|
+
// always page by sequence.
|
|
758
|
+
const startTime = sinceMs === undefined ? undefined : new Date(Date.now() - sinceMs);
|
|
759
|
+
let startSeq = 1;
|
|
760
|
+
let first = true;
|
|
761
|
+
pages: for (;;) {
|
|
762
|
+
let last = 0;
|
|
763
|
+
let got = 0;
|
|
764
|
+
try {
|
|
765
|
+
const query = first && startTime !== undefined
|
|
766
|
+
? { start_time: startTime, next_by_subj: subject, batch: 256 }
|
|
767
|
+
: { seq: startSeq, next_by_subj: subject, batch: 256 };
|
|
768
|
+
first = false;
|
|
769
|
+
const iter = await this.jsm.direct.getBatch(chatStream(this.space), query);
|
|
770
|
+
for await (const sm of iter) {
|
|
771
|
+
got++;
|
|
772
|
+
if (sm.seq > upToSeq)
|
|
773
|
+
break pages; // crossed the frontier — the tail owns the rest
|
|
774
|
+
last = sm.seq;
|
|
775
|
+
let msg;
|
|
776
|
+
try {
|
|
777
|
+
msg = sm.json();
|
|
778
|
+
}
|
|
779
|
+
catch {
|
|
780
|
+
continue; // skip undecodable
|
|
781
|
+
}
|
|
782
|
+
// Same authenticity guard as the tail; skip our own echoes in history.
|
|
783
|
+
const parsed = parseSubject(sm.subject);
|
|
784
|
+
if (!parsed || msg.from?.id !== parsed.sender || msg.from.id === this.card.id)
|
|
785
|
+
continue;
|
|
786
|
+
collected.push({ msg, seq: sm.seq });
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
catch (e) {
|
|
790
|
+
// Batch Direct Get raises a 404 ("message not found") when no message matches from
|
|
791
|
+
// `start` — the normal "no more history" signal (empty channel or last page), not a
|
|
792
|
+
// fault. Anything else is real.
|
|
793
|
+
if (e.code === 404)
|
|
794
|
+
break;
|
|
795
|
+
this.emit("error", e);
|
|
796
|
+
break;
|
|
797
|
+
}
|
|
798
|
+
if (got === 0 || last === 0)
|
|
799
|
+
break; // drained
|
|
800
|
+
startSeq = last + 1;
|
|
801
|
+
}
|
|
802
|
+
const noop = { ack: () => { }, nak: () => { } };
|
|
803
|
+
for (const { msg } of collected)
|
|
804
|
+
// Backfill only ever pages the chat stream, so the authenticated class is always "channel".
|
|
805
|
+
this.emit("message", msg, noop, { historical: true, kind: "channel" });
|
|
806
|
+
return collected.length;
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
|
|
810
|
+
* focus-recall read behind `cotal_inbox`. Returns the messages (NOT emitted — this is a pull,
|
|
811
|
+
* not a push into context) plus `dropped: true` when the channel's earliest *retained* message
|
|
812
|
+
* is already newer than the watermark, i.e. some ambient aged out of the per-subject window and
|
|
813
|
+
* the caller must say so rather than silently short the window.
|
|
814
|
+
*
|
|
815
|
+
* Honors the **same** per-channel replay gate as join-backfill ({@link joinPolicyFresh}): a
|
|
816
|
+
* `replay=off` channel returns nothing, so `focus` can't become a history bypass for a channel
|
|
817
|
+
* that denies replay to everyone else (chat is `allow_direct` with no broker-level ACL, so this
|
|
818
|
+
* app gate is the entire boundary).
|
|
819
|
+
*/
|
|
820
|
+
async recallChannel(channel, sinceSeq) {
|
|
821
|
+
if (!this.jsm)
|
|
822
|
+
throw new Error("endpoint not started");
|
|
823
|
+
if (!isConcreteChannel(channel))
|
|
824
|
+
return { messages: [], dropped: false };
|
|
825
|
+
const policy = await this.joinPolicyFresh(channel);
|
|
826
|
+
if (!policy.replay)
|
|
827
|
+
return { messages: [], dropped: false };
|
|
828
|
+
const subject = chatSubject(this.space, "*", channel);
|
|
829
|
+
const collected = [];
|
|
830
|
+
let startSeq = sinceSeq + 1;
|
|
831
|
+
pages: for (;;) {
|
|
832
|
+
let last = 0;
|
|
833
|
+
let got = 0;
|
|
834
|
+
try {
|
|
835
|
+
const iter = await this.jsm.direct.getBatch(chatStream(this.space), {
|
|
836
|
+
seq: startSeq,
|
|
837
|
+
next_by_subj: subject,
|
|
838
|
+
batch: 256,
|
|
839
|
+
});
|
|
840
|
+
for await (const sm of iter) {
|
|
841
|
+
got++;
|
|
842
|
+
last = sm.seq;
|
|
843
|
+
let msg;
|
|
844
|
+
try {
|
|
845
|
+
msg = sm.json();
|
|
846
|
+
}
|
|
847
|
+
catch {
|
|
848
|
+
continue; // skip undecodable
|
|
849
|
+
}
|
|
850
|
+
// Same authenticity guard as the tail/backfill; skip our own echoes.
|
|
851
|
+
const parsed = parseSubject(sm.subject);
|
|
852
|
+
if (!parsed || msg.from?.id !== parsed.sender || msg.from.id === this.card.id)
|
|
853
|
+
continue;
|
|
854
|
+
collected.push(msg);
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
catch (e) {
|
|
858
|
+
if (e.code === 404)
|
|
859
|
+
break; // no more history (empty or last page)
|
|
860
|
+
this.emit("error", e);
|
|
861
|
+
break;
|
|
862
|
+
}
|
|
863
|
+
if (got === 0 || last === 0)
|
|
864
|
+
break;
|
|
865
|
+
startSeq = last + 1;
|
|
866
|
+
}
|
|
867
|
+
const dropped = await this.channelDropped(subject, sinceSeq);
|
|
868
|
+
return { messages: collected, dropped };
|
|
869
|
+
}
|
|
870
|
+
/** Did focus recall on `subject` miss ambient that aged out past the watermark? Ambient is only
|
|
871
|
+
* ever discarded once a sender-subject reaches {@link MAX_MSGS_PER_SUBJECT} (`DiscardPolicy.Old`);
|
|
872
|
+
* below the cap nothing was evicted, so the window is complete — return false without crying
|
|
873
|
+
* wolf. At the cap, the surviving oldest seq decides: if it already postdates the watermark, the
|
|
874
|
+
* eviction reached into the "since you focused" window. (Avoids the false positive of comparing a
|
|
875
|
+
* per-subject oldest against the stream-global frontier, which fires on any other channel's
|
|
876
|
+
* traffic.) */
|
|
877
|
+
async channelDropped(subject, sinceSeq) {
|
|
878
|
+
if (!this.jsm)
|
|
879
|
+
return false;
|
|
880
|
+
let maxPerSubject = 0;
|
|
881
|
+
try {
|
|
882
|
+
const info = await this.jsm.streams.info(chatStream(this.space), { subjects_filter: subject });
|
|
883
|
+
for (const count of Object.values(info.state.subjects ?? {}))
|
|
884
|
+
maxPerSubject = Math.max(maxPerSubject, count);
|
|
885
|
+
}
|
|
886
|
+
catch (e) {
|
|
887
|
+
if (e.code !== 404)
|
|
888
|
+
this.emit("error", e);
|
|
889
|
+
return false; // stream/subject missing — nothing retained, nothing dropped
|
|
890
|
+
}
|
|
891
|
+
if (maxPerSubject < MAX_MSGS_PER_SUBJECT)
|
|
892
|
+
return false; // never hit the cap ⇒ never evicted
|
|
893
|
+
const oldest = await this.channelOldestSeq(subject);
|
|
894
|
+
return oldest !== undefined && oldest > sinceSeq + 1;
|
|
895
|
+
}
|
|
896
|
+
/** Sequence of the earliest message still retained on a channel subject (any sender), or
|
|
897
|
+
* undefined if nothing is retained. One 1-message Direct Get — used for the recall drop marker. */
|
|
898
|
+
async channelOldestSeq(subject) {
|
|
899
|
+
if (!this.jsm)
|
|
900
|
+
return undefined;
|
|
901
|
+
try {
|
|
902
|
+
const iter = await this.jsm.direct.getBatch(chatStream(this.space), {
|
|
903
|
+
seq: 1,
|
|
904
|
+
next_by_subj: subject,
|
|
905
|
+
batch: 1,
|
|
906
|
+
});
|
|
907
|
+
for await (const sm of iter)
|
|
908
|
+
return sm.seq;
|
|
909
|
+
return undefined;
|
|
910
|
+
}
|
|
911
|
+
catch (e) {
|
|
912
|
+
if (e.code !== 404)
|
|
913
|
+
this.emit("error", e);
|
|
914
|
+
return undefined; // 404 = nothing retained on this subject (normal)
|
|
915
|
+
}
|
|
916
|
+
}
|
|
500
917
|
async publishPresence() {
|
|
501
918
|
if (!this.kv)
|
|
502
919
|
return;
|
|
@@ -517,6 +934,43 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
517
934
|
this.handleKvEntry(e);
|
|
518
935
|
})().catch((e) => this.emit("error", e));
|
|
519
936
|
}
|
|
937
|
+
/** Watch the channel registry: replay existing keys, then stream updates, into the local
|
|
938
|
+
* cache. Best-effort — a registry the endpoint can't read leaves the cache empty (effective
|
|
939
|
+
* policy then falls back to the default), never a fault. */
|
|
940
|
+
async startChannelWatch() {
|
|
941
|
+
if (!this.channelKv)
|
|
942
|
+
return;
|
|
943
|
+
const iter = await this.channelKv.watch();
|
|
944
|
+
void (async () => {
|
|
945
|
+
for await (const e of iter)
|
|
946
|
+
this.handleChannelEntry(e);
|
|
947
|
+
})().catch((e) => this.emit("error", e));
|
|
948
|
+
}
|
|
949
|
+
handleChannelEntry(e) {
|
|
950
|
+
const gone = e.operation === "DEL" || e.operation === "PURGE";
|
|
951
|
+
if (e.key === CHANNEL_DEFAULTS_KEY) {
|
|
952
|
+
if (gone)
|
|
953
|
+
this.channelDefaults = {};
|
|
954
|
+
else
|
|
955
|
+
try {
|
|
956
|
+
this.channelDefaults = e.json();
|
|
957
|
+
}
|
|
958
|
+
catch {
|
|
959
|
+
/* keep last good */
|
|
960
|
+
}
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
if (gone) {
|
|
964
|
+
this.channelConfigs.delete(e.key);
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
try {
|
|
968
|
+
this.channelConfigs.set(e.key, e.json());
|
|
969
|
+
}
|
|
970
|
+
catch {
|
|
971
|
+
/* keep last good */
|
|
972
|
+
}
|
|
973
|
+
}
|
|
520
974
|
handleKvEntry(e) {
|
|
521
975
|
if (e.operation === "DEL" || e.operation === "PURGE") {
|
|
522
976
|
this.markOffline(e.key);
|
|
@@ -584,6 +1038,35 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
584
1038
|
this.emit("roster", this.getRoster());
|
|
585
1039
|
}
|
|
586
1040
|
}
|
|
1041
|
+
/** The id token of a chat-stream durable, or null if it isn't one — the inverse of
|
|
1042
|
+
* `chatDurable` (`chat_<token(id)>`). token() is lossy, so this returns the token, not the
|
|
1043
|
+
* original id; callers match it forward against `token(card.id)`. */
|
|
1044
|
+
function chatDurableToken(durable) {
|
|
1045
|
+
const prefix = "chat_";
|
|
1046
|
+
return durable.startsWith(prefix) ? durable.slice(prefix.length) : null;
|
|
1047
|
+
}
|
|
1048
|
+
/** Map an authenticated parsed-subject kind to the message class surfaced to "message" listeners.
|
|
1049
|
+
* Throws on `ctl` (control-plane is request/reply, never a "message") — per repo convention, no
|
|
1050
|
+
* silent default: an unexpected delivering kind is a bug, not something to swallow. */
|
|
1051
|
+
function kindFromParsed(kind) {
|
|
1052
|
+
switch (kind) {
|
|
1053
|
+
case "chat":
|
|
1054
|
+
return "channel";
|
|
1055
|
+
case "inst":
|
|
1056
|
+
return "dm";
|
|
1057
|
+
case "svc":
|
|
1058
|
+
return "anycast";
|
|
1059
|
+
default:
|
|
1060
|
+
throw new Error(`cannot derive a message kind from subject kind "${kind}"`);
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
/** Set equality over two subject lists (order/duplicate-insensitive). */
|
|
1064
|
+
function sameSet(a, b) {
|
|
1065
|
+
if (a.length !== b.length)
|
|
1066
|
+
return false;
|
|
1067
|
+
const s = new Set(a);
|
|
1068
|
+
return b.every((x) => s.has(x));
|
|
1069
|
+
}
|
|
587
1070
|
function authOpts(a) {
|
|
588
1071
|
const tls = a.tls ? {} : undefined;
|
|
589
1072
|
// creds (JWT/nkey) are mutually exclusive with token/user/pass — reject rather than
|