@coasys/ad4m 0.13.1-staging.2 → 0.13.1-staging.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/lib/index.cjs +610 -208
  2. package/lib/index.js +607 -209
  3. package/lib/src/Ad4mClient.d.ts +7 -0
  4. package/lib/src/Ad4mClient.js +9 -0
  5. package/lib/src/agent/AgentClient.d.ts +16 -1
  6. package/lib/src/agent/AgentClient.js +21 -2
  7. package/lib/src/ai/AITypes.d.ts +6 -2
  8. package/lib/src/apiClient.d.ts +13 -0
  9. package/lib/src/apiClient.js +67 -12
  10. package/lib/src/generated/api/Apps.d.ts +4 -0
  11. package/lib/src/generated/api/AuthInfo.d.ts +4 -0
  12. package/lib/src/generated/api/Events.d.ts +10 -12
  13. package/lib/src/generated/api/FlowStateChangedEvent.d.ts +3 -2
  14. package/lib/src/generated/api/Model.d.ts +4 -0
  15. package/lib/src/generated/api/ModelApi.d.ts +5 -1
  16. package/lib/src/generated/api/ModelApiInput.d.ts +7 -1
  17. package/lib/src/generated/api/Notification.d.ts +5 -1
  18. package/lib/src/generated/api/NotificationInput.d.ts +4 -0
  19. package/lib/src/generated/api/PerspectiveEvent.d.ts +8 -0
  20. package/lib/src/generated/api/{PerspectiveLinkWithOwner.d.ts → PerspectiveLinkEvent.d.ts} +4 -2
  21. package/lib/src/generated/api/{PerspectiveLinkUpdatedWithOwner.d.ts → PerspectiveLinkUpdatedEvent.d.ts} +4 -2
  22. package/lib/src/generated/api/PerspectiveRemovedEvent.d.ts +7 -0
  23. package/lib/src/generated/api/RequestCapabilityRequest.d.ts +1 -0
  24. package/lib/src/generated/api/RpcMethods.d.ts +10 -1
  25. package/lib/src/generated/api/RuntimeInfo.d.ts +5 -0
  26. package/lib/src/generated/api/RuntimeUpdateNotificationParams.d.ts +6 -1
  27. package/lib/src/generated/api/UsersRevokeAllSessionsParams.d.ts +7 -0
  28. package/{src/generated/api/PerspectiveRemovedWithOwner.ts → lib/src/generated/api/UsersRevokeAllSessionsParams.js} +1 -2
  29. package/lib/src/index.d.ts +1 -0
  30. package/lib/src/index.js +1 -0
  31. package/lib/src/model/Ad4mModel.js +45 -18
  32. package/lib/src/model/ModelQueryBuilder.d.ts +7 -3
  33. package/lib/src/model/ModelQueryBuilder.js +9 -4
  34. package/lib/src/model/query-common.js +7 -1
  35. package/lib/src/model/types.d.ts +45 -2
  36. package/lib/src/model/types.js +8 -0
  37. package/lib/src/perspectives/Container.d.ts +41 -0
  38. package/lib/src/perspectives/Container.js +62 -0
  39. package/lib/src/perspectives/PerspectiveProxy.d.ts +6 -3
  40. package/lib/src/perspectives/PerspectiveProxy.js +10 -4
  41. package/lib/src/perspectives/WakerSubscriptionManager.d.ts +94 -12
  42. package/lib/src/perspectives/WakerSubscriptionManager.js +276 -75
  43. package/lib/src/runtime/RuntimeClient.d.ts +6 -3
  44. package/lib/src/runtime/RuntimeClient.js +2 -0
  45. package/lib/src/runtime/RuntimeTypes.d.ts +7 -1
  46. package/package.json +2 -2
  47. package/src/Ad4mClient.test.ts +54 -19
  48. package/src/Ad4mClient.ts +10 -0
  49. package/src/agent/AgentClient.ts +26 -2
  50. package/src/ai/AITypes.ts +6 -2
  51. package/src/apiClient.test.ts +128 -3
  52. package/src/apiClient.ts +64 -11
  53. package/src/generated/api/Apps.ts +6 -1
  54. package/src/generated/api/AuthInfo.ts +6 -1
  55. package/src/generated/api/Events.ts +10 -12
  56. package/src/generated/api/FlowStateChangedEvent.ts +3 -2
  57. package/src/generated/api/Model.ts +4 -0
  58. package/src/generated/api/ModelApi.ts +5 -1
  59. package/src/generated/api/ModelApiInput.ts +8 -1
  60. package/src/generated/api/Notification.ts +5 -1
  61. package/src/generated/api/NotificationInput.ts +6 -1
  62. package/src/generated/api/PerspectiveEvent.ts +7 -0
  63. package/src/generated/api/{PerspectiveLinkUpdatedWithOwner.ts → PerspectiveLinkEvent.ts} +4 -1
  64. package/src/generated/api/PerspectiveLinkUpdatedEvent.ts +7 -0
  65. package/src/generated/api/PerspectiveRemovedEvent.ts +6 -0
  66. package/src/generated/api/RequestCapabilityRequest.ts +1 -1
  67. package/src/generated/api/RpcMethods.ts +4 -1
  68. package/src/generated/api/RuntimeInfo.ts +6 -1
  69. package/src/generated/api/RuntimeUpdateNotificationParams.ts +7 -1
  70. package/src/generated/api/UsersRevokeAllSessionsParams.ts +8 -0
  71. package/src/index.ts +1 -0
  72. package/src/model/Ad4mModel.ts +51 -19
  73. package/src/model/ModelQueryBuilder.ts +9 -6
  74. package/src/model/containment-scope.test.ts +104 -0
  75. package/src/model/query-common.ts +10 -1
  76. package/src/model/types.ts +56 -2
  77. package/src/perspectives/Container.test.ts +65 -0
  78. package/src/perspectives/Container.ts +46 -0
  79. package/src/perspectives/PerspectiveProxy.test.ts +11 -0
  80. package/src/perspectives/PerspectiveProxy.ts +10 -4
  81. package/src/perspectives/WakerSubscriptionManager.test.ts +400 -82
  82. package/src/perspectives/WakerSubscriptionManager.ts +321 -85
  83. package/src/perspectives/link-updated-type.test.ts +2 -2
  84. package/src/perspectives/listener-leaks.test.ts +3 -3
  85. package/src/runtime/RuntimeClient.ts +6 -3
  86. package/src/runtime/RuntimeTypes.ts +10 -1
  87. package/lib/src/generated/api/MessageReceivedEvent.d.ts +0 -7
  88. package/lib/src/generated/api/PerspectiveRemovedWithOwner.d.ts +0 -5
  89. package/lib/src/generated/api/PerspectiveWithOwner.d.ts +0 -6
  90. package/lib/src/generated/api/PerspectiveWithOwner.js +0 -1
  91. package/src/generated/api/MessageReceivedEvent.ts +0 -7
  92. package/src/generated/api/PerspectiveLinkWithOwner.ts +0 -4
  93. package/src/generated/api/PerspectiveWithOwner.ts +0 -4
  94. /package/lib/src/generated/api/{MessageReceivedEvent.js → PerspectiveEvent.js} +0 -0
  95. /package/lib/src/generated/api/{PerspectiveLinkUpdatedWithOwner.js → PerspectiveLinkEvent.js} +0 -0
  96. /package/lib/src/generated/api/{PerspectiveLinkWithOwner.js → PerspectiveLinkUpdatedEvent.js} +0 -0
  97. /package/lib/src/generated/api/{PerspectiveRemovedWithOwner.js → PerspectiveRemovedEvent.js} +0 -0
@@ -12,15 +12,143 @@ export interface WakerSubscription {
12
12
  type: "mention" | "channel-messages";
13
13
  perspective: string;
14
14
  channel: string;
15
+ /** SPARQL, or with `className` a model query (JSON) over that class. */
15
16
  query: string;
17
+ /**
18
+ * The class `query` reads, as a model query. A mention subscription has
19
+ * one, from the executor's `get_mention_waker_config`: the built-in
20
+ * `ad4m://TextContent`; a children subscription has the built-in
21
+ * `ad4m://Container`, from `generate_waker_query` (#1446). Without it
22
+ * `query` is SPARQL.
23
+ */
24
+ className?: string;
25
+ /**
26
+ * A mention subscription's terms (profile names and the DID), from
27
+ * `get_mention_waker_config`. The query matches substrings, so a record
28
+ * wakes only when its text holds one of these as a whole word
29
+ * ({@link holdsAWholeWordTerm}); without them a mention subscription is
30
+ * refused.
31
+ */
32
+ terms?: string[];
16
33
  neighbourhood?: string;
17
34
  }
18
35
 
36
+ /** A letter, combining mark, digit or underscore: what a term must not touch. */
37
+ const WORD_CHAR = "[\\p{L}\\p{M}\\p{N}_]";
38
+
39
+ /**
40
+ * The elements a message's HTML is made of. A `<` before any other name is text:
41
+ * We stores plain text, where "<hey Data> can you look" is words.
42
+ */
43
+ const ELEMENT =
44
+ "a|abbr|b|bdi|bdo|blockquote|br|caption|center|cite|code|col|colgroup|dd|del|details|dfn|div|dl|dt|em|" +
45
+ "figcaption|figure|font|h[1-6]|hr|i|img|ins|kbd|label|li|mark|ol|p|pre|q|s|samp|section|article|small|" +
46
+ "span|strike|strong|sub|summary|sup|table|tbody|td|tfoot|th|thead|time|tr|u|ul|var|wbr|script|style";
47
+ /** An attribute: a name, then optionally `=` and a quoted value (which may hold `>`) or an unquoted one. */
48
+ const ATTRIBUTE = `\\s+[A-Za-z_:][-\\w:.]*(?:\\s*=\\s*(?:"[^"]*"|'[^']*'|[^\\s"'=<>\`]+))?`;
49
+ /**
50
+ * Markup: a comment, a `<script>`/`<style>` element with its contents, or a tag of
51
+ * a known element whose rest is a valid attribute list. "a < b" and
52
+ * "if a <b then ask Data, else c -> d" are text: the comma is no attribute.
53
+ */
54
+ const TAG = new RegExp(
55
+ `<!--[\\s\\S]*?-->` +
56
+ `|<(script|style)(?=[\\s/>])(?:${ATTRIBUTE})*\\s*>[\\s\\S]*?<\\/\\1\\s*>` +
57
+ `|<(?:${ELEMENT})(?=[\\s/>])(?:${ATTRIBUTE})*\\s*\\/?>` +
58
+ `|<\\/(?:${ELEMENT})\\s*>`,
59
+ "gi",
60
+ );
61
+ /** Tags that break a line: replaced by a space, so `<p>hi</p><p>Data</p>` keeps "Data" a word. */
62
+ const BREAKING_TAG = /^<\/?(?:p|div|br|hr|li|ul|ol|h[1-6]|blockquote|pre|tr|td|th|table|section|article)\b/i;
63
+ /**
64
+ * The attributes a mention puts its target's DID in: Flux's `<a data-type="mention"
65
+ * href="did:…">`, older Flux messages' `<span data-id="did:…">`, and We's editor chip
66
+ * `<span class="we-mention" data-did="did:…">`.
67
+ */
68
+ const ATTRIBUTES = new RegExp(ATTRIBUTE, "g");
69
+ const MENTION_TARGET = /^\s+(?:href|data-id|data-did)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))$/i;
70
+
71
+ /**
72
+ * How much of a record's text the word check reads. Dropping markup costs time
73
+ * quadratic in a run of unclosed `<script>`, `<style>` or `<!--` openers, so a
74
+ * very long text could hold the event loop for seconds, and a row that holds
75
+ * no term is checked again on every result while it is in the window. A term
76
+ * past this many characters does not wake.
77
+ */
78
+ export const MENTION_TEXT_LIMIT = 16_384;
79
+
80
+ const ENTITIES: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
81
+
82
+ function decodeEntities(s: string): string {
83
+ return s.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z]+);/gi, (whole, ref: string) => {
84
+ if (ref[0] !== "#") return ENTITIES[ref.toLowerCase()] ?? whole;
85
+ const code = ref[1] === "x" || ref[1] === "X" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);
86
+ return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
87
+ });
88
+ }
89
+
90
+ /**
91
+ * What a reader sees of `text`, and the DIDs its mentions point at. `text` may be
92
+ * HTML (TextContent allows it): tag names and attributes are markup, not words, so
93
+ * a bot named "Data" must not wake on `data-type="mention"` in a mention of
94
+ * someone else. Plain text passes through unchanged, apart from entities.
95
+ */
96
+ function readableParts(text: string): { visible: string; targets: string[] } {
97
+ const targets: string[] = [];
98
+ const visible = text.replace(TAG, (tag, scriptOrStyle?: string) => {
99
+ if (scriptOrStyle) return " ";
100
+ if (tag.startsWith("<!--")) return "";
101
+ // Parse complete attributes first: a DID-shaped string inside `title`
102
+ // is not a target attribute and must not wake an agent.
103
+ for (const attribute of tag.matchAll(ATTRIBUTES)) {
104
+ const target = MENTION_TARGET.exec(attribute[0]);
105
+ if (target) targets.push(decodeEntities(target[1] ?? target[2] ?? target[3]));
106
+ }
107
+ return BREAKING_TAG.test(tag) ? " " : "";
108
+ });
109
+ return { visible: decodeEntities(visible), targets };
110
+ }
111
+
112
+ /**
113
+ * Whether `text` holds one of `terms` as a whole word, case-insensitively:
114
+ * "Data" is in "hi Data!", "@data" and "Data's turn", not in "metadata" or
115
+ * "database". A term is matched literally (regex characters, quotes) and must
116
+ * not touch a letter, mark, digit or underscore on either side, so a name
117
+ * that starts or ends with punctuation still matches. Same rule as #1459's
118
+ * SPARQL `(?:^|\W)term(?:\W|$)`.
119
+ *
120
+ * The rule runs on the visible text only: in HTML, tags and attributes are
121
+ * dropped and entities decoded first. A mention's target attribute
122
+ * (`href`, `data-id`, `data-did`) matches an exact, case-sensitive DID term, which
123
+ * is how a mention by DID is found when the DID is not in the visible text.
124
+ *
125
+ * The mention query's `contains` matches substrings (model query has no word
126
+ * match); this is the check that keeps "data" from waking on "metadata".
127
+ * It reads the first {@link MENTION_TEXT_LIMIT} characters only.
128
+ */
129
+ export function holdsAWholeWordTerm(text: string, terms: string[]): boolean {
130
+ const { visible, targets } = readableParts(text.slice(0, MENTION_TEXT_LIMIT));
131
+ return terms.some((term) => {
132
+ if (!term) return false;
133
+ if (term.startsWith("did:") && targets.includes(term)) return true;
134
+ const escaped = term.replace(/[\\^$.*+?()[\]{}|\/]/g, "\\$&");
135
+ return new RegExp(`(?<!${WORD_CHAR})${escaped}(?!${WORD_CHAR})`, "iu").test(visible);
136
+ });
137
+ }
138
+
139
+ /** The built-in `Container` (`Container.builtinUri`; not imported, so the
140
+ * plugin bundle does not pull in the model layer). */
141
+ const CONTAINER = "ad4m://Container";
142
+
19
143
  /** Per-message parent resolution result for mention subscriptions. */
20
144
  export interface MentionMessage {
21
- /** The message's expression address (source of the body link). */
145
+ /** The mentioned record's id. */
22
146
  address: string;
23
- /** All parent addresses this message belongs to (channels, conversations, etc.). */
147
+ /**
148
+ * Every container holding it (channels, threads, calls, blocks): whatever
149
+ * links it with `ad4m://has_child`, and every instance of a class whose
150
+ * `children` relation links it. A record can sit in several.
151
+ */
24
152
  parents: string[];
25
153
  /**
26
154
  * Everything above the parents, nearest first: their parents, theirs, and so on up to the
@@ -50,9 +178,17 @@ export interface WakerSubscriptionManagerOptions {
50
178
  * what the levels hold. 0 reports parents only.
51
179
  */
52
180
  ancestorDepth?: number;
53
- /** Called when a subscription fires (after debounce). Return value is ignored.
54
- * For mention subs, `mentions` contains per-message parent info. */
55
- onWake: (sub: WakerSubscription, result: any, mentions?: MentionMessage[]) => void;
181
+ /**
182
+ * Called when a subscription fires (after debounce). For mention subs, `mentions` holds every
183
+ * mention not yet delivered, with its parent info. Resolve `false` (or reject) when the wake did
184
+ * not reach the agent: those mentions stay out of the persisted seen set and are offered again
185
+ * after `retryWakeMs`, so a wake lost to a restarting gateway is not lost for good.
186
+ */
187
+ onWake: (
188
+ sub: WakerSubscription,
189
+ result: any,
190
+ mentions?: MentionMessage[],
191
+ ) => void | boolean | Promise<void | boolean>;
56
192
  /** Called when the active subscription list changes (for persistence). */
57
193
  onPersist?: (subscriptions: WakerSubscription[], seenMessages: Record<string, string[]>) => void;
58
194
  /** Previously persisted seen message IDs per subscription. Seeds seenMessages on resubscribe to avoid duplicate wakes. */
@@ -71,6 +207,9 @@ export interface WakerSubscriptionManagerOptions {
71
207
  * so the re-attempt cycle never saw it (#1016).
72
208
  */
73
209
  subscribeTimeoutMs?: number;
210
+ /** First delay before offering an undelivered mention wake again (default 5000); doubles per
211
+ * failed attempt, up to 60000. */
212
+ retryWakeMs?: number;
74
213
  }
75
214
 
76
215
  /**
@@ -82,13 +221,13 @@ export function hintFor(msg: string): string {
82
221
  // "Executor is locked" is the current executor's message; "main key not
83
222
  // found" is what older executors (and unrelated key lookups) say for the
84
223
  // same locked state. Either way the caller cannot fix it themselves: keys
85
- // live in memory only, so every restart locks the node again and only its
224
+ // live in memory only, so every restart locks the executor again and only its
86
225
  // operator can unlock it.
87
226
  if (/executor is locked/i.test(msg) || /main key not found/i.test(msg)) {
88
- return " — the executor is locked (its keys live in memory only, so every restart locks it again). Only the node's operator can unlock it: contact them and ask for unlockAgent, then subscribe again — a pending subscription re-attempts on its own once the node is unlocked";
227
+ return " — the executor is locked (its keys live in memory only, so every restart locks it again). Only the executor's operator can unlock it: contact them and ask for unlockAgent, then subscribe again — a pending subscription re-attempts on its own once the executor is unlocked";
89
228
  }
90
229
  if (/timed out after \d+ms/i.test(msg)) {
91
- return " — the executor accepted the waker's connection but never answered; check that the waker is pointed at the right executor host and API port (executorUrl in the plugin config) and that the node is unlocked";
230
+ return " — the executor accepted the waker's connection but never answered; check that the waker is pointed at the right executor host and API port (executorUrl in the plugin config) and that the executor is unlocked";
92
231
  }
93
232
  return "";
94
233
  }
@@ -121,12 +260,13 @@ export class WakerSubscriptionManager {
121
260
  private logger: WakerLogger;
122
261
  private debounceMs: number;
123
262
  private ancestorDepth: number;
124
- private onWake: (sub: WakerSubscription, result: any, mentions?: MentionMessage[]) => void;
263
+ private onWake: WakerSubscriptionManagerOptions["onWake"];
125
264
  private onPersist?: (subscriptions: WakerSubscription[], seenMessages: Record<string, string[]>) => void;
126
265
  private QuerySubscriptionProxyCtor: any;
127
266
  private previousSeenMessages: Record<string, string[]>;
128
267
  private retryPendingMs: number;
129
268
  private subscribeTimeoutMs: number;
269
+ private retryWakeMs: number;
130
270
 
131
271
  private proxies = new Map<string, any>();
132
272
  private activeSubscriptions = new Map<string, WakerSubscription>();
@@ -149,6 +289,20 @@ export class WakerSubscriptionManager {
149
289
  /** Per-subscription set of already-processed message addresses (for mention subs)
150
290
  * or serialized result hash (for channel-messages subs). */
151
291
  private seenMessages = new Map<string, Set<string>>();
292
+ /**
293
+ * Per mention subscription: the addresses whose wake was delivered, or that were history when
294
+ * the subscription opened. This, not `seenMessages`, is what gets persisted: an address joins
295
+ * `seenMessages` as soon as it is read, while its parents are still resolving and before any
296
+ * wake, so a restart in between must not take it as done.
297
+ */
298
+ private delivered = new Map<string, Set<string>>();
299
+ /**
300
+ * Per mention subscription: mentions read but whose wake has not been delivered yet, keyed by
301
+ * address. They accumulate across a debounce window, so one wake names all of them.
302
+ */
303
+ private undelivered = new Map<string, Map<string, MentionMessage>>();
304
+ /** Failed wake attempts in a row per subscription, for the retry backoff. */
305
+ private wakeFailures = new Map<string, number>();
152
306
 
153
307
  constructor(options: WakerSubscriptionManagerOptions) {
154
308
  this.perspectiveClient = options.perspectiveClient;
@@ -161,6 +315,7 @@ export class WakerSubscriptionManager {
161
315
  this.previousSeenMessages = options.previousSeenMessages ?? {};
162
316
  this.retryPendingMs = options.retryPendingMs ?? 30000;
163
317
  this.subscribeTimeoutMs = options.subscribeTimeoutMs ?? 15000;
318
+ this.retryWakeMs = options.retryWakeMs ?? 5000;
164
319
  }
165
320
 
166
321
  /** A token identifying this attempt at subscribing `id`. */
@@ -176,7 +331,7 @@ export class WakerSubscriptionManager {
176
331
  }
177
332
 
178
333
  /**
179
- * Create a live SPARQL subscription.
334
+ * Create a live subscription: SPARQL, or a model query when `sub.className` is set.
180
335
  * If a subscription with the same id already exists, it is disposed first.
181
336
  */
182
337
  async subscribe(sub: WakerSubscription): Promise<void> {
@@ -188,17 +343,22 @@ export class WakerSubscriptionManager {
188
343
  throw new Error("WakerSubscriptionManager: QuerySubscriptionProxy must be provided via constructor options");
189
344
  }
190
345
  const QuerySubscriptionProxy = this.QuerySubscriptionProxyCtor;
346
+ if (sub.type === "mention" && sub.className && !sub.terms?.length) {
347
+ // Without terms the whole-word check cannot run, and every substring
348
+ // ("metadata" for "data") would wake the agent.
349
+ throw new Error(
350
+ `Waker subscription ${sub.id}: a mention subscription needs its terms (get_mention_waker_config returns them)`,
351
+ );
352
+ }
191
353
 
192
354
  this.logger.info(
193
355
  `[waker] ${sub.id}: creating subscription (perspective=${sub.perspective}, type=${sub.type})`,
194
356
  );
195
- this.logger.info(`[waker] ${sub.id}: SPARQL query:\n${sub.query}`);
357
+ this.logger.info(`[waker] ${sub.id}: ${sub.className ? `model query over ${sub.className}` : "SPARQL query"}:\n${sub.query}`);
196
358
 
197
- const proxy = new QuerySubscriptionProxy(
198
- sub.perspective,
199
- sub.query,
200
- this.perspectiveClient,
201
- );
359
+ const proxy = sub.className
360
+ ? new QuerySubscriptionProxy(sub.perspective, sub.query, this.perspectiveClient, sub.className)
361
+ : new QuerySubscriptionProxy(sub.perspective, sub.query, this.perspectiveClient);
202
362
  // Suppress unhandled rejection from proxy.initialized — QuerySubscriptionProxy
203
363
  // rejects this promise internally when subscribe() fails, and if nobody catches
204
364
  // it before the next microtask it crashes the process.
@@ -238,6 +398,7 @@ export class WakerSubscriptionManager {
238
398
  // reported as active or persisted as if it were.
239
399
  this.activeSubscriptions.delete(sub.id);
240
400
  this.seenMessages.delete(sub.id);
401
+ this.delivered.delete(sub.id);
241
402
  this.persist();
242
403
  // Keep it pending and re-attempt: the usual cause (locked wallet, executor
243
404
  // mid-restart, a handshake the executor never answered) clears on its
@@ -262,46 +423,76 @@ export class WakerSubscriptionManager {
262
423
  this.logger.info(`[waker] ${sub.id}: subscription initialized successfully`);
263
424
 
264
425
  // Seed from persisted seen messages so we don't re-wake after restart
426
+ const restored = Object.prototype.hasOwnProperty.call(this.previousSeenMessages, sub.id);
265
427
  const seen = new Set<string>(this.previousSeenMessages[sub.id] ?? []);
266
428
  this.seenMessages.set(sub.id, seen);
429
+ this.delivered.set(sub.id, new Set(seen));
267
430
  if (seen.size > 0) {
268
431
  this.logger.info(`[waker] ${sub.id}: seeded ${seen.size} seen message(s) from persisted state`);
269
432
  }
270
433
 
271
- proxy.onResult(async (result: any) => {
434
+ // `opening` is the result the subscription holds when it opens (`proxy.result`): `onResult`
435
+ // only reports later changes, so this is the one chance to see what was written meanwhile.
436
+ const handleResult = async (result: any, opening = false) => {
272
437
  this.logger.info(
273
- `[waker] ${sub.id}: onResult fired — type=${typeof result}, isArray=${Array.isArray(result)}, value=${String(JSON.stringify(result)).substring(0, 500)}`,
438
+ `[waker] ${sub.id}: ${opening ? "opening result" : "onResult fired"} — type=${typeof result}, isArray=${Array.isArray(result)}, value=${String(JSON.stringify(result)).substring(0, 500)}`,
274
439
  );
275
440
 
276
- // Query engine can deliver non-array values (e.g. false) on disconnect/reconnect — ignore them
277
- if (!Array.isArray(result)) {
441
+ // A model subscription delivers `{ instances }`, SPARQL an array of
442
+ // rows. Anything else (e.g. `false` on a disconnect/reconnect) is
443
+ // ignored.
444
+ const rows: unknown = sub.className ? result?.instances : result;
445
+ if (!Array.isArray(rows)) {
278
446
  this.logger.warn(
279
- `[waker] ${sub.id}: ignoring non-array result: ${JSON.stringify(result)}`,
447
+ `[waker] ${sub.id}: ignoring result without ${sub.className ? "instances" : "rows"}: ${JSON.stringify(result)}`,
280
448
  );
281
449
  return;
282
450
  }
283
451
 
284
452
  if (sub.type === "mention") {
285
- // For mention subscriptions: track seen message addresses.
286
- // Only wake for messages we haven't seen before.
453
+ // For mention subscriptions: track seen record ids.
454
+ // Only wake for records we haven't seen before.
287
455
  const currentSeen = this.seenMessages.get(sub.id) ?? new Set<string>();
288
456
  const newMessages: string[] = [];
289
457
 
290
- for (const item of result) {
291
- if (item && item.source && !currentSeen.has(item.source)) {
292
- newMessages.push(item.source);
458
+ const terms = sub.terms ?? [];
459
+ for (const item of rows as any[]) {
460
+ // A SPARQL row names the record as `source`, a model row as `id`.
461
+ const address = sub.className ? item?.id : item?.source;
462
+ if (typeof address !== "string" || !address || currentSeen.has(address)) continue;
463
+ if (terms.length) {
464
+ // The mention query matched a substring; wake only on a whole word.
465
+ const text = Array.isArray(item.text) ? item.text.join(" ") : item.text;
466
+ if (typeof text !== "string" || !holdsAWholeWordTerm(text, terms)) {
467
+ this.logger.debug(`[waker] ${sub.id}: ${address} holds no term as a whole word, skipping`);
468
+ continue;
469
+ }
293
470
  }
471
+ currentSeen.add(address);
472
+ newMessages.push(address);
294
473
  }
295
474
 
296
475
  if (newMessages.length === 0) {
297
476
  this.logger.info(
298
- `[waker] ${sub.id}: all ${result.length} items already seen, skipping`,
477
+ `[waker] ${sub.id}: no new mention among ${rows.length} items, skipping`,
478
+ );
479
+ return;
480
+ }
481
+
482
+ if (opening && !restored) {
483
+ // A new subscription: what it opens on is history, readable on demand, not news.
484
+ this.seenMessages.set(sub.id, currentSeen);
485
+ const done = this.delivered.get(sub.id) ?? new Set<string>();
486
+ for (const address of newMessages) done.add(address);
487
+ this.delivered.set(sub.id, done);
488
+ this.logger.info(
489
+ `[waker] ${sub.id}: ${newMessages.length} existing mention(s) taken as history, not woken`,
299
490
  );
300
491
  return;
301
492
  }
302
493
 
303
494
  this.logger.info(
304
- `[waker] ${sub.id}: ${newMessages.length} new message(s) out of ${result.length} total`,
495
+ `[waker] ${sub.id}: ${newMessages.length} new message(s) out of ${rows.length} total`,
305
496
  );
306
497
 
307
498
  // Resolve parents, and the chain above them, per new message
@@ -317,29 +508,19 @@ export class WakerSubscriptionManager {
317
508
  mentions.push({ address: msgAddr, parents, ancestors });
318
509
  }
319
510
 
320
- // Mark all new messages as seen
321
- for (const addr of newMessages) {
322
- currentSeen.add(addr);
323
- }
324
511
  this.seenMessages.set(sub.id, currentSeen);
512
+ const pending = this.undelivered.get(sub.id) ?? new Map<string, MentionMessage>();
513
+ for (const mention of mentions) pending.set(mention.address, mention);
514
+ this.undelivered.set(sub.id, pending);
325
515
 
326
- // Debounce the wake callback
327
- const existing = this.debounceTimers.get(sub.id);
328
- if (existing) clearTimeout(existing);
329
-
330
- this.debounceTimers.set(
331
- sub.id,
332
- setTimeout(() => {
333
- this.onWake(sub, result, mentions);
334
- this.debounceTimers.delete(sub.id);
335
- this.persist();
336
- }, this.debounceMs),
337
- );
516
+ this.scheduleMentionWake(sub, result, this.debounceMs);
338
517
  } else {
518
+ // A container's contents when the subscription opens are not an update.
519
+ if (opening) return;
339
520
  // For channel-messages: always notify the agent on every subscription push.
340
521
  // No deduplication — the agent reads recent messages itself.
341
522
  this.logger.info(
342
- `[waker] ${sub.id}: channel update (${result.length} items)`,
523
+ `[waker] ${sub.id}: channel update (${rows.length} items)`,
343
524
  );
344
525
 
345
526
  const existing = this.debounceTimers.get(sub.id);
@@ -353,7 +534,8 @@ export class WakerSubscriptionManager {
353
534
  }, this.debounceMs),
354
535
  );
355
536
  }
356
- });
537
+ };
538
+ proxy.onResult((result: any) => handleResult(result));
357
539
 
358
540
  if (this.isStale(sub.id, token)) {
359
541
  try { proxy.dispose(); } catch {}
@@ -361,6 +543,15 @@ export class WakerSubscriptionManager {
361
543
  }
362
544
  this.proxies.set(sub.id, proxy);
363
545
  this.activeSubscriptions.set(sub.id, sub);
546
+ // After a restart this wakes on mentions written while the waker was down; on a new
547
+ // subscription it records the existing ones as history.
548
+ if (proxy.result !== undefined) {
549
+ try {
550
+ await handleResult(proxy.result, true);
551
+ } catch (e: any) {
552
+ this.logger.warn(`[waker] ${sub.id}: reading the opening result failed: ${e?.message ?? e}`);
553
+ }
554
+ }
364
555
  this.persist();
365
556
 
366
557
  this.logger.info(
@@ -439,9 +630,63 @@ export class WakerSubscriptionManager {
439
630
  this.activeSubscriptions.delete(id);
440
631
  this.pendingSubscriptions.delete(id);
441
632
  this.seenMessages.delete(id);
633
+ this.delivered.delete(id);
634
+ this.undelivered.delete(id);
635
+ this.wakeFailures.delete(id);
442
636
  if (persist) this.persist();
443
637
  }
444
638
 
639
+ /** (Re)start the timer that offers `sub`'s undelivered mentions to `onWake` after `delayMs`. */
640
+ private scheduleMentionWake(sub: WakerSubscription, result: any, delayMs: number): void {
641
+ const existing = this.debounceTimers.get(sub.id);
642
+ if (existing) clearTimeout(existing);
643
+ this.debounceTimers.set(
644
+ sub.id,
645
+ setTimeout(() => {
646
+ this.debounceTimers.delete(sub.id);
647
+ void this.deliverMentions(sub, result);
648
+ }, delayMs),
649
+ );
650
+ }
651
+
652
+ /**
653
+ * Offer every undelivered mention of `sub` in one wake. On delivery they join the persisted
654
+ * seen set; on failure they stay undelivered and are offered again with backoff.
655
+ */
656
+ private async deliverMentions(sub: WakerSubscription, result: any): Promise<void> {
657
+ const pending = this.undelivered.get(sub.id);
658
+ if (!pending?.size || !this.activeSubscriptions.has(sub.id)) return;
659
+ const mentions = Array.from(pending.values());
660
+ let delivered: boolean;
661
+ try {
662
+ delivered = (await this.onWake(sub, result, mentions)) !== false;
663
+ } catch (e: any) {
664
+ this.logger.warn(`[waker] ${sub.id}: wake callback threw: ${e?.message ?? e}`);
665
+ delivered = false;
666
+ }
667
+ // Disposed while the wake was in flight: nothing left to retry or persist.
668
+ if (this.undelivered.get(sub.id) !== pending) return;
669
+ if (!delivered) {
670
+ const failures = (this.wakeFailures.get(sub.id) ?? 0) + 1;
671
+ this.wakeFailures.set(sub.id, failures);
672
+ const delay = Math.min(this.retryWakeMs * 2 ** (failures - 1), 60_000);
673
+ this.logger.warn(
674
+ `[waker] ${sub.id}: wake for ${mentions.length} mention(s) not delivered, offering again in ${delay}ms`,
675
+ );
676
+ // A newer result arriving meanwhile restarts the timer and joins this wake.
677
+ if (!this.debounceTimers.has(sub.id)) this.scheduleMentionWake(sub, result, delay);
678
+ return;
679
+ }
680
+ this.wakeFailures.delete(sub.id);
681
+ const done = this.delivered.get(sub.id) ?? new Set<string>();
682
+ for (const mention of mentions) {
683
+ pending.delete(mention.address);
684
+ done.add(mention.address);
685
+ }
686
+ this.delivered.set(sub.id, done);
687
+ this.persist();
688
+ }
689
+
445
690
  /** Dispose all active subscriptions and drop anything still pending. */
446
691
  disposeAll(): void {
447
692
  // Bump the epoch first: in-flight subscribe() calls have no proxy in the
@@ -467,58 +712,49 @@ export class WakerSubscriptionManager {
467
712
  return this.activeSubscriptions.has(id);
468
713
  }
469
714
 
470
- /** Resolve one containment level in one query; skip unsafe IRI tokens and treat failures as none. */
715
+ /**
716
+ * Everything holding any of `addresses`, in one model query: a `Container`
717
+ * step in (#1446). The executor follows `ad4m://has_child` on any node and
718
+ * each class's own `children` relation on its instances, so the waker
719
+ * looks up no predicate. Failures count as none.
720
+ */
471
721
  private async parentsOf(sub: WakerSubscription, addresses: string[]): Promise<string[]> {
472
- const safe = [...new Set(addresses)].filter((address) => {
473
- if (/[<>"{}|^`\\\s\x00-\x20]/.test(address)) {
474
- this.logger.warn(`[waker] ${sub.id}: skipping parent lookup for unsafe address ${JSON.stringify(address)}`);
475
- return false;
476
- }
477
- return true;
478
- });
479
- if (!safe.length) return [];
480
- try {
481
- const query = safe.length === 1
482
- ? `SELECT ?source WHERE { ?source <ad4m://has_child> <${safe[0]}> . }`
483
- : `SELECT ?source WHERE { VALUES ?child { ${safe.map((address) => `<${address}>`).join(" ")} } ?source <ad4m://has_child> ?child . }`;
484
- const rows: unknown = await this.perspectiveClient.querySparql(sub.perspective, query);
485
- if (!Array.isArray(rows)) return [];
486
- // A row omits a variable left unbound in its solution.
487
- return rows.map((row) => row?.source).filter((source): source is string => typeof source === "string");
488
- } catch (err: any) {
489
- this.logger.warn(`[waker] ${sub.id}: parent resolution failed for ${JSON.stringify(safe)} — ${err?.message ?? err}`);
490
- return [];
491
- }
722
+ if (!addresses.length) return [];
723
+ return this.containers(sub, { ids: [...new Set(addresses)], direction: "in" }, addresses);
492
724
  }
493
725
 
494
726
  /**
495
- * The chain above an item's parents, a level at a time, nearest first, up to `ancestorDepth`.
496
- *
497
- * Breadth-first with a visited set: containment is a graph, not a tree — an item can sit in two
498
- * places, and nothing stops a cycle — so each address is reported once and the walk always ends.
499
- * Each level uses one batched parent lookup, bounded by the depth.
727
+ * The chain above an item's parents, nearest first, up to `ancestorDepth` levels: one
728
+ * transitive `Container` walk in from the parents, bounded by `maxDepth`. The executor walks
729
+ * containment as a graph, so each address comes once however many paths reach it, and a
730
+ * cycle ends. The item and its parents are never reported as ancestors.
500
731
  */
501
732
  private async ancestorsOf(sub: WakerSubscription, address: string, parents: string[]): Promise<string[]> {
733
+ if (!parents.length || this.ancestorDepth <= 0) return [];
734
+ const scope = { ids: parents, direction: "in", transitive: true, maxDepth: this.ancestorDepth };
502
735
  const seen = new Set<string>([address, ...parents]);
503
- const ancestors: string[] = [];
504
- let level = parents;
505
- for (let depth = 0; depth < this.ancestorDepth && level.length; depth++) {
506
- const next: string[] = [];
507
- for (const parent of await this.parentsOf(sub, level)) {
508
- if (seen.has(parent)) continue;
509
- seen.add(parent);
510
- next.push(parent);
511
- }
512
- ancestors.push(...next);
513
- level = next;
736
+ return (await this.containers(sub, scope, parents)).filter((id) => !seen.has(id));
737
+ }
738
+
739
+ /** The ids of the `Container` rows `parent` scopes to, in the executor's order. */
740
+ private async containers(sub: WakerSubscription, parent: Record<string, unknown>, from: string[]): Promise<string[]> {
741
+ try {
742
+ const query = JSON.stringify({ parent, properties: [] });
743
+ const result = await this.perspectiveClient.modelQuery(sub.perspective, CONTAINER, query);
744
+ const instances: unknown = result?.instances;
745
+ if (!Array.isArray(instances)) return [];
746
+ return instances.map((row) => row?.id).filter((id): id is string => typeof id === "string");
747
+ } catch (err: any) {
748
+ this.logger.warn(`[waker] ${sub.id}: containment lookup failed for ${JSON.stringify(from)} — ${err?.message ?? err}`);
749
+ return [];
514
750
  }
515
- return ancestors;
516
751
  }
517
752
 
518
753
  private persist(): void {
519
754
  if (this.onPersist) {
520
755
  const seen: Record<string, string[]> = {};
521
- for (const [id, set] of this.seenMessages) {
756
+ // Only what was delivered: anything else must wake again after a restart.
757
+ for (const [id, set] of this.delivered) {
522
758
  seen[id] = Array.from(set);
523
759
  }
524
760
  this.onPersist(this.getActive(), seen);
@@ -8,7 +8,7 @@ import { PerspectiveClient } from './PerspectiveClient';
8
8
  import { PerspectiveHandle } from './PerspectiveHandle';
9
9
  import { PerspectiveProxy } from './PerspectiveProxy';
10
10
  import type { DecoratedLinkExpression } from '../generated/api/DecoratedLinkExpression';
11
- import type { PerspectiveLinkUpdatedWithOwner } from '../generated/api/PerspectiveLinkUpdatedWithOwner';
11
+ import type { PerspectiveLinkUpdatedEvent } from '../generated/api/PerspectiveLinkUpdatedEvent';
12
12
 
13
13
  /** Captures the socket ApiClient opens, so the test can push an event. */
14
14
  class FakeWebSocket {
@@ -47,7 +47,7 @@ describe('link-updated handler type', () => {
47
47
  // link-added keeps its `{ link }` payload
48
48
  proxy.on('link-added', ({ link }: { link: DecoratedLinkExpression }) => {});
49
49
 
50
- const event: PerspectiveLinkUpdatedWithOwner = { perspectiveUuid: 'test-uuid', owner: 'did:test:1', oldLink: link('old'), newLink: link('new') };
50
+ const event: PerspectiveLinkUpdatedEvent = { perspectiveUuid: 'test-uuid', oldLink: link('old'), newLink: link('new') };
51
51
  FakeWebSocket.last.onmessage?.({ data: JSON.stringify({ type: 'link-updated', ...event }) });
52
52
  expect(seen).toEqual(['old->new']);
53
53
  api.closeAll();
@@ -52,11 +52,11 @@ const link = (source: string) => ({
52
52
  });
53
53
  const handle = (uuid: string) => ({ uuid, name: 'test', neighbourhood: null, sharedUrl: null, state: PerspectiveState.Synced, owners: null });
54
54
  const linkAdded = (perspectiveUuid: string, source: string) =>
55
- ({ type: 'link-added', perspectiveUuid, owner: 'did:test', link: link(source) });
55
+ ({ type: 'link-added', perspectiveUuid, link: link(source) });
56
56
  const linkRemoved = (perspectiveUuid: string, source: string) =>
57
- ({ type: 'link-removed', perspectiveUuid, owner: 'did:test', link: link(source) });
57
+ ({ type: 'link-removed', perspectiveUuid, link: link(source) });
58
58
  const linkUpdated = (perspectiveUuid: string, oldSource: string, newSource: string) =>
59
- ({ type: 'link-updated', perspectiveUuid, owner: 'did:test', oldLink: link(oldSource), newLink: link(newSource) });
59
+ ({ type: 'link-updated', perspectiveUuid, oldLink: link(oldSource), newLink: link(newSource) });
60
60
  const syncStateChange = (perspectiveUuid: string, state: PerspectiveState) =>
61
61
  ({ type: 'sync-state-change', perspectiveUuid, state, perspective: handle(perspectiveUuid) });
62
62
  const autoProcessorEvent = (perspectiveUuid: string) => ({
@@ -1,7 +1,8 @@
1
1
  import {ApiClient, CallOptions } from '../apiClient'
2
2
  import { Perspective, PerspectiveExpression } from "../perspectives/Perspective"
3
- import { RuntimeInfo, SentMessage, NotificationInput, Notification, ImportResult, UserStatistics } from "./RuntimeTypes"
3
+ import { RuntimeInfo, SentMessage, NotificationInput, NotificationUpdate, Notification, ImportResult, UserStatistics } from "./RuntimeTypes"
4
4
  import type { HostRate, UnytVersionInfo } from "../generated/api"
5
+ import { LanguageMeta } from "../language/LanguageMeta"
5
6
 
6
7
  export class RuntimeClient {
7
8
  #apiClient: ApiClient
@@ -46,7 +47,9 @@ export class RuntimeClient {
46
47
  return this.#apiClient.call('runtime.removeLinkLanguageTemplates', { addresses })
47
48
  }
48
49
 
49
- async knownLinkLanguageTemplates(): Promise<string[]> {
50
+ /** Every link-language template the executor knows, with its meta (name, description,
51
+ * `possibleTemplateParams`). A template whose meta cannot be read is left out. */
52
+ async knownLinkLanguageTemplates(): Promise<LanguageMeta[]> {
50
53
  return this.#apiClient.call('runtime.linkLanguageTemplates', {})
51
54
  }
52
55
 
@@ -127,7 +130,7 @@ export class RuntimeClient {
127
130
  return this.#apiClient.call('runtime.notifications', {})
128
131
  }
129
132
 
130
- async updateNotification(id: string, notification: NotificationInput): Promise<boolean> {
133
+ async updateNotification(id: string, notification: NotificationUpdate): Promise<boolean> {
131
134
  return this.#apiClient.call('runtime.updateNotification', { ...notification, id })
132
135
  }
133
136