@ixo/oracle-runtime-workers 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +32 -10
  2. package/package.json +13 -8
  3. package/src/artifacts/config.ts +158 -0
  4. package/src/artifacts/crypto.ts +94 -0
  5. package/src/artifacts/routes.ts +73 -0
  6. package/src/artifacts/store.ts +238 -0
  7. package/src/artifacts/sweep.ts +200 -0
  8. package/src/artifacts/tool.ts +103 -0
  9. package/src/artifacts/viewer-page.ts +614 -0
  10. package/src/attachments/download.ts +168 -45
  11. package/src/attachments/index.ts +9 -0
  12. package/src/attachments/magic.ts +33 -0
  13. package/src/attachments/pipeline.ts +105 -49
  14. package/src/attachments/sandbox-archive.ts +9 -8
  15. package/src/attachments/view-cache.ts +142 -0
  16. package/src/attachments/view.ts +53 -12
  17. package/src/channels/auth.ts +267 -0
  18. package/src/channels/contract.ts +137 -0
  19. package/src/channels/turns.ts +270 -0
  20. package/src/core/budgeted-llm.ts +223 -0
  21. package/src/core/capability-router.ts +273 -0
  22. package/src/core/context-budget.ts +97 -10
  23. package/src/core/env.ts +111 -0
  24. package/src/core/execution-profile.ts +17 -0
  25. package/src/core/index.ts +267 -9
  26. package/src/core/llm.ts +30 -9
  27. package/src/core/main-agent-types.ts +41 -1
  28. package/src/core/main-agent.ts +367 -65
  29. package/src/core/manifest.ts +34 -0
  30. package/src/core/meta-tools.ts +128 -18
  31. package/src/core/middlewares/capability-gate.ts +130 -12
  32. package/src/core/middlewares/context-guard.ts +55 -24
  33. package/src/core/middlewares/index.ts +20 -2
  34. package/src/core/middlewares/page-context.ts +46 -2
  35. package/src/core/middlewares/return-direct-first.ts +47 -0
  36. package/src/core/middlewares/summarization.ts +152 -10
  37. package/src/core/middlewares/tool-execution.ts +323 -0
  38. package/src/core/middlewares/tool-repetition-guard.ts +202 -35
  39. package/src/core/middlewares/turn-boundary.ts +162 -0
  40. package/src/core/openrouter-pricing.ts +155 -22
  41. package/src/core/prompt-composer.ts +112 -7
  42. package/src/core/registries.ts +253 -48
  43. package/src/core/request-admission.ts +159 -0
  44. package/src/core/runtime-context.ts +107 -8
  45. package/src/core/state.ts +1 -1
  46. package/src/core/sub-agent-fallback.ts +21 -2
  47. package/src/core/subagent-as-tool.ts +78 -43
  48. package/src/core/tool-access.ts +189 -0
  49. package/src/core/tool-scheduler.ts +125 -0
  50. package/src/core/turn-budget.ts +205 -0
  51. package/src/core/turn-time-note.ts +43 -0
  52. package/src/core/user-kv.ts +446 -0
  53. package/src/core/wrap-plugin-tool.ts +20 -0
  54. package/src/delivery/plan.ts +278 -0
  55. package/src/delivery/profile.ts +161 -0
  56. package/src/delivery/prompt.ts +27 -0
  57. package/src/delivery/schema.ts +61 -0
  58. package/src/delivery/shaper.ts +389 -0
  59. package/src/delivery/types.ts +84 -0
  60. package/src/do/ambient.ts +51 -15
  61. package/src/do/contracts.ts +170 -6
  62. package/src/do/idle-eviction.ts +38 -0
  63. package/src/do/matrix-turn-ledger.ts +24 -5
  64. package/src/do/result-store-test-do.ts +137 -3
  65. package/src/do/result-store.ts +239 -58
  66. package/src/do/room-mirror.ts +29 -0
  67. package/src/do/run-buffer.ts +64 -9
  68. package/src/do/run-coordinator.ts +451 -76
  69. package/src/do/run-request.ts +2 -0
  70. package/src/do/run-store-test-do.ts +301 -1
  71. package/src/do/run-store.ts +385 -43
  72. package/src/do/secrets-adapter.ts +22 -3
  73. package/src/do/sse-stream.ts +60 -9
  74. package/src/do/transcript.ts +48 -5
  75. package/src/do/turn-body.ts +105 -5
  76. package/src/do/ucan-service.ts +204 -34
  77. package/src/do/user-oracle-do.ts +1883 -264
  78. package/src/feedback/config.ts +127 -0
  79. package/src/feedback/contract.ts +148 -0
  80. package/src/feedback/linear-sink.ts +285 -0
  81. package/src/feedback/privacy.ts +150 -0
  82. package/src/feedback/reservation.ts +298 -0
  83. package/src/feedback/submit.ts +247 -0
  84. package/src/index.ts +138 -6
  85. package/src/llm/byo-adapter.ts +27 -6
  86. package/src/llm/byo-catalog.ts +13 -0
  87. package/src/llm/byo-model-fallback.ts +248 -0
  88. package/src/llm/byo-service.ts +45 -8
  89. package/src/llm/provider-error.ts +139 -2
  90. package/src/matrix/gateway-do.ts +504 -108
  91. package/src/matrix/group-chat.ts +66 -23
  92. package/src/matrix/inbox-store.ts +16 -1
  93. package/src/matrix/ingest.ts +79 -14
  94. package/src/matrix/media-deadline.ts +130 -0
  95. package/src/matrix/replay-format.ts +5 -3
  96. package/src/matrix/reply-parts.ts +92 -0
  97. package/src/matrix/room-alias.ts +96 -0
  98. package/src/matrix/user-homeserver.ts +166 -7
  99. package/src/owner-store/ixo-vfs-store.ts +170 -43
  100. package/src/owner-store/migrating-store.ts +31 -15
  101. package/src/owner-store/types.ts +18 -3
  102. package/src/plugin-api/index.ts +43 -2
  103. package/src/plugin-api/oracle-plugin.ts +35 -0
  104. package/src/plugin-api/request-admission.ts +41 -0
  105. package/src/plugin-api/tool-helper.ts +25 -1
  106. package/src/plugin-api/tool-plane.ts +92 -0
  107. package/src/plugin-api/types.ts +194 -6
  108. package/src/plugins/agui/agui.plugin.ts +31 -16
  109. package/src/plugins/composio/composio-tools.ts +57 -6
  110. package/src/plugins/composio/composio-ucan.ts +3 -2
  111. package/src/plugins/composio/composio.plugin.ts +2 -0
  112. package/src/plugins/delegated-capabilities.ts +25 -0
  113. package/src/plugins/editor/content-session.ts +173 -83
  114. package/src/plugins/editor/content-tools.ts +76 -17
  115. package/src/plugins/editor/editor-mx.ts +28 -9
  116. package/src/plugins/editor/editor.plugin.ts +3 -1
  117. package/src/plugins/editor/failures.ts +28 -7
  118. package/src/plugins/editor/prompts.ts +2 -1
  119. package/src/plugins/editor/provider.ts +553 -40
  120. package/src/plugins/editor/standalone-editor-tool.ts +61 -18
  121. package/src/plugins/firecrawl/firecrawl-tools.ts +13 -5
  122. package/src/plugins/flows/action-metadata.ts +111 -11
  123. package/src/plugins/flows/actions.ts +18 -0
  124. package/src/plugins/flows/decision-context.ts +31 -0
  125. package/src/plugins/flows/edit.ts +223 -52
  126. package/src/plugins/flows/errors.ts +22 -1
  127. package/src/plugins/flows/flow-doc.ts +92 -9
  128. package/src/plugins/flows/flows.plugin.ts +11 -5
  129. package/src/plugins/flows/forms.ts +34 -0
  130. package/src/plugins/flows/index.ts +2 -0
  131. package/src/plugins/flows/input-policy.ts +140 -0
  132. package/src/plugins/flows/prompts.ts +14 -11
  133. package/src/plugins/flows/read.ts +49 -9
  134. package/src/plugins/flows/ref-syntax.ts +129 -0
  135. package/src/plugins/flows/test-homeserver.ts +245 -0
  136. package/src/plugins/flows/tools/authoring.ts +216 -112
  137. package/src/plugins/flows/tools/discovery.ts +4 -1
  138. package/src/plugins/flows/tools/forms.ts +21 -2
  139. package/src/plugins/flows/tools/settings.ts +34 -13
  140. package/src/plugins/flows/translator.ts +13 -10
  141. package/src/plugins/flows/types.ts +167 -42
  142. package/src/plugins/index.ts +11 -1
  143. package/src/plugins/ixo-transaction/chain-ids.ts +43 -0
  144. package/src/plugins/ixo-transaction/index.ts +2 -0
  145. package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +119 -0
  146. package/src/plugins/ixo-transaction/receipts.ts +99 -0
  147. package/src/plugins/ixo-transaction/tools.ts +329 -0
  148. package/src/plugins/ixo-transaction/wallet-signing.ts +202 -0
  149. package/src/plugins/mcp-tool-adapter.ts +23 -2
  150. package/src/plugins/memory/memory-tools.ts +7 -2
  151. package/src/plugins/memory/memory-ucan.ts +3 -2
  152. package/src/plugins/memory/memory.plugin.ts +2 -0
  153. package/src/plugins/pod-creator/blueprint-store.ts +163 -0
  154. package/src/plugins/pod-creator/blueprint-types.ts +61 -0
  155. package/src/plugins/pod-creator/bounded-map.ts +87 -0
  156. package/src/plugins/pod-creator/capsule-content-client.ts +285 -0
  157. package/src/plugins/pod-creator/chain-gateway.ts +110 -0
  158. package/src/plugins/pod-creator/config.ts +85 -0
  159. package/src/plugins/pod-creator/create-session-store.ts +180 -0
  160. package/src/plugins/pod-creator/create-tools.ts +538 -0
  161. package/src/plugins/pod-creator/design-pod-roles.ts +111 -0
  162. package/src/plugins/pod-creator/index.ts +65 -0
  163. package/src/plugins/pod-creator/orchestration-tools.ts +172 -0
  164. package/src/plugins/pod-creator/pod-creator.plugin.ts +190 -0
  165. package/src/plugins/pod-creator/stage.ts +201 -0
  166. package/src/plugins/pod-creator/sub-agents.ts +189 -0
  167. package/src/plugins/portal/action-log.ts +21 -1
  168. package/src/plugins/portal/declared-tools.ts +90 -0
  169. package/src/plugins/portal/portal.plugin.ts +42 -16
  170. package/src/plugins/sandbox/sandbox-bridge.ts +79 -10
  171. package/src/plugins/sandbox/sandbox-mcp.ts +3 -2
  172. package/src/plugins/sandbox/sandbox-write-blob.ts +11 -1
  173. package/src/plugins/sandbox/sandbox.plugin.ts +8 -16
  174. package/src/plugins/tasks/manifest.ts +1 -1
  175. package/src/plugins/tasks/middleware.ts +20 -7
  176. package/src/plugins/tasks/tasks-tools.ts +30 -3
  177. package/src/plugins/tasks/tasks.plugin.ts +64 -11
  178. package/src/plugins/user-preferences/schema.ts +12 -1
  179. package/src/plugins/user-preferences/user-preferences-store.ts +38 -13
  180. package/src/plugins/user-preferences/user-preferences-tool.ts +3 -2
  181. package/src/plugins/vfs/vfs-client.ts +250 -71
  182. package/src/plugins/vfs/vfs-content.ts +30 -11
  183. package/src/plugins/vfs/vfs-errors.ts +22 -0
  184. package/src/plugins/vfs/vfs-sandbox-tools.ts +12 -5
  185. package/src/plugins/vfs/vfs-tools.ts +60 -20
  186. package/src/realtime/frontend-call-registry.ts +228 -81
  187. package/src/realtime/realtime-endpoint.ts +215 -44
  188. package/src/realtime/session-socket-hub.ts +47 -2
  189. package/src/secrets/secrets-service.ts +9 -3
  190. package/src/shell/app.ts +529 -82
  191. package/src/shell/auth.ts +131 -37
  192. package/src/shell/turn-body-cap.ts +11 -13
  193. package/src/sqlite/blob-compactor.ts +61 -28
  194. package/src/sqlite/database.ts +156 -90
  195. package/src/sqlite/do-vfs.ts +416 -129
  196. package/src/sqlite/page-tier.ts +133 -40
  197. package/src/sqlite/serialization.ts +40 -0
  198. package/src/sqlite/sessions-store.ts +10 -2
  199. package/src/sqlite/sqlite-saver.ts +139 -19
  200. package/src/sqlite/user-kv-store.ts +317 -0
  201. package/src/tasks/schedule.ts +201 -12
  202. package/src/tasks/scheduler.ts +586 -98
  203. package/src/tasks/spec.ts +67 -3
  204. package/src/tasks/store.ts +292 -27
  205. package/src/tasks/topic-deliverables.ts +70 -0
@@ -1,16 +1,385 @@
1
1
  import { MatrixProvider } from '@ixo/matrix-crdt';
2
- import type { MatrixClient } from 'matrix-js-sdk';
2
+ import { MatrixError, type MatrixClient } from 'matrix-js-sdk';
3
3
  import * as Y from 'yjs';
4
4
 
5
5
  import { createConsoleLogger } from '../../core/utils';
6
6
 
7
7
  const logger = createConsoleLogger({ module: 'editor-provider' });
8
8
 
9
- const wait = (ms: number) =>
9
+ /** Resolve after `ms`, or as soon as `signal` aborts (the timer is cleared). */
10
+ const wait = (ms: number, signal: AbortSignal) =>
10
11
  new Promise<void>((resolve) => {
11
- setTimeout(resolve, ms);
12
+ if (signal.aborted) return resolve();
13
+ const done = () => {
14
+ clearTimeout(timer);
15
+ signal.removeEventListener('abort', done);
16
+ resolve();
17
+ };
18
+ const timer = setTimeout(done, ms);
19
+ signal.addEventListener('abort', done, { once: true });
12
20
  });
13
21
 
22
+ /** The event type matrix-crdt sends Y.Doc updates as (its translator default). */
23
+ export const DOC_UPDATE_EVENT_TYPE = 'matrix-crdt.doc_update';
24
+
25
+ /** Sends of one batch of document changes before the write is abandoned. */
26
+ export const DOC_WRITE_MAX_ATTEMPTS = 4;
27
+ /** Time from a batch's first failed send until the write is abandoned. */
28
+ export const DOC_WRITE_RETRY_BUDGET_MS = 15_000;
29
+ /** Back-off before the second send; doubled for each further one. */
30
+ export const DOC_WRITE_BASE_BACKOFF_MS = 250;
31
+ /**
32
+ * How long a caller waits for pending changes to reach the homeserver. Longer
33
+ * than the retry budget, so a write that is still being retried is given the
34
+ * whole budget before the caller reports it unsent.
35
+ */
36
+ export const DOC_FLUSH_TIMEOUT_MS = DOC_WRITE_RETRY_BUDGET_MS + 5_000;
37
+
38
+ /** Why changes made to a document did not reach the room. */
39
+ export type DocWriteFailure =
40
+ | {
41
+ /** The homeserver refused the write (`M_FORBIDDEN`): no permission. */
42
+ kind: 'forbidden';
43
+ detail: string;
44
+ }
45
+ | {
46
+ /** The write was not acknowledged by the homeserver. */
47
+ kind: 'unsent';
48
+ detail: string;
49
+ /**
50
+ * False only when every failed send was refused with a client error
51
+ * (400, 401, 413, 429 …) — the homeserver answered and stored nothing.
52
+ * True when a send failed with a server error, a timeout or a network
53
+ * failure, when the flush wait ran out, or when a send was dropped
54
+ * after the document closed: the change may or may not have landed.
55
+ */
56
+ mayHaveLanded: boolean;
57
+ };
58
+
59
+ export interface DocWriteRetryOptions {
60
+ maxAttempts?: number;
61
+ budgetMs?: number;
62
+ baseBackoffMs?: number;
63
+ }
64
+
65
+ export interface DocWriteGuardOptions extends DocWriteRetryOptions {
66
+ /**
67
+ * What further writes do once one has been abandoned. `reject`: they fail
68
+ * as `M_FORBIDDEN`, so matrix-crdt's writer stops retrying and its
69
+ * `canWrite` turns false (callers that only look at `canWrite` still fail
70
+ * closed). `drain`: they resolve without being sent, so the writer empties
71
+ * its queue; the caller must read {@link DocWriteGuard.failure}.
72
+ */
73
+ onGiveUp: 'reject' | 'drain';
74
+ }
75
+
76
+ /** A document write abandoned by {@link DocWriteGuard}; matrix-crdt reads `errcode`. */
77
+ class DocWriteAbandonedError extends Error {
78
+ readonly errcode = 'M_FORBIDDEN';
79
+
80
+ constructor(detail: string) {
81
+ super(`Document write abandoned: ${detail}`);
82
+ this.name = 'DocWriteAbandonedError';
83
+ }
84
+ }
85
+
86
+ function ownField(source: unknown, key: string): unknown {
87
+ if (!source || typeof source !== 'object') return undefined;
88
+ return Object.getOwnPropertyDescriptor(source, key)?.value;
89
+ }
90
+
91
+ function errorDetail(error: unknown): string {
92
+ return error instanceof Error ? error.message : String(error);
93
+ }
94
+
95
+ /** The homeserver's requested wait before the next send, if it sent one. */
96
+ function retryAfterMs(error: unknown): number | undefined {
97
+ if (error instanceof MatrixError) {
98
+ try {
99
+ const fromServer = error.getRetryAfterMs();
100
+ if (typeof fromServer === 'number' && fromServer >= 0) return fromServer;
101
+ } catch {
102
+ // A malformed Retry-After header — fall back to the body field.
103
+ }
104
+ }
105
+ const fromBody = ownField(ownField(error, 'data'), 'retry_after_ms');
106
+ return typeof fromBody === 'number' && fromBody >= 0 ? fromBody : undefined;
107
+ }
108
+
109
+ /**
110
+ * Whether a failed send may still have been stored: a server error, a request
111
+ * timeout, or a failure with no HTTP answer at all (network, abort). A client
112
+ * error is the homeserver's definite refusal of that request.
113
+ */
114
+ function sendMayHaveLanded(error: unknown): boolean {
115
+ const status = ownField(error, 'httpStatus');
116
+ if (typeof status !== 'number') return true;
117
+ return status === 408 || status >= 500;
118
+ }
119
+
120
+ /**
121
+ * Whether sending the same batch again can succeed: rate limits, timeouts,
122
+ * server errors and network failures can; any other client error (a 413 for
123
+ * an oversized event, a 401 for a revoked token, a 400) cannot.
124
+ */
125
+ function isRetryableSendError(error: unknown): boolean {
126
+ const status = ownField(error, 'httpStatus');
127
+ if (typeof status !== 'number') return true;
128
+ return status === 408 || status === 429 || status >= 500;
129
+ }
130
+
131
+ /**
132
+ * Bounds what matrix-crdt does with a document write that fails.
133
+ *
134
+ * matrix-crdt's writer re-queues a failed batch and sends it again after its
135
+ * flush interval, forever, unless the homeserver answers `M_FORBIDDEN`; while
136
+ * it does, `waitForFlush()` never resolves. Its retry timer also survives
137
+ * `MatrixProvider.dispose()`, which does not dispose the writer. This guard
138
+ * sits between the writer and the Matrix client (`client` is the client to
139
+ * hand to the provider) and, for document updates only:
140
+ *
141
+ * - waits before each re-send — the homeserver's `retry_after_ms` when it
142
+ * sends one, else an exponential back-off — so a failing write is not
143
+ * re-sent every few milliseconds;
144
+ * - abandons the write after {@link DOC_WRITE_MAX_ATTEMPTS} sends, after
145
+ * {@link DOC_WRITE_RETRY_BUDGET_MS}, or at once for an error that cannot
146
+ * succeed on retry, and records why in {@link failure};
147
+ * - records an `M_FORBIDDEN` as a `forbidden` failure;
148
+ * - after {@link close}, completes every send without the network, so the
149
+ * writer drains and its timer stops.
150
+ *
151
+ * It also attaches an abort signal to the client's direct `http.authedRequest`
152
+ * calls (matrix-crdt's `/events` long-poll and snapshot media downloads),
153
+ * which {@link close} aborts, so no long-poll outlives the document.
154
+ */
155
+ export class DocWriteGuard {
156
+ readonly client: MatrixClient;
157
+ /** Document update sends that reached the network. */
158
+ sends = 0;
159
+
160
+ private recorded: DocWriteFailure | undefined;
161
+ private closed = false;
162
+ private failedSends = 0;
163
+ /** A failed send in the current streak may have been stored. */
164
+ private streakMayHaveLanded = false;
165
+ private streakStartedAt: number | undefined;
166
+ private nextSendAt = 0;
167
+ private readonly abortController = new AbortController();
168
+ private readonly wakers = new Set<() => void>();
169
+ private readonly maxAttempts: number;
170
+ private readonly budgetMs: number;
171
+ private readonly baseBackoffMs: number;
172
+
173
+ constructor(
174
+ target: MatrixClient,
175
+ private readonly opts: DocWriteGuardOptions,
176
+ ) {
177
+ this.maxAttempts = Math.max(1, opts.maxAttempts ?? DOC_WRITE_MAX_ATTEMPTS);
178
+ this.budgetMs = Math.max(0, opts.budgetMs ?? DOC_WRITE_RETRY_BUDGET_MS);
179
+ this.baseBackoffMs = Math.max(
180
+ 0,
181
+ opts.baseBackoffMs ?? DOC_WRITE_BASE_BACKOFF_MS,
182
+ );
183
+
184
+ const signal = this.abortController.signal;
185
+ const http = target.http;
186
+ const authedRequest: typeof http.authedRequest = (
187
+ method,
188
+ path,
189
+ queryParams,
190
+ body,
191
+ paramOpts,
192
+ ) =>
193
+ http.authedRequest(
194
+ method,
195
+ path,
196
+ queryParams,
197
+ body,
198
+ paramOpts?.abortSignal
199
+ ? paramOpts
200
+ : Object.assign({}, paramOpts, { abortSignal: signal }),
201
+ );
202
+ const guardedHttp = new Proxy(http, {
203
+ get: (source, prop) => {
204
+ if (prop === 'authedRequest') return authedRequest;
205
+ const value: unknown = Reflect.get(source, prop, source);
206
+ return typeof value === 'function' ? value.bind(source) : value;
207
+ },
208
+ });
209
+
210
+ const sendEvent = (...args: unknown[]): Promise<unknown> => {
211
+ const send = (): Promise<unknown> =>
212
+ Promise.resolve(Reflect.apply(target.sendEvent, target, args));
213
+ if (args[1] !== DOC_UPDATE_EVENT_TYPE) return send();
214
+ return this.sendUpdate(send, () => forgetJoinedRoom(target, args[0]));
215
+ };
216
+
217
+ // Every other member is the real client's, bound to it, so the SDK's
218
+ // internals keep running against the real instance.
219
+ this.client = new Proxy(target, {
220
+ get: (source, prop) => {
221
+ if (prop === 'sendEvent') return sendEvent;
222
+ if (prop === 'http') return guardedHttp;
223
+ const value: unknown = Reflect.get(source, prop, source);
224
+ return typeof value === 'function' ? value.bind(source) : value;
225
+ },
226
+ set: (source, prop, value) => Reflect.set(source, prop, value, source),
227
+ });
228
+ }
229
+
230
+ /** Why a write was abandoned, or `undefined` while every write landed. */
231
+ get failure(): DocWriteFailure | undefined {
232
+ return this.recorded;
233
+ }
234
+
235
+ /** Stop: abort direct requests and complete further sends without sending. */
236
+ close(): void {
237
+ if (this.closed) return;
238
+ this.closed = true;
239
+ this.abortController.abort(new Error('document closed'));
240
+ for (const wake of [...this.wakers]) wake();
241
+ }
242
+
243
+ private async sendUpdate(
244
+ send: () => Promise<unknown>,
245
+ onForbidden: () => void,
246
+ ): Promise<unknown> {
247
+ if (this.recorded) return this.giveUp(this.recorded.detail);
248
+ await this.waitUntil(this.nextSendAt);
249
+ if (this.closed) {
250
+ // A batch dropped after close was never acknowledged; earlier sends of
251
+ // it may have landed, so it is reported as possibly saved.
252
+ this.recorded ??= {
253
+ kind: 'unsent',
254
+ detail: 'the document closed before the change was acknowledged',
255
+ mayHaveLanded: true,
256
+ };
257
+ return {};
258
+ }
259
+ try {
260
+ this.sends += 1;
261
+ const result = await send();
262
+ this.failedSends = 0;
263
+ this.streakMayHaveLanded = false;
264
+ this.streakStartedAt = undefined;
265
+ this.nextSendAt = 0;
266
+ return result;
267
+ } catch (error) {
268
+ return this.onSendFailed(error, onForbidden);
269
+ }
270
+ }
271
+
272
+ private onSendFailed(error: unknown, onForbidden: () => void): unknown {
273
+ const detail = errorDetail(error);
274
+ if (ownField(error, 'errcode') === 'M_FORBIDDEN') {
275
+ this.recorded = { kind: 'forbidden', detail };
276
+ // The oracle may have been removed: the next load must join again.
277
+ onForbidden();
278
+ if (this.opts.onGiveUp === 'drain') return {};
279
+ throw error;
280
+ }
281
+
282
+ const now = Date.now();
283
+ this.failedSends += 1;
284
+ this.streakMayHaveLanded ||= sendMayHaveLanded(error);
285
+ this.streakStartedAt ??= now;
286
+ const delay = Math.max(
287
+ retryAfterMs(error) ?? 0,
288
+ this.baseBackoffMs * 2 ** (this.failedSends - 1),
289
+ );
290
+ const exhausted =
291
+ !isRetryableSendError(error) ||
292
+ this.failedSends >= this.maxAttempts ||
293
+ now + delay - this.streakStartedAt > this.budgetMs;
294
+ if (exhausted) {
295
+ this.recorded = {
296
+ kind: 'unsent',
297
+ detail: `${detail} (after ${this.failedSends} attempt${this.failedSends === 1 ? '' : 's'})`,
298
+ mayHaveLanded: this.streakMayHaveLanded,
299
+ };
300
+ logger.warn(`Abandoning document write: ${this.recorded.detail}`);
301
+ return this.giveUp(this.recorded.detail);
302
+ }
303
+ this.nextSendAt = now + delay;
304
+ throw error;
305
+ }
306
+
307
+ private giveUp(detail: string): unknown {
308
+ if (this.closed || this.opts.onGiveUp === 'drain') return {};
309
+ throw new DocWriteAbandonedError(detail);
310
+ }
311
+
312
+ private waitUntil(at: number): Promise<void> {
313
+ const ms = at - Date.now();
314
+ if (ms <= 0 || this.closed) return Promise.resolve();
315
+ return new Promise((resolve) => {
316
+ const done = () => {
317
+ clearTimeout(timer);
318
+ this.wakers.delete(done);
319
+ resolve();
320
+ };
321
+ const timer = setTimeout(done, ms);
322
+ this.wakers.add(done);
323
+ });
324
+ }
325
+ }
326
+
327
+ /** How long a successful join lets later loads of the room skip `POST /join`. */
328
+ export const JOIN_REUSE_WINDOW_MS = 60_000;
329
+
330
+ /**
331
+ * When each client last joined each room, in this isolate. A load skips
332
+ * `POST /join` only within {@link JOIN_REUSE_WINDOW_MS} of the last join, so
333
+ * a burst of tool calls shares one join while a removal from the room is
334
+ * noticed within that window: a removed member can still read the history
335
+ * up to its removal, so only the join (refused) reveals it. The entry is
336
+ * dropped as soon as a write to the room is refused with `M_FORBIDDEN`, and
337
+ * a load refused with 403 after a skipped join forgets it, joins and retries
338
+ * once.
339
+ */
340
+ const joinedRooms = new WeakMap<MatrixClient, Map<string, number>>();
341
+
342
+ function forgetJoinedRoom(client: MatrixClient, roomId: unknown): void {
343
+ if (typeof roomId === 'string') joinedRooms.get(client)?.delete(roomId);
344
+ }
345
+
346
+ /** Whether a failed load was the homeserver refusing access (403). */
347
+ function isAccessRefusal(error: unknown): boolean {
348
+ return ownField(error, 'httpStatus') === 403;
349
+ }
350
+
351
+ /**
352
+ * Whether a failed document load can succeed on another attempt. The room
353
+ * being unreachable for this account, a client error from the homeserver, or
354
+ * a history whose only snapshot is unreadable will fail the same way again.
355
+ */
356
+ function isRetryableLoadError(error: unknown): boolean {
357
+ if (error instanceof RoomNotAccessibleError) return false;
358
+ if (error instanceof Error && error.name === 'SnapshotUnavailableError')
359
+ return false;
360
+ const status = ownField(error, 'httpStatus');
361
+ if (typeof status === 'number' && status >= 400 && status < 500)
362
+ return status === 408 || status === 429;
363
+ return true;
364
+ }
365
+
366
+ /** Reject with `message` when `promise` has not settled within `ms`. */
367
+ async function withDeadline<T>(
368
+ promise: Promise<T>,
369
+ ms: number,
370
+ message: string,
371
+ ): Promise<T> {
372
+ let timer: ReturnType<typeof setTimeout> | undefined;
373
+ const deadline = new Promise<never>((_resolve, reject) => {
374
+ timer = setTimeout(() => reject(new Error(message)), ms);
375
+ });
376
+ try {
377
+ return await Promise.race([promise, deadline]);
378
+ } finally {
379
+ clearTimeout(timer);
380
+ }
381
+ }
382
+
14
383
  type RoomDescriptor =
15
384
  | { type: 'id'; id: string }
16
385
  | { type: 'alias'; alias: string };
@@ -34,6 +403,8 @@ export interface ProviderInitResult {
34
403
  doc: Y.Doc;
35
404
  awareness?: unknown;
36
405
  provider: MatrixProvider;
406
+ /** The write guard between the provider and Matrix; see {@link DocWriteGuard}. */
407
+ writes: DocWriteGuard;
37
408
  }
38
409
  export interface MatrixRoomById {
39
410
  type: 'id';
@@ -55,6 +426,8 @@ export interface ProviderConfig {
55
426
  retryIfForbiddenInterval?: number;
56
427
  /** How many rejected writes to retry before giving up (0 = unlimited). */
57
428
  maxForbiddenRetries?: number;
429
+ /** Bounds on re-sending a failed write; defaults to the `DOC_WRITE_*` constants. */
430
+ writeRetry?: DocWriteRetryOptions;
58
431
  }
59
432
 
60
433
  export type MatrixRoomConfig = MatrixRoomById | MatrixRoomByAlias;
@@ -102,12 +475,66 @@ export class MatrixProviderManager {
102
475
  private documentAvailable = false;
103
476
  private disposed = false;
104
477
  private availabilityResolvers: Array<() => void> = [];
478
+ /** Aborted by `dispose()`: ends a retry back-off and pending availability waits. */
479
+ private readonly disposal = new AbortController();
480
+ /** True once the current provider finished loading the room's history. */
481
+ private providerInitialized = false;
482
+ /** A flush already ran out of time; disposal does not wait a second time. */
483
+ private flushTimedOut = false;
484
+ /** The room id the current load resolved, and whether it skipped the join. */
485
+ private loadingRoomId: string | undefined;
486
+ private joinSkipped = false;
487
+ /** Incremented per load attempt; a superseded attempt abandons its provider. */
488
+ private attempt = 0;
489
+ private readonly writes: DocWriteGuard;
490
+
491
+ private readonly now: () => number;
105
492
 
106
493
  constructor(
107
494
  private readonly matrixClient: MatrixClient,
108
495
  private readonly cfg: AppConfig,
496
+ options: { now?: () => number } = {},
109
497
  ) {
498
+ this.now = options.now ?? Date.now;
110
499
  this.doc = new Y.Doc();
500
+ this.writes = new DocWriteGuard(matrixClient, {
501
+ onGiveUp: 'reject',
502
+ ...cfg.provider.writeRetry,
503
+ });
504
+ }
505
+
506
+ /** Why a write to this document did not reach the room, if one did not. */
507
+ public get writeFailure(): DocWriteFailure | undefined {
508
+ if (this.writes.failure) return this.writes.failure;
509
+ if (this.provider && !this.provider.canWrite)
510
+ return { kind: 'forbidden', detail: 'the homeserver rejected the write' };
511
+ return undefined;
512
+ }
513
+
514
+ /**
515
+ * Wait (bounded by {@link DOC_FLUSH_TIMEOUT_MS}) until every change made to
516
+ * the doc so far has been sent, and report a write that did not land.
517
+ */
518
+ public async flush(): Promise<DocWriteFailure | undefined> {
519
+ const provider = this.provider;
520
+ if (!provider || !this.providerInitialized) return this.writeFailure;
521
+ try {
522
+ await withDeadline(
523
+ provider.waitForFlush(),
524
+ DOC_FLUSH_TIMEOUT_MS,
525
+ `changes were not sent within ${DOC_FLUSH_TIMEOUT_MS}ms`,
526
+ );
527
+ } catch (error) {
528
+ this.flushTimedOut = true;
529
+ return (
530
+ this.writeFailure ?? {
531
+ kind: 'unsent',
532
+ detail: errorDetail(error),
533
+ mayHaveLanded: true,
534
+ }
535
+ );
536
+ }
537
+ return this.writeFailure;
111
538
  }
112
539
 
113
540
  public get ydoc(): Y.Doc {
@@ -132,13 +559,23 @@ export class MatrixProviderManager {
132
559
  const attempts = Math.max(1, this.cfg.provider.retryAttempts);
133
560
  const delayMs = Math.max(0, this.cfg.provider.retryDelayMs);
134
561
 
562
+ let rejoined = false;
135
563
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
564
+ if (this.disposed)
565
+ throw new Error('MatrixProviderManager was disposed during the load');
136
566
  try {
137
- // Initialize the provider (creates MatrixProvider, fetches room history)
138
- await this.initializeProvider();
139
-
140
- // Wait for document to become available
141
- await this.waitForAvailability(this.cfg.matrix.initialSyncTimeoutMs);
567
+ // Join, history walk and availability share one deadline: a request
568
+ // that never answers must fail the attempt, not hang it.
569
+ const timeoutMs = this.cfg.matrix.initialSyncTimeoutMs;
570
+ this.attempt = attempt;
571
+ await withDeadline(
572
+ (async () => {
573
+ await this.initializeProvider(attempt);
574
+ await this.waitForAvailability(timeoutMs);
575
+ })(),
576
+ timeoutMs,
577
+ `Document did not load within ${timeoutMs}ms. Check room permissions or connectivity.`,
578
+ );
142
579
 
143
580
  logger.log('Matrix provider initialized', { attempt });
144
581
  logger.log(`📄 Y.Doc GUID: ${this.doc.guid}`);
@@ -148,18 +585,38 @@ export class MatrixProviderManager {
148
585
  doc: this.doc,
149
586
  awareness: this.provider?.awarenessInstance,
150
587
  provider: this.ensureProvider(),
588
+ writes: this.writes,
151
589
  };
152
590
  } catch (error) {
153
591
  logger.warn(`Matrix provider init attempt ${attempt} failed`, error);
154
592
  await this.cleanupProvider();
155
593
 
156
- if (attempt === attempts) {
594
+ this.attempt = 0;
595
+ // Refused after skipping the join: the remembered membership is
596
+ // stale. Join and run this attempt again, once.
597
+ if (
598
+ !rejoined &&
599
+ !this.disposed &&
600
+ this.joinSkipped &&
601
+ this.loadingRoomId &&
602
+ isAccessRefusal(error)
603
+ ) {
604
+ rejoined = true;
605
+ forgetJoinedRoom(this.matrixClient, this.loadingRoomId);
606
+ attempt -= 1;
607
+ continue;
608
+ }
609
+ if (
610
+ attempt === attempts ||
611
+ this.disposed ||
612
+ !isRetryableLoadError(error)
613
+ ) {
157
614
  throw error;
158
615
  }
159
616
 
160
617
  const backoff = delayMs * attempt;
161
618
  logger.log(`Retrying provider initialization in ${backoff}ms`);
162
- await wait(backoff);
619
+ await wait(backoff, this.disposal.signal);
163
620
  }
164
621
  }
165
622
 
@@ -172,6 +629,17 @@ export class MatrixProviderManager {
172
629
  */
173
630
  private async ensureRoomAvailable(roomId: string): Promise<void> {
174
631
  const room = this.matrixClient.getRoom(roomId);
632
+ const joined = joinedRooms.get(this.matrixClient);
633
+ const joinedAt = joined?.get(roomId);
634
+ this.loadingRoomId = roomId;
635
+ this.joinSkipped =
636
+ !room &&
637
+ joinedAt !== undefined &&
638
+ this.now() - joinedAt < JOIN_REUSE_WINDOW_MS;
639
+ if (this.joinSkipped) {
640
+ logger.debug?.(`Room ${roomId} joined by this client moments ago`);
641
+ return;
642
+ }
175
643
 
176
644
  if (!room) {
177
645
  logger.warn(
@@ -182,6 +650,9 @@ export class MatrixProviderManager {
182
650
  // Try joining first — peek fails when room previews are disabled
183
651
  await this.matrixClient.joinRoom(roomId);
184
652
  logger.log(`Successfully joined room ${roomId}`);
653
+ if (joined) joined.set(roomId, this.now());
654
+ else
655
+ joinedRooms.set(this.matrixClient, new Map([[roomId, this.now()]]));
185
656
  } catch (joinError) {
186
657
  logger.warn(joinError);
187
658
  // Fall back to peek if join fails (e.g. already joined but not synced)
@@ -199,7 +670,12 @@ export class MatrixProviderManager {
199
670
  }
200
671
  }
201
672
 
202
- private async initializeProvider() {
673
+ /** True when this load attempt was superseded (timed out) or the manager disposed. */
674
+ private isStale(attempt: number): boolean {
675
+ return this.disposed || this.attempt !== attempt;
676
+ }
677
+
678
+ private async initializeProvider(attempt: number) {
203
679
  this.cleanupProviderListeners();
204
680
 
205
681
  const roomDescriptor = this.resolveRoomDescriptor();
@@ -219,12 +695,15 @@ export class MatrixProviderManager {
219
695
  // Ensure the room is in the client's store before proceeding
220
696
  // This is critical for MatrixProvider to fetch room history correctly
221
697
  await this.ensureRoomAvailable(roomId);
698
+ if (this.isStale(attempt))
699
+ throw new Error('Document load was cancelled or timed out');
222
700
 
223
701
  logger.log('Creating MatrixProvider', roomDescriptor);
224
702
 
703
+ this.providerInitialized = false;
225
704
  this.provider = new MatrixProvider(
226
705
  this.doc,
227
- this.matrixClient,
706
+ this.writes.client,
228
707
  roomDescriptor,
229
708
  {
230
709
  enableAwareness: this.cfg.provider.enableAwareness,
@@ -245,9 +724,17 @@ export class MatrixProviderManager {
245
724
  },
246
725
  );
247
726
 
248
- this.registerProviderListeners(this.provider);
727
+ const provider = this.provider;
728
+ this.registerProviderListeners(provider);
249
729
 
250
- await this.provider.initialize();
730
+ await provider.initialize();
731
+ if (this.isStale(attempt)) {
732
+ // The attempt timed out (or the manager closed) while the history was
733
+ // loading; this provider is no longer tracked, so stop it here.
734
+ provider.dispose();
735
+ throw new Error('Document load was cancelled or timed out');
736
+ }
737
+ this.providerInitialized = true;
251
738
  }
252
739
 
253
740
  private waitForAvailability(timeoutMs: number): Promise<void> {
@@ -256,22 +743,32 @@ export class MatrixProviderManager {
256
743
  }
257
744
 
258
745
  return new Promise((resolve, reject) => {
259
- const timeoutHandle = setTimeout(() => {
746
+ const signal = this.disposal.signal;
747
+ const settle = (error?: Error) => {
748
+ clearTimeout(timeoutHandle);
749
+ signal.removeEventListener('abort', onDispose);
260
750
  this.availabilityResolvers = this.availabilityResolvers.filter(
261
751
  (candidate) => candidate !== resolver,
262
752
  );
263
- reject(
264
- new Error(
265
- `Document did not become available within ${timeoutMs}ms. Check room permissions or connectivity.`,
266
- ),
267
- );
268
- }, timeoutMs);
269
-
270
- const resolver = () => {
271
- clearTimeout(timeoutHandle);
272
- resolve();
753
+ if (error) reject(error);
754
+ else resolve();
273
755
  };
756
+ const timeoutHandle = setTimeout(
757
+ () =>
758
+ settle(
759
+ new Error(
760
+ `Document did not become available within ${timeoutMs}ms. Check room permissions or connectivity.`,
761
+ ),
762
+ ),
763
+ timeoutMs,
764
+ );
765
+ // Disposal ends the wait at once instead of leaving its timer running.
766
+ const onDispose = () =>
767
+ settle(new Error('MatrixProviderManager was disposed'));
768
+ const resolver = () => settle();
274
769
 
770
+ if (signal.aborted) return onDispose();
771
+ signal.addEventListener('abort', onDispose, { once: true });
275
772
  this.availabilityResolvers.push(resolver);
276
773
  });
277
774
  }
@@ -279,18 +776,13 @@ export class MatrixProviderManager {
279
776
  private registerProviderListeners(provider: MatrixProvider) {
280
777
  this.disposables.push(
281
778
  provider.onDocumentAvailable(() => {
779
+ // matrix-crdt applies the whole catch-up history to the doc before it
780
+ // fires this event, so the content is already in place.
282
781
  logger.log('Matrix document available');
283
-
284
- // Wait briefly for Y.Doc to apply initial sync
285
- // Can't rely on onReceivedEvents - it only fires for NEW events,
286
- // not when connecting to an already-synced client
287
- setTimeout(() => {
288
- logger.log('Marking document as available after sync delay');
289
- this.documentAvailable = true;
290
- const resolvers = [...this.availabilityResolvers];
291
- this.availabilityResolvers = [];
292
- resolvers.forEach((resolver) => resolver());
293
- }, 500); // Short delay for Y.Doc sync
782
+ this.documentAvailable = true;
783
+ const resolvers = [...this.availabilityResolvers];
784
+ this.availabilityResolvers = [];
785
+ resolvers.forEach((resolver) => resolver());
294
786
  }),
295
787
  );
296
788
 
@@ -351,29 +843,50 @@ export class MatrixProviderManager {
351
843
 
352
844
  logger.log('Disposing Matrix provider manager');
353
845
 
846
+ // Mark first so an initialisation still in flight stops at its next step.
847
+ this.disposed = true;
848
+ this.disposal.abort();
849
+
354
850
  // Clean up provider and Y.Doc
355
851
  await this.cleanupProvider();
852
+ this.writes.close();
356
853
  this.doc.destroy();
357
854
 
358
855
  // NOTE: We NEVER call matrixClient.stopClient() here!
359
856
  // The Matrix client is managed by EditorMatrixClient singleton
360
857
  // and is shared across all rooms. Stopping it would break
361
858
  // all other active providers.
362
-
363
- this.disposed = true;
364
859
  }
365
860
 
861
+ /**
862
+ * Dispose the current provider. Pending writes are flushed first (bounded)
863
+ * only when the provider finished loading: before that, matrix-crdt's
864
+ * `waitForFlush()` waits for an initialisation that will never complete.
865
+ */
366
866
  private async cleanupProvider() {
367
867
  this.cleanupProviderListeners();
368
868
 
369
869
  if (this.provider) {
870
+ const provider = this.provider;
871
+ try {
872
+ // Once a write has failed or a flush has timed out, waiting again
873
+ // only delays disposal; closing the guard drops what is left.
874
+ if (
875
+ this.providerInitialized &&
876
+ !this.flushTimedOut &&
877
+ !this.writeFailure
878
+ )
879
+ await this.flush();
880
+ } catch (error) {
881
+ logger.warn('Error while flushing MatrixProvider', error);
882
+ }
370
883
  try {
371
- await this.provider.waitForFlush();
372
- this.provider.dispose();
884
+ provider.dispose();
373
885
  } catch (error) {
374
886
  logger.warn('Error while disposing MatrixProvider', error);
375
887
  } finally {
376
888
  this.provider = undefined;
889
+ this.providerInitialized = false;
377
890
  }
378
891
  }
379
892