@openclaw/crabline 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/README.md +47 -22
  2. package/assets/crabline-banner.svg +20 -0
  3. package/dist/src/bin/crabline.js.map +1 -1
  4. package/dist/src/cli/program.d.ts +25 -1
  5. package/dist/src/cli/program.js +579 -82
  6. package/dist/src/cli/program.js.map +1 -1
  7. package/dist/src/config/load.js +59 -6
  8. package/dist/src/config/load.js.map +1 -1
  9. package/dist/src/config/schema.d.ts +116 -92
  10. package/dist/src/config/schema.js +179 -73
  11. package/dist/src/config/schema.js.map +1 -1
  12. package/dist/src/core/errors.js +9 -2
  13. package/dist/src/core/errors.js.map +1 -1
  14. package/dist/src/core/exit-codes.js.map +1 -1
  15. package/dist/src/core/matcher.js +7 -6
  16. package/dist/src/core/matcher.js.map +1 -1
  17. package/dist/src/core/message-template.js.map +1 -1
  18. package/dist/src/core/nonces.d.ts +1 -0
  19. package/dist/src/core/nonces.js +4 -2
  20. package/dist/src/core/nonces.js.map +1 -1
  21. package/dist/src/core/reporters.d.ts +1 -0
  22. package/dist/src/core/reporters.js +42 -4
  23. package/dist/src/core/reporters.js.map +1 -1
  24. package/dist/src/core/run.d.ts +3 -0
  25. package/dist/src/core/run.js +415 -107
  26. package/dist/src/core/run.js.map +1 -1
  27. package/dist/src/core/safe-regex.d.ts +3 -0
  28. package/dist/src/core/safe-regex.js +21 -0
  29. package/dist/src/core/safe-regex.js.map +1 -0
  30. package/dist/src/index.d.ts +2 -2
  31. package/dist/src/index.js +1 -1
  32. package/dist/src/index.js.map +1 -1
  33. package/dist/src/openclaw/artifact-generation.d.ts +43 -0
  34. package/dist/src/openclaw/artifact-generation.js +363 -0
  35. package/dist/src/openclaw/artifact-generation.js.map +1 -0
  36. package/dist/src/openclaw/bridges/matrix.js +97 -12
  37. package/dist/src/openclaw/bridges/matrix.js.map +1 -1
  38. package/dist/src/openclaw/bridges/mattermost.js +32 -10
  39. package/dist/src/openclaw/bridges/mattermost.js.map +1 -1
  40. package/dist/src/openclaw/bridges/signal.js +55 -18
  41. package/dist/src/openclaw/bridges/signal.js.map +1 -1
  42. package/dist/src/openclaw/bridges/slack.js +85 -7
  43. package/dist/src/openclaw/bridges/slack.js.map +1 -1
  44. package/dist/src/openclaw/bridges/telegram.js +124 -30
  45. package/dist/src/openclaw/bridges/telegram.js.map +1 -1
  46. package/dist/src/openclaw/bridges/whatsapp.js +78 -17
  47. package/dist/src/openclaw/bridges/whatsapp.js.map +1 -1
  48. package/dist/src/openclaw/bridges/zalo.js +14 -5
  49. package/dist/src/openclaw/bridges/zalo.js.map +1 -1
  50. package/dist/src/openclaw/outbound-contract.d.ts +5 -0
  51. package/dist/src/openclaw/outbound-contract.js +40 -0
  52. package/dist/src/openclaw/outbound-contract.js.map +1 -0
  53. package/dist/src/openclaw/private-file.d.ts +28 -0
  54. package/dist/src/openclaw/private-file.js +465 -0
  55. package/dist/src/openclaw/private-file.js.map +1 -0
  56. package/dist/src/openclaw/shared.d.ts +21 -2
  57. package/dist/src/openclaw/shared.js +115 -18
  58. package/dist/src/openclaw/shared.js.map +1 -1
  59. package/dist/src/openclaw/smoke-lock.d.ts +63 -0
  60. package/dist/src/openclaw/smoke-lock.js +1004 -0
  61. package/dist/src/openclaw/smoke-lock.js.map +1 -0
  62. package/dist/src/openclaw.d.ts +16 -8
  63. package/dist/src/openclaw.js +334 -52
  64. package/dist/src/openclaw.js.map +1 -1
  65. package/dist/src/providers/builtin/discord.d.ts +10 -1
  66. package/dist/src/providers/builtin/discord.js +144 -18
  67. package/dist/src/providers/builtin/discord.js.map +1 -1
  68. package/dist/src/providers/builtin/external-webhook-auth.d.ts +11 -0
  69. package/dist/src/providers/builtin/external-webhook-auth.js +10 -0
  70. package/dist/src/providers/builtin/external-webhook-auth.js.map +1 -0
  71. package/dist/src/providers/builtin/feishu.d.ts +19 -2
  72. package/dist/src/providers/builtin/feishu.js +307 -25
  73. package/dist/src/providers/builtin/feishu.js.map +1 -1
  74. package/dist/src/providers/builtin/googlechat.d.ts +20 -1
  75. package/dist/src/providers/builtin/googlechat.js +178 -19
  76. package/dist/src/providers/builtin/googlechat.js.map +1 -1
  77. package/dist/src/providers/builtin/imessage.d.ts +4 -0
  78. package/dist/src/providers/builtin/imessage.js +26 -11
  79. package/dist/src/providers/builtin/imessage.js.map +1 -1
  80. package/dist/src/providers/builtin/loopback.js +92 -30
  81. package/dist/src/providers/builtin/loopback.js.map +1 -1
  82. package/dist/src/providers/builtin/matrix.d.ts +11 -1
  83. package/dist/src/providers/builtin/matrix.js +53 -29
  84. package/dist/src/providers/builtin/matrix.js.map +1 -1
  85. package/dist/src/providers/builtin/mattermost.d.ts +10 -0
  86. package/dist/src/providers/builtin/mattermost.js +38 -15
  87. package/dist/src/providers/builtin/mattermost.js.map +1 -1
  88. package/dist/src/providers/builtin/msteams.d.ts +17 -1
  89. package/dist/src/providers/builtin/msteams.js +113 -14
  90. package/dist/src/providers/builtin/msteams.js.map +1 -1
  91. package/dist/src/providers/builtin/native-local-mock.d.ts +2 -9
  92. package/dist/src/providers/builtin/native-local-mock.js +12 -35
  93. package/dist/src/providers/builtin/native-local-mock.js.map +1 -1
  94. package/dist/src/providers/builtin/script.d.ts +11 -15
  95. package/dist/src/providers/builtin/script.js +566 -104
  96. package/dist/src/providers/builtin/script.js.map +1 -1
  97. package/dist/src/providers/builtin/slack.d.ts +6 -0
  98. package/dist/src/providers/builtin/slack.js +99 -40
  99. package/dist/src/providers/builtin/slack.js.map +1 -1
  100. package/dist/src/providers/builtin/telegram.d.ts +6 -0
  101. package/dist/src/providers/builtin/telegram.js +64 -43
  102. package/dist/src/providers/builtin/telegram.js.map +1 -1
  103. package/dist/src/providers/builtin/whatsapp.d.ts +28 -8
  104. package/dist/src/providers/builtin/whatsapp.js +567 -30
  105. package/dist/src/providers/builtin/whatsapp.js.map +1 -1
  106. package/dist/src/providers/builtin/zalo.js +30 -19
  107. package/dist/src/providers/builtin/zalo.js.map +1 -1
  108. package/dist/src/providers/catalog.d.ts +3 -3
  109. package/dist/src/providers/catalog.js +3 -3
  110. package/dist/src/providers/catalog.js.map +1 -1
  111. package/dist/src/providers/local-mock.d.ts +11 -2
  112. package/dist/src/providers/local-mock.js +322 -96
  113. package/dist/src/providers/local-mock.js.map +1 -1
  114. package/dist/src/providers/native-ids.d.ts +1 -0
  115. package/dist/src/providers/native-ids.js +6 -1
  116. package/dist/src/providers/native-ids.js.map +1 -1
  117. package/dist/src/providers/recorder.d.ts +30 -2
  118. package/dist/src/providers/recorder.js +549 -70
  119. package/dist/src/providers/recorder.js.map +1 -1
  120. package/dist/src/providers/registry.d.ts +25 -1
  121. package/dist/src/providers/registry.js +280 -40
  122. package/dist/src/providers/registry.js.map +1 -1
  123. package/dist/src/providers/signed-jwt.d.ts +29 -0
  124. package/dist/src/providers/signed-jwt.js +202 -0
  125. package/dist/src/providers/signed-jwt.js.map +1 -0
  126. package/dist/src/providers/slack-ids.js.map +1 -1
  127. package/dist/src/providers/target-normalizers.d.ts +26 -0
  128. package/dist/src/providers/target-normalizers.js +276 -0
  129. package/dist/src/providers/target-normalizers.js.map +1 -0
  130. package/dist/src/providers/types.d.ts +5 -0
  131. package/dist/src/providers/types.js.map +1 -1
  132. package/dist/src/providers/webhook-server.d.ts +2 -0
  133. package/dist/src/providers/webhook-server.js +72 -25
  134. package/dist/src/providers/webhook-server.js.map +1 -1
  135. package/dist/src/servers/http.d.ts +23 -3
  136. package/dist/src/servers/http.js +213 -20
  137. package/dist/src/servers/http.js.map +1 -1
  138. package/dist/src/servers/index.js +20 -21
  139. package/dist/src/servers/index.js.map +1 -1
  140. package/dist/src/servers/matrix.js +465 -51
  141. package/dist/src/servers/matrix.js.map +1 -1
  142. package/dist/src/servers/mattermost.d.ts +5 -0
  143. package/dist/src/servers/mattermost.js +306 -58
  144. package/dist/src/servers/mattermost.js.map +1 -1
  145. package/dist/src/servers/pending-events.d.ts +2 -0
  146. package/dist/src/servers/pending-events.js +11 -0
  147. package/dist/src/servers/pending-events.js.map +1 -0
  148. package/dist/src/servers/recorder.d.ts +5 -0
  149. package/dist/src/servers/recorder.js +262 -3
  150. package/dist/src/servers/recorder.js.map +1 -1
  151. package/dist/src/servers/signal.d.ts +2 -0
  152. package/dist/src/servers/signal.js +444 -56
  153. package/dist/src/servers/signal.js.map +1 -1
  154. package/dist/src/servers/slack.d.ts +5 -0
  155. package/dist/src/servers/slack.js +515 -49
  156. package/dist/src/servers/slack.js.map +1 -1
  157. package/dist/src/servers/telegram.d.ts +111 -0
  158. package/dist/src/servers/telegram.js +830 -158
  159. package/dist/src/servers/telegram.js.map +1 -1
  160. package/dist/src/servers/webhook-target.d.ts +40 -0
  161. package/dist/src/servers/webhook-target.js +349 -0
  162. package/dist/src/servers/webhook-target.js.map +1 -0
  163. package/dist/src/servers/websocket.d.ts +2 -0
  164. package/dist/src/servers/websocket.js +29 -0
  165. package/dist/src/servers/websocket.js.map +1 -0
  166. package/dist/src/servers/whatsapp-baileys-websocket.d.ts +65 -1
  167. package/dist/src/servers/whatsapp-baileys-websocket.js +723 -99
  168. package/dist/src/servers/whatsapp-baileys-websocket.js.map +1 -1
  169. package/dist/src/servers/whatsapp-jid.d.ts +5 -0
  170. package/dist/src/servers/whatsapp-jid.js +32 -0
  171. package/dist/src/servers/whatsapp-jid.js.map +1 -0
  172. package/dist/src/servers/whatsapp-wire/binary-node.d.ts +5 -0
  173. package/dist/src/servers/whatsapp-wire/binary-node.js +87 -20
  174. package/dist/src/servers/whatsapp-wire/binary-node.js.map +1 -1
  175. package/dist/src/servers/whatsapp-wire/crypto.d.ts +6 -1
  176. package/dist/src/servers/whatsapp-wire/crypto.js +99 -45
  177. package/dist/src/servers/whatsapp-wire/crypto.js.map +1 -1
  178. package/dist/src/servers/whatsapp-wire/handshake.js +39 -4
  179. package/dist/src/servers/whatsapp-wire/handshake.js.map +1 -1
  180. package/dist/src/servers/whatsapp-wire/signal.js.map +1 -1
  181. package/dist/src/servers/whatsapp.d.ts +12 -7
  182. package/dist/src/servers/whatsapp.js +269 -107
  183. package/dist/src/servers/whatsapp.js.map +1 -1
  184. package/dist/src/servers/zalo.d.ts +14 -0
  185. package/dist/src/servers/zalo.js +521 -88
  186. package/dist/src/servers/zalo.js.map +1 -1
  187. package/docs/channel-setup.md +197 -19
  188. package/fixtures/examples/crabline.example.yaml +10 -0
  189. package/package.json +20 -15
@@ -1,12 +1,382 @@
1
- import { appendFile, mkdir, open, readFile } from "node:fs/promises";
1
+ import { lstat, mkdir, open, readFile, readlink, realpath } from "node:fs/promises";
2
2
  import path from "node:path";
3
- function sleep(ms) {
4
- return new Promise((resolve) => setTimeout(resolve, ms));
3
+ import { lock } from "proper-lockfile";
4
+ function sleep(ms, signal) {
5
+ if (signal?.aborted) {
6
+ return Promise.resolve();
7
+ }
8
+ return new Promise((resolve) => {
9
+ const done = () => {
10
+ clearTimeout(timeout);
11
+ signal?.removeEventListener("abort", done);
12
+ resolve();
13
+ };
14
+ const timeout = setTimeout(done, ms);
15
+ signal?.addEventListener("abort", done, { once: true });
16
+ });
17
+ }
18
+ const pendingAppends = new Map();
19
+ const recordIdentityIndexes = new Map();
20
+ const MAX_RECORD_IDENTITY_INDEXES = 128;
21
+ const MAX_RECENT_RECORD_KEYS = 4096;
22
+ const RECORDER_BATCH_VERSION = 1;
23
+ const RECORDER_LOCK_STALE_MS = 30_000;
24
+ const RECORDER_LOCK_UPDATE_MS = 10_000;
25
+ const RECORDER_ROTATION_ATTEMPTS = 3;
26
+ class RecorderRotatedError extends Error {
27
+ }
28
+ const CONTINUITY_BYTES = 4096;
29
+ const MAX_INCREMENTAL_READ_BYTES = 256 * 1024;
30
+ const MAX_PENDING_RECORD_BYTES = 4 * 1024 * 1024;
31
+ function createIncrementalReadState() {
32
+ return {
33
+ caughtUp: true,
34
+ continuity: Buffer.alloc(0),
35
+ generation: 0,
36
+ identity: undefined,
37
+ offset: 0,
38
+ pending: Buffer.alloc(0),
39
+ };
40
+ }
41
+ export function createRecordedInboundCursor() {
42
+ return {
43
+ buffered: [],
44
+ readState: createIncrementalReadState(),
45
+ seen: new Set(),
46
+ };
47
+ }
48
+ export function cloneRecordedInboundCursor(cursor) {
49
+ return {
50
+ buffered: [...cursor.buffered],
51
+ readState: {
52
+ caughtUp: cursor.readState.caughtUp,
53
+ continuity: Buffer.from(cursor.readState.continuity),
54
+ generation: cursor.readState.generation,
55
+ identity: cursor.readState.identity ? { ...cursor.readState.identity } : undefined,
56
+ offset: cursor.readState.offset,
57
+ pending: Buffer.from(cursor.readState.pending),
58
+ },
59
+ seen: new Set(cursor.seen),
60
+ };
61
+ }
62
+ function rememberRecentRecord(seen, event) {
63
+ const key = JSON.stringify([
64
+ event.provider,
65
+ event.threadId,
66
+ event.id,
67
+ recordedDirectionOf(event),
68
+ ]);
69
+ if (seen.has(key)) {
70
+ return false;
71
+ }
72
+ seen.add(key);
73
+ if (seen.size > MAX_RECENT_RECORD_KEYS) {
74
+ seen.delete(seen.values().next().value);
75
+ }
76
+ return true;
77
+ }
78
+ function recordedDirectionOf(event) {
79
+ if (event.recordedDirection) {
80
+ return event.recordedDirection;
81
+ }
82
+ return event.raw !== null &&
83
+ typeof event.raw === "object" &&
84
+ "direction" in event.raw &&
85
+ event.raw.direction === "outbound"
86
+ ? "outbound"
87
+ : "inbound";
88
+ }
89
+ function requireNonEmptyString(value, field) {
90
+ if (typeof value !== "string" || value.length === 0) {
91
+ throw new Error(`Recorded inbound envelope ${field} must be a non-empty string.`);
92
+ }
93
+ return value;
94
+ }
95
+ function requireString(value, field) {
96
+ if (typeof value !== "string") {
97
+ throw new Error(`Recorded inbound envelope ${field} must be a string.`);
98
+ }
99
+ return value;
100
+ }
101
+ function parseRecordedEnvelope(value) {
102
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
103
+ throw new Error("Recorded inbound envelope must be an object.");
104
+ }
105
+ const record = value;
106
+ if (record.author !== "assistant" && record.author !== "system" && record.author !== "user") {
107
+ throw new Error("Recorded inbound envelope author must be assistant, system, or user.");
108
+ }
109
+ if (record.recordedDirection !== undefined &&
110
+ record.recordedDirection !== "inbound" &&
111
+ record.recordedDirection !== "outbound") {
112
+ throw new Error("Recorded inbound envelope recordedDirection must be inbound or outbound.");
113
+ }
114
+ return {
115
+ author: record.author,
116
+ id: requireNonEmptyString(record.id, "id"),
117
+ provider: requireNonEmptyString(record.provider, "provider"),
118
+ ...(record.raw !== undefined ? { raw: record.raw } : {}),
119
+ recordedAt: requireNonEmptyString(record.recordedAt, "recordedAt"),
120
+ ...(record.recordedDirection !== undefined
121
+ ? { recordedDirection: record.recordedDirection }
122
+ : {}),
123
+ sentAt: requireNonEmptyString(record.sentAt, "sentAt"),
124
+ text: requireString(record.text, "text"),
125
+ threadId: requireNonEmptyString(record.threadId, "threadId"),
126
+ };
127
+ }
128
+ function parseRecordedLine(line) {
129
+ const parsed = JSON.parse(line);
130
+ if (typeof parsed === "object" &&
131
+ parsed !== null &&
132
+ !Array.isArray(parsed) &&
133
+ "recordType" in parsed &&
134
+ parsed.recordType === "crabline.recorder.batch" &&
135
+ "recorderBatchVersion" in parsed &&
136
+ parsed.recorderBatchVersion === RECORDER_BATCH_VERSION &&
137
+ "events" in parsed &&
138
+ Array.isArray(parsed.events)) {
139
+ return parsed.events.map(parseRecordedEnvelope);
140
+ }
141
+ return [parseRecordedEnvelope(parsed)];
142
+ }
143
+ function resetIncrementalReadState(state) {
144
+ if (state.identity !== undefined ||
145
+ state.offset > 0 ||
146
+ state.pending.length > 0 ||
147
+ state.continuity.length > 0) {
148
+ state.generation += 1;
149
+ }
150
+ state.caughtUp = true;
151
+ state.continuity = Buffer.alloc(0);
152
+ state.offset = 0;
153
+ state.pending = Buffer.alloc(0);
154
+ }
155
+ async function readBufferAt(handle, length, position) {
156
+ const buffer = Buffer.alloc(length);
157
+ let bytesRead = 0;
158
+ while (bytesRead < length) {
159
+ const result = await handle.read(buffer, bytesRead, length - bytesRead, position + bytesRead);
160
+ if (result.bytesRead === 0) {
161
+ break;
162
+ }
163
+ bytesRead += result.bytesRead;
164
+ }
165
+ return buffer.subarray(0, bytesRead);
166
+ }
167
+ function consumeRecordedChunk(state, chunk) {
168
+ const continuity = Buffer.concat([state.continuity, chunk]);
169
+ state.continuity = Buffer.from(continuity.subarray(-CONTINUITY_BYTES));
170
+ const raw = state.pending.length > 0 ? Buffer.concat([state.pending, chunk]) : chunk;
171
+ const lastNewline = raw.lastIndexOf(0x0a);
172
+ if (lastNewline < 0) {
173
+ if (raw.length > MAX_PENDING_RECORD_BYTES) {
174
+ throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
175
+ }
176
+ state.pending = Buffer.from(raw);
177
+ return [];
178
+ }
179
+ state.pending = Buffer.from(raw.subarray(lastNewline + 1));
180
+ if (state.pending.length > MAX_PENDING_RECORD_BYTES) {
181
+ throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
182
+ }
183
+ const events = [];
184
+ for (const line of raw.subarray(0, lastNewline).toString("utf8").split("\n")) {
185
+ if (line.trim()) {
186
+ events.push(...parseRecordedLine(line));
187
+ }
188
+ }
189
+ return events;
190
+ }
191
+ async function appendJsonLine(filePath, line) {
192
+ for (let attempt = 0;; attempt++) {
193
+ try {
194
+ await serializeAppend(filePath, async (publicationPath, logicalPath) => {
195
+ await appendCommittedLine(publicationPath, logicalPath, line, true);
196
+ });
197
+ return;
198
+ }
199
+ catch (error) {
200
+ if (!(error instanceof RecorderRotatedError) || attempt + 1 >= RECORDER_ROTATION_ATTEMPTS) {
201
+ throw error;
202
+ }
203
+ }
204
+ }
205
+ }
206
+ async function readRecorderFileIdentity(filePath) {
207
+ try {
208
+ const stats = await lstat(filePath, { bigint: true });
209
+ if (!stats.isFile()) {
210
+ throw new Error(`Recorder path is not a regular file: ${filePath}`);
211
+ }
212
+ return {
213
+ dev: stats.dev,
214
+ ino: stats.ino,
215
+ };
216
+ }
217
+ catch (error) {
218
+ if (error.code === "ENOENT") {
219
+ return undefined;
220
+ }
221
+ throw error;
222
+ }
223
+ }
224
+ function sameRecorderFileIdentity(left, right) {
225
+ return left?.dev === right?.dev && left?.ino === right?.ino;
226
+ }
227
+ async function resolveRecorderPublicationPath(filePath) {
228
+ try {
229
+ return await realpath(filePath);
230
+ }
231
+ catch (error) {
232
+ if (error.code !== "ENOENT") {
233
+ throw error;
234
+ }
235
+ }
236
+ try {
237
+ const stats = await lstat(filePath);
238
+ if (stats.isSymbolicLink()) {
239
+ const target = await readlink(filePath);
240
+ return await resolveRecorderPublicationPath(path.resolve(path.dirname(filePath), target));
241
+ }
242
+ }
243
+ catch (error) {
244
+ if (error.code !== "ENOENT") {
245
+ throw error;
246
+ }
247
+ }
248
+ return path.join(await realpath(path.dirname(filePath)), path.basename(filePath));
249
+ }
250
+ async function prepareRecorderTailForAppend(handle) {
251
+ const stats = await handle.stat();
252
+ if (stats.size === 0) {
253
+ return false;
254
+ }
255
+ const finalByte = await readBufferAt(handle, 1, stats.size - 1);
256
+ if (finalByte[0] === 0x0a) {
257
+ return false;
258
+ }
259
+ const windowSize = Math.min(stats.size, MAX_PENDING_RECORD_BYTES + 1);
260
+ const tailWindow = await readBufferAt(handle, windowSize, stats.size - windowSize);
261
+ const lastNewline = tailWindow.lastIndexOf(0x0a);
262
+ if (lastNewline < 0 && stats.size > MAX_PENDING_RECORD_BYTES) {
263
+ throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
264
+ }
265
+ const tailStart = stats.size - windowSize + lastNewline + 1;
266
+ const tail = tailWindow.subarray(lastNewline + 1).toString("utf8");
267
+ try {
268
+ JSON.parse(tail);
269
+ await handle.writeFile("\n", "utf8");
270
+ return true;
271
+ }
272
+ catch (error) {
273
+ if (!(error instanceof SyntaxError)) {
274
+ throw error;
275
+ }
276
+ await handle.truncate(tailStart);
277
+ return true;
278
+ }
279
+ }
280
+ async function prepareRecorderPathForAppend(publicationPath, logicalPath, durable) {
281
+ const handle = await open(publicationPath, "a+");
282
+ const identity = await handle.stat({ bigint: true });
283
+ const recorderIdentity = { dev: identity.dev, ino: identity.ino };
284
+ try {
285
+ const changed = await prepareRecorderTailForAppend(handle);
286
+ if (changed && durable) {
287
+ await handle.sync();
288
+ }
289
+ }
290
+ finally {
291
+ await handle.close();
292
+ }
293
+ if (!sameRecorderFileIdentity(recorderIdentity, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
294
+ throw new RecorderRotatedError("Recorder rotated while preparing a committed line.");
295
+ }
296
+ return recorderIdentity;
297
+ }
298
+ async function appendCommittedLine(publicationPath, logicalPath, line, durable, expectedIdentity) {
299
+ const handle = await open(publicationPath, "a+");
300
+ const identity = await handle.stat({ bigint: true });
301
+ const recorderIdentity = { dev: identity.dev, ino: identity.ino };
302
+ try {
303
+ if (expectedIdentity !== undefined &&
304
+ !sameRecorderFileIdentity(expectedIdentity, recorderIdentity)) {
305
+ throw new RecorderRotatedError("Recorder rotated before appending a committed line.");
306
+ }
307
+ await prepareRecorderTailForAppend(handle);
308
+ await handle.writeFile(line, "utf8");
309
+ if (durable) {
310
+ await handle.sync();
311
+ }
312
+ }
313
+ finally {
314
+ await handle.close();
315
+ }
316
+ if (!sameRecorderFileIdentity(recorderIdentity, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
317
+ throw new RecorderRotatedError("Recorder rotated while appending a committed line.");
318
+ }
319
+ }
320
+ function recorderLockReleaseError(filePath, operationError, releaseError) {
321
+ return new AggregateError([operationError, releaseError], `Recorder append and lock release both failed for "${filePath}".`, { cause: operationError });
322
+ }
323
+ async function withRecorderLock(filePath, operation) {
324
+ const release = await lock(filePath, {
325
+ realpath: false,
326
+ retries: {
327
+ factor: 1,
328
+ maxTimeout: 10,
329
+ minTimeout: 10,
330
+ retries: 500,
331
+ },
332
+ stale: RECORDER_LOCK_STALE_MS,
333
+ update: RECORDER_LOCK_UPDATE_MS,
334
+ });
335
+ let operationFailed = false;
336
+ let operationError;
337
+ let result;
338
+ try {
339
+ result = await operation();
340
+ }
341
+ catch (error) {
342
+ operationFailed = true;
343
+ operationError = error;
344
+ }
345
+ try {
346
+ await release();
347
+ }
348
+ catch (releaseError) {
349
+ if (operationFailed) {
350
+ throw recorderLockReleaseError(filePath, operationError, releaseError);
351
+ }
352
+ throw releaseError;
353
+ }
354
+ if (operationFailed) {
355
+ throw operationError;
356
+ }
357
+ return result;
5
358
  }
6
- function toRecordKey(event) {
7
- return `${event.provider}:${event.threadId}:${event.id}`;
359
+ async function serializeAppend(filePath, operation) {
360
+ const logicalPath = path.resolve(filePath);
361
+ const key = await resolveRecorderPublicationPath(logicalPath);
362
+ const previous = pendingAppends.get(key) ?? Promise.resolve();
363
+ let result;
364
+ const current = previous
365
+ .catch(() => { })
366
+ .then(async () => {
367
+ result = await withRecorderLock(key, async () => await operation(key, logicalPath));
368
+ });
369
+ pendingAppends.set(key, current);
370
+ try {
371
+ await current;
372
+ return result;
373
+ }
374
+ finally {
375
+ if (pendingAppends.get(key) === current) {
376
+ pendingAppends.delete(key);
377
+ }
378
+ }
8
379
  }
9
- const MAX_WATCH_SEEN_KEYS = 4096;
10
380
  async function readRecordedInboundAppend(filePath, state) {
11
381
  let handle;
12
382
  try {
@@ -15,54 +385,49 @@ async function readRecordedInboundAppend(filePath, state) {
15
385
  catch (error) {
16
386
  const code = error.code;
17
387
  if (code === "ENOENT") {
18
- state.offset = 0;
19
- state.pending = Buffer.alloc(0);
388
+ resetIncrementalReadState(state);
389
+ state.identity = undefined;
20
390
  return [];
21
391
  }
22
392
  throw error;
23
393
  }
24
394
  try {
25
- const size = (await handle.stat()).size;
26
- if (size < state.offset) {
27
- state.offset = 0;
28
- state.pending = Buffer.alloc(0);
395
+ const stats = await handle.stat();
396
+ const identity = { dev: stats.dev, ino: stats.ino };
397
+ const sameFile = state.identity?.dev === identity.dev && state.identity.ino === identity.ino;
398
+ const rotated = state.identity !== undefined && !sameFile;
399
+ let hasContinuity = stats.size >= state.offset;
400
+ if (hasContinuity && state.offset > 0 && state.continuity.length === 0) {
401
+ hasContinuity = sameFile;
402
+ }
403
+ else if (hasContinuity && state.continuity.length > 0) {
404
+ const actual = await readBufferAt(handle, state.continuity.length, state.offset - state.continuity.length);
405
+ hasContinuity = actual.equals(state.continuity);
406
+ }
407
+ if (!hasContinuity) {
408
+ resetIncrementalReadState(state);
409
+ }
410
+ else if (rotated) {
411
+ state.generation += 1;
29
412
  }
30
- const chunks = [];
413
+ state.identity = identity;
414
+ const events = [];
31
415
  let position = state.offset;
32
- while (position < size) {
33
- const chunk = Buffer.alloc(Math.min(64 * 1024, size - position));
416
+ let remainingBatchBytes = MAX_INCREMENTAL_READ_BYTES;
417
+ let reachedUnexpectedEof = false;
418
+ while (position < stats.size && remainingBatchBytes > 0) {
419
+ const chunk = Buffer.alloc(Math.min(64 * 1024, stats.size - position, remainingBatchBytes));
34
420
  const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
35
421
  if (bytesRead === 0) {
422
+ reachedUnexpectedEof = true;
36
423
  break;
37
424
  }
38
- chunks.push(chunk.subarray(0, bytesRead));
39
425
  position += bytesRead;
426
+ state.offset = position;
427
+ remainingBatchBytes -= bytesRead;
428
+ events.push(...consumeRecordedChunk(state, chunk.subarray(0, bytesRead)));
40
429
  }
41
- state.offset = position;
42
- const raw = Buffer.concat([state.pending, ...chunks]);
43
- const lastNewline = raw.lastIndexOf(0x0a);
44
- if (lastNewline < 0) {
45
- state.pending = raw;
46
- return [];
47
- }
48
- state.pending = raw.subarray(lastNewline + 1);
49
- const lines = raw
50
- .subarray(0, lastNewline)
51
- .toString("utf8")
52
- .split("\n")
53
- .filter((line) => line.trim().length > 0);
54
- const events = [];
55
- for (const [index, line] of lines.entries()) {
56
- try {
57
- events.push(JSON.parse(line));
58
- }
59
- catch (error) {
60
- if (index === lines.length - 1 && state.pending.length === 0) {
61
- continue;
62
- }
63
- throw error;
64
- }
65
- }
430
+ state.caughtUp = reachedUnexpectedEof || position >= stats.size;
66
431
  return events;
67
432
  }
68
433
  finally {
@@ -75,9 +440,92 @@ export async function appendRecordedInbound(filePath, event) {
75
440
  ...event,
76
441
  recordedAt: new Date().toISOString(),
77
442
  };
78
- await appendFile(filePath, `${JSON.stringify(recorded)}\n`, "utf8");
443
+ const line = `${JSON.stringify(recorded)}\n`;
444
+ if (Buffer.byteLength(line) > MAX_PENDING_RECORD_BYTES) {
445
+ throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
446
+ }
447
+ await appendJsonLine(filePath, line);
79
448
  return recorded;
80
449
  }
450
+ export async function appendRecordedInboundBatch(filePath, events) {
451
+ if (events.length === 0) {
452
+ return [];
453
+ }
454
+ await mkdir(path.dirname(filePath), { recursive: true });
455
+ for (let attempt = 0;; attempt++) {
456
+ try {
457
+ return await serializeAppend(filePath, async (publicationPath, logicalPath) => {
458
+ const generation = await prepareRecorderPathForAppend(publicationPath, logicalPath, true);
459
+ const seen = await syncRecordIdentityIndex(publicationPath);
460
+ const pendingIdentities = new Set();
461
+ const recorded = [];
462
+ for (const event of events) {
463
+ const identity = recordIdentity(event);
464
+ if (seen.has(identity) || pendingIdentities.has(identity)) {
465
+ continue;
466
+ }
467
+ pendingIdentities.add(identity);
468
+ recorded.push({
469
+ ...event,
470
+ recordedAt: new Date().toISOString(),
471
+ });
472
+ }
473
+ if (recorded.length > 0) {
474
+ const batch = {
475
+ events: recorded,
476
+ recordType: "crabline.recorder.batch",
477
+ recorderBatchVersion: RECORDER_BATCH_VERSION,
478
+ };
479
+ const line = `${JSON.stringify(batch)}\n`;
480
+ if (Buffer.byteLength(line) > MAX_PENDING_RECORD_BYTES) {
481
+ throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
482
+ }
483
+ await appendCommittedLine(publicationPath, logicalPath, line, true, generation);
484
+ await syncRecordIdentityIndex(publicationPath);
485
+ }
486
+ else if (!sameRecorderFileIdentity(generation, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
487
+ throw new RecorderRotatedError("Recorder rotated before confirming a duplicate batch.");
488
+ }
489
+ return recorded;
490
+ });
491
+ }
492
+ catch (error) {
493
+ if (!(error instanceof RecorderRotatedError) || attempt + 1 >= RECORDER_ROTATION_ATTEMPTS) {
494
+ throw error;
495
+ }
496
+ }
497
+ }
498
+ }
499
+ function recordIdentity(event) {
500
+ return JSON.stringify([event.provider, event.threadId, event.id, recordedDirectionOf(event)]);
501
+ }
502
+ async function syncRecordIdentityIndex(filePath) {
503
+ const key = path.resolve(filePath);
504
+ let index = recordIdentityIndexes.get(key);
505
+ if (!index) {
506
+ index = { readState: createIncrementalReadState(), seen: new Set() };
507
+ recordIdentityIndexes.set(key, index);
508
+ if (recordIdentityIndexes.size > MAX_RECORD_IDENTITY_INDEXES) {
509
+ recordIdentityIndexes.delete(recordIdentityIndexes.keys().next().value);
510
+ }
511
+ }
512
+ else {
513
+ recordIdentityIndexes.delete(key);
514
+ recordIdentityIndexes.set(key, index);
515
+ }
516
+ let generation = index.readState.generation;
517
+ do {
518
+ const appended = await readRecordedInboundAppend(filePath, index.readState);
519
+ if (index.readState.generation !== generation) {
520
+ index.seen.clear();
521
+ generation = index.readState.generation;
522
+ }
523
+ for (const event of appended) {
524
+ rememberRecentRecord(index.seen, event);
525
+ }
526
+ } while (!index.readState.caughtUp);
527
+ return index.seen;
528
+ }
81
529
  export async function readRecordedInbound(filePath) {
82
530
  let raw = "";
83
531
  try {
@@ -90,62 +538,90 @@ export async function readRecordedInbound(filePath) {
90
538
  }
91
539
  throw error;
92
540
  }
93
- const lines = raw.split("\n").filter((line) => line.trim().length > 0);
541
+ const lastNewline = raw.lastIndexOf("\n");
542
+ const completed = lastNewline >= 0 ? raw.slice(0, lastNewline) : "";
543
+ const tail = raw.slice(lastNewline + 1);
94
544
  const events = [];
95
- for (const [index, line] of lines.entries()) {
545
+ for (const line of completed.split("\n")) {
546
+ if (line.trim()) {
547
+ events.push(...parseRecordedLine(line));
548
+ }
549
+ }
550
+ if (tail.trim()) {
96
551
  try {
97
- events.push(JSON.parse(line));
552
+ events.push(...parseRecordedLine(tail));
98
553
  }
99
554
  catch (error) {
100
- if (index === lines.length - 1) {
101
- continue;
555
+ if (!(error instanceof SyntaxError)) {
556
+ throw error;
102
557
  }
103
- throw error;
558
+ // Ignore a syntactically partial final append; completed lines remain strict.
104
559
  }
105
560
  }
106
561
  return events;
107
562
  }
108
563
  export async function waitForRecordedInbound(params) {
109
564
  const deadline = Date.now() + params.timeoutMs;
110
- const seen = new Set();
111
- while (Date.now() <= deadline) {
112
- const events = await readRecordedInbound(params.filePath);
113
- for (const event of events) {
114
- const key = toRecordKey(event);
115
- if (seen.has(key)) {
565
+ const cursor = params.cursor ?? createRecordedInboundCursor();
566
+ while (!params.signal?.aborted && Date.now() <= deadline) {
567
+ const generation = cursor.readState.generation;
568
+ const events = cursor.buffered.length > 0
569
+ ? cursor.buffered.splice(0)
570
+ : await readRecordedInboundAppend(params.filePath, cursor.readState);
571
+ if (cursor.readState.generation !== generation) {
572
+ cursor.seen.clear();
573
+ }
574
+ for (const [index, event] of events.entries()) {
575
+ if (params.recordedDirection && recordedDirectionOf(event) !== params.recordedDirection) {
576
+ continue;
577
+ }
578
+ // Incremental read state owns progress; this bounded window only filters appended retries.
579
+ if (!rememberRecentRecord(cursor.seen, event)) {
116
580
  continue;
117
581
  }
118
- seen.add(key);
119
582
  if (params.since && new Date(event.sentAt).getTime() < new Date(params.since).getTime()) {
120
583
  continue;
121
584
  }
122
585
  if (params.matches(event)) {
586
+ cursor.buffered.push(...events.slice(index + 1));
123
587
  return event;
124
588
  }
125
589
  }
126
- await sleep(params.pollMs ?? 200);
590
+ if (!cursor.readState.caughtUp) {
591
+ if (Date.now() >= deadline) {
592
+ return null;
593
+ }
594
+ continue;
595
+ }
596
+ const remainingMs = deadline - Date.now();
597
+ if (remainingMs <= 0) {
598
+ return null;
599
+ }
600
+ await sleep(Math.min(params.pollMs ?? 200, remainingMs), params.signal);
127
601
  }
128
602
  return null;
129
603
  }
130
604
  export async function* watchRecordedInbound(params) {
131
- const state = {
132
- offset: 0,
133
- pending: Buffer.alloc(0),
134
- };
605
+ const state = createIncrementalReadState();
135
606
  const seen = new Set();
136
- while (true) {
607
+ while (!params.signal?.aborted) {
608
+ const generation = state.generation;
137
609
  const events = await readRecordedInboundAppend(params.filePath, state);
610
+ if (state.generation !== generation) {
611
+ seen.clear();
612
+ }
613
+ if (params.signal?.aborted) {
614
+ return;
615
+ }
138
616
  for (const event of events) {
139
- const key = toRecordKey(event);
140
- if (seen.has(key)) {
617
+ if (params.signal?.aborted) {
618
+ return;
619
+ }
620
+ if (!rememberRecentRecord(seen, event)) {
141
621
  continue;
142
622
  }
143
- seen.add(key);
144
- if (seen.size > MAX_WATCH_SEEN_KEYS) {
145
- const oldest = seen.values().next().value;
146
- if (oldest !== undefined) {
147
- seen.delete(oldest);
148
- }
623
+ if (params.recordedDirection && recordedDirectionOf(event) !== params.recordedDirection) {
624
+ continue;
149
625
  }
150
626
  if (params.since && new Date(event.sentAt).getTime() < new Date(params.since).getTime()) {
151
627
  continue;
@@ -154,7 +630,10 @@ export async function* watchRecordedInbound(params) {
154
630
  yield event;
155
631
  }
156
632
  }
157
- await sleep(params.pollMs ?? 250);
633
+ if (!state.caughtUp) {
634
+ continue;
635
+ }
636
+ await sleep(params.pollMs ?? 250, params.signal);
158
637
  }
159
638
  }
160
639
  //# sourceMappingURL=recorder.js.map