@openclaw/crabline 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -22
- package/assets/crabline-banner.svg +20 -0
- package/dist/src/bin/crabline.js.map +1 -1
- package/dist/src/cli/program.d.ts +25 -1
- package/dist/src/cli/program.js +579 -82
- package/dist/src/cli/program.js.map +1 -1
- package/dist/src/config/load.js +59 -6
- package/dist/src/config/load.js.map +1 -1
- package/dist/src/config/schema.d.ts +116 -92
- package/dist/src/config/schema.js +179 -73
- package/dist/src/config/schema.js.map +1 -1
- package/dist/src/core/errors.js +9 -2
- package/dist/src/core/errors.js.map +1 -1
- package/dist/src/core/exit-codes.js.map +1 -1
- package/dist/src/core/matcher.js +7 -6
- package/dist/src/core/matcher.js.map +1 -1
- package/dist/src/core/message-template.js.map +1 -1
- package/dist/src/core/nonces.d.ts +1 -0
- package/dist/src/core/nonces.js +4 -2
- package/dist/src/core/nonces.js.map +1 -1
- package/dist/src/core/reporters.d.ts +1 -0
- package/dist/src/core/reporters.js +42 -4
- package/dist/src/core/reporters.js.map +1 -1
- package/dist/src/core/run.d.ts +3 -0
- package/dist/src/core/run.js +415 -107
- package/dist/src/core/run.js.map +1 -1
- package/dist/src/core/safe-regex.d.ts +3 -0
- package/dist/src/core/safe-regex.js +21 -0
- package/dist/src/core/safe-regex.js.map +1 -0
- package/dist/src/index.d.ts +2 -2
- package/dist/src/index.js +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/openclaw/artifact-generation.d.ts +43 -0
- package/dist/src/openclaw/artifact-generation.js +363 -0
- package/dist/src/openclaw/artifact-generation.js.map +1 -0
- package/dist/src/openclaw/bridges/matrix.js +97 -12
- package/dist/src/openclaw/bridges/matrix.js.map +1 -1
- package/dist/src/openclaw/bridges/mattermost.js +32 -10
- package/dist/src/openclaw/bridges/mattermost.js.map +1 -1
- package/dist/src/openclaw/bridges/signal.js +55 -18
- package/dist/src/openclaw/bridges/signal.js.map +1 -1
- package/dist/src/openclaw/bridges/slack.js +85 -7
- package/dist/src/openclaw/bridges/slack.js.map +1 -1
- package/dist/src/openclaw/bridges/telegram.js +124 -30
- package/dist/src/openclaw/bridges/telegram.js.map +1 -1
- package/dist/src/openclaw/bridges/whatsapp.js +78 -17
- package/dist/src/openclaw/bridges/whatsapp.js.map +1 -1
- package/dist/src/openclaw/bridges/zalo.js +14 -5
- package/dist/src/openclaw/bridges/zalo.js.map +1 -1
- package/dist/src/openclaw/outbound-contract.d.ts +5 -0
- package/dist/src/openclaw/outbound-contract.js +40 -0
- package/dist/src/openclaw/outbound-contract.js.map +1 -0
- package/dist/src/openclaw/private-file.d.ts +28 -0
- package/dist/src/openclaw/private-file.js +465 -0
- package/dist/src/openclaw/private-file.js.map +1 -0
- package/dist/src/openclaw/shared.d.ts +21 -2
- package/dist/src/openclaw/shared.js +115 -18
- package/dist/src/openclaw/shared.js.map +1 -1
- package/dist/src/openclaw/smoke-lock.d.ts +63 -0
- package/dist/src/openclaw/smoke-lock.js +1004 -0
- package/dist/src/openclaw/smoke-lock.js.map +1 -0
- package/dist/src/openclaw.d.ts +16 -8
- package/dist/src/openclaw.js +334 -52
- package/dist/src/openclaw.js.map +1 -1
- package/dist/src/providers/builtin/discord.d.ts +10 -1
- package/dist/src/providers/builtin/discord.js +144 -18
- package/dist/src/providers/builtin/discord.js.map +1 -1
- package/dist/src/providers/builtin/external-webhook-auth.d.ts +11 -0
- package/dist/src/providers/builtin/external-webhook-auth.js +10 -0
- package/dist/src/providers/builtin/external-webhook-auth.js.map +1 -0
- package/dist/src/providers/builtin/feishu.d.ts +19 -2
- package/dist/src/providers/builtin/feishu.js +307 -25
- package/dist/src/providers/builtin/feishu.js.map +1 -1
- package/dist/src/providers/builtin/googlechat.d.ts +20 -1
- package/dist/src/providers/builtin/googlechat.js +178 -19
- package/dist/src/providers/builtin/googlechat.js.map +1 -1
- package/dist/src/providers/builtin/imessage.d.ts +4 -0
- package/dist/src/providers/builtin/imessage.js +26 -11
- package/dist/src/providers/builtin/imessage.js.map +1 -1
- package/dist/src/providers/builtin/loopback.js +92 -30
- package/dist/src/providers/builtin/loopback.js.map +1 -1
- package/dist/src/providers/builtin/matrix.d.ts +11 -1
- package/dist/src/providers/builtin/matrix.js +53 -29
- package/dist/src/providers/builtin/matrix.js.map +1 -1
- package/dist/src/providers/builtin/mattermost.d.ts +10 -0
- package/dist/src/providers/builtin/mattermost.js +38 -15
- package/dist/src/providers/builtin/mattermost.js.map +1 -1
- package/dist/src/providers/builtin/msteams.d.ts +17 -1
- package/dist/src/providers/builtin/msteams.js +113 -14
- package/dist/src/providers/builtin/msteams.js.map +1 -1
- package/dist/src/providers/builtin/native-local-mock.d.ts +2 -9
- package/dist/src/providers/builtin/native-local-mock.js +12 -35
- package/dist/src/providers/builtin/native-local-mock.js.map +1 -1
- package/dist/src/providers/builtin/script.d.ts +11 -15
- package/dist/src/providers/builtin/script.js +566 -104
- package/dist/src/providers/builtin/script.js.map +1 -1
- package/dist/src/providers/builtin/slack.d.ts +6 -0
- package/dist/src/providers/builtin/slack.js +99 -40
- package/dist/src/providers/builtin/slack.js.map +1 -1
- package/dist/src/providers/builtin/telegram.d.ts +6 -0
- package/dist/src/providers/builtin/telegram.js +64 -43
- package/dist/src/providers/builtin/telegram.js.map +1 -1
- package/dist/src/providers/builtin/whatsapp.d.ts +28 -8
- package/dist/src/providers/builtin/whatsapp.js +567 -30
- package/dist/src/providers/builtin/whatsapp.js.map +1 -1
- package/dist/src/providers/builtin/zalo.js +30 -19
- package/dist/src/providers/builtin/zalo.js.map +1 -1
- package/dist/src/providers/catalog.d.ts +3 -3
- package/dist/src/providers/catalog.js +3 -3
- package/dist/src/providers/catalog.js.map +1 -1
- package/dist/src/providers/local-mock.d.ts +11 -2
- package/dist/src/providers/local-mock.js +322 -96
- package/dist/src/providers/local-mock.js.map +1 -1
- package/dist/src/providers/native-ids.d.ts +1 -0
- package/dist/src/providers/native-ids.js +6 -1
- package/dist/src/providers/native-ids.js.map +1 -1
- package/dist/src/providers/recorder.d.ts +30 -2
- package/dist/src/providers/recorder.js +549 -70
- package/dist/src/providers/recorder.js.map +1 -1
- package/dist/src/providers/registry.d.ts +25 -1
- package/dist/src/providers/registry.js +280 -40
- package/dist/src/providers/registry.js.map +1 -1
- package/dist/src/providers/signed-jwt.d.ts +29 -0
- package/dist/src/providers/signed-jwt.js +202 -0
- package/dist/src/providers/signed-jwt.js.map +1 -0
- package/dist/src/providers/slack-ids.js.map +1 -1
- package/dist/src/providers/target-normalizers.d.ts +26 -0
- package/dist/src/providers/target-normalizers.js +276 -0
- package/dist/src/providers/target-normalizers.js.map +1 -0
- package/dist/src/providers/types.d.ts +5 -0
- package/dist/src/providers/types.js.map +1 -1
- package/dist/src/providers/webhook-server.d.ts +2 -0
- package/dist/src/providers/webhook-server.js +72 -25
- package/dist/src/providers/webhook-server.js.map +1 -1
- package/dist/src/servers/http.d.ts +23 -3
- package/dist/src/servers/http.js +213 -20
- package/dist/src/servers/http.js.map +1 -1
- package/dist/src/servers/index.js +20 -21
- package/dist/src/servers/index.js.map +1 -1
- package/dist/src/servers/matrix.js +465 -51
- package/dist/src/servers/matrix.js.map +1 -1
- package/dist/src/servers/mattermost.d.ts +5 -0
- package/dist/src/servers/mattermost.js +306 -58
- package/dist/src/servers/mattermost.js.map +1 -1
- package/dist/src/servers/pending-events.d.ts +2 -0
- package/dist/src/servers/pending-events.js +11 -0
- package/dist/src/servers/pending-events.js.map +1 -0
- package/dist/src/servers/recorder.d.ts +5 -0
- package/dist/src/servers/recorder.js +262 -3
- package/dist/src/servers/recorder.js.map +1 -1
- package/dist/src/servers/signal.d.ts +2 -0
- package/dist/src/servers/signal.js +444 -56
- package/dist/src/servers/signal.js.map +1 -1
- package/dist/src/servers/slack.d.ts +5 -0
- package/dist/src/servers/slack.js +515 -49
- package/dist/src/servers/slack.js.map +1 -1
- package/dist/src/servers/telegram.d.ts +111 -0
- package/dist/src/servers/telegram.js +830 -158
- package/dist/src/servers/telegram.js.map +1 -1
- package/dist/src/servers/webhook-target.d.ts +40 -0
- package/dist/src/servers/webhook-target.js +349 -0
- package/dist/src/servers/webhook-target.js.map +1 -0
- package/dist/src/servers/websocket.d.ts +2 -0
- package/dist/src/servers/websocket.js +29 -0
- package/dist/src/servers/websocket.js.map +1 -0
- package/dist/src/servers/whatsapp-baileys-websocket.d.ts +65 -1
- package/dist/src/servers/whatsapp-baileys-websocket.js +723 -99
- package/dist/src/servers/whatsapp-baileys-websocket.js.map +1 -1
- package/dist/src/servers/whatsapp-jid.d.ts +5 -0
- package/dist/src/servers/whatsapp-jid.js +32 -0
- package/dist/src/servers/whatsapp-jid.js.map +1 -0
- package/dist/src/servers/whatsapp-wire/binary-node.d.ts +5 -0
- package/dist/src/servers/whatsapp-wire/binary-node.js +87 -20
- package/dist/src/servers/whatsapp-wire/binary-node.js.map +1 -1
- package/dist/src/servers/whatsapp-wire/crypto.d.ts +6 -1
- package/dist/src/servers/whatsapp-wire/crypto.js +99 -45
- package/dist/src/servers/whatsapp-wire/crypto.js.map +1 -1
- package/dist/src/servers/whatsapp-wire/handshake.js +39 -4
- package/dist/src/servers/whatsapp-wire/handshake.js.map +1 -1
- package/dist/src/servers/whatsapp-wire/signal.js.map +1 -1
- package/dist/src/servers/whatsapp.d.ts +12 -7
- package/dist/src/servers/whatsapp.js +269 -107
- package/dist/src/servers/whatsapp.js.map +1 -1
- package/dist/src/servers/zalo.d.ts +14 -0
- package/dist/src/servers/zalo.js +521 -88
- package/dist/src/servers/zalo.js.map +1 -1
- package/docs/channel-setup.md +197 -19
- package/fixtures/examples/crabline.example.yaml +10 -0
- package/package.json +20 -15
|
@@ -1,12 +1,382 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { lstat, mkdir, open, readFile, readlink, realpath } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
import { lock } from "proper-lockfile";
|
|
4
|
+
function sleep(ms, signal) {
|
|
5
|
+
if (signal?.aborted) {
|
|
6
|
+
return Promise.resolve();
|
|
7
|
+
}
|
|
8
|
+
return new Promise((resolve) => {
|
|
9
|
+
const done = () => {
|
|
10
|
+
clearTimeout(timeout);
|
|
11
|
+
signal?.removeEventListener("abort", done);
|
|
12
|
+
resolve();
|
|
13
|
+
};
|
|
14
|
+
const timeout = setTimeout(done, ms);
|
|
15
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
const pendingAppends = new Map();
|
|
19
|
+
const recordIdentityIndexes = new Map();
|
|
20
|
+
const MAX_RECORD_IDENTITY_INDEXES = 128;
|
|
21
|
+
const MAX_RECENT_RECORD_KEYS = 4096;
|
|
22
|
+
const RECORDER_BATCH_VERSION = 1;
|
|
23
|
+
const RECORDER_LOCK_STALE_MS = 30_000;
|
|
24
|
+
const RECORDER_LOCK_UPDATE_MS = 10_000;
|
|
25
|
+
const RECORDER_ROTATION_ATTEMPTS = 3;
|
|
26
|
+
class RecorderRotatedError extends Error {
|
|
27
|
+
}
|
|
28
|
+
const CONTINUITY_BYTES = 4096;
|
|
29
|
+
const MAX_INCREMENTAL_READ_BYTES = 256 * 1024;
|
|
30
|
+
const MAX_PENDING_RECORD_BYTES = 4 * 1024 * 1024;
|
|
31
|
+
function createIncrementalReadState() {
|
|
32
|
+
return {
|
|
33
|
+
caughtUp: true,
|
|
34
|
+
continuity: Buffer.alloc(0),
|
|
35
|
+
generation: 0,
|
|
36
|
+
identity: undefined,
|
|
37
|
+
offset: 0,
|
|
38
|
+
pending: Buffer.alloc(0),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export function createRecordedInboundCursor() {
|
|
42
|
+
return {
|
|
43
|
+
buffered: [],
|
|
44
|
+
readState: createIncrementalReadState(),
|
|
45
|
+
seen: new Set(),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
export function cloneRecordedInboundCursor(cursor) {
|
|
49
|
+
return {
|
|
50
|
+
buffered: [...cursor.buffered],
|
|
51
|
+
readState: {
|
|
52
|
+
caughtUp: cursor.readState.caughtUp,
|
|
53
|
+
continuity: Buffer.from(cursor.readState.continuity),
|
|
54
|
+
generation: cursor.readState.generation,
|
|
55
|
+
identity: cursor.readState.identity ? { ...cursor.readState.identity } : undefined,
|
|
56
|
+
offset: cursor.readState.offset,
|
|
57
|
+
pending: Buffer.from(cursor.readState.pending),
|
|
58
|
+
},
|
|
59
|
+
seen: new Set(cursor.seen),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function rememberRecentRecord(seen, event) {
|
|
63
|
+
const key = JSON.stringify([
|
|
64
|
+
event.provider,
|
|
65
|
+
event.threadId,
|
|
66
|
+
event.id,
|
|
67
|
+
recordedDirectionOf(event),
|
|
68
|
+
]);
|
|
69
|
+
if (seen.has(key)) {
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
seen.add(key);
|
|
73
|
+
if (seen.size > MAX_RECENT_RECORD_KEYS) {
|
|
74
|
+
seen.delete(seen.values().next().value);
|
|
75
|
+
}
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
function recordedDirectionOf(event) {
|
|
79
|
+
if (event.recordedDirection) {
|
|
80
|
+
return event.recordedDirection;
|
|
81
|
+
}
|
|
82
|
+
return event.raw !== null &&
|
|
83
|
+
typeof event.raw === "object" &&
|
|
84
|
+
"direction" in event.raw &&
|
|
85
|
+
event.raw.direction === "outbound"
|
|
86
|
+
? "outbound"
|
|
87
|
+
: "inbound";
|
|
88
|
+
}
|
|
89
|
+
function requireNonEmptyString(value, field) {
|
|
90
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
91
|
+
throw new Error(`Recorded inbound envelope ${field} must be a non-empty string.`);
|
|
92
|
+
}
|
|
93
|
+
return value;
|
|
94
|
+
}
|
|
95
|
+
function requireString(value, field) {
|
|
96
|
+
if (typeof value !== "string") {
|
|
97
|
+
throw new Error(`Recorded inbound envelope ${field} must be a string.`);
|
|
98
|
+
}
|
|
99
|
+
return value;
|
|
100
|
+
}
|
|
101
|
+
function parseRecordedEnvelope(value) {
|
|
102
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
103
|
+
throw new Error("Recorded inbound envelope must be an object.");
|
|
104
|
+
}
|
|
105
|
+
const record = value;
|
|
106
|
+
if (record.author !== "assistant" && record.author !== "system" && record.author !== "user") {
|
|
107
|
+
throw new Error("Recorded inbound envelope author must be assistant, system, or user.");
|
|
108
|
+
}
|
|
109
|
+
if (record.recordedDirection !== undefined &&
|
|
110
|
+
record.recordedDirection !== "inbound" &&
|
|
111
|
+
record.recordedDirection !== "outbound") {
|
|
112
|
+
throw new Error("Recorded inbound envelope recordedDirection must be inbound or outbound.");
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
author: record.author,
|
|
116
|
+
id: requireNonEmptyString(record.id, "id"),
|
|
117
|
+
provider: requireNonEmptyString(record.provider, "provider"),
|
|
118
|
+
...(record.raw !== undefined ? { raw: record.raw } : {}),
|
|
119
|
+
recordedAt: requireNonEmptyString(record.recordedAt, "recordedAt"),
|
|
120
|
+
...(record.recordedDirection !== undefined
|
|
121
|
+
? { recordedDirection: record.recordedDirection }
|
|
122
|
+
: {}),
|
|
123
|
+
sentAt: requireNonEmptyString(record.sentAt, "sentAt"),
|
|
124
|
+
text: requireString(record.text, "text"),
|
|
125
|
+
threadId: requireNonEmptyString(record.threadId, "threadId"),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function parseRecordedLine(line) {
|
|
129
|
+
const parsed = JSON.parse(line);
|
|
130
|
+
if (typeof parsed === "object" &&
|
|
131
|
+
parsed !== null &&
|
|
132
|
+
!Array.isArray(parsed) &&
|
|
133
|
+
"recordType" in parsed &&
|
|
134
|
+
parsed.recordType === "crabline.recorder.batch" &&
|
|
135
|
+
"recorderBatchVersion" in parsed &&
|
|
136
|
+
parsed.recorderBatchVersion === RECORDER_BATCH_VERSION &&
|
|
137
|
+
"events" in parsed &&
|
|
138
|
+
Array.isArray(parsed.events)) {
|
|
139
|
+
return parsed.events.map(parseRecordedEnvelope);
|
|
140
|
+
}
|
|
141
|
+
return [parseRecordedEnvelope(parsed)];
|
|
142
|
+
}
|
|
143
|
+
function resetIncrementalReadState(state) {
|
|
144
|
+
if (state.identity !== undefined ||
|
|
145
|
+
state.offset > 0 ||
|
|
146
|
+
state.pending.length > 0 ||
|
|
147
|
+
state.continuity.length > 0) {
|
|
148
|
+
state.generation += 1;
|
|
149
|
+
}
|
|
150
|
+
state.caughtUp = true;
|
|
151
|
+
state.continuity = Buffer.alloc(0);
|
|
152
|
+
state.offset = 0;
|
|
153
|
+
state.pending = Buffer.alloc(0);
|
|
154
|
+
}
|
|
155
|
+
async function readBufferAt(handle, length, position) {
|
|
156
|
+
const buffer = Buffer.alloc(length);
|
|
157
|
+
let bytesRead = 0;
|
|
158
|
+
while (bytesRead < length) {
|
|
159
|
+
const result = await handle.read(buffer, bytesRead, length - bytesRead, position + bytesRead);
|
|
160
|
+
if (result.bytesRead === 0) {
|
|
161
|
+
break;
|
|
162
|
+
}
|
|
163
|
+
bytesRead += result.bytesRead;
|
|
164
|
+
}
|
|
165
|
+
return buffer.subarray(0, bytesRead);
|
|
166
|
+
}
|
|
167
|
+
function consumeRecordedChunk(state, chunk) {
|
|
168
|
+
const continuity = Buffer.concat([state.continuity, chunk]);
|
|
169
|
+
state.continuity = Buffer.from(continuity.subarray(-CONTINUITY_BYTES));
|
|
170
|
+
const raw = state.pending.length > 0 ? Buffer.concat([state.pending, chunk]) : chunk;
|
|
171
|
+
const lastNewline = raw.lastIndexOf(0x0a);
|
|
172
|
+
if (lastNewline < 0) {
|
|
173
|
+
if (raw.length > MAX_PENDING_RECORD_BYTES) {
|
|
174
|
+
throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
|
|
175
|
+
}
|
|
176
|
+
state.pending = Buffer.from(raw);
|
|
177
|
+
return [];
|
|
178
|
+
}
|
|
179
|
+
state.pending = Buffer.from(raw.subarray(lastNewline + 1));
|
|
180
|
+
if (state.pending.length > MAX_PENDING_RECORD_BYTES) {
|
|
181
|
+
throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
|
|
182
|
+
}
|
|
183
|
+
const events = [];
|
|
184
|
+
for (const line of raw.subarray(0, lastNewline).toString("utf8").split("\n")) {
|
|
185
|
+
if (line.trim()) {
|
|
186
|
+
events.push(...parseRecordedLine(line));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return events;
|
|
190
|
+
}
|
|
191
|
+
async function appendJsonLine(filePath, line) {
|
|
192
|
+
for (let attempt = 0;; attempt++) {
|
|
193
|
+
try {
|
|
194
|
+
await serializeAppend(filePath, async (publicationPath, logicalPath) => {
|
|
195
|
+
await appendCommittedLine(publicationPath, logicalPath, line, true);
|
|
196
|
+
});
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
if (!(error instanceof RecorderRotatedError) || attempt + 1 >= RECORDER_ROTATION_ATTEMPTS) {
|
|
201
|
+
throw error;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
async function readRecorderFileIdentity(filePath) {
|
|
207
|
+
try {
|
|
208
|
+
const stats = await lstat(filePath, { bigint: true });
|
|
209
|
+
if (!stats.isFile()) {
|
|
210
|
+
throw new Error(`Recorder path is not a regular file: ${filePath}`);
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
dev: stats.dev,
|
|
214
|
+
ino: stats.ino,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
if (error.code === "ENOENT") {
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function sameRecorderFileIdentity(left, right) {
|
|
225
|
+
return left?.dev === right?.dev && left?.ino === right?.ino;
|
|
226
|
+
}
|
|
227
|
+
async function resolveRecorderPublicationPath(filePath) {
|
|
228
|
+
try {
|
|
229
|
+
return await realpath(filePath);
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
if (error.code !== "ENOENT") {
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
try {
|
|
237
|
+
const stats = await lstat(filePath);
|
|
238
|
+
if (stats.isSymbolicLink()) {
|
|
239
|
+
const target = await readlink(filePath);
|
|
240
|
+
return await resolveRecorderPublicationPath(path.resolve(path.dirname(filePath), target));
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
if (error.code !== "ENOENT") {
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return path.join(await realpath(path.dirname(filePath)), path.basename(filePath));
|
|
249
|
+
}
|
|
250
|
+
async function prepareRecorderTailForAppend(handle) {
|
|
251
|
+
const stats = await handle.stat();
|
|
252
|
+
if (stats.size === 0) {
|
|
253
|
+
return false;
|
|
254
|
+
}
|
|
255
|
+
const finalByte = await readBufferAt(handle, 1, stats.size - 1);
|
|
256
|
+
if (finalByte[0] === 0x0a) {
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
const windowSize = Math.min(stats.size, MAX_PENDING_RECORD_BYTES + 1);
|
|
260
|
+
const tailWindow = await readBufferAt(handle, windowSize, stats.size - windowSize);
|
|
261
|
+
const lastNewline = tailWindow.lastIndexOf(0x0a);
|
|
262
|
+
if (lastNewline < 0 && stats.size > MAX_PENDING_RECORD_BYTES) {
|
|
263
|
+
throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
|
|
264
|
+
}
|
|
265
|
+
const tailStart = stats.size - windowSize + lastNewline + 1;
|
|
266
|
+
const tail = tailWindow.subarray(lastNewline + 1).toString("utf8");
|
|
267
|
+
try {
|
|
268
|
+
JSON.parse(tail);
|
|
269
|
+
await handle.writeFile("\n", "utf8");
|
|
270
|
+
return true;
|
|
271
|
+
}
|
|
272
|
+
catch (error) {
|
|
273
|
+
if (!(error instanceof SyntaxError)) {
|
|
274
|
+
throw error;
|
|
275
|
+
}
|
|
276
|
+
await handle.truncate(tailStart);
|
|
277
|
+
return true;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
async function prepareRecorderPathForAppend(publicationPath, logicalPath, durable) {
|
|
281
|
+
const handle = await open(publicationPath, "a+");
|
|
282
|
+
const identity = await handle.stat({ bigint: true });
|
|
283
|
+
const recorderIdentity = { dev: identity.dev, ino: identity.ino };
|
|
284
|
+
try {
|
|
285
|
+
const changed = await prepareRecorderTailForAppend(handle);
|
|
286
|
+
if (changed && durable) {
|
|
287
|
+
await handle.sync();
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
finally {
|
|
291
|
+
await handle.close();
|
|
292
|
+
}
|
|
293
|
+
if (!sameRecorderFileIdentity(recorderIdentity, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
|
|
294
|
+
throw new RecorderRotatedError("Recorder rotated while preparing a committed line.");
|
|
295
|
+
}
|
|
296
|
+
return recorderIdentity;
|
|
297
|
+
}
|
|
298
|
+
async function appendCommittedLine(publicationPath, logicalPath, line, durable, expectedIdentity) {
|
|
299
|
+
const handle = await open(publicationPath, "a+");
|
|
300
|
+
const identity = await handle.stat({ bigint: true });
|
|
301
|
+
const recorderIdentity = { dev: identity.dev, ino: identity.ino };
|
|
302
|
+
try {
|
|
303
|
+
if (expectedIdentity !== undefined &&
|
|
304
|
+
!sameRecorderFileIdentity(expectedIdentity, recorderIdentity)) {
|
|
305
|
+
throw new RecorderRotatedError("Recorder rotated before appending a committed line.");
|
|
306
|
+
}
|
|
307
|
+
await prepareRecorderTailForAppend(handle);
|
|
308
|
+
await handle.writeFile(line, "utf8");
|
|
309
|
+
if (durable) {
|
|
310
|
+
await handle.sync();
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
finally {
|
|
314
|
+
await handle.close();
|
|
315
|
+
}
|
|
316
|
+
if (!sameRecorderFileIdentity(recorderIdentity, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
|
|
317
|
+
throw new RecorderRotatedError("Recorder rotated while appending a committed line.");
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
function recorderLockReleaseError(filePath, operationError, releaseError) {
|
|
321
|
+
return new AggregateError([operationError, releaseError], `Recorder append and lock release both failed for "${filePath}".`, { cause: operationError });
|
|
322
|
+
}
|
|
323
|
+
async function withRecorderLock(filePath, operation) {
|
|
324
|
+
const release = await lock(filePath, {
|
|
325
|
+
realpath: false,
|
|
326
|
+
retries: {
|
|
327
|
+
factor: 1,
|
|
328
|
+
maxTimeout: 10,
|
|
329
|
+
minTimeout: 10,
|
|
330
|
+
retries: 500,
|
|
331
|
+
},
|
|
332
|
+
stale: RECORDER_LOCK_STALE_MS,
|
|
333
|
+
update: RECORDER_LOCK_UPDATE_MS,
|
|
334
|
+
});
|
|
335
|
+
let operationFailed = false;
|
|
336
|
+
let operationError;
|
|
337
|
+
let result;
|
|
338
|
+
try {
|
|
339
|
+
result = await operation();
|
|
340
|
+
}
|
|
341
|
+
catch (error) {
|
|
342
|
+
operationFailed = true;
|
|
343
|
+
operationError = error;
|
|
344
|
+
}
|
|
345
|
+
try {
|
|
346
|
+
await release();
|
|
347
|
+
}
|
|
348
|
+
catch (releaseError) {
|
|
349
|
+
if (operationFailed) {
|
|
350
|
+
throw recorderLockReleaseError(filePath, operationError, releaseError);
|
|
351
|
+
}
|
|
352
|
+
throw releaseError;
|
|
353
|
+
}
|
|
354
|
+
if (operationFailed) {
|
|
355
|
+
throw operationError;
|
|
356
|
+
}
|
|
357
|
+
return result;
|
|
5
358
|
}
|
|
6
|
-
function
|
|
7
|
-
|
|
359
|
+
async function serializeAppend(filePath, operation) {
|
|
360
|
+
const logicalPath = path.resolve(filePath);
|
|
361
|
+
const key = await resolveRecorderPublicationPath(logicalPath);
|
|
362
|
+
const previous = pendingAppends.get(key) ?? Promise.resolve();
|
|
363
|
+
let result;
|
|
364
|
+
const current = previous
|
|
365
|
+
.catch(() => { })
|
|
366
|
+
.then(async () => {
|
|
367
|
+
result = await withRecorderLock(key, async () => await operation(key, logicalPath));
|
|
368
|
+
});
|
|
369
|
+
pendingAppends.set(key, current);
|
|
370
|
+
try {
|
|
371
|
+
await current;
|
|
372
|
+
return result;
|
|
373
|
+
}
|
|
374
|
+
finally {
|
|
375
|
+
if (pendingAppends.get(key) === current) {
|
|
376
|
+
pendingAppends.delete(key);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
8
379
|
}
|
|
9
|
-
const MAX_WATCH_SEEN_KEYS = 4096;
|
|
10
380
|
async function readRecordedInboundAppend(filePath, state) {
|
|
11
381
|
let handle;
|
|
12
382
|
try {
|
|
@@ -15,54 +385,49 @@ async function readRecordedInboundAppend(filePath, state) {
|
|
|
15
385
|
catch (error) {
|
|
16
386
|
const code = error.code;
|
|
17
387
|
if (code === "ENOENT") {
|
|
18
|
-
state
|
|
19
|
-
state.
|
|
388
|
+
resetIncrementalReadState(state);
|
|
389
|
+
state.identity = undefined;
|
|
20
390
|
return [];
|
|
21
391
|
}
|
|
22
392
|
throw error;
|
|
23
393
|
}
|
|
24
394
|
try {
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
395
|
+
const stats = await handle.stat();
|
|
396
|
+
const identity = { dev: stats.dev, ino: stats.ino };
|
|
397
|
+
const sameFile = state.identity?.dev === identity.dev && state.identity.ino === identity.ino;
|
|
398
|
+
const rotated = state.identity !== undefined && !sameFile;
|
|
399
|
+
let hasContinuity = stats.size >= state.offset;
|
|
400
|
+
if (hasContinuity && state.offset > 0 && state.continuity.length === 0) {
|
|
401
|
+
hasContinuity = sameFile;
|
|
402
|
+
}
|
|
403
|
+
else if (hasContinuity && state.continuity.length > 0) {
|
|
404
|
+
const actual = await readBufferAt(handle, state.continuity.length, state.offset - state.continuity.length);
|
|
405
|
+
hasContinuity = actual.equals(state.continuity);
|
|
406
|
+
}
|
|
407
|
+
if (!hasContinuity) {
|
|
408
|
+
resetIncrementalReadState(state);
|
|
409
|
+
}
|
|
410
|
+
else if (rotated) {
|
|
411
|
+
state.generation += 1;
|
|
29
412
|
}
|
|
30
|
-
|
|
413
|
+
state.identity = identity;
|
|
414
|
+
const events = [];
|
|
31
415
|
let position = state.offset;
|
|
32
|
-
|
|
33
|
-
|
|
416
|
+
let remainingBatchBytes = MAX_INCREMENTAL_READ_BYTES;
|
|
417
|
+
let reachedUnexpectedEof = false;
|
|
418
|
+
while (position < stats.size && remainingBatchBytes > 0) {
|
|
419
|
+
const chunk = Buffer.alloc(Math.min(64 * 1024, stats.size - position, remainingBatchBytes));
|
|
34
420
|
const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
|
|
35
421
|
if (bytesRead === 0) {
|
|
422
|
+
reachedUnexpectedEof = true;
|
|
36
423
|
break;
|
|
37
424
|
}
|
|
38
|
-
chunks.push(chunk.subarray(0, bytesRead));
|
|
39
425
|
position += bytesRead;
|
|
426
|
+
state.offset = position;
|
|
427
|
+
remainingBatchBytes -= bytesRead;
|
|
428
|
+
events.push(...consumeRecordedChunk(state, chunk.subarray(0, bytesRead)));
|
|
40
429
|
}
|
|
41
|
-
state.
|
|
42
|
-
const raw = Buffer.concat([state.pending, ...chunks]);
|
|
43
|
-
const lastNewline = raw.lastIndexOf(0x0a);
|
|
44
|
-
if (lastNewline < 0) {
|
|
45
|
-
state.pending = raw;
|
|
46
|
-
return [];
|
|
47
|
-
}
|
|
48
|
-
state.pending = raw.subarray(lastNewline + 1);
|
|
49
|
-
const lines = raw
|
|
50
|
-
.subarray(0, lastNewline)
|
|
51
|
-
.toString("utf8")
|
|
52
|
-
.split("\n")
|
|
53
|
-
.filter((line) => line.trim().length > 0);
|
|
54
|
-
const events = [];
|
|
55
|
-
for (const [index, line] of lines.entries()) {
|
|
56
|
-
try {
|
|
57
|
-
events.push(JSON.parse(line));
|
|
58
|
-
}
|
|
59
|
-
catch (error) {
|
|
60
|
-
if (index === lines.length - 1 && state.pending.length === 0) {
|
|
61
|
-
continue;
|
|
62
|
-
}
|
|
63
|
-
throw error;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
430
|
+
state.caughtUp = reachedUnexpectedEof || position >= stats.size;
|
|
66
431
|
return events;
|
|
67
432
|
}
|
|
68
433
|
finally {
|
|
@@ -75,9 +440,92 @@ export async function appendRecordedInbound(filePath, event) {
|
|
|
75
440
|
...event,
|
|
76
441
|
recordedAt: new Date().toISOString(),
|
|
77
442
|
};
|
|
78
|
-
|
|
443
|
+
const line = `${JSON.stringify(recorded)}\n`;
|
|
444
|
+
if (Buffer.byteLength(line) > MAX_PENDING_RECORD_BYTES) {
|
|
445
|
+
throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
|
|
446
|
+
}
|
|
447
|
+
await appendJsonLine(filePath, line);
|
|
79
448
|
return recorded;
|
|
80
449
|
}
|
|
450
|
+
export async function appendRecordedInboundBatch(filePath, events) {
|
|
451
|
+
if (events.length === 0) {
|
|
452
|
+
return [];
|
|
453
|
+
}
|
|
454
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
455
|
+
for (let attempt = 0;; attempt++) {
|
|
456
|
+
try {
|
|
457
|
+
return await serializeAppend(filePath, async (publicationPath, logicalPath) => {
|
|
458
|
+
const generation = await prepareRecorderPathForAppend(publicationPath, logicalPath, true);
|
|
459
|
+
const seen = await syncRecordIdentityIndex(publicationPath);
|
|
460
|
+
const pendingIdentities = new Set();
|
|
461
|
+
const recorded = [];
|
|
462
|
+
for (const event of events) {
|
|
463
|
+
const identity = recordIdentity(event);
|
|
464
|
+
if (seen.has(identity) || pendingIdentities.has(identity)) {
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
pendingIdentities.add(identity);
|
|
468
|
+
recorded.push({
|
|
469
|
+
...event,
|
|
470
|
+
recordedAt: new Date().toISOString(),
|
|
471
|
+
});
|
|
472
|
+
}
|
|
473
|
+
if (recorded.length > 0) {
|
|
474
|
+
const batch = {
|
|
475
|
+
events: recorded,
|
|
476
|
+
recordType: "crabline.recorder.batch",
|
|
477
|
+
recorderBatchVersion: RECORDER_BATCH_VERSION,
|
|
478
|
+
};
|
|
479
|
+
const line = `${JSON.stringify(batch)}\n`;
|
|
480
|
+
if (Buffer.byteLength(line) > MAX_PENDING_RECORD_BYTES) {
|
|
481
|
+
throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
|
|
482
|
+
}
|
|
483
|
+
await appendCommittedLine(publicationPath, logicalPath, line, true, generation);
|
|
484
|
+
await syncRecordIdentityIndex(publicationPath);
|
|
485
|
+
}
|
|
486
|
+
else if (!sameRecorderFileIdentity(generation, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
|
|
487
|
+
throw new RecorderRotatedError("Recorder rotated before confirming a duplicate batch.");
|
|
488
|
+
}
|
|
489
|
+
return recorded;
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
catch (error) {
|
|
493
|
+
if (!(error instanceof RecorderRotatedError) || attempt + 1 >= RECORDER_ROTATION_ATTEMPTS) {
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
function recordIdentity(event) {
|
|
500
|
+
return JSON.stringify([event.provider, event.threadId, event.id, recordedDirectionOf(event)]);
|
|
501
|
+
}
|
|
502
|
+
async function syncRecordIdentityIndex(filePath) {
|
|
503
|
+
const key = path.resolve(filePath);
|
|
504
|
+
let index = recordIdentityIndexes.get(key);
|
|
505
|
+
if (!index) {
|
|
506
|
+
index = { readState: createIncrementalReadState(), seen: new Set() };
|
|
507
|
+
recordIdentityIndexes.set(key, index);
|
|
508
|
+
if (recordIdentityIndexes.size > MAX_RECORD_IDENTITY_INDEXES) {
|
|
509
|
+
recordIdentityIndexes.delete(recordIdentityIndexes.keys().next().value);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
else {
|
|
513
|
+
recordIdentityIndexes.delete(key);
|
|
514
|
+
recordIdentityIndexes.set(key, index);
|
|
515
|
+
}
|
|
516
|
+
let generation = index.readState.generation;
|
|
517
|
+
do {
|
|
518
|
+
const appended = await readRecordedInboundAppend(filePath, index.readState);
|
|
519
|
+
if (index.readState.generation !== generation) {
|
|
520
|
+
index.seen.clear();
|
|
521
|
+
generation = index.readState.generation;
|
|
522
|
+
}
|
|
523
|
+
for (const event of appended) {
|
|
524
|
+
rememberRecentRecord(index.seen, event);
|
|
525
|
+
}
|
|
526
|
+
} while (!index.readState.caughtUp);
|
|
527
|
+
return index.seen;
|
|
528
|
+
}
|
|
81
529
|
export async function readRecordedInbound(filePath) {
|
|
82
530
|
let raw = "";
|
|
83
531
|
try {
|
|
@@ -90,62 +538,90 @@ export async function readRecordedInbound(filePath) {
|
|
|
90
538
|
}
|
|
91
539
|
throw error;
|
|
92
540
|
}
|
|
93
|
-
const
|
|
541
|
+
const lastNewline = raw.lastIndexOf("\n");
|
|
542
|
+
const completed = lastNewline >= 0 ? raw.slice(0, lastNewline) : "";
|
|
543
|
+
const tail = raw.slice(lastNewline + 1);
|
|
94
544
|
const events = [];
|
|
95
|
-
for (const
|
|
545
|
+
for (const line of completed.split("\n")) {
|
|
546
|
+
if (line.trim()) {
|
|
547
|
+
events.push(...parseRecordedLine(line));
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
if (tail.trim()) {
|
|
96
551
|
try {
|
|
97
|
-
events.push(
|
|
552
|
+
events.push(...parseRecordedLine(tail));
|
|
98
553
|
}
|
|
99
554
|
catch (error) {
|
|
100
|
-
if (
|
|
101
|
-
|
|
555
|
+
if (!(error instanceof SyntaxError)) {
|
|
556
|
+
throw error;
|
|
102
557
|
}
|
|
103
|
-
|
|
558
|
+
// Ignore a syntactically partial final append; completed lines remain strict.
|
|
104
559
|
}
|
|
105
560
|
}
|
|
106
561
|
return events;
|
|
107
562
|
}
|
|
108
563
|
export async function waitForRecordedInbound(params) {
|
|
109
564
|
const deadline = Date.now() + params.timeoutMs;
|
|
110
|
-
const
|
|
111
|
-
while (Date.now() <= deadline) {
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
565
|
+
const cursor = params.cursor ?? createRecordedInboundCursor();
|
|
566
|
+
while (!params.signal?.aborted && Date.now() <= deadline) {
|
|
567
|
+
const generation = cursor.readState.generation;
|
|
568
|
+
const events = cursor.buffered.length > 0
|
|
569
|
+
? cursor.buffered.splice(0)
|
|
570
|
+
: await readRecordedInboundAppend(params.filePath, cursor.readState);
|
|
571
|
+
if (cursor.readState.generation !== generation) {
|
|
572
|
+
cursor.seen.clear();
|
|
573
|
+
}
|
|
574
|
+
for (const [index, event] of events.entries()) {
|
|
575
|
+
if (params.recordedDirection && recordedDirectionOf(event) !== params.recordedDirection) {
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
// Incremental read state owns progress; this bounded window only filters appended retries.
|
|
579
|
+
if (!rememberRecentRecord(cursor.seen, event)) {
|
|
116
580
|
continue;
|
|
117
581
|
}
|
|
118
|
-
seen.add(key);
|
|
119
582
|
if (params.since && new Date(event.sentAt).getTime() < new Date(params.since).getTime()) {
|
|
120
583
|
continue;
|
|
121
584
|
}
|
|
122
585
|
if (params.matches(event)) {
|
|
586
|
+
cursor.buffered.push(...events.slice(index + 1));
|
|
123
587
|
return event;
|
|
124
588
|
}
|
|
125
589
|
}
|
|
126
|
-
|
|
590
|
+
if (!cursor.readState.caughtUp) {
|
|
591
|
+
if (Date.now() >= deadline) {
|
|
592
|
+
return null;
|
|
593
|
+
}
|
|
594
|
+
continue;
|
|
595
|
+
}
|
|
596
|
+
const remainingMs = deadline - Date.now();
|
|
597
|
+
if (remainingMs <= 0) {
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
await sleep(Math.min(params.pollMs ?? 200, remainingMs), params.signal);
|
|
127
601
|
}
|
|
128
602
|
return null;
|
|
129
603
|
}
|
|
130
604
|
export async function* watchRecordedInbound(params) {
|
|
131
|
-
const state =
|
|
132
|
-
offset: 0,
|
|
133
|
-
pending: Buffer.alloc(0),
|
|
134
|
-
};
|
|
605
|
+
const state = createIncrementalReadState();
|
|
135
606
|
const seen = new Set();
|
|
136
|
-
while (
|
|
607
|
+
while (!params.signal?.aborted) {
|
|
608
|
+
const generation = state.generation;
|
|
137
609
|
const events = await readRecordedInboundAppend(params.filePath, state);
|
|
610
|
+
if (state.generation !== generation) {
|
|
611
|
+
seen.clear();
|
|
612
|
+
}
|
|
613
|
+
if (params.signal?.aborted) {
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
138
616
|
for (const event of events) {
|
|
139
|
-
|
|
140
|
-
|
|
617
|
+
if (params.signal?.aborted) {
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
if (!rememberRecentRecord(seen, event)) {
|
|
141
621
|
continue;
|
|
142
622
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const oldest = seen.values().next().value;
|
|
146
|
-
if (oldest !== undefined) {
|
|
147
|
-
seen.delete(oldest);
|
|
148
|
-
}
|
|
623
|
+
if (params.recordedDirection && recordedDirectionOf(event) !== params.recordedDirection) {
|
|
624
|
+
continue;
|
|
149
625
|
}
|
|
150
626
|
if (params.since && new Date(event.sentAt).getTime() < new Date(params.since).getTime()) {
|
|
151
627
|
continue;
|
|
@@ -154,7 +630,10 @@ export async function* watchRecordedInbound(params) {
|
|
|
154
630
|
yield event;
|
|
155
631
|
}
|
|
156
632
|
}
|
|
157
|
-
|
|
633
|
+
if (!state.caughtUp) {
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
await sleep(params.pollMs ?? 250, params.signal);
|
|
158
637
|
}
|
|
159
638
|
}
|
|
160
639
|
//# sourceMappingURL=recorder.js.map
|