message-cp 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +45 -2
- package/dist/accounts.js +279 -0
- package/dist/agents.js +507 -0
- package/dist/approve.js +260 -0
- package/dist/channels.js +262 -0
- package/dist/cli.js +523 -0
- package/dist/client-metadata.js +167 -0
- package/dist/connect.js +85 -0
- package/dist/dashboard.js +4961 -0
- package/dist/db.js +605 -0
- package/dist/http.js +1933 -0
- package/dist/index.js +21 -0
- package/dist/init.js +960 -0
- package/dist/login.js +530 -0
- package/dist/messages.js +513 -0
- package/dist/oauth.js +384 -0
- package/dist/server.js +420 -0
- package/dist/sessions.js +53 -0
- package/dist/settings.js +13 -0
- package/dist/wait.js +137 -0
- package/dist/webhooks.js +464 -0
- package/package.json +44 -4
package/dist/webhooks.js
ADDED
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
+
import { NAME_PATTERN, WAIT_MARK_MS } from "./agents.js";
|
|
3
|
+
import { MAX_INBOX_LIMIT } from "./messages.js";
|
|
4
|
+
// Outgoing webhooks. The operator manages them with `message-cp webhook` on the server; no agent or token can.
|
|
5
|
+
// Messages and posts a webhook is for are queued in `webhook_deliveries` when they are stored, and `serve` sends them
|
|
6
|
+
// with `sendPending`: each body signed with the webhook's secret, a timeout and a few attempts. A message or mention
|
|
7
|
+
// queued while its agent is marked waiting for mail (`markWaiting`) is held until the mark ends, and dropped once the
|
|
8
|
+
// agent has read it. The stdio server and
|
|
9
|
+
// `message-cp wait` send nothing. A webhook's secret is shown once, when it is added, and stored as is so that
|
|
10
|
+
// deliveries can sign with it. A webhook may also carry one header the owner sets, such as `Authorization`, sent on each
|
|
11
|
+
// of its deliveries; its value is stored as is too, and never printed, logged or returned. And it may carry a limit on
|
|
12
|
+
// how many delivery requests it gets per window of 60 minutes; deliveries past it wait in the queue for a later window.
|
|
13
|
+
// And it may carry a fixed JSON body the owner sets, sent in place of the event body, signed the same way, with its one
|
|
14
|
+
// `{{delivery}}` replaced by an id of that delivery that every retry repeats; the body is never printed or logged.
|
|
15
|
+
export class WebhookError extends Error {
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* The URL a webhook may send to, as `new URL(...).href`: https, or http only to a literal loopback address, with no
|
|
19
|
+
* username, password, query or fragment. Errors never repeat the URL, since it may carry a credential.
|
|
20
|
+
*/
|
|
21
|
+
export function webhookUrl(raw) {
|
|
22
|
+
let url;
|
|
23
|
+
try {
|
|
24
|
+
url = new URL(raw);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw new WebhookError("Invalid --url: give an absolute https URL.");
|
|
28
|
+
}
|
|
29
|
+
const loopback = /^127(\.\d{1,3}){3}$/.test(url.hostname) || url.hostname === "[::1]";
|
|
30
|
+
if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) {
|
|
31
|
+
throw new WebhookError("--url must be https (plain http only for 127.x.x.x or [::1]): deliveries would cross the network in the clear.");
|
|
32
|
+
}
|
|
33
|
+
if (url.username || url.password || url.search || url.hash) {
|
|
34
|
+
throw new WebhookError("--url must not carry a username, password, query or fragment.");
|
|
35
|
+
}
|
|
36
|
+
// Drops a bare `?` or `#`, which leave `search` and `hash` empty but stay in `href`.
|
|
37
|
+
url.search = "";
|
|
38
|
+
url.hash = "";
|
|
39
|
+
return url.href;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Parses `--on`: `#<channel>`, `mention:<agent>` or `agent:<agent>`, each name following the agent-name rules. An agent
|
|
43
|
+
* cannot be named `all`, so `mention:all` and `agent:all` are refused; `#all` is an ordinary channel.
|
|
44
|
+
*/
|
|
45
|
+
export function webhookOn(on) {
|
|
46
|
+
const [, prefix, name] = /^(#|mention:|agent:)(.*)$/s.exec(on) ?? [];
|
|
47
|
+
if (prefix === undefined || !NAME_PATTERN.test(name)) {
|
|
48
|
+
throw new WebhookError("Invalid --on: use #<channel>, mention:<agent> or agent:<agent>, names of 1-32 letters, digits, '-' and '_'.");
|
|
49
|
+
}
|
|
50
|
+
if (prefix !== "#" && name.toLowerCase() === "all") {
|
|
51
|
+
throw new WebhookError("Invalid --on: 'all' is reserved and names no agent.");
|
|
52
|
+
}
|
|
53
|
+
return { event: prefix === "#" ? "channel" : prefix.slice(0, -1), name: name };
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Header names message-cp sets on a delivery itself, that `fetch` controls or refuses to send, or that describe the body
|
|
57
|
+
* (which message-cp builds for each delivery), in lower case. Every `Content-*` name is refused as well.
|
|
58
|
+
*/
|
|
59
|
+
const RESERVED_HEADERS = [
|
|
60
|
+
"x-message-cp-signature",
|
|
61
|
+
"host",
|
|
62
|
+
"transfer-encoding",
|
|
63
|
+
"connection",
|
|
64
|
+
"keep-alive",
|
|
65
|
+
"upgrade",
|
|
66
|
+
"expect",
|
|
67
|
+
"digest",
|
|
68
|
+
"repr-digest",
|
|
69
|
+
];
|
|
70
|
+
/**
|
|
71
|
+
* Parses the one header line `--header-stdin` reads, `<Name>: <value>`: the name an HTTP token that message-cp does not
|
|
72
|
+
* set itself, the value trimmed of spaces and tabs, not empty, with no control character and nothing past U+00FF.
|
|
73
|
+
* Errors never repeat the value.
|
|
74
|
+
*/
|
|
75
|
+
export function webhookHeader(line) {
|
|
76
|
+
const colon = line.indexOf(":");
|
|
77
|
+
if (colon === -1)
|
|
78
|
+
throw new WebhookError("Invalid header: give one line '<Name>: <value>'.");
|
|
79
|
+
const name = line.slice(0, colon);
|
|
80
|
+
const value = line.slice(colon + 1).replace(/^[ \t]+|[ \t]+$/g, "");
|
|
81
|
+
if (!/^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$/.test(name)) {
|
|
82
|
+
throw new WebhookError("Invalid header name: use letters, digits and !#$%&'*+-.^_`|~ only, before the ':'.");
|
|
83
|
+
}
|
|
84
|
+
if (RESERVED_HEADERS.includes(name.toLowerCase()) || /^content-/i.test(name)) {
|
|
85
|
+
throw new WebhookError(`Invalid header: message-cp controls ${name} itself; choose another header.`);
|
|
86
|
+
}
|
|
87
|
+
if (value === "")
|
|
88
|
+
throw new WebhookError("Invalid header: its value is empty.");
|
|
89
|
+
// Tab included: inside a value it is a control character like CR and LF. So are U+0080-U+009F.
|
|
90
|
+
if (/[\x00-\x1f\x7f-\x9f]/.test(value))
|
|
91
|
+
throw new WebhookError("Invalid header: its value contains a control character.");
|
|
92
|
+
// `fetch` refuses a header value with a character past U+00FF, so every delivery would fail.
|
|
93
|
+
if (/[^\x00-\xff]/.test(value))
|
|
94
|
+
throw new WebhookError("Invalid header: its value has a character past U+00FF, which HTTP cannot send.");
|
|
95
|
+
return { name, value };
|
|
96
|
+
}
|
|
97
|
+
/** The highest `--max-per-hour`. */
|
|
98
|
+
export const WEBHOOK_MAX_PER_HOUR = 10000;
|
|
99
|
+
const isMaxPerHour = (n) => Number.isSafeInteger(n) && n >= 1 && n <= WEBHOOK_MAX_PER_HOUR;
|
|
100
|
+
/** Parses `--max-per-hour`: a whole number from 1 to 10000, in decimal digits only. */
|
|
101
|
+
export function webhookMaxPerHour(raw) {
|
|
102
|
+
if (!/^[0-9]+$/.test(raw) || !isMaxPerHour(Number(raw))) {
|
|
103
|
+
throw new WebhookError(`Invalid --max-per-hour: give a whole number from 1 to ${WEBHOOK_MAX_PER_HOUR}.`);
|
|
104
|
+
}
|
|
105
|
+
return Number(raw);
|
|
106
|
+
}
|
|
107
|
+
/** What a fixed body's placeholder is replaced with: the delivery's id, the same on every attempt of it. */
|
|
108
|
+
export const WEBHOOK_PLACEHOLDER = "{{delivery}}";
|
|
109
|
+
/** The longest `--body`, in UTF-8 bytes. */
|
|
110
|
+
export const WEBHOOK_MAX_BODY_BYTES = 4096;
|
|
111
|
+
/** How many times `text` contains the placeholder. */
|
|
112
|
+
const placeholders = (text) => text.split(WEBHOOK_PLACEHOLDER).length - 1;
|
|
113
|
+
/**
|
|
114
|
+
* Checks `--body`: a JSON object of at most `WEBHOOK_MAX_BODY_BYTES`, with `WEBHOOK_PLACEHOLDER` written as is exactly
|
|
115
|
+
* once in the text, inside a string value that parsing keeps (not in a key, and not dropped by a later duplicate key),
|
|
116
|
+
* and nowhere else in any parsed string or key, `\u` escapes included. Returns it unchanged, so a delivery sends the
|
|
117
|
+
* owner's own bytes with only the placeholder replaced. Errors never repeat the body.
|
|
118
|
+
*/
|
|
119
|
+
export function webhookBody(raw) {
|
|
120
|
+
if (Buffer.byteLength(raw, "utf8") > WEBHOOK_MAX_BODY_BYTES) {
|
|
121
|
+
throw new WebhookError(`Invalid --body: it is over ${WEBHOOK_MAX_BODY_BYTES} bytes.`);
|
|
122
|
+
}
|
|
123
|
+
let parsed;
|
|
124
|
+
try {
|
|
125
|
+
parsed = JSON.parse(raw);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
throw new WebhookError("Invalid --body: it is not JSON.");
|
|
129
|
+
}
|
|
130
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
131
|
+
throw new WebhookError("Invalid --body: give a JSON object.");
|
|
132
|
+
}
|
|
133
|
+
const misplaced = new WebhookError(`Invalid --body: put ${WEBHOOK_PLACEHOLDER} exactly once, inside a JSON string value.`);
|
|
134
|
+
if (placeholders(raw) !== 1)
|
|
135
|
+
throw misplaced;
|
|
136
|
+
// Parsed again with the literal placeholder swapped for a value found nowhere else, as a delivery swaps it, so the
|
|
137
|
+
// count is of what a receiver parses: one sentinel in a string value, and no placeholder left in any string or key.
|
|
138
|
+
const sentinel = randomUUID();
|
|
139
|
+
let sentinels = 0;
|
|
140
|
+
let others = 0;
|
|
141
|
+
const walk = (v) => {
|
|
142
|
+
if (typeof v === "string") {
|
|
143
|
+
sentinels += v.split(sentinel).length - 1;
|
|
144
|
+
others += placeholders(v);
|
|
145
|
+
}
|
|
146
|
+
else if (Array.isArray(v))
|
|
147
|
+
v.forEach(walk);
|
|
148
|
+
else if (typeof v === "object" && v !== null) {
|
|
149
|
+
for (const [k, x] of Object.entries(v)) {
|
|
150
|
+
others += placeholders(k) + k.split(sentinel).length - 1;
|
|
151
|
+
walk(x);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
walk(JSON.parse(raw.replace(WEBHOOK_PLACEHOLDER, () => sentinel)));
|
|
156
|
+
if (sentinels !== 1 || others !== 0)
|
|
157
|
+
throw misplaced;
|
|
158
|
+
return raw;
|
|
159
|
+
}
|
|
160
|
+
/** How `--on` names a webhook's event, for printing. */
|
|
161
|
+
export const formatOn = (w) => (w.event === "channel" ? `#${w.name}` : `${w.event}:${w.name}`);
|
|
162
|
+
/**
|
|
163
|
+
* Adds a webhook to the account named `account`, with a new secret, with the header line `header` if given, limited
|
|
164
|
+
* to `maxPerHour` delivery requests per window if given, and sending the fixed body `body` if given.
|
|
165
|
+
*/
|
|
166
|
+
export const addWebhook = (db, account, url, on, includeText, header, maxPerHour, body) => db.transaction(() => {
|
|
167
|
+
const href = webhookUrl(url);
|
|
168
|
+
const { event, name } = webhookOn(on);
|
|
169
|
+
const extra = header === undefined ? undefined : webhookHeader(header);
|
|
170
|
+
if (maxPerHour !== undefined && !isMaxPerHour(maxPerHour)) {
|
|
171
|
+
throw new WebhookError(`Invalid --max-per-hour: give a whole number from 1 to ${WEBHOOK_MAX_PER_HOUR}.`);
|
|
172
|
+
}
|
|
173
|
+
// A fixed body never carries the text, so --no-text with it would mean nothing.
|
|
174
|
+
if (body !== undefined && !includeText)
|
|
175
|
+
throw new WebhookError("--body and --no-text cannot go together: a fixed body never carries the text.");
|
|
176
|
+
const fixed = body === undefined ? null : webhookBody(body);
|
|
177
|
+
const row = db.prepare("SELECT id, name FROM accounts WHERE name = ?").get(account);
|
|
178
|
+
if (!row)
|
|
179
|
+
throw new WebhookError(`No account named '${account}'.`);
|
|
180
|
+
const secret = randomBytes(32).toString("hex");
|
|
181
|
+
const { id } = db
|
|
182
|
+
.prepare(`INSERT INTO webhooks (account_id, url, event, name, include_text, secret, header_name, header_value, max_per_hour, fixed_body, created_at)
|
|
183
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`)
|
|
184
|
+
.get(row.id, href, event, name, includeText ? 1 : 0, secret, extra?.name ?? null, extra?.value ?? null, maxPerHour ?? null, fixed, Date.now());
|
|
185
|
+
return {
|
|
186
|
+
id,
|
|
187
|
+
account: row.name,
|
|
188
|
+
url: href,
|
|
189
|
+
event,
|
|
190
|
+
name,
|
|
191
|
+
includeText,
|
|
192
|
+
headerName: extra?.name ?? null,
|
|
193
|
+
maxPerHour: maxPerHour ?? null,
|
|
194
|
+
fixedBody: fixed !== null,
|
|
195
|
+
secret,
|
|
196
|
+
};
|
|
197
|
+
}).immediate();
|
|
198
|
+
/** The webhooks of the account named `account`, or of every account without it; never their secrets, header values or bodies. */
|
|
199
|
+
export function listWebhooks(db, account) {
|
|
200
|
+
if (account !== undefined && !db.prepare("SELECT 1 FROM accounts WHERE name = ?").get(account)) {
|
|
201
|
+
throw new WebhookError(`No account named '${account}'.`);
|
|
202
|
+
}
|
|
203
|
+
const rows = db
|
|
204
|
+
.prepare(`SELECT webhooks.id, accounts.name AS account, url, event, webhooks.name, include_text, header_name, max_per_hour,
|
|
205
|
+
fixed_body IS NOT NULL AS fixed
|
|
206
|
+
FROM webhooks JOIN accounts ON accounts.id = webhooks.account_id
|
|
207
|
+
WHERE ? IS NULL OR accounts.name = ? ORDER BY webhooks.id`)
|
|
208
|
+
.all(account ?? null, account ?? null);
|
|
209
|
+
return rows.map(({ include_text, header_name, max_per_hour, fixed, ...w }) => ({
|
|
210
|
+
...w,
|
|
211
|
+
includeText: include_text === 1,
|
|
212
|
+
headerName: header_name,
|
|
213
|
+
maxPerHour: max_per_hour,
|
|
214
|
+
fixedBody: fixed === 1,
|
|
215
|
+
}));
|
|
216
|
+
}
|
|
217
|
+
/** Deletes the webhook `id`. Returns the account it belonged to. */
|
|
218
|
+
export function removeWebhook(db, id) {
|
|
219
|
+
const row = db
|
|
220
|
+
.prepare("DELETE FROM webhooks WHERE id = ? RETURNING (SELECT name FROM accounts WHERE accounts.id = webhooks.account_id) AS account")
|
|
221
|
+
.get(id);
|
|
222
|
+
if (!row)
|
|
223
|
+
throw new WebhookError(`No webhook with id ${id}.`);
|
|
224
|
+
return row.account;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* One page of the deliveries queued for the webhooks of `account`, newest first. Scoped through the webhook's account,
|
|
228
|
+
* so another account's deliveries are never counted or returned. Reads only: no row is claimed or changed.
|
|
229
|
+
*/
|
|
230
|
+
export function listAccountDeliveries(db, account, maxAttempts, limit, offset) {
|
|
231
|
+
const rows = db
|
|
232
|
+
.prepare(`SELECT d.id, d.webhook_id, w.event, w.name, d.created_at, d.attempts, d.delivered_at
|
|
233
|
+
FROM webhook_deliveries d JOIN webhooks w ON w.id = d.webhook_id
|
|
234
|
+
WHERE w.account_id = ? ORDER BY d.id DESC LIMIT ? OFFSET ?`)
|
|
235
|
+
.all(account, Math.min(limit, MAX_INBOX_LIMIT), offset);
|
|
236
|
+
return rows.map((r) => ({
|
|
237
|
+
id: r.id,
|
|
238
|
+
webhook: r.webhook_id,
|
|
239
|
+
on: formatOn(r),
|
|
240
|
+
queuedAt: new Date(r.created_at).toISOString(),
|
|
241
|
+
attempts: r.attempts,
|
|
242
|
+
deliveredAt: r.delivered_at === null ? null : new Date(r.delivered_at).toISOString(),
|
|
243
|
+
// As `sendPending` picks what is still pending: undelivered with attempts left.
|
|
244
|
+
status: r.delivered_at !== null ? "delivered" : r.attempts < maxAttempts ? "pending" : "failed",
|
|
245
|
+
}));
|
|
246
|
+
}
|
|
247
|
+
/** The body of the queued delivery `id`, built from its message or post as it is now, or undefined if there is none. */
|
|
248
|
+
export function deliveryBody(db, id) {
|
|
249
|
+
const row = db
|
|
250
|
+
.prepare(`SELECT w.event, w.include_text, c.name AS channel,
|
|
251
|
+
CASE WHEN a.removed_at IS NULL THEN a.name ELSE a.removed_name END AS recipient,
|
|
252
|
+
COALESCE(m.sender, p.sender) AS sender, COALESCE(m.created_at, p.created_at) AS created_at, COALESCE(m.body, p.body) AS body
|
|
253
|
+
FROM webhook_deliveries d
|
|
254
|
+
JOIN webhooks w ON w.id = d.webhook_id
|
|
255
|
+
LEFT JOIN messages m ON m.id = d.message_id
|
|
256
|
+
LEFT JOIN agents a ON a.id = m.recipient_id
|
|
257
|
+
LEFT JOIN channel_posts p ON p.id = d.post_id
|
|
258
|
+
LEFT JOIN channels c ON c.id = p.channel_id
|
|
259
|
+
WHERE d.id = ?`)
|
|
260
|
+
.get(id);
|
|
261
|
+
if (!row)
|
|
262
|
+
return undefined;
|
|
263
|
+
return {
|
|
264
|
+
event: row.event,
|
|
265
|
+
...(row.channel !== null ? { channel: row.channel } : { recipient: row.recipient }),
|
|
266
|
+
sender: row.sender,
|
|
267
|
+
sentAt: new Date(row.created_at).toISOString(),
|
|
268
|
+
...(row.include_text === 1 ? { text: row.body } : {}),
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* True while the agent a held delivery is for (`held_for`) is still waiting for mail; its parameters are the time now and
|
|
273
|
+
* `WAIT_MARK_MS` after it, so a mark set before the clock was put back holds nothing longer than a mark set now. A removed
|
|
274
|
+
* agent waits for nothing, so its held deliveries go out at once.
|
|
275
|
+
*/
|
|
276
|
+
const WAITING = "SELECT 1 FROM agents a WHERE a.id = held_for AND a.waiting_until > ? AND a.waiting_until <= ? AND a.removed_at IS NULL";
|
|
277
|
+
/** True once the agent a held delivery is for has read its message or mention; read_at is never cleared. */
|
|
278
|
+
const READ = `(EXISTS (SELECT 1 FROM messages m WHERE m.id = message_id AND m.recipient_id = held_for AND m.read_at IS NOT NULL)
|
|
279
|
+
OR EXISTS (SELECT 1 FROM mentions x WHERE x.agent_id = held_for AND x.post_id = webhook_deliveries.post_id AND x.read_at IS NOT NULL))`;
|
|
280
|
+
/** A delivery request not finished in this long is aborted and counts as a failed attempt. */
|
|
281
|
+
export const WEBHOOK_TIMEOUT_MS = 5000;
|
|
282
|
+
/** Attempts a delivery gets in all; after the last failed one it is left undelivered and not tried again. */
|
|
283
|
+
export const WEBHOOK_MAX_ATTEMPTS = 3;
|
|
284
|
+
/** Most deliveries one pass sends. */
|
|
285
|
+
export const WEBHOOK_BATCH = 20;
|
|
286
|
+
/** How long a webhook's window lasts: `--max-per-hour` caps the delivery requests in one. */
|
|
287
|
+
export const WEBHOOK_WINDOW_MS = 60 * 60 * 1000;
|
|
288
|
+
/** Requests a pass has in flight at once. */
|
|
289
|
+
const WEBHOOK_CONCURRENCY = 4;
|
|
290
|
+
/** `options` with defaults filled in; throws for a value that is not a positive integer, so a pass is always bounded. */
|
|
291
|
+
export function sendOptions(options) {
|
|
292
|
+
const resolved = {
|
|
293
|
+
timeoutMs: options.timeoutMs ?? WEBHOOK_TIMEOUT_MS,
|
|
294
|
+
maxAttempts: options.maxAttempts ?? WEBHOOK_MAX_ATTEMPTS,
|
|
295
|
+
batch: options.batch ?? WEBHOOK_BATCH,
|
|
296
|
+
};
|
|
297
|
+
for (const [key, value] of Object.entries(resolved)) {
|
|
298
|
+
if (!Number.isSafeInteger(value) || value < 1)
|
|
299
|
+
throw new Error(`${key} must be a positive integer`);
|
|
300
|
+
}
|
|
301
|
+
return resolved;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* One pass over the queue: claims up to `batch` pending deliveries, oldest first, and POSTs each one's body to its own
|
|
305
|
+
* webhook's URL, signed with its secret in `X-Message-CP-Signature`, with the webhook's own header if it has one. A
|
|
306
|
+
* webhook with a fixed body sends that in place of the event body, its placeholder replaced by the row's
|
|
307
|
+
* `delivery_uuid`, set on the first attempt that sends it and read back on each retry. A 2xx answer delivers it; anything else, a
|
|
308
|
+
* network error or the timeout is a failed attempt, retried on a later pass. Redirects are not followed.
|
|
309
|
+
* An attempt is counted when the row is claimed, before the request, so two senders on one file never send the same
|
|
310
|
+
* attempt. The claim also records until when that request may still be in flight (twice the claiming sender's own
|
|
311
|
+
* `timeoutMs`, leaving room to record the answer), and no sender, whatever its own timeout, takes the row before then.
|
|
312
|
+
* A webhook with a limit gets at most that many requests, retries included, per fixed window of `WEBHOOK_WINDOW_MS`: a
|
|
313
|
+
* window starts with the first request claimed after the last one ended. The pass leaves out the rows a webhook has no
|
|
314
|
+
* allowance left for, so its batch fills with other webhooks' rows; those rows stay pending, with no attempt used, and go
|
|
315
|
+
* out in a later window, oldest first. The allowance is checked again, and used, in the transaction that claims the row,
|
|
316
|
+
* so two senders on one file never go past it together.
|
|
317
|
+
* Nothing about a delivery is logged: its body, secret, URL and header value are all private. Nor is a webhook at its limit.
|
|
318
|
+
*/
|
|
319
|
+
export async function sendPending(db, options = {}) {
|
|
320
|
+
const { timeoutMs, maxAttempts, batch } = sendOptions(options);
|
|
321
|
+
const now = Date.now();
|
|
322
|
+
// Each webhook's allowance: null for no limit, all of it for a window that has ended or never started.
|
|
323
|
+
const webhooks = db
|
|
324
|
+
.prepare(`SELECT id, url, secret, header_name, header_value, fixed_body,
|
|
325
|
+
CASE WHEN max_per_hour IS NULL THEN NULL
|
|
326
|
+
WHEN window_started_at IS NULL OR window_started_at + ? <= ? THEN max_per_hour
|
|
327
|
+
ELSE max_per_hour - window_count END AS allowance
|
|
328
|
+
FROM webhooks`)
|
|
329
|
+
.all(WEBHOOK_WINDOW_MS, now);
|
|
330
|
+
// A webhook's oldest claimable rows, through `webhook_deliveries_pending_by_webhook`; at most `LIMIT` are read, so a
|
|
331
|
+
// backlog held behind a limit costs nothing. A row held for an agent that is still waiting is passed over until that
|
|
332
|
+
// agent reads it, when it is taken to be dropped, so what is passed over is only the waiting agent's unread mail.
|
|
333
|
+
const pendingOf = db.prepare(`SELECT id, attempts FROM webhook_deliveries
|
|
334
|
+
WHERE webhook_id = ? AND delivered_at IS NULL AND attempts < ? AND (claimed_until IS NULL OR claimed_until <= ?)
|
|
335
|
+
AND (held_for IS NULL OR NOT EXISTS (${WAITING}) OR ${READ})
|
|
336
|
+
ORDER BY id LIMIT ?`);
|
|
337
|
+
const rows = [];
|
|
338
|
+
for (const w of webhooks) {
|
|
339
|
+
const take = w.allowance === null ? batch : Math.min(w.allowance, batch);
|
|
340
|
+
if (take <= 0)
|
|
341
|
+
continue;
|
|
342
|
+
for (const d of pendingOf.all(w.id, maxAttempts, now, now, now + WAIT_MARK_MS, take)) {
|
|
343
|
+
rows.push({
|
|
344
|
+
id: d.id,
|
|
345
|
+
attempts: d.attempts,
|
|
346
|
+
webhook_id: w.id,
|
|
347
|
+
url: w.url,
|
|
348
|
+
secret: w.secret,
|
|
349
|
+
header_name: w.header_name,
|
|
350
|
+
header_value: w.header_value,
|
|
351
|
+
fixed_body: w.fixed_body,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
rows.sort((a, b) => a.id - b.id);
|
|
356
|
+
rows.length = Math.min(rows.length, batch);
|
|
357
|
+
const claim = db.prepare("UPDATE webhook_deliveries SET attempts = attempts + 1, claimed_until = ? WHERE id = ? AND attempts = ? AND delivered_at IS NULL");
|
|
358
|
+
const windowOf = db.prepare("SELECT max_per_hour, window_started_at, window_count FROM webhooks WHERE id = ?");
|
|
359
|
+
// A held row whose message or mention its agent has read, which nobody needs woken for any more, and a held row whose
|
|
360
|
+
// agent is still waiting. Neither is ever true of a row that was not held.
|
|
361
|
+
const dropRead = db.prepare(`DELETE FROM webhook_deliveries WHERE id = ? AND attempts = ? AND delivered_at IS NULL AND held_for IS NOT NULL AND ${READ}`);
|
|
362
|
+
const stillHeld = db.prepare(`SELECT 1 FROM webhook_deliveries WHERE id = ? AND held_for IS NOT NULL AND EXISTS (${WAITING})`);
|
|
363
|
+
const use = db.prepare("UPDATE webhooks SET window_started_at = ?, window_count = ? WHERE id = ?");
|
|
364
|
+
// Claims the row if its webhook has allowance left, using one request of it. Counted at the claim, as
|
|
365
|
+
// attempts are, so a claim that sends nothing (a stored URL that no longer passes, a deleted message) uses one too.
|
|
366
|
+
// `fresh` must match the `allowance` CASE in the webhooks query above. Returns the claim's `claimed_until`, or null.
|
|
367
|
+
// A held row is deleted unsent once its agent has read it, and left alone while the agent still waits; in either case no
|
|
368
|
+
// attempt or allowance is used. Otherwise, the wait over and the mail unread, it is sent as any other row.
|
|
369
|
+
const claimWithin = db.transaction((row) => {
|
|
370
|
+
const w = windowOf.get(row.webhook_id);
|
|
371
|
+
if (!w)
|
|
372
|
+
return null;
|
|
373
|
+
const at = Date.now();
|
|
374
|
+
if (dropRead.run(row.id, row.attempts).changes === 1)
|
|
375
|
+
return null;
|
|
376
|
+
if (stillHeld.get(row.id, at, at + WAIT_MARK_MS))
|
|
377
|
+
return null;
|
|
378
|
+
const fresh = w.window_started_at === null || w.window_started_at + WEBHOOK_WINDOW_MS <= at;
|
|
379
|
+
if (w.max_per_hour !== null && !fresh && w.window_count >= w.max_per_hour)
|
|
380
|
+
return null;
|
|
381
|
+
const until = at + 2 * timeoutMs;
|
|
382
|
+
if (claim.run(until, row.id, row.attempts).changes !== 1)
|
|
383
|
+
return null;
|
|
384
|
+
if (w.max_per_hour !== null)
|
|
385
|
+
use.run(fresh ? at : w.window_started_at, fresh ? 1 : w.window_count + 1, row.webhook_id);
|
|
386
|
+
return until;
|
|
387
|
+
});
|
|
388
|
+
// Both match the row as this sender claimed it, not only its id: if the delivery is deleted while its request is being
|
|
389
|
+
// built or sent and a new one queued takes the same id, the new one neither takes its id nor counts as delivered.
|
|
390
|
+
const mine = "id = ? AND webhook_id = ? AND attempts = ? AND claimed_until = ?";
|
|
391
|
+
// The delivery's id for a fixed body: a new one on the first request sent for it, the stored one on every retry.
|
|
392
|
+
const deliveryId = db.prepare(`UPDATE webhook_deliveries SET delivery_uuid = COALESCE(delivery_uuid, ?) WHERE ${mine} RETURNING delivery_uuid`);
|
|
393
|
+
const done = db.prepare(`UPDATE webhook_deliveries SET delivered_at = ? WHERE ${mine}`);
|
|
394
|
+
let delivered = 0;
|
|
395
|
+
let failed = 0;
|
|
396
|
+
const send = async (row) => {
|
|
397
|
+
const until = claimWithin.immediate(row);
|
|
398
|
+
if (until === null)
|
|
399
|
+
return;
|
|
400
|
+
const claimed = [row.id, row.webhook_id, row.attempts + 1, until];
|
|
401
|
+
let href;
|
|
402
|
+
try {
|
|
403
|
+
href = webhookUrl(row.url);
|
|
404
|
+
}
|
|
405
|
+
catch {
|
|
406
|
+
// Stored before the check was what it is now: the attempt is used without a request.
|
|
407
|
+
failed++;
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
const body = deliveryBody(db, row.id);
|
|
411
|
+
if (!body)
|
|
412
|
+
return;
|
|
413
|
+
let text;
|
|
414
|
+
if (row.fixed_body === null)
|
|
415
|
+
text = JSON.stringify(body);
|
|
416
|
+
else {
|
|
417
|
+
const id = deliveryId.get(randomUUID(), ...claimed);
|
|
418
|
+
if (!id)
|
|
419
|
+
return;
|
|
420
|
+
// `webhookBody` checked that the placeholder is there once, inside a string, and a UUID needs no escaping.
|
|
421
|
+
text = row.fixed_body.replace(WEBHOOK_PLACEHOLDER, () => id.delivery_uuid);
|
|
422
|
+
}
|
|
423
|
+
let ok = false;
|
|
424
|
+
try {
|
|
425
|
+
const res = await fetch(href, {
|
|
426
|
+
method: "POST",
|
|
427
|
+
headers: {
|
|
428
|
+
...(row.header_name !== null ? { [row.header_name]: row.header_value } : {}),
|
|
429
|
+
"Content-Type": "application/json",
|
|
430
|
+
"X-Message-CP-Signature": `sha256=${createHmac("sha256", row.secret).update(text).digest("hex")}`,
|
|
431
|
+
},
|
|
432
|
+
body: text,
|
|
433
|
+
redirect: "manual",
|
|
434
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
435
|
+
});
|
|
436
|
+
ok = res.status >= 200 && res.status < 300;
|
|
437
|
+
await res.body?.cancel();
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
// A network error or the timeout; the answer, if any, is only read for its status.
|
|
441
|
+
}
|
|
442
|
+
if (!ok)
|
|
443
|
+
failed++;
|
|
444
|
+
// Not counted when the delivery it was for is gone: there is nothing left to mark.
|
|
445
|
+
else if (done.run(Date.now(), ...claimed).changes === 1)
|
|
446
|
+
delivered++;
|
|
447
|
+
};
|
|
448
|
+
let next = 0;
|
|
449
|
+
let error;
|
|
450
|
+
const worker = async () => {
|
|
451
|
+
while (next < rows.length && error === undefined && !options.signal?.aborted) {
|
|
452
|
+
try {
|
|
453
|
+
await send(rows[next++]);
|
|
454
|
+
}
|
|
455
|
+
catch (err) {
|
|
456
|
+
error ??= err;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
await Promise.all(Array.from({ length: Math.min(WEBHOOK_CONCURRENCY, rows.length) }, worker));
|
|
461
|
+
if (error !== undefined)
|
|
462
|
+
throw error;
|
|
463
|
+
return { delivered, failed };
|
|
464
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,46 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "message-cp",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "A messaging service for AI agents: register a name, find each other, send messages and be woken when mail arrives",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"files": [
|
|
8
|
+
"dist"
|
|
9
|
+
],
|
|
10
|
+
"scripts": {
|
|
11
|
+
"build": "tsc",
|
|
12
|
+
"prepack": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && npm run build",
|
|
13
|
+
"start": "node dist/index.js",
|
|
14
|
+
"test": "vitest run",
|
|
15
|
+
"e2e": "node scripts/e2e.mjs"
|
|
16
|
+
},
|
|
17
|
+
"type": "module",
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/colinhallett/message-cp.git"
|
|
21
|
+
},
|
|
22
|
+
"bugs": {
|
|
23
|
+
"url": "https://github.com/colinhallett/message-cp/issues"
|
|
24
|
+
},
|
|
25
|
+
"homepage": "https://github.com/colinhallett/message-cp#readme",
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@fontsource/instrument-sans": "^5.3.0",
|
|
28
|
+
"@fontsource/jetbrains-mono": "^5.3.0",
|
|
29
|
+
"@modelcontextprotocol/sdk": "^1.32.1",
|
|
30
|
+
"better-sqlite3": "^13.0.3",
|
|
31
|
+
"zod": "^4.6.7"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@types/better-sqlite3": "^9.6.0",
|
|
35
|
+
"@types/node": "^26.6.5",
|
|
36
|
+
"tsx": "^4.23.15",
|
|
37
|
+
"typescript": "^7.0.2",
|
|
38
|
+
"vitest": "^5.0.4"
|
|
39
|
+
},
|
|
40
|
+
"engines": {
|
|
41
|
+
"node": ">=22.14"
|
|
42
|
+
},
|
|
43
|
+
"bin": {
|
|
44
|
+
"message-cp": "dist/cli.js"
|
|
45
|
+
}
|
|
46
|
+
}
|