hookarmor 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +47 -21
- package/bin/hookarmor.js +46 -10
- package/package.json +1 -1
- package/src/dispatcher.js +83 -19
- package/src/public/index.html +5 -2
- package/src/public/landing.html +2 -2
- package/src/server.js +166 -18
- package/src/storage.js +184 -29
- package/src/worker.js +8 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 HookArmor Team
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# 🛡️ HookArmor
|
|
2
2
|
|
|
3
|
-
> **
|
|
3
|
+
> **Durable at-least-once Webhook Dead-Letter Queue (DLQ), Reliability Proxy, and Replay Gateway for Stripe, Shopify, Clerk, and modern B2B SaaS.**
|
|
4
4
|
|
|
5
5
|
[](https://opensource.org/licenses/MIT)
|
|
6
|
-
[]()
|
|
7
|
+
[]()
|
|
8
8
|
|
|
9
9
|
---
|
|
10
10
|
|
|
@@ -26,8 +26,8 @@ Every developer using Stripe, Shopify, GitHub, Clerk, Paddle, or custom webhooks
|
|
|
26
26
|
▼
|
|
27
27
|
┌───────────────────────────────────────┐
|
|
28
28
|
│ HookArmor Ingress Edge │
|
|
29
|
-
│ - Returns immediate 200 OK
|
|
30
|
-
│ - Stores raw payload in SQLite
|
|
29
|
+
│ - Returns immediate 200 OK │
|
|
30
|
+
│ - Stores raw payload in SQLite WAL │
|
|
31
31
|
└───────────────────┬───────────────────┘
|
|
32
32
|
│
|
|
33
33
|
▼
|
|
@@ -48,12 +48,12 @@ Every developer using Stripe, Shopify, GitHub, Clerk, Paddle, or custom webhooks
|
|
|
48
48
|
└─────────────────────────────────────┘
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
-
1. **
|
|
51
|
+
1. **Immediate Durable Ingest**: HookArmor returns a `200 OK` once committed to durable SQLite storage so Stripe or Shopify never abandons the webhook.
|
|
52
52
|
2. **Cryptographic Header & Signature Preservation**: Forwards exact raw bytes, `stripe-signature`, `x-shopify-hmac-sha256`, and timestamps.
|
|
53
53
|
3. **Dead-Letter Queue (DLQ)**: If your server returns 500, 502, 504, 429, or times out, HookArmor safely preserves the raw event with full error diagnostics.
|
|
54
54
|
4. **Instant Alerts**: Sends immediate Slack / Discord webhooks when an endpoint begins failing.
|
|
55
|
-
5. **1-Click Bulk Replay**: As soon as you push your code fix, hit **Replay All Dead-Letter** in the Web UI or run `hookarmor replay --failed` to
|
|
56
|
-
6. **Local Dev
|
|
55
|
+
5. **1-Click Bulk Replay**: As soon as you push your code fix, hit **Replay All Dead-Letter** in the Web UI or run `hookarmor replay --failed` to re-deliver customer transactions.
|
|
56
|
+
6. **Local Dev Relay**: `hookarmor listen http://localhost:3000/api/webhooks` to proxy webhooks straight into local dev servers with automatic re-signing.
|
|
57
57
|
|
|
58
58
|
---
|
|
59
59
|
|
|
@@ -105,7 +105,7 @@ npx hookarmor replay evt_1758513516086_m8r0e7
|
|
|
105
105
|
|
|
106
106
|
## 📊 Verification Test Suite
|
|
107
107
|
|
|
108
|
-
HookArmor includes an 8-scenario verification suite (`test/verify.js`) plus a
|
|
108
|
+
HookArmor includes an 8-scenario verification suite (`test/verify.js`) plus a 39-case regression suite (`test/regression.js`) covering retry scheduling, crash recovery, multi-instance lease claims, cascading deletion, duplicate suppression, signature verification for every supported provider, outbound address validation, authentication, and dashboard escaping:
|
|
109
109
|
```bash
|
|
110
110
|
npm test
|
|
111
111
|
```
|
|
@@ -222,22 +222,47 @@ HookArmor returns an immediate `200 OK` to Stripe and Shopify in `<10ms` to prot
|
|
|
222
222
|
|
|
223
223
|
---
|
|
224
224
|
|
|
225
|
-
##
|
|
225
|
+
## 🏗️ Architecture & Operational Scope
|
|
226
226
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
227
|
+
HookArmor is 100% free and open-source under the MIT license, architected specifically as a **single-node, low-footprint buffer and sidecar** for monolithic and containerized applications.
|
|
228
|
+
|
|
229
|
+
* **Single-Process Simplicity**: Operates entirely in a single Node.js process using embedded SQLite with Write-Ahead Logging (`WAL`) and `synchronous=FULL`. It consumes <50MB RAM and eliminates the operational overhead of running external message brokers (Kafka, RabbitMQ, SQS, or Redis).
|
|
230
|
+
* **Scope & Boundaries**: HookArmor is intended to run on the same VPS, container host, or private network cluster as your downstream web application. It is **not** a distributed multi-region cluster broker.
|
|
231
|
+
* **Persistent Disk Required**: Because events are durably acknowledged to providers in `<5ms`, your container volume (`/app/data`) must be backed by a persistent disk or volume mount.
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## 📊 Prometheus & Grafana Metrics
|
|
236
|
+
|
|
237
|
+
HookArmor includes a native, zero-dependency Prometheus exposition endpoint at `GET /metrics`. Scrape this endpoint into your existing Prometheus or VictoriaMetrics instance to monitor webhook health:
|
|
238
|
+
|
|
239
|
+
```text
|
|
240
|
+
# Scraping: http://localhost:4000/metrics
|
|
241
|
+
hookarmor_uptime_seconds 8432
|
|
242
|
+
hookarmor_events_total{status="delivered"} 1420
|
|
243
|
+
hookarmor_events_total{status="failed"} 12
|
|
244
|
+
hookarmor_events_total{status="pending"} 0
|
|
245
|
+
hookarmor_endpoints_total 4
|
|
246
|
+
hookarmor_endpoints_unverified_total 1
|
|
247
|
+
hookarmor_delivery_latency_ms_avg 34.2
|
|
248
|
+
```
|
|
238
249
|
|
|
239
250
|
---
|
|
240
251
|
|
|
252
|
+
## 🔒 Secret Encryption at Rest (AES-256-GCM)
|
|
253
|
+
|
|
254
|
+
By default, endpoint signing secrets are stored in SQLite. For hardened environments, set `HOOKARMOR_ENCRYPTION_KEY` to enable transparent **AES-256-GCM authenticated envelope encryption** at rest:
|
|
255
|
+
|
|
256
|
+
```bash
|
|
257
|
+
# Generate a 256-bit encryption key
|
|
258
|
+
openssl rand -hex 32
|
|
259
|
+
|
|
260
|
+
# Set in your environment:
|
|
261
|
+
export HOOKARMOR_ENCRYPTION_KEY=your_64_char_hex_key
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
When enabled, all provider webhook secrets are encrypted with a unique 96-bit random IV and 128-bit authentication tag before being written to disk (`enc:v1:iv:tag:ciphertext`), preventing secret extraction even if database files or backups are compromised.
|
|
265
|
+
|
|
241
266
|
## 🔐 Production Security & Admin Authentication
|
|
242
267
|
|
|
243
268
|
When running HookArmor locally on your laptop (`localhost`), management endpoints and the replay dashboard operate without authentication for fast developer onboarding.
|
|
@@ -273,6 +298,7 @@ When `HOOKARMOR_API_KEY` is present:
|
|
|
273
298
|
| Variable | Default | Purpose |
|
|
274
299
|
|---|---|---|
|
|
275
300
|
| `HOOKARMOR_API_KEY` | none (required in production) | Admin key for the management API, dashboard and CLI |
|
|
301
|
+
| `HOOKARMOR_ENCRYPTION_KEY` | none | 256-bit key enabling AES-256-GCM at-rest encryption for endpoint secrets |
|
|
276
302
|
| `HOOKARMOR_STRICT_SSRF` | `true` in production, `false` locally | Block loopback, private-network and link-local delivery targets (checked on the resolved IP at delivery time). Cloud metadata addresses are always blocked |
|
|
277
303
|
| `HOOKARMOR_SIGNING_SECRET` | none | Enables Mode B internal signatures |
|
|
278
304
|
| `HOOKARMOR_DATA_DIR` | `./data` (current directory) | Where `hookarmor.db` is stored |
|
package/bin/hookarmor.js
CHANGED
|
@@ -22,6 +22,28 @@ function startServerOrExit() {
|
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
function setupGracefulShutdown(server, dispatcher, storage) {
|
|
26
|
+
let shuttingDown = false;
|
|
27
|
+
const shutdown = async (signal) => {
|
|
28
|
+
if (shuttingDown) return;
|
|
29
|
+
shuttingDown = true;
|
|
30
|
+
console.log(chalk.yellow(`\n⏻ ${signal} received: stopping incoming traffic and draining in-flight webhooks...`));
|
|
31
|
+
server.close();
|
|
32
|
+
if (dispatcher && dispatcher.drain) {
|
|
33
|
+
await dispatcher.drain(15000);
|
|
34
|
+
}
|
|
35
|
+
if (storage && storage.db) {
|
|
36
|
+
try {
|
|
37
|
+
storage.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
38
|
+
} catch (_) {}
|
|
39
|
+
}
|
|
40
|
+
console.log(chalk.green('✔ Shutdown complete.'));
|
|
41
|
+
process.exit(0);
|
|
42
|
+
};
|
|
43
|
+
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
|
44
|
+
process.on('SIGINT', () => shutdown('SIGINT'));
|
|
45
|
+
}
|
|
46
|
+
|
|
25
47
|
function apiHeaders(apiKey) {
|
|
26
48
|
const headers = { 'Content-Type': 'application/json' };
|
|
27
49
|
if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
|
|
@@ -41,9 +63,12 @@ program
|
|
|
41
63
|
.command('start')
|
|
42
64
|
.description('Start HookArmor server with web dashboard and ingress proxy')
|
|
43
65
|
.option('-p, --port <number>', 'Port to listen on', process.env.PORT || '4000')
|
|
66
|
+
.option('-H, --host <host>', 'Host address to bind to', process.env.HOOKARMOR_HOST || process.env.HOST || (process.env.HOOKARMOR_API_KEY || process.env.NODE_ENV === 'production' ? '0.0.0.0' : '127.0.0.1'))
|
|
44
67
|
.action((options) => {
|
|
45
68
|
const port = parseInt(options.port, 10);
|
|
46
|
-
const
|
|
69
|
+
const host = options.host;
|
|
70
|
+
const { server, storage, dispatcher, mockEnabled } = startServerOrExit();
|
|
71
|
+
setupGracefulShutdown(server, dispatcher, storage);
|
|
47
72
|
|
|
48
73
|
// Ensure a default mock endpoint exists for first-time onboarding (local simulation only)
|
|
49
74
|
if (mockEnabled && !storage.getEndpoint('demo-stripe')) {
|
|
@@ -57,13 +82,14 @@ program
|
|
|
57
82
|
});
|
|
58
83
|
}
|
|
59
84
|
|
|
60
|
-
server.listen(port, () => {
|
|
85
|
+
server.listen(port, host, () => {
|
|
61
86
|
console.log(chalk.blue.bold('\n🛡️ HookArmor Webhook Sentinel is LIVE!'));
|
|
62
87
|
console.log(chalk.gray('─────────────────────────────────────────'));
|
|
63
|
-
|
|
64
|
-
console.log(
|
|
88
|
+
const displayHost = (host === '0.0.0.0' || host === '127.0.0.1') ? 'localhost' : host;
|
|
89
|
+
console.log(`📡 Ingress Gateway : ${chalk.cyan(`http://${displayHost}:${port}/in/:endpointId`)}`);
|
|
90
|
+
console.log(`📊 Web Dashboard : ${chalk.green.bold(`http://${displayHost}:${port}/dashboard`)}`);
|
|
65
91
|
if (mockEnabled) {
|
|
66
|
-
console.log(`⚙️ Mock Receiver : ${chalk.yellow(`http
|
|
92
|
+
console.log(`⚙️ Mock Receiver : ${chalk.yellow(`http://${displayHost}:${port}/mock/target`)}`);
|
|
67
93
|
}
|
|
68
94
|
console.log(chalk.gray('─────────────────────────────────────────'));
|
|
69
95
|
console.log(chalk.white('Press Ctrl+C to stop.\n'));
|
|
@@ -75,11 +101,17 @@ program
|
|
|
75
101
|
.description('Tunnel incoming webhooks directly to a local or remote target')
|
|
76
102
|
.argument('<targetUrl>', 'Destination target URL (e.g. http://localhost:3000/api/webhook)')
|
|
77
103
|
.option('-p, --port <number>', 'Proxy port', '4000')
|
|
104
|
+
.option('-H, --host <host>', 'Host address to bind to', '127.0.0.1')
|
|
78
105
|
.option('-e, --endpoint <id>', 'Endpoint ID slug', 'local-dev')
|
|
79
106
|
.option('-s, --secret <secret>', 'Provider signing secret: verify on ingress and re-sign on delivery')
|
|
80
107
|
.action(async (targetUrl, options) => {
|
|
81
108
|
const port = parseInt(options.port, 10);
|
|
109
|
+
const host = options.host || '127.0.0.1';
|
|
110
|
+
if (host === '0.0.0.0') {
|
|
111
|
+
console.warn(chalk.yellow('\n[Security Warning] hookarmor listen is bound to 0.0.0.0 — reachable by anyone on your local network. Use 127.0.0.1 for local isolation.'));
|
|
112
|
+
}
|
|
82
113
|
const { server, storage, dispatcher } = startServerOrExit();
|
|
114
|
+
setupGracefulShutdown(server, dispatcher, storage);
|
|
83
115
|
|
|
84
116
|
storage.createEndpoint({
|
|
85
117
|
id: options.endpoint,
|
|
@@ -100,10 +132,10 @@ program
|
|
|
100
132
|
}
|
|
101
133
|
});
|
|
102
134
|
|
|
103
|
-
server.listen(port, () => {
|
|
135
|
+
server.listen(port, host, () => {
|
|
104
136
|
console.log(chalk.cyan.bold(`\n⚡ HookArmor Listen Active`));
|
|
105
|
-
console.log(`Forwarding: ${chalk.yellow(`http
|
|
106
|
-
console.log(`Dashboard: ${chalk.cyan(`http
|
|
137
|
+
console.log(`Forwarding: ${chalk.yellow(`http://${host}:${port}/in/${options.endpoint}`)} ➔ ${chalk.green(targetUrl)}`);
|
|
138
|
+
console.log(`Dashboard: ${chalk.cyan(`http://${host}:${port}/dashboard`)}\n`);
|
|
107
139
|
});
|
|
108
140
|
});
|
|
109
141
|
|
|
@@ -117,7 +149,7 @@ program
|
|
|
117
149
|
.action(async (eventId, options) => {
|
|
118
150
|
const baseUrl = options.url.replace(/\/$/, '');
|
|
119
151
|
try {
|
|
120
|
-
if (options.failed
|
|
152
|
+
if (options.failed) {
|
|
121
153
|
console.log(chalk.yellow('🔄 Replaying all dead-letter events...'));
|
|
122
154
|
const res = await fetch(`${baseUrl}/api/events/replay-all`, { method: 'POST', headers: apiHeaders(options.apiKey), body: '{}' });
|
|
123
155
|
const data = await readJson(res);
|
|
@@ -125,7 +157,7 @@ program
|
|
|
125
157
|
if (data.remaining > 0) {
|
|
126
158
|
console.log(chalk.yellow(` ${data.remaining} failed events still queued; run the command again to continue.`));
|
|
127
159
|
}
|
|
128
|
-
} else {
|
|
160
|
+
} else if (eventId) {
|
|
129
161
|
console.log(chalk.yellow(`🔄 Replaying event ${eventId}...`));
|
|
130
162
|
const res = await fetch(`${baseUrl}/api/events/${encodeURIComponent(eventId)}/replay`, { method: 'POST', headers: apiHeaders(options.apiKey), body: '{}' });
|
|
131
163
|
const data = await readJson(res);
|
|
@@ -135,6 +167,10 @@ program
|
|
|
135
167
|
console.log(chalk.red(`❌ Replay failed: HTTP ${data.statusCode} (${data.errorMessage || 'Target rejected'})`));
|
|
136
168
|
process.exitCode = 1;
|
|
137
169
|
}
|
|
170
|
+
} else {
|
|
171
|
+
console.error(chalk.red('✖ Missing argument: specify an [eventId] or pass --failed to replay all dead-letter events.'));
|
|
172
|
+
console.log(chalk.gray(' Usage: hookarmor replay <eventId> OR hookarmor replay --failed\n'));
|
|
173
|
+
process.exitCode = 1;
|
|
138
174
|
}
|
|
139
175
|
} catch (err) {
|
|
140
176
|
console.error(chalk.red(`Replay request to ${baseUrl} failed: ${err.message}`));
|
package/package.json
CHANGED
package/src/dispatcher.js
CHANGED
|
@@ -17,9 +17,30 @@ function svixKey(secret) {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
function svixSignature(secret, msgId, timestamp, rawBody) {
|
|
20
|
-
|
|
20
|
+
const hmac = crypto.createHmac('sha256', svixKey(secret));
|
|
21
|
+
hmac.update(Buffer.from(`${msgId}.${timestamp}.`, 'utf8'));
|
|
22
|
+
hmac.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'));
|
|
23
|
+
return hmac.digest('base64');
|
|
21
24
|
}
|
|
22
25
|
|
|
26
|
+
const STRIPPED_INBOUND_HEADERS = new Set([
|
|
27
|
+
'forwarded',
|
|
28
|
+
'x-real-ip',
|
|
29
|
+
'x-original-url',
|
|
30
|
+
'x-rewrite-url',
|
|
31
|
+
'x-http-method-override',
|
|
32
|
+
'cf-connecting-ip',
|
|
33
|
+
'true-client-ip',
|
|
34
|
+
'proxy-authorization',
|
|
35
|
+
'x-forwarded-for',
|
|
36
|
+
'x-forwarded-host',
|
|
37
|
+
'x-forwarded-proto',
|
|
38
|
+
'x-forwarded-port',
|
|
39
|
+
'x-forwarded-server',
|
|
40
|
+
'x-forwarded-prefix',
|
|
41
|
+
'x-forwarded-ssl'
|
|
42
|
+
]);
|
|
43
|
+
|
|
23
44
|
class Dispatcher extends EventEmitter {
|
|
24
45
|
constructor(storage, options = {}) {
|
|
25
46
|
super();
|
|
@@ -30,6 +51,7 @@ class Dispatcher extends EventEmitter {
|
|
|
30
51
|
this.defaultTimeoutMs = options.defaultTimeoutMs || 25000;
|
|
31
52
|
this.strictSSRF = Boolean(options.strictSSRF);
|
|
32
53
|
this.signingSecret = options.signingSecret || null;
|
|
54
|
+
this.activeDeliveries = 0;
|
|
33
55
|
}
|
|
34
56
|
|
|
35
57
|
// Verify signature at ingress (preventing HookArmor from acting as an open signature oracle).
|
|
@@ -62,10 +84,10 @@ class Dispatcher extends EventEmitter {
|
|
|
62
84
|
return { valid: false, reason: `Timestamp outside tolerance (${toleranceSec}s)` };
|
|
63
85
|
}
|
|
64
86
|
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
87
|
+
const hmac = crypto.createHmac('sha256', secret);
|
|
88
|
+
hmac.update(Buffer.from(`${timestampStr}.`, 'utf8'));
|
|
89
|
+
hmac.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'));
|
|
90
|
+
const expectedSig = hmac.digest('hex');
|
|
69
91
|
|
|
70
92
|
if (!signatures.some((sig) => safeEqual(expectedSig, sig))) {
|
|
71
93
|
return { valid: false, reason: 'Signature mismatch' };
|
|
@@ -79,7 +101,7 @@ class Dispatcher extends EventEmitter {
|
|
|
79
101
|
|
|
80
102
|
const expectedSig = crypto
|
|
81
103
|
.createHmac('sha256', secret)
|
|
82
|
-
.update(rawBody)
|
|
104
|
+
.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'))
|
|
83
105
|
.digest('base64');
|
|
84
106
|
|
|
85
107
|
if (!safeEqual(expectedSig, hmac)) {
|
|
@@ -92,7 +114,10 @@ class Dispatcher extends EventEmitter {
|
|
|
92
114
|
const sig = headers['x-hub-signature-256'];
|
|
93
115
|
if (!sig) return { valid: false, reason: 'Missing X-Hub-Signature-256 header' };
|
|
94
116
|
|
|
95
|
-
const expectedSig = 'sha256=' + crypto
|
|
117
|
+
const expectedSig = 'sha256=' + crypto
|
|
118
|
+
.createHmac('sha256', secret)
|
|
119
|
+
.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'))
|
|
120
|
+
.digest('hex');
|
|
96
121
|
if (!safeEqual(expectedSig, sig)) {
|
|
97
122
|
return { valid: false, reason: 'GitHub signature mismatch' };
|
|
98
123
|
}
|
|
@@ -137,19 +162,22 @@ class Dispatcher extends EventEmitter {
|
|
|
137
162
|
try {
|
|
138
163
|
if (provider === 'stripe') {
|
|
139
164
|
const freshTimestamp = Math.floor(Date.now() / 1000);
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
165
|
+
const hmac = crypto.createHmac('sha256', secret);
|
|
166
|
+
hmac.update(Buffer.from(`${freshTimestamp}.`, 'utf8'));
|
|
167
|
+
hmac.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'));
|
|
168
|
+
const freshSignature = hmac.digest('hex');
|
|
144
169
|
modified['stripe-signature'] = `t=${freshTimestamp},v1=${freshSignature}`;
|
|
145
170
|
} else if (provider === 'shopify') {
|
|
146
171
|
const freshHmac = crypto
|
|
147
172
|
.createHmac('sha256', secret)
|
|
148
|
-
.update(rawBody)
|
|
173
|
+
.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'))
|
|
149
174
|
.digest('base64');
|
|
150
175
|
modified['x-shopify-hmac-sha256'] = freshHmac;
|
|
151
176
|
} else if (provider === 'github') {
|
|
152
|
-
const freshSig = 'sha256=' + crypto
|
|
177
|
+
const freshSig = 'sha256=' + crypto
|
|
178
|
+
.createHmac('sha256', secret)
|
|
179
|
+
.update(Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8'))
|
|
180
|
+
.digest('hex');
|
|
153
181
|
modified['x-hub-signature-256'] = freshSig;
|
|
154
182
|
} else if (provider === 'clerk/svix' && modified['svix-id']) {
|
|
155
183
|
const freshTimestamp = String(Math.floor(Date.now() / 1000));
|
|
@@ -195,14 +223,29 @@ class Dispatcher extends EventEmitter {
|
|
|
195
223
|
return new Date(Date.now() + delaySec * 1000).toISOString().slice(0, 19).replace('T', ' ');
|
|
196
224
|
}
|
|
197
225
|
|
|
226
|
+
pendingDeliveries() {
|
|
227
|
+
return this.activeDeliveries;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async drain(timeoutMs = 15000) {
|
|
231
|
+
const start = Date.now();
|
|
232
|
+
while (this.pendingDeliveries() > 0) {
|
|
233
|
+
if (Date.now() - start > timeoutMs) break;
|
|
234
|
+
await new Promise(r => setTimeout(r, 100));
|
|
235
|
+
}
|
|
236
|
+
return this.pendingDeliveries() === 0;
|
|
237
|
+
}
|
|
238
|
+
|
|
198
239
|
// Callers must have claimed the event (status 'replaying') before dispatching
|
|
199
240
|
async dispatch(event, endpoint, { replay = false } = {}) {
|
|
200
241
|
const limit = endpoint.concurrency_limit || this.defaultConcurrency;
|
|
201
242
|
await this.acquireSlot(endpoint.id, limit);
|
|
243
|
+
this.activeDeliveries++;
|
|
202
244
|
|
|
203
245
|
try {
|
|
204
246
|
return await this._executeDispatch(event, endpoint, replay);
|
|
205
247
|
} finally {
|
|
248
|
+
this.activeDeliveries = Math.max(0, this.activeDeliveries - 1);
|
|
206
249
|
this.releaseSlot(endpoint.id, limit);
|
|
207
250
|
}
|
|
208
251
|
}
|
|
@@ -213,9 +256,16 @@ class Dispatcher extends EventEmitter {
|
|
|
213
256
|
|
|
214
257
|
let headers = { ...event.headers };
|
|
215
258
|
|
|
216
|
-
// Never forward sender-supplied HookArmor headers
|
|
259
|
+
// Never forward sender-supplied HookArmor headers or spoofable routing headers
|
|
217
260
|
for (const name of Object.keys(headers)) {
|
|
218
|
-
|
|
261
|
+
const lower = name.toLowerCase();
|
|
262
|
+
if (
|
|
263
|
+
lower.startsWith('x-hookarmor-') ||
|
|
264
|
+
lower.startsWith('x-forwarded-') ||
|
|
265
|
+
STRIPPED_INBOUND_HEADERS.has(lower)
|
|
266
|
+
) {
|
|
267
|
+
delete headers[name];
|
|
268
|
+
}
|
|
219
269
|
}
|
|
220
270
|
|
|
221
271
|
// Preserve original raw signatures before re-signing for audit and forensic trace
|
|
@@ -229,15 +279,22 @@ class Dispatcher extends EventEmitter {
|
|
|
229
279
|
headers['x-hookarmor-original-svix-signature'] = event.headers['svix-signature'];
|
|
230
280
|
}
|
|
231
281
|
|
|
282
|
+
const rawBuffer = Buffer.isBuffer(event.raw_body)
|
|
283
|
+
? event.raw_body
|
|
284
|
+
: Buffer.from(String(event.raw_body || ''), 'utf8');
|
|
285
|
+
|
|
232
286
|
// Re-sign outbound headers if endpoint secret is configured (Transparent Zero-Code-Change Mode)
|
|
233
287
|
if (endpoint.secret) {
|
|
234
|
-
headers = this.signHeaders(event.provider,
|
|
288
|
+
headers = this.signHeaders(event.provider, rawBuffer, headers, endpoint.secret);
|
|
235
289
|
}
|
|
236
290
|
|
|
237
291
|
// Isolated Trust Domain mode: vouch only for events whose signature was verified at ingress
|
|
238
292
|
if (this.signingSecret && event.verified) {
|
|
239
293
|
const freshTs = Math.floor(Date.now() / 1000);
|
|
240
|
-
const
|
|
294
|
+
const hmac = crypto.createHmac('sha256', this.signingSecret);
|
|
295
|
+
hmac.update(Buffer.from(`${freshTs}.`, 'utf8'));
|
|
296
|
+
hmac.update(rawBuffer);
|
|
297
|
+
const internalSig = hmac.digest('hex');
|
|
241
298
|
headers['x-hookarmor-signature'] = `t=${freshTs},v1=${internalSig}`;
|
|
242
299
|
}
|
|
243
300
|
|
|
@@ -249,13 +306,20 @@ class Dispatcher extends EventEmitter {
|
|
|
249
306
|
delete headers['transfer-encoding'];
|
|
250
307
|
delete headers['expect'];
|
|
251
308
|
|
|
252
|
-
const rawBuffer = Buffer.from(event.raw_body, 'utf8');
|
|
253
309
|
headers['content-length'] = String(rawBuffer.length);
|
|
254
310
|
headers['x-hookarmor-delivery-id'] = event.id;
|
|
255
311
|
headers['x-hookarmor-attempt'] = String(event.attempts + 1);
|
|
256
|
-
|
|
312
|
+
// Strict ISO-8601 UTC timestamp with Z to prevent client-side local timezone parsing skew
|
|
313
|
+
const createdAtStr = String(event.created_at || '');
|
|
314
|
+
const originalTimestamp = createdAtStr.includes('T')
|
|
315
|
+
? (createdAtStr.endsWith('Z') ? createdAtStr : `${createdAtStr}Z`)
|
|
316
|
+
: (createdAtStr ? `${createdAtStr.replace(' ', 'T')}Z` : new Date().toISOString());
|
|
317
|
+
headers['x-hookarmor-original-timestamp'] = originalTimestamp;
|
|
257
318
|
const isReplay = replay || event.attempts > 0;
|
|
258
319
|
headers['x-hookarmor-is-replay'] = isReplay ? 'true' : 'false';
|
|
320
|
+
if (!event.verified) {
|
|
321
|
+
headers['x-hookarmor-unverified'] = 'true';
|
|
322
|
+
}
|
|
259
323
|
|
|
260
324
|
return new Promise((resolve) => {
|
|
261
325
|
let settled = false;
|
package/src/public/index.html
CHANGED
|
@@ -900,13 +900,16 @@
|
|
|
900
900
|
|
|
901
901
|
container.innerHTML = cachedEndpoints.map(ep => `
|
|
902
902
|
<div class="endpoint-item">
|
|
903
|
-
<div class="endpoint-name"
|
|
903
|
+
<div class="endpoint-name" style="display: flex; align-items: center; justify-content: space-between;">
|
|
904
|
+
<span>${esc(ep.name)}</span>
|
|
905
|
+
${!ep.has_secret ? '<span class="badge" style="background: rgba(245, 158, 11, 0.15); color: #f59e0b; border: 1px solid rgba(245, 158, 11, 0.3); font-size: 0.68rem; padding: 0.1rem 0.4rem;">⚠️ No Secret</span>' : ''}
|
|
906
|
+
</div>
|
|
904
907
|
<div class="endpoint-url" title="Ingress URL">${esc(window.location.origin)}/in/${esc(ep.id)}</div>
|
|
905
908
|
<div class="target-url">🎯 <strong>Target:</strong> ${esc(ep.target_url)}</div>
|
|
906
909
|
<div style="font-size: 0.72rem; color: var(--text-muted); display: flex; gap: 0.75rem;">
|
|
907
910
|
<span>Auto-Retry: <strong>${ep.auto_retry ? 'Yes' : 'No'}</strong></span>
|
|
908
911
|
<span>Max: <strong>${esc(ep.max_retries)} attempts</strong></span>
|
|
909
|
-
<span>Signed: <strong>${ep.has_secret ? 'Yes' : 'No'}</strong></span>
|
|
912
|
+
<span>Signed: <strong>${ep.has_secret ? 'Yes' : '<span style="color: #f59e0b;">No (⚠️ Unverified)</span>'}</strong></span>
|
|
910
913
|
</div>
|
|
911
914
|
</div>
|
|
912
915
|
`).join('');
|
package/src/public/landing.html
CHANGED
|
@@ -464,7 +464,7 @@
|
|
|
464
464
|
<div class="card-icon" style="background: rgba(56, 189, 248, 0.1); color: #38bdf8;">💻</div>
|
|
465
465
|
<h3>Zero-Friction Local Tunnel</h3>
|
|
466
466
|
<p>
|
|
467
|
-
Run <code>
|
|
467
|
+
Run <code>hookarmor listen http://localhost:3000/api/webhook</code> to proxy and replay captured webhook events straight into your local server with automatic signature verification.
|
|
468
468
|
</p>
|
|
469
469
|
</div>
|
|
470
470
|
</div>
|
|
@@ -515,7 +515,7 @@
|
|
|
515
515
|
<li>✓ <strong>250,000 events / month</strong></li>
|
|
516
516
|
<li>✓ <strong>90-day retention</strong> in DLQ</li>
|
|
517
517
|
<li>✓ Unlimited Endpoints</li>
|
|
518
|
-
<li>✓
|
|
518
|
+
<li>✓ Per-Endpoint Concurrency Limiting</li>
|
|
519
519
|
<li>✓ Team Workspaces (Beta Roadmap)</li>
|
|
520
520
|
<li>✓ High-Concurrency Queue Worker</li>
|
|
521
521
|
</ul>
|
package/src/server.js
CHANGED
|
@@ -90,15 +90,53 @@ function createServer(options = {}) {
|
|
|
90
90
|
const app = express();
|
|
91
91
|
const server = http.createServer(app);
|
|
92
92
|
|
|
93
|
+
app.disable('x-powered-by');
|
|
94
|
+
|
|
95
|
+
app.use((req, res, next) => {
|
|
96
|
+
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
97
|
+
res.setHeader('X-Frame-Options', 'DENY');
|
|
98
|
+
res.setHeader('Referrer-Policy', 'no-referrer');
|
|
99
|
+
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
|
|
100
|
+
res.setHeader(
|
|
101
|
+
'Content-Security-Policy',
|
|
102
|
+
"default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self' ws: wss:; frame-ancestors 'none';"
|
|
103
|
+
);
|
|
104
|
+
const isHttps = req.secure || req.headers['x-forwarded-proto'] === 'https';
|
|
105
|
+
if (isHttps) {
|
|
106
|
+
res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains');
|
|
107
|
+
}
|
|
108
|
+
next();
|
|
109
|
+
});
|
|
110
|
+
|
|
93
111
|
const trustProxy = process.env.HOOKARMOR_TRUST_PROXY || ((process.env.RAILWAY_ENVIRONMENT || process.env.RENDER) ? '1' : null);
|
|
94
112
|
if (trustProxy) app.set('trust proxy', /^\d+$/.test(trustProxy) ? parseInt(trustProxy, 10) : trustProxy);
|
|
95
113
|
|
|
96
|
-
const storage = new Storage(options.dbPath
|
|
97
|
-
|
|
114
|
+
const storage = new Storage(options.dbPath, {
|
|
115
|
+
encryptionKey: options.encryptionKey || process.env.HOOKARMOR_ENCRYPTION_KEY
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const instanceId = options.instanceId || `inst_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
|
|
119
|
+
storage.registerInstance({
|
|
120
|
+
id: instanceId,
|
|
121
|
+
hostname: require('os').hostname(),
|
|
122
|
+
pid: process.pid
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const recovered = storage.recoverInterruptedDeliveries(instanceId);
|
|
98
126
|
if (recovered > 0) {
|
|
99
127
|
console.warn(`[HookArmor] Re-queued ${recovered} event(s) that were mid-delivery when the process last stopped.`);
|
|
100
128
|
}
|
|
101
129
|
|
|
130
|
+
// Security audit check: warn about endpoints without secrets on startup
|
|
131
|
+
try {
|
|
132
|
+
const initialEndpoints = storage.listEndpoints();
|
|
133
|
+
for (const ep of initialEndpoints) {
|
|
134
|
+
if (!ep.secret) {
|
|
135
|
+
console.warn(`[Security Warning] Endpoint '${ep.id}' has no signing secret. Incoming webhooks cannot be cryptographically verified against spoofing!`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
} catch (_) {}
|
|
139
|
+
|
|
102
140
|
const dispatcher = new Dispatcher(storage, {
|
|
103
141
|
defaultConcurrency: options.defaultConcurrency || 5,
|
|
104
142
|
defaultTimeoutMs: options.defaultTimeoutMs || 25000,
|
|
@@ -108,7 +146,8 @@ function createServer(options = {}) {
|
|
|
108
146
|
|
|
109
147
|
const worker = new RetryWorker(storage, dispatcher, {
|
|
110
148
|
intervalMs: options.retryIntervalMs || 5000,
|
|
111
|
-
retentionDays
|
|
149
|
+
retentionDays,
|
|
150
|
+
instanceId
|
|
112
151
|
});
|
|
113
152
|
if (options.autoStartWorker !== false) {
|
|
114
153
|
worker.start();
|
|
@@ -135,7 +174,12 @@ function createServer(options = {}) {
|
|
|
135
174
|
path: '/ws',
|
|
136
175
|
maxPayload: 4096,
|
|
137
176
|
// Without an API key the feed is only served to localhost (blocks DNS-rebinding pages)
|
|
138
|
-
verifyClient: (info) =>
|
|
177
|
+
verifyClient: (info) => {
|
|
178
|
+
if (apiKey) return true;
|
|
179
|
+
const remote = info.req.socket && info.req.socket.remoteAddress;
|
|
180
|
+
const isLoopbackRemote = !remote || remote === '127.0.0.1' || remote === '::1' || remote === '::ffff:127.0.0.1';
|
|
181
|
+
return isLoopbackRemote && LOCAL_HOSTNAMES.has(hostnameOf(info.req.headers.host));
|
|
182
|
+
}
|
|
139
183
|
});
|
|
140
184
|
|
|
141
185
|
wss.on('connection', (socket) => {
|
|
@@ -184,8 +228,24 @@ function createServer(options = {}) {
|
|
|
184
228
|
// Serve static UI assets
|
|
185
229
|
app.use('/static', express.static(path.join(__dirname, 'public')));
|
|
186
230
|
|
|
231
|
+
const ingressLimiter = createRateLimiter({
|
|
232
|
+
limit: clampInt(process.env.HOOKARMOR_INGRESS_RATE_LIMIT, 600, 1, 50000),
|
|
233
|
+
windowMs: 60000
|
|
234
|
+
});
|
|
235
|
+
const maxPendingEvents = clampInt(process.env.HOOKARMOR_MAX_PENDING_EVENTS, 50000, 100, 1000000);
|
|
236
|
+
|
|
187
237
|
// Ingress endpoint for webhooks - uses raw body parser to preserve raw bytes
|
|
188
238
|
app.post('/in/:endpointId', express.raw({ type: '*/*', limit: maxBody }), (req, res) => {
|
|
239
|
+
// Ingress rate limiting per IP
|
|
240
|
+
if (!ingressLimiter(req.ip)) {
|
|
241
|
+
return res.status(429).json({ error: 'Too many ingress requests from this IP; rate limit exceeded' });
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Storage capacity backpressure protection
|
|
245
|
+
if (storage.getPendingCount && storage.getPendingCount() >= maxPendingEvents) {
|
|
246
|
+
return res.status(503).json({ error: 'Ingress queue capacity reached. System is under high load; retry shortly.' });
|
|
247
|
+
}
|
|
248
|
+
|
|
189
249
|
const endpointId = req.params.endpointId;
|
|
190
250
|
const endpoint = storage.getEndpoint(endpointId);
|
|
191
251
|
|
|
@@ -193,8 +253,11 @@ function createServer(options = {}) {
|
|
|
193
253
|
return res.status(404).json({ error: 'HookArmor endpoint not found' });
|
|
194
254
|
}
|
|
195
255
|
|
|
196
|
-
const rawBodyBuffer = Buffer.isBuffer(req.body) ? req.body : Buffer.from('');
|
|
197
|
-
|
|
256
|
+
const rawBodyBuffer = Buffer.isBuffer(req.body) ? req.body : Buffer.from(req.body || '');
|
|
257
|
+
let rawBody = '';
|
|
258
|
+
try {
|
|
259
|
+
rawBody = rawBodyBuffer.toString('utf8');
|
|
260
|
+
} catch (_) {}
|
|
198
261
|
const headers = { ...req.headers };
|
|
199
262
|
|
|
200
263
|
// Detect provider & idempotency key
|
|
@@ -238,7 +301,7 @@ function createServer(options = {}) {
|
|
|
238
301
|
// 0. Verify signature on ingress if endpoint secret is configured
|
|
239
302
|
let verified = false;
|
|
240
303
|
if (endpoint.secret) {
|
|
241
|
-
const verification = Dispatcher.verifyIngressSignature(provider,
|
|
304
|
+
const verification = Dispatcher.verifyIngressSignature(provider, rawBodyBuffer, headers, endpoint.secret);
|
|
242
305
|
if (!verification.valid) {
|
|
243
306
|
return res.status(400).json({
|
|
244
307
|
error: 'Webhook signature verification failed',
|
|
@@ -247,6 +310,9 @@ function createServer(options = {}) {
|
|
|
247
310
|
});
|
|
248
311
|
}
|
|
249
312
|
verified = true;
|
|
313
|
+
} else {
|
|
314
|
+
console.warn(`[Ingress Warning] Ingested webhook for endpoint '${endpoint.id}' without signature verification (no secret configured).`);
|
|
315
|
+
headers['x-hookarmor-unverified'] = 'true';
|
|
250
316
|
}
|
|
251
317
|
|
|
252
318
|
let savedEvent;
|
|
@@ -274,8 +340,9 @@ function createServer(options = {}) {
|
|
|
274
340
|
provider,
|
|
275
341
|
eventType,
|
|
276
342
|
headers,
|
|
277
|
-
rawBody,
|
|
343
|
+
rawBody: rawBodyBuffer,
|
|
278
344
|
status: 'replaying',
|
|
345
|
+
claimedBy: instanceId,
|
|
279
346
|
verified
|
|
280
347
|
});
|
|
281
348
|
} catch (err) {
|
|
@@ -319,10 +386,12 @@ function createServer(options = {}) {
|
|
|
319
386
|
|
|
320
387
|
if (!apiKey) {
|
|
321
388
|
if (!isDemoMode) {
|
|
322
|
-
// Open local-dev mode: only answer to
|
|
323
|
-
//
|
|
324
|
-
|
|
325
|
-
|
|
389
|
+
// Open local-dev mode: only answer to loopback connections and localhost host names,
|
|
390
|
+
// so LAN attackers or web pages through DNS rebinding cannot reach this API
|
|
391
|
+
const remote = req.socket && req.socket.remoteAddress;
|
|
392
|
+
const isLoopbackRemote = !remote || remote === '127.0.0.1' || remote === '::1' || remote === '::ffff:127.0.0.1';
|
|
393
|
+
if (!isLoopbackRemote || !LOCAL_HOSTNAMES.has(hostnameOf(req.headers.host))) {
|
|
394
|
+
return res.status(403).json({ error: 'Without HOOKARMOR_API_KEY the management API is only accessible locally from loopback interfaces (localhost/127.0.0.1)' });
|
|
326
395
|
}
|
|
327
396
|
return next();
|
|
328
397
|
}
|
|
@@ -352,7 +421,9 @@ function createServer(options = {}) {
|
|
|
352
421
|
app.use('/api', (req, res, next) => {
|
|
353
422
|
const contentType = String(req.headers['content-type'] || '').split(';')[0].trim().toLowerCase();
|
|
354
423
|
const isJson = contentType === 'application/json' || contentType.endsWith('+json');
|
|
355
|
-
|
|
424
|
+
const isBodyMethod = req.method === 'POST' || req.method === 'PUT' || req.method === 'PATCH';
|
|
425
|
+
const hasBody = Boolean(req.headers['content-length'] && req.headers['content-length'] !== '0');
|
|
426
|
+
if ((isBodyMethod || hasBody) && !isJson) {
|
|
356
427
|
return res.status(415).json({ error: 'Content-Type must be application/json' });
|
|
357
428
|
}
|
|
358
429
|
next();
|
|
@@ -406,6 +477,13 @@ function createServer(options = {}) {
|
|
|
406
477
|
res.json(publicEndpoint(endpoint));
|
|
407
478
|
});
|
|
408
479
|
|
|
480
|
+
// Delete an endpoint
|
|
481
|
+
app.delete('/api/endpoints/:id', (req, res) => {
|
|
482
|
+
const deleted = storage.deleteEndpoint(req.params.id);
|
|
483
|
+
if (!deleted) return res.status(404).json({ error: 'Endpoint not found' });
|
|
484
|
+
res.json({ success: true, message: 'Endpoint deleted' });
|
|
485
|
+
});
|
|
486
|
+
|
|
409
487
|
// List Events
|
|
410
488
|
app.get('/api/events', (req, res) => {
|
|
411
489
|
const { endpointId, status, limit, offset } = req.query;
|
|
@@ -415,7 +493,10 @@ function createServer(options = {}) {
|
|
|
415
493
|
limit: clampInt(limit, 50, 1, 500),
|
|
416
494
|
offset: clampInt(offset, 0, 0, Number.MAX_SAFE_INTEGER)
|
|
417
495
|
});
|
|
418
|
-
res.json(events
|
|
496
|
+
res.json(events.map(e => ({
|
|
497
|
+
...e,
|
|
498
|
+
raw_body: Buffer.isBuffer(e.raw_body) ? e.raw_body.toString('utf8') : String(e.raw_body || '')
|
|
499
|
+
})));
|
|
419
500
|
});
|
|
420
501
|
|
|
421
502
|
// Get single event with attempts
|
|
@@ -423,7 +504,18 @@ function createServer(options = {}) {
|
|
|
423
504
|
const event = storage.getEvent(req.params.id);
|
|
424
505
|
if (!event) return res.status(404).json({ error: 'Event not found' });
|
|
425
506
|
const attempts = storage.getAttempts(req.params.id);
|
|
426
|
-
res.json({
|
|
507
|
+
res.json({
|
|
508
|
+
...event,
|
|
509
|
+
raw_body: Buffer.isBuffer(event.raw_body) ? event.raw_body.toString('utf8') : String(event.raw_body || ''),
|
|
510
|
+
attempts
|
|
511
|
+
});
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
// Delete a specific event and its attempts
|
|
515
|
+
app.delete('/api/events/:id', (req, res) => {
|
|
516
|
+
const deleted = storage.deleteEvent(req.params.id);
|
|
517
|
+
if (!deleted) return res.status(404).json({ error: 'Event not found' });
|
|
518
|
+
res.json({ success: true, message: 'Event deleted' });
|
|
427
519
|
});
|
|
428
520
|
|
|
429
521
|
// Replay a specific event
|
|
@@ -473,13 +565,29 @@ function createServer(options = {}) {
|
|
|
473
565
|
responseBody: { status: 'mock_processed' }
|
|
474
566
|
};
|
|
475
567
|
|
|
568
|
+
const mockAuthMiddleware = (req, res, next) => {
|
|
569
|
+
if (apiKey) {
|
|
570
|
+
const header = req.headers['authorization'] || '';
|
|
571
|
+
const token = header.startsWith('Bearer ') ? header.slice(7) : (req.headers['x-api-key'] || req.query.key || '');
|
|
572
|
+
if (!tokenMatches(token)) {
|
|
573
|
+
return res.status(401).json({ error: 'Unauthorized: valid API key required for /mock/config' });
|
|
574
|
+
}
|
|
575
|
+
} else {
|
|
576
|
+
const host = hostnameOf(req.headers['host']);
|
|
577
|
+
if (!LOCAL_HOSTNAMES.has(host) && !allowNoAuth) {
|
|
578
|
+
return res.status(403).json({ error: 'Forbidden: loopback access only' });
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
next();
|
|
582
|
+
};
|
|
583
|
+
|
|
476
584
|
app.post('/mock/target', (req, res) => {
|
|
477
585
|
setTimeout(() => {
|
|
478
586
|
res.status(mockTargetBehavior.statusCode).json(mockTargetBehavior.responseBody);
|
|
479
587
|
}, mockTargetBehavior.delayMs);
|
|
480
588
|
});
|
|
481
589
|
|
|
482
|
-
app.post('/mock/config', (req, res) => {
|
|
590
|
+
app.post('/mock/config', mockAuthMiddleware, (req, res) => {
|
|
483
591
|
const body = req.body || {};
|
|
484
592
|
mockTargetBehavior = {
|
|
485
593
|
statusCode: clampInt(body.statusCode, mockTargetBehavior.statusCode, 100, 599),
|
|
@@ -489,11 +597,51 @@ function createServer(options = {}) {
|
|
|
489
597
|
res.json({ message: 'Mock target configuration updated', config: mockTargetBehavior });
|
|
490
598
|
});
|
|
491
599
|
|
|
492
|
-
app.get('/mock/config', (req, res) => {
|
|
600
|
+
app.get('/mock/config', mockAuthMiddleware, (req, res) => {
|
|
493
601
|
res.json(mockTargetBehavior);
|
|
494
602
|
});
|
|
495
603
|
}
|
|
496
604
|
|
|
605
|
+
// Prometheus Metrics Exposition Endpoint
|
|
606
|
+
app.get('/metrics', (req, res) => {
|
|
607
|
+
try {
|
|
608
|
+
const stats = storage.getStats();
|
|
609
|
+
const endpoints = storage.listEndpoints();
|
|
610
|
+
const unverifiedCount = endpoints.filter((e) => !e.secret).length;
|
|
611
|
+
const uptime = Math.floor(process.uptime());
|
|
612
|
+
|
|
613
|
+
const lines = [
|
|
614
|
+
'# HELP hookarmor_uptime_seconds Total process uptime in seconds',
|
|
615
|
+
'# TYPE hookarmor_uptime_seconds counter',
|
|
616
|
+
`hookarmor_uptime_seconds ${uptime}`,
|
|
617
|
+
'',
|
|
618
|
+
'# HELP hookarmor_events_total Total number of events by status',
|
|
619
|
+
'# TYPE hookarmor_events_total gauge',
|
|
620
|
+
`hookarmor_events_total{status="delivered"} ${stats.delivered || 0}`,
|
|
621
|
+
`hookarmor_events_total{status="failed"} ${stats.failed || 0}`,
|
|
622
|
+
`hookarmor_events_total{status="pending"} ${stats.pending || 0}`,
|
|
623
|
+
`hookarmor_events_total{status="total"} ${stats.total || 0}`,
|
|
624
|
+
'',
|
|
625
|
+
'# HELP hookarmor_endpoints_total Total number of configured endpoints',
|
|
626
|
+
'# TYPE hookarmor_endpoints_total gauge',
|
|
627
|
+
`hookarmor_endpoints_total ${endpoints.length}`,
|
|
628
|
+
'',
|
|
629
|
+
'# HELP hookarmor_endpoints_unverified_total Number of endpoints without a cryptographic signing secret',
|
|
630
|
+
'# TYPE hookarmor_endpoints_unverified_total gauge',
|
|
631
|
+
`hookarmor_endpoints_unverified_total ${unverifiedCount}`,
|
|
632
|
+
'',
|
|
633
|
+
'# HELP hookarmor_delivery_latency_ms_avg Average delivery latency in milliseconds',
|
|
634
|
+
'# TYPE hookarmor_delivery_latency_ms_avg gauge',
|
|
635
|
+
`hookarmor_delivery_latency_ms_avg ${stats.avgLatencyMs || 0}`
|
|
636
|
+
];
|
|
637
|
+
|
|
638
|
+
res.setHeader('Content-Type', 'text/plain; version=0.0.4; charset=utf-8');
|
|
639
|
+
res.end(lines.join('\n') + '\n');
|
|
640
|
+
} catch (err) {
|
|
641
|
+
res.status(500).setHeader('Content-Type', 'text/plain').end(`# Error collecting metrics: ${err.message}\n`);
|
|
642
|
+
}
|
|
643
|
+
});
|
|
644
|
+
|
|
497
645
|
// SEO & Web Crawler Discovery
|
|
498
646
|
app.get('/robots.txt', (req, res) => {
|
|
499
647
|
res.type('text/plain');
|
|
@@ -538,7 +686,7 @@ function createServer(options = {}) {
|
|
|
538
686
|
res.redirect('/blog');
|
|
539
687
|
});
|
|
540
688
|
|
|
541
|
-
return { app, server, storage, dispatcher, worker, mockEnabled, strictSSRF };
|
|
689
|
+
return { app, server, storage, dispatcher, worker, mockEnabled, strictSSRF, instanceId };
|
|
542
690
|
}
|
|
543
691
|
|
|
544
692
|
module.exports = { createServer };
|
package/src/storage.js
CHANGED
|
@@ -1,14 +1,52 @@
|
|
|
1
1
|
const Database = require('better-sqlite3');
|
|
2
2
|
const path = require('path');
|
|
3
3
|
const fs = require('fs');
|
|
4
|
+
const crypto = require('crypto');
|
|
4
5
|
|
|
5
6
|
// SQLite datetime() format (UTC, "YYYY-MM-DD HH:MM:SS") so stored times compare correctly with datetime('now')
|
|
6
7
|
function toSqliteTime(date) {
|
|
7
8
|
return date.toISOString().slice(0, 19).replace('T', ' ');
|
|
8
9
|
}
|
|
9
10
|
|
|
11
|
+
function deriveKey(keyInput) {
|
|
12
|
+
if (!keyInput) return null;
|
|
13
|
+
if (Buffer.isBuffer(keyInput) && keyInput.length === 32) return keyInput;
|
|
14
|
+
return crypto.createHash('sha256').update(String(keyInput)).digest();
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function encryptSecret(plaintext, key) {
|
|
18
|
+
if (!plaintext || typeof plaintext !== 'string') return plaintext || '';
|
|
19
|
+
if (!key) return plaintext;
|
|
20
|
+
const iv = crypto.randomBytes(12);
|
|
21
|
+
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
|
|
22
|
+
const encrypted = Buffer.concat([cipher.update(Buffer.from(plaintext, 'utf8')), cipher.final()]);
|
|
23
|
+
const tag = cipher.getAuthTag();
|
|
24
|
+
return `enc:v1:${iv.toString('hex')}:${tag.toString('hex')}:${encrypted.toString('hex')}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function decryptSecret(stored, key) {
|
|
28
|
+
if (!stored || typeof stored !== 'string') return stored || '';
|
|
29
|
+
if (!stored.startsWith('enc:v1:')) {
|
|
30
|
+
return stored; // Plaintext legacy or fallback
|
|
31
|
+
}
|
|
32
|
+
if (!key) {
|
|
33
|
+
throw new Error('Endpoint secret is encrypted with AES-256-GCM, but HOOKARMOR_ENCRYPTION_KEY is not configured');
|
|
34
|
+
}
|
|
35
|
+
const parts = stored.split(':');
|
|
36
|
+
if (parts.length !== 5) {
|
|
37
|
+
throw new Error('Malformed encrypted secret format in database');
|
|
38
|
+
}
|
|
39
|
+
const iv = Buffer.from(parts[2], 'hex');
|
|
40
|
+
const tag = Buffer.from(parts[3], 'hex');
|
|
41
|
+
const ciphertext = Buffer.from(parts[4], 'hex');
|
|
42
|
+
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
|
|
43
|
+
decipher.setAuthTag(tag);
|
|
44
|
+
const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
|
45
|
+
return decrypted.toString('utf8');
|
|
46
|
+
}
|
|
47
|
+
|
|
10
48
|
class Storage {
|
|
11
|
-
constructor(dbPath) {
|
|
49
|
+
constructor(dbPath, options = {}) {
|
|
12
50
|
if (!dbPath) {
|
|
13
51
|
const dataDir = process.env.HOOKARMOR_DATA_DIR || path.join(process.cwd(), 'data');
|
|
14
52
|
if (!fs.existsSync(dataDir)) {
|
|
@@ -16,6 +54,16 @@ class Storage {
|
|
|
16
54
|
}
|
|
17
55
|
dbPath = path.join(dataDir, 'hookarmor.db');
|
|
18
56
|
}
|
|
57
|
+
const envKey = process.env.HOOKARMOR_ENCRYPTION_KEY;
|
|
58
|
+
this.encryptionKey = options.encryptionKey !== undefined
|
|
59
|
+
? deriveKey(options.encryptionKey)
|
|
60
|
+
: deriveKey(envKey);
|
|
61
|
+
|
|
62
|
+
if (!this.encryptionKey && !Storage._warnedNoKey && process.env.NODE_ENV !== 'test') {
|
|
63
|
+
console.warn('[Security Warning] HOOKARMOR_ENCRYPTION_KEY is not set. Endpoint secrets are stored in plaintext. Set a 32+ character key to enable AES-256-GCM at-rest encryption.');
|
|
64
|
+
Storage._warnedNoKey = true;
|
|
65
|
+
}
|
|
66
|
+
|
|
19
67
|
this.db = new Database(dbPath);
|
|
20
68
|
this.init();
|
|
21
69
|
}
|
|
@@ -54,6 +102,8 @@ class Storage {
|
|
|
54
102
|
last_error TEXT,
|
|
55
103
|
last_latency_ms INTEGER,
|
|
56
104
|
next_retry_at TEXT,
|
|
105
|
+
claimed_by TEXT,
|
|
106
|
+
claimed_at TEXT,
|
|
57
107
|
created_at TEXT DEFAULT (datetime('now')),
|
|
58
108
|
updated_at TEXT DEFAULT (datetime('now')),
|
|
59
109
|
FOREIGN KEY (endpoint_id) REFERENCES endpoints(id)
|
|
@@ -70,6 +120,14 @@ class Storage {
|
|
|
70
120
|
FOREIGN KEY (event_id) REFERENCES events(id)
|
|
71
121
|
);
|
|
72
122
|
|
|
123
|
+
CREATE TABLE IF NOT EXISTS instances (
|
|
124
|
+
id TEXT PRIMARY KEY,
|
|
125
|
+
hostname TEXT,
|
|
126
|
+
pid INTEGER,
|
|
127
|
+
started_at TEXT DEFAULT (datetime('now')),
|
|
128
|
+
last_heartbeat TEXT DEFAULT (datetime('now'))
|
|
129
|
+
);
|
|
130
|
+
|
|
73
131
|
CREATE TABLE IF NOT EXISTS waitlist (
|
|
74
132
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
75
133
|
email TEXT UNIQUE NOT NULL,
|
|
@@ -85,11 +143,24 @@ class Storage {
|
|
|
85
143
|
CREATE INDEX IF NOT EXISTS idx_attempts_event ON delivery_attempts(event_id);
|
|
86
144
|
`);
|
|
87
145
|
|
|
88
|
-
//
|
|
146
|
+
// Migrations for existing databases
|
|
89
147
|
const eventColumns = this.db.prepare('PRAGMA table_info(events)').all().map((c) => c.name);
|
|
90
148
|
if (!eventColumns.includes('verified')) {
|
|
91
149
|
this.db.exec('ALTER TABLE events ADD COLUMN verified INTEGER DEFAULT 0');
|
|
92
150
|
}
|
|
151
|
+
if (!eventColumns.includes('claimed_by')) {
|
|
152
|
+
this.db.exec('ALTER TABLE events ADD COLUMN claimed_by TEXT');
|
|
153
|
+
}
|
|
154
|
+
if (!eventColumns.includes('claimed_at')) {
|
|
155
|
+
this.db.exec('ALTER TABLE events ADD COLUMN claimed_at TEXT');
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
this.db.exec(`
|
|
160
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_events_idem_unique
|
|
161
|
+
ON events(endpoint_id, idempotency_key) WHERE idempotency_key IS NOT NULL;
|
|
162
|
+
`);
|
|
163
|
+
} catch (_) {}
|
|
93
164
|
}
|
|
94
165
|
|
|
95
166
|
// Upsert. Omitting secret or alertWebhookUrl (undefined) keeps the stored value; '' clears it.
|
|
@@ -97,6 +168,7 @@ class Storage {
|
|
|
97
168
|
const existing = this.getEndpoint(id);
|
|
98
169
|
const finalSecret = secret !== undefined ? secret : (existing ? existing.secret : '');
|
|
99
170
|
const finalAlert = alertWebhookUrl !== undefined ? alertWebhookUrl : (existing ? existing.alert_webhook_url : '');
|
|
171
|
+
const encryptedSecret = encryptSecret(finalSecret, this.encryptionKey);
|
|
100
172
|
this.db.prepare(`
|
|
101
173
|
INSERT INTO endpoints (id, name, target_url, secret, alert_webhook_url, auto_retry, max_retries, concurrency_limit)
|
|
102
174
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
@@ -108,16 +180,32 @@ class Storage {
|
|
|
108
180
|
auto_retry = excluded.auto_retry,
|
|
109
181
|
max_retries = excluded.max_retries,
|
|
110
182
|
concurrency_limit = excluded.concurrency_limit
|
|
111
|
-
`).run(id, name, targetUrl,
|
|
183
|
+
`).run(id, name, targetUrl, encryptedSecret || '', finalAlert || '', autoRetry ? 1 : 0, maxRetries, concurrencyLimit || 5);
|
|
112
184
|
return this.getEndpoint(id);
|
|
113
185
|
}
|
|
114
186
|
|
|
115
187
|
getEndpoint(id) {
|
|
116
|
-
|
|
188
|
+
const row = this.db.prepare('SELECT * FROM endpoints WHERE id = ?').get(id);
|
|
189
|
+
if (!row) return null;
|
|
190
|
+
return { ...row, secret: decryptSecret(row.secret, this.encryptionKey) };
|
|
117
191
|
}
|
|
118
192
|
|
|
119
193
|
listEndpoints() {
|
|
120
|
-
|
|
194
|
+
const rows = this.db.prepare('SELECT * FROM endpoints ORDER BY created_at DESC').all();
|
|
195
|
+
return rows.map((r) => ({ ...r, secret: decryptSecret(r.secret, this.encryptionKey) }));
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
deleteEndpoint(id) {
|
|
199
|
+
const deleteTx = this.db.transaction((epId) => {
|
|
200
|
+
this.db.prepare(`
|
|
201
|
+
DELETE FROM delivery_attempts WHERE event_id IN (
|
|
202
|
+
SELECT id FROM events WHERE endpoint_id = ?
|
|
203
|
+
)
|
|
204
|
+
`).run(epId);
|
|
205
|
+
this.db.prepare('DELETE FROM events WHERE endpoint_id = ?').run(epId);
|
|
206
|
+
return this.db.prepare('DELETE FROM endpoints WHERE id = ?').run(epId).changes > 0;
|
|
207
|
+
});
|
|
208
|
+
return deleteTx(id);
|
|
121
209
|
}
|
|
122
210
|
|
|
123
211
|
// Any stored copy counts: once HookArmor has custody, a provider re-send is redundant
|
|
@@ -128,23 +216,37 @@ class Storage {
|
|
|
128
216
|
return { ...row, headers: JSON.parse(row.headers) };
|
|
129
217
|
}
|
|
130
218
|
|
|
131
|
-
saveEvent({ id, endpointId, idempotencyKey = null, provider, eventType, headers, rawBody, status = 'pending', verified = false }) {
|
|
219
|
+
saveEvent({ id, endpointId, idempotencyKey = null, provider, eventType, headers, rawBody, status = 'pending', claimedBy = null, verified = false }) {
|
|
220
|
+
const rawBuffer = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody || ''), 'utf8');
|
|
132
221
|
const stmt = this.db.prepare(`
|
|
133
|
-
INSERT INTO events (id, endpoint_id, idempotency_key, provider, event_type, headers, raw_body, status, attempts, verified, created_at, updated_at)
|
|
134
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, datetime('now'), datetime('now'))
|
|
222
|
+
INSERT INTO events (id, endpoint_id, idempotency_key, provider, event_type, headers, raw_body, status, attempts, claimed_by, claimed_at, verified, created_at, updated_at)
|
|
223
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ${claimedBy ? "datetime('now')" : 'NULL'}, ?, datetime('now'), datetime('now'))
|
|
135
224
|
`);
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
225
|
+
try {
|
|
226
|
+
stmt.run(
|
|
227
|
+
id,
|
|
228
|
+
endpointId,
|
|
229
|
+
idempotencyKey,
|
|
230
|
+
provider,
|
|
231
|
+
eventType,
|
|
232
|
+
JSON.stringify(headers),
|
|
233
|
+
rawBuffer,
|
|
234
|
+
status,
|
|
235
|
+
claimedBy,
|
|
236
|
+
verified ? 1 : 0
|
|
237
|
+
);
|
|
238
|
+
return this.getEvent(id);
|
|
239
|
+
} catch (err) {
|
|
240
|
+
if (idempotencyKey && (err.code === 'SQLITE_CONSTRAINT_UNIQUE' || (err.message && err.message.includes('UNIQUE constraint failed')))) {
|
|
241
|
+
const existing = this.findEventByIdempotencyKey(endpointId, idempotencyKey);
|
|
242
|
+
if (existing) return existing;
|
|
243
|
+
}
|
|
244
|
+
throw err;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
getPendingCount() {
|
|
249
|
+
return this.db.prepare("SELECT count(*) as count FROM events WHERE status IN ('pending', 'replaying')").get().count;
|
|
148
250
|
}
|
|
149
251
|
|
|
150
252
|
recordAttempt({ eventId, statusCode, responseBody = '', errorMessage = '', latencyMs = 0, nextRetryAt = null }) {
|
|
@@ -188,6 +290,14 @@ class Storage {
|
|
|
188
290
|
return this.db.prepare('SELECT * FROM delivery_attempts WHERE event_id = ? ORDER BY id DESC').all(eventId);
|
|
189
291
|
}
|
|
190
292
|
|
|
293
|
+
deleteEvent(id) {
|
|
294
|
+
const deleteTx = this.db.transaction((eventId) => {
|
|
295
|
+
this.db.prepare('DELETE FROM delivery_attempts WHERE event_id = ?').run(eventId);
|
|
296
|
+
return this.db.prepare('DELETE FROM events WHERE id = ?').run(eventId).changes > 0;
|
|
297
|
+
});
|
|
298
|
+
return deleteTx(id);
|
|
299
|
+
}
|
|
300
|
+
|
|
191
301
|
listEvents({ endpointId = null, status = null, limit = 50, offset = 0 } = {}) {
|
|
192
302
|
let sql = 'SELECT * FROM events WHERE 1=1';
|
|
193
303
|
const params = [];
|
|
@@ -272,35 +382,77 @@ class Storage {
|
|
|
272
382
|
}));
|
|
273
383
|
}
|
|
274
384
|
|
|
275
|
-
|
|
385
|
+
registerInstance({ id, hostname, pid }) {
|
|
386
|
+
if (!id) return;
|
|
387
|
+
this.db.prepare(`
|
|
388
|
+
INSERT INTO instances (id, hostname, pid, started_at, last_heartbeat)
|
|
389
|
+
VALUES (?, ?, ?, datetime('now'), datetime('now'))
|
|
390
|
+
ON CONFLICT(id) DO UPDATE SET last_heartbeat = datetime('now'), pid = excluded.pid, hostname = excluded.hostname
|
|
391
|
+
`).run(id, hostname || null, pid || null);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
heartbeatInstance(id) {
|
|
395
|
+
if (!id) return;
|
|
396
|
+
this.db.prepare(`
|
|
397
|
+
UPDATE instances SET last_heartbeat = datetime('now') WHERE id = ?
|
|
398
|
+
`).run(id);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
reapStaleInstances(timeoutSeconds = 300) {
|
|
402
|
+
return this.db.prepare(`
|
|
403
|
+
DELETE FROM instances WHERE datetime(last_heartbeat) < datetime('now', '-' || ? || ' seconds')
|
|
404
|
+
`).run(timeoutSeconds).changes;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
claimEventForRetry(eventId, instanceId = null) {
|
|
276
408
|
const info = this.db.prepare(`
|
|
277
409
|
UPDATE events
|
|
278
|
-
SET status = 'replaying',
|
|
410
|
+
SET status = 'replaying',
|
|
411
|
+
claimed_by = ?,
|
|
412
|
+
claimed_at = datetime('now'),
|
|
413
|
+
updated_at = datetime('now')
|
|
279
414
|
WHERE id = ? AND status IN ('failed', 'pending')
|
|
280
|
-
`).run(eventId);
|
|
415
|
+
`).run(instanceId, eventId);
|
|
281
416
|
return info.changes > 0;
|
|
282
417
|
}
|
|
283
418
|
|
|
284
419
|
// Manual replay may re-send delivered events, but never one that is already in flight
|
|
285
|
-
claimEventForManualReplay(eventId) {
|
|
420
|
+
claimEventForManualReplay(eventId, instanceId = null) {
|
|
286
421
|
const info = this.db.prepare(`
|
|
287
422
|
UPDATE events
|
|
288
|
-
SET status = 'replaying',
|
|
423
|
+
SET status = 'replaying',
|
|
424
|
+
claimed_by = ?,
|
|
425
|
+
claimed_at = datetime('now'),
|
|
426
|
+
updated_at = datetime('now')
|
|
289
427
|
WHERE id = ? AND status IN ('failed', 'pending', 'delivered')
|
|
290
|
-
`).run(eventId);
|
|
428
|
+
`).run(instanceId, eventId);
|
|
291
429
|
return info.changes > 0;
|
|
292
430
|
}
|
|
293
431
|
|
|
294
|
-
// Run
|
|
432
|
+
// Run at startup or worker tick:
|
|
433
|
+
// Reclaim events if:
|
|
434
|
+
// 1. claimed_by is NULL, OR
|
|
435
|
+
// 2. claimed_by is not active in instances table (last_heartbeat < 45 seconds ago), OR
|
|
436
|
+
// 3. claimed_at is older than 180 seconds (worker hung)
|
|
295
437
|
recoverInterruptedDeliveries() {
|
|
296
|
-
|
|
438
|
+
const sql = `
|
|
297
439
|
UPDATE events
|
|
298
440
|
SET status = 'failed',
|
|
299
441
|
next_retry_at = datetime('now'),
|
|
300
442
|
last_error = COALESCE(last_error, 'Delivery interrupted by restart'),
|
|
443
|
+
claimed_by = NULL,
|
|
444
|
+
claimed_at = NULL,
|
|
301
445
|
updated_at = datetime('now')
|
|
302
446
|
WHERE status = 'replaying'
|
|
303
|
-
|
|
447
|
+
AND (
|
|
448
|
+
claimed_by IS NULL
|
|
449
|
+
OR claimed_by NOT IN (
|
|
450
|
+
SELECT id FROM instances WHERE datetime(last_heartbeat) >= datetime('now', '-45 seconds')
|
|
451
|
+
)
|
|
452
|
+
OR (claimed_at IS NOT NULL AND datetime(claimed_at) < datetime('now', '-180 seconds'))
|
|
453
|
+
)
|
|
454
|
+
`;
|
|
455
|
+
return this.db.prepare(sql).run().changes;
|
|
304
456
|
}
|
|
305
457
|
|
|
306
458
|
// Delete delivered events (and their attempt logs) older than `days`. Failed events are kept.
|
|
@@ -333,5 +485,8 @@ class Storage {
|
|
|
333
485
|
}
|
|
334
486
|
|
|
335
487
|
Storage.toSqliteTime = toSqliteTime;
|
|
488
|
+
Storage.deriveKey = deriveKey;
|
|
489
|
+
Storage.encryptSecret = encryptSecret;
|
|
490
|
+
Storage.decryptSecret = decryptSecret;
|
|
336
491
|
|
|
337
492
|
module.exports = Storage;
|
package/src/worker.js
CHANGED
|
@@ -9,6 +9,7 @@ class RetryWorker extends EventEmitter {
|
|
|
9
9
|
this.retentionDays = options.retentionDays || 0;
|
|
10
10
|
this.pruneIntervalMs = options.pruneIntervalMs || 60 * 60 * 1000;
|
|
11
11
|
this.lastPruneAt = 0;
|
|
12
|
+
this.instanceId = options.instanceId || null;
|
|
12
13
|
this.timer = null;
|
|
13
14
|
this.isProcessing = false;
|
|
14
15
|
}
|
|
@@ -41,13 +42,19 @@ class RetryWorker extends EventEmitter {
|
|
|
41
42
|
this.isProcessing = true;
|
|
42
43
|
|
|
43
44
|
try {
|
|
45
|
+
if (this.instanceId) {
|
|
46
|
+
this.storage.heartbeatInstance(this.instanceId);
|
|
47
|
+
}
|
|
48
|
+
this.storage.recoverInterruptedDeliveries(this.instanceId);
|
|
49
|
+
this.storage.reapStaleInstances(300);
|
|
50
|
+
|
|
44
51
|
this.prune();
|
|
45
52
|
|
|
46
53
|
const eventsDue = this.storage.getEventsDueForRetry(25);
|
|
47
54
|
if (!eventsDue || eventsDue.length === 0) return;
|
|
48
55
|
|
|
49
56
|
for (const event of eventsDue) {
|
|
50
|
-
const claimed = this.storage.claimEventForRetry(event.id);
|
|
57
|
+
const claimed = this.storage.claimEventForRetry(event.id, this.instanceId);
|
|
51
58
|
if (!claimed) continue; // Another process or manual replay already claimed it
|
|
52
59
|
|
|
53
60
|
const endpoint = this.storage.getEndpoint(event.endpoint_id);
|