@wolpertingerlabs/drawlatch 1.0.0-alpha.3 → 1.0.0-alpha.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONNECTIONS.md +3 -0
- package/README.md +393 -465
- package/bin/drawlatch.js +1151 -49
- package/dist/auth/auth.d.ts +10 -0
- package/dist/auth/auth.js +146 -0
- package/dist/auth/env-writer.d.ts +6 -0
- package/dist/auth/env-writer.js +52 -0
- package/dist/auth/password.d.ts +8 -0
- package/dist/auth/password.js +28 -0
- package/dist/auth/sessions.d.ts +13 -0
- package/dist/auth/sessions.js +85 -0
- package/dist/cli/generate-keys.d.ts +4 -4
- package/dist/cli/generate-keys.js +30 -25
- package/dist/connections/{anthropic.json → ai/anthropic.json} +19 -1
- package/dist/connections/{devin.json → ai/devin.json} +7 -1
- package/dist/connections/{google-ai.json → ai/google-ai.json} +7 -1
- package/dist/connections/{openai.json → ai/openai.json} +7 -1
- package/dist/connections/{openrouter.json → ai/openrouter.json} +7 -1
- package/dist/connections/developer-tools/github.json +138 -0
- package/dist/connections/{hex.json → developer-tools/hex.json} +7 -1
- package/dist/connections/developer-tools/linear.json +75 -0
- package/dist/connections/{lichess.json → gaming/lichess.json} +7 -1
- package/dist/connections/messaging/agentmail.json +21 -0
- package/dist/connections/messaging/discord-bot.json +114 -0
- package/dist/connections/{discord-oauth.json → messaging/discord-oauth.json} +7 -1
- package/dist/connections/messaging/slack.json +75 -0
- package/dist/connections/messaging/telegram.json +66 -0
- package/dist/connections/{google.json → productivity/google.json} +7 -1
- package/dist/connections/productivity/notion.json +75 -0
- package/dist/connections/productivity/stripe.json +74 -0
- package/dist/connections/productivity/trello.json +113 -0
- package/dist/connections/{bluesky.json → social-media/bluesky.json} +41 -0
- package/dist/connections/social-media/mastodon.json +65 -0
- package/dist/connections/social-media/reddit.json +80 -0
- package/dist/connections/{twitch.json → social-media/twitch.json} +40 -0
- package/dist/connections/social-media/x.json +67 -0
- package/dist/mcp/server.js +544 -31
- package/dist/remote/admin-mutations.d.ts +43 -0
- package/dist/remote/admin-mutations.js +321 -0
- package/dist/remote/admin-types.d.ts +153 -0
- package/dist/remote/admin-types.js +11 -0
- package/dist/remote/admin.d.ts +37 -0
- package/dist/remote/admin.js +317 -0
- package/dist/remote/caller-bootstrap.d.ts +121 -0
- package/dist/remote/caller-bootstrap.js +270 -0
- package/dist/remote/ingestors/base-ingestor.d.ts +19 -3
- package/dist/remote/ingestors/base-ingestor.js +27 -6
- package/dist/remote/ingestors/discord/discord-gateway.d.ts +2 -2
- package/dist/remote/ingestors/discord/discord-gateway.js +29 -6
- package/dist/remote/ingestors/e2e/setup.d.ts +69 -0
- package/dist/remote/ingestors/e2e/setup.js +147 -0
- package/dist/remote/ingestors/manager.d.ts +110 -10
- package/dist/remote/ingestors/manager.js +449 -42
- package/dist/remote/ingestors/poll/poll-ingestor.d.ts +4 -2
- package/dist/remote/ingestors/poll/poll-ingestor.js +26 -5
- package/dist/remote/ingestors/registry.d.ts +2 -2
- package/dist/remote/ingestors/registry.js +2 -2
- package/dist/remote/ingestors/slack/socket-mode.d.ts +2 -2
- package/dist/remote/ingestors/slack/socket-mode.js +28 -6
- package/dist/remote/ingestors/types.d.ts +25 -0
- package/dist/remote/ingestors/webhook/base-webhook-ingestor.d.ts +46 -7
- package/dist/remote/ingestors/webhook/base-webhook-ingestor.js +115 -10
- package/dist/remote/ingestors/webhook/github-webhook-ingestor.d.ts +30 -0
- package/dist/remote/ingestors/webhook/github-webhook-ingestor.js +73 -2
- package/dist/remote/ingestors/webhook/lifecycle-types.d.ts +63 -0
- package/dist/remote/ingestors/webhook/lifecycle-types.js +12 -0
- package/dist/remote/ingestors/webhook/stripe-webhook-ingestor.js +2 -2
- package/dist/remote/ingestors/webhook/trello-webhook-ingestor.d.ts +17 -5
- package/dist/remote/ingestors/webhook/trello-webhook-ingestor.js +32 -26
- package/dist/remote/ingestors/webhook/webhook-lifecycle-manager.d.ts +46 -0
- package/dist/remote/ingestors/webhook/webhook-lifecycle-manager.js +261 -0
- package/dist/remote/server.d.ts +25 -33
- package/dist/remote/server.js +596 -187
- package/dist/remote/tool-dispatch.d.ts +84 -0
- package/dist/remote/tool-dispatch.js +910 -0
- package/dist/remote/triggers/rule-engine.d.ts +40 -0
- package/dist/remote/triggers/rule-engine.js +228 -0
- package/dist/remote/triggers/types.d.ts +69 -0
- package/dist/remote/triggers/types.js +10 -0
- package/dist/remote/tunnel-state.d.ts +14 -0
- package/dist/remote/tunnel-state.js +20 -0
- package/dist/remote/tunnel.d.ts +13 -0
- package/dist/remote/tunnel.js +33 -0
- package/dist/shared/config.d.ts +104 -25
- package/dist/shared/config.js +49 -36
- package/dist/shared/connections.d.ts +21 -5
- package/dist/shared/connections.js +56 -14
- package/dist/shared/crypto/index.d.ts +1 -0
- package/dist/shared/crypto/index.js +1 -0
- package/dist/shared/crypto/key-manager.d.ts +81 -0
- package/dist/shared/crypto/key-manager.js +174 -0
- package/dist/shared/env-utils.d.ts +42 -0
- package/dist/shared/env-utils.js +150 -0
- package/dist/shared/listener-config.d.ts +157 -0
- package/dist/shared/listener-config.js +10 -0
- package/dist/shared/migrations.d.ts +40 -0
- package/dist/shared/migrations.js +122 -0
- package/dist/shared/protocol/caller-bundle-crypto.d.ts +37 -0
- package/dist/shared/protocol/caller-bundle-crypto.js +62 -0
- package/dist/shared/protocol/caller-bundle.d.ts +75 -0
- package/dist/shared/protocol/caller-bundle.js +24 -0
- package/dist/shared/protocol/handshake.js +14 -1
- package/dist/shared/protocol/index.d.ts +2 -0
- package/dist/shared/protocol/index.js +2 -0
- package/dist/shared/protocol/sync-client.d.ts +52 -0
- package/dist/shared/protocol/sync-client.js +99 -0
- package/dist/shared/protocol/sync.d.ts +71 -0
- package/dist/shared/protocol/sync.js +176 -0
- package/frontend/dist/assets/index-BdCSPSZK.js +282 -0
- package/frontend/dist/assets/index-BmK26bY2.css +1 -0
- package/frontend/dist/index.html +15 -0
- package/package.json +65 -13
- package/dist/connections/discord-bot.json +0 -24
- package/dist/connections/github.json +0 -25
- package/dist/connections/linear.json +0 -29
- package/dist/connections/mastodon.json +0 -25
- package/dist/connections/notion.json +0 -33
- package/dist/connections/reddit.json +0 -28
- package/dist/connections/slack.json +0 -23
- package/dist/connections/stripe.json +0 -25
- package/dist/connections/telegram.json +0 -26
- package/dist/connections/trello.json +0 -25
- package/dist/connections/x.json +0 -27
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Caller provisioning — drawlatch is the sole issuer of caller key material.
|
|
3
|
+
*
|
|
4
|
+
* One primitive, two delivery modes (see plans/caller-credential-issuance.md):
|
|
5
|
+
* - `issueCallerBundle()` — mint a keypair IN MEMORY, persist only the public
|
|
6
|
+
* half, and return the credential bundle (download via the admin API, or the
|
|
7
|
+
* `drawlatch issue-caller` CLI). The caller PRIVATE key never touches
|
|
8
|
+
* drawlatch disk; it lives only inside the returned bundle.
|
|
9
|
+
* - `issueLocalCaller()` — same primitive, but write the UNPACKED key files
|
|
10
|
+
* straight into a co-located callboard's keys dir over the shared filesystem.
|
|
11
|
+
* The same-host write IS the trust proof (replaces the old enroll-token dance).
|
|
12
|
+
*
|
|
13
|
+
* `createCallerWithKeys()` (priv+pub on drawlatch disk) is retained for the
|
|
14
|
+
* password-gated `POST /api/admin/callers` create endpoint and tests.
|
|
15
|
+
*/
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import path from 'node:path';
|
|
18
|
+
import { loadRemoteConfig, saveRemoteConfig, getCallerKeysDir, } from '../shared/config.js';
|
|
19
|
+
import { createCaller, exportServerPublicKeys, serverFingerprint, saveCallerPublicKeys, } from '../shared/crypto/key-manager.js';
|
|
20
|
+
import { generateKeyBundle, serializeKeyBundle, extractPublicKeys, serializePublicKeys, fingerprint, } from '../shared/crypto/keys.js';
|
|
21
|
+
import { generateBundleSalt, deriveBundleKey, encryptBundleField, DEFAULT_SCRYPT_PARAMS, } from '../shared/protocol/caller-bundle-crypto.js';
|
|
22
|
+
import { setEnvVars } from '../shared/env-utils.js';
|
|
23
|
+
import { createLogger } from '../shared/logger.js';
|
|
24
|
+
const log = createLogger('caller-bootstrap');
|
|
25
|
+
/** Valid caller alias: starts with a letter/number, then letters/numbers/-/_. */
|
|
26
|
+
export const CALLER_ALIAS_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
|
|
27
|
+
function assertValidAlias(alias) {
|
|
28
|
+
if (!CALLER_ALIAS_REGEX.test(alias)) {
|
|
29
|
+
throw new Error('Invalid alias: must start with a letter or number and contain only ' +
|
|
30
|
+
'letters, numbers, hyphens, and underscores');
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/** Clone the `default` caller's connections (the new-caller default access). */
|
|
34
|
+
function defaultConnections(config) {
|
|
35
|
+
const defaultCaller = config.callers.default;
|
|
36
|
+
return [...(defaultCaller?.connections ?? [])];
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Create a caller WITH a fresh keypair on drawlatch disk (priv+pub), registered
|
|
40
|
+
* in remote.config.json. Used by the dashboard "New caller" action.
|
|
41
|
+
*
|
|
42
|
+
* Throws on invalid alias, or if the caller (config entry or key dir) exists.
|
|
43
|
+
*/
|
|
44
|
+
export function createCallerWithKeys(alias, opts = {}) {
|
|
45
|
+
assertValidAlias(alias);
|
|
46
|
+
const config = loadRemoteConfig();
|
|
47
|
+
if (alias in config.callers || callerKeyDirExists(alias)) {
|
|
48
|
+
throw new Error(`Caller "${alias}" already exists`);
|
|
49
|
+
}
|
|
50
|
+
// Generate the caller's keypair under keys/callers/<alias>/.
|
|
51
|
+
const { publicKeys, fingerprint: fp } = createCaller(alias);
|
|
52
|
+
const name = opts.name ?? alias;
|
|
53
|
+
const connections = opts.connections ?? defaultConnections(config);
|
|
54
|
+
config.callers[alias] = { name, connections };
|
|
55
|
+
saveRemoteConfig(config);
|
|
56
|
+
log.info(`Created caller "${alias}" with keypair (${connections.length} connection(s), fingerprint ${fp})`);
|
|
57
|
+
return {
|
|
58
|
+
alias,
|
|
59
|
+
name,
|
|
60
|
+
fingerprint: fp,
|
|
61
|
+
publicKeys,
|
|
62
|
+
keysDir: path.join(getCallerKeysDir(), alias),
|
|
63
|
+
connections,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Whether a caller key directory already holds key material. */
|
|
67
|
+
function callerKeyDirExists(alias) {
|
|
68
|
+
const dir = path.join(getCallerKeysDir(), alias);
|
|
69
|
+
return (fs.existsSync(path.join(dir, 'signing.pub.pem')) ||
|
|
70
|
+
fs.existsSync(path.join(dir, 'signing.key.pem')));
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Issue (or re-issue / rotate) a caller credential bundle.
|
|
74
|
+
*
|
|
75
|
+
* 1. Generate the Ed25519 + X25519 keypair IN MEMORY.
|
|
76
|
+
* 2. Register/update the caller in remote.config.json (connections, name,
|
|
77
|
+
* source='bundle-issued').
|
|
78
|
+
* 3. Persist ONLY the public keys to keys/callers/<alias>/ (the private key
|
|
79
|
+
* never touches drawlatch disk).
|
|
80
|
+
* 4. Assemble the v1 bundle (caller priv+pub, server pub-only, endpoint,
|
|
81
|
+
* fingerprints, connections), optionally passphrase-wrapping the two
|
|
82
|
+
* private PEMs.
|
|
83
|
+
* 5. Zero the derived key material from memory and return the bundle.
|
|
84
|
+
*
|
|
85
|
+
* Re-issuing an existing caller mints a FRESH keypair and overwrites the stored
|
|
86
|
+
* public key — invalidating the prior credential (the old private key can no
|
|
87
|
+
* longer authenticate). The bundle is returned once; drawlatch keeps no copy of
|
|
88
|
+
* the private material, so it is unrecoverable afterward.
|
|
89
|
+
*/
|
|
90
|
+
export function issueCallerBundle(opts) {
|
|
91
|
+
const { alias, endpointUrl, passphrase } = opts;
|
|
92
|
+
assertValidAlias(alias);
|
|
93
|
+
if (typeof endpointUrl !== 'string' || endpointUrl.trim() === '') {
|
|
94
|
+
throw new Error('endpointUrl is required');
|
|
95
|
+
}
|
|
96
|
+
const config = loadRemoteConfig();
|
|
97
|
+
const existing = config.callers[alias];
|
|
98
|
+
// Mint the keypair in memory and serialize before any disk writes.
|
|
99
|
+
const bundleKeys = generateKeyBundle();
|
|
100
|
+
const serialized = serializeKeyBundle(bundleKeys);
|
|
101
|
+
const publicKeys = serializePublicKeys(extractPublicKeys(bundleKeys));
|
|
102
|
+
const callerFp = fingerprint(extractPublicKeys(bundleKeys));
|
|
103
|
+
// Resolve name + connections (preserve existing values on re-issue).
|
|
104
|
+
const name = opts.name ?? existing?.name ?? alias;
|
|
105
|
+
const connections = opts.connections ?? existing?.connections ?? defaultConnections(config);
|
|
106
|
+
// Register/update the caller, preserving any unrelated fields (env, overrides…).
|
|
107
|
+
config.callers[alias] = {
|
|
108
|
+
...(existing ?? {}),
|
|
109
|
+
name,
|
|
110
|
+
connections,
|
|
111
|
+
source: 'bundle-issued',
|
|
112
|
+
};
|
|
113
|
+
saveRemoteConfig(config);
|
|
114
|
+
// Persist ONLY the public keys (drops any stale private files from a prior mint).
|
|
115
|
+
saveCallerPublicKeys(alias, publicKeys);
|
|
116
|
+
const serverPub = exportServerPublicKeys();
|
|
117
|
+
const serverFp = serverFingerprint();
|
|
118
|
+
// Optionally passphrase-wrap the two private PEMs. The derived key is zeroed
|
|
119
|
+
// immediately after use; the plaintext PEMs are only referenced transiently.
|
|
120
|
+
let encryption = null;
|
|
121
|
+
let callerSigningPriv = serialized.signing.privateKey;
|
|
122
|
+
let callerExchangePriv = serialized.exchange.privateKey;
|
|
123
|
+
if (passphrase !== undefined && passphrase !== '') {
|
|
124
|
+
encryption = { kdf: 'scrypt', salt: generateBundleSalt(), ...DEFAULT_SCRYPT_PARAMS, alg: 'aes-256-gcm' };
|
|
125
|
+
const key = deriveBundleKey(passphrase, encryption);
|
|
126
|
+
callerSigningPriv = encryptBundleField(serialized.signing.privateKey, key);
|
|
127
|
+
callerExchangePriv = encryptBundleField(serialized.exchange.privateKey, key);
|
|
128
|
+
key.fill(0);
|
|
129
|
+
}
|
|
130
|
+
const bundle = {
|
|
131
|
+
version: 1,
|
|
132
|
+
callerAlias: alias,
|
|
133
|
+
fingerprint: callerFp,
|
|
134
|
+
createdAt: opts.createdAt ?? new Date().toISOString(),
|
|
135
|
+
expiresAt: null,
|
|
136
|
+
endpointUrl,
|
|
137
|
+
serverKeyFingerprint: serverFp,
|
|
138
|
+
connections,
|
|
139
|
+
caller: {
|
|
140
|
+
signing: { priv: callerSigningPriv, pub: serialized.signing.publicKey },
|
|
141
|
+
exchange: { priv: callerExchangePriv, pub: serialized.exchange.publicKey },
|
|
142
|
+
},
|
|
143
|
+
server: {
|
|
144
|
+
signing: { pub: serverPub.signing },
|
|
145
|
+
exchange: { pub: serverPub.exchange },
|
|
146
|
+
},
|
|
147
|
+
encryption,
|
|
148
|
+
};
|
|
149
|
+
// Best-effort: drop our references to the in-memory private material. (JS
|
|
150
|
+
// strings can't be explicitly zeroed; the guarantee that matters — the private
|
|
151
|
+
// key is never written to drawlatch disk — is enforced above.)
|
|
152
|
+
callerSigningPriv = '';
|
|
153
|
+
callerExchangePriv = '';
|
|
154
|
+
log.info(`Issued caller "${alias}" (${connections.length} connection(s), fingerprint ${callerFp}` +
|
|
155
|
+
`${encryption ? ', passphrase-wrapped' : ''}${existing ? ', re-issue' : ''})`);
|
|
156
|
+
return bundle;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Issue a caller and write the UNPACKED key files straight into a co-located
|
|
160
|
+
* callboard's keys dir over the shared filesystem (same-host write is the trust
|
|
161
|
+
* proof). drawlatch keeps only the public key; callboard gets the private key on
|
|
162
|
+
* its own disk — bit-for-bit the steady state of the old pairing flow, with no
|
|
163
|
+
* token dance.
|
|
164
|
+
*
|
|
165
|
+
* Writes, under `<keysDir>`:
|
|
166
|
+
* - callers/<alias>/{signing,exchange}.{key,pub}.pem (0600 / 0644)
|
|
167
|
+
* - server/{signing,exchange}.pub.pem (0644)
|
|
168
|
+
*/
|
|
169
|
+
export function issueLocalCaller(opts) {
|
|
170
|
+
const alias = opts.alias ?? 'callboard-local';
|
|
171
|
+
assertValidAlias(alias);
|
|
172
|
+
const config = loadRemoteConfig();
|
|
173
|
+
const existing = config.callers[alias];
|
|
174
|
+
const bundleKeys = generateKeyBundle();
|
|
175
|
+
const serialized = serializeKeyBundle(bundleKeys);
|
|
176
|
+
const publicKeys = serializePublicKeys(extractPublicKeys(bundleKeys));
|
|
177
|
+
const callerFp = fingerprint(extractPublicKeys(bundleKeys));
|
|
178
|
+
const connections = opts.connections ?? existing?.connections ?? defaultConnections(config);
|
|
179
|
+
const name = opts.name ?? existing?.name ?? alias;
|
|
180
|
+
config.callers[alias] = {
|
|
181
|
+
...(existing ?? {}),
|
|
182
|
+
name,
|
|
183
|
+
connections,
|
|
184
|
+
source: 'local-auto',
|
|
185
|
+
};
|
|
186
|
+
saveRemoteConfig(config);
|
|
187
|
+
// drawlatch keeps only the public half.
|
|
188
|
+
saveCallerPublicKeys(alias, publicKeys);
|
|
189
|
+
// Write the unpacked key files into callboard's keys dir.
|
|
190
|
+
const callerKeysDir = path.join(opts.keysDir, 'callers', alias);
|
|
191
|
+
fs.mkdirSync(callerKeysDir, { recursive: true, mode: 0o700 });
|
|
192
|
+
fs.writeFileSync(path.join(callerKeysDir, 'signing.pub.pem'), serialized.signing.publicKey, { mode: 0o644 });
|
|
193
|
+
fs.writeFileSync(path.join(callerKeysDir, 'exchange.pub.pem'), serialized.exchange.publicKey, { mode: 0o644 });
|
|
194
|
+
fs.writeFileSync(path.join(callerKeysDir, 'signing.key.pem'), serialized.signing.privateKey, { mode: 0o600 });
|
|
195
|
+
fs.writeFileSync(path.join(callerKeysDir, 'exchange.key.pem'), serialized.exchange.privateKey, { mode: 0o600 });
|
|
196
|
+
// And the server's public keys so callboard can pin the server identity.
|
|
197
|
+
const serverPub = exportServerPublicKeys();
|
|
198
|
+
const serverDir = path.join(opts.keysDir, 'server');
|
|
199
|
+
fs.mkdirSync(serverDir, { recursive: true, mode: 0o700 });
|
|
200
|
+
fs.writeFileSync(path.join(serverDir, 'signing.pub.pem'), serverPub.signing, { mode: 0o644 });
|
|
201
|
+
fs.writeFileSync(path.join(serverDir, 'exchange.pub.pem'), serverPub.exchange, { mode: 0o644 });
|
|
202
|
+
log.info(`Auto-shared local caller "${alias}" to ${callerKeysDir} ` +
|
|
203
|
+
`(${connections.length} connection(s), fingerprint ${callerFp})`);
|
|
204
|
+
return { alias, fingerprint: callerFp, callerKeysDir, connections };
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* First-boot hook: when drawlatch is supervised by a co-located callboard (which
|
|
208
|
+
* sets DRAWLATCH_LOCAL_CALLER_KEYS_DIR to its own keys dir), auto-share a default
|
|
209
|
+
* caller into that dir — but only once (skipped if the caller already exists).
|
|
210
|
+
*
|
|
211
|
+
* Replaces the retired enroll-token / `/sync/auto-enroll` path.
|
|
212
|
+
*/
|
|
213
|
+
export function maybeIssueLocalCaller() {
|
|
214
|
+
const keysDir = process.env.DRAWLATCH_LOCAL_CALLER_KEYS_DIR;
|
|
215
|
+
if (!keysDir)
|
|
216
|
+
return null;
|
|
217
|
+
const alias = process.env.DRAWLATCH_LOCAL_CALLER_ALIAS ?? 'callboard-local';
|
|
218
|
+
const config = loadRemoteConfig();
|
|
219
|
+
if (alias in config.callers && callerKeyDirExists(alias)) {
|
|
220
|
+
// Already provisioned on a prior boot — nothing to do.
|
|
221
|
+
return null;
|
|
222
|
+
}
|
|
223
|
+
const connectionsEnv = process.env.DRAWLATCH_LOCAL_CALLER_CONNECTIONS;
|
|
224
|
+
const connections = connectionsEnv
|
|
225
|
+
? connectionsEnv.split(',').map((c) => c.trim()).filter(Boolean)
|
|
226
|
+
: undefined;
|
|
227
|
+
try {
|
|
228
|
+
return issueLocalCaller({ alias, keysDir, ...(connections && { connections }) });
|
|
229
|
+
}
|
|
230
|
+
catch (err) {
|
|
231
|
+
log.error(`Could not auto-share local caller "${alias}":`, err);
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
// ── Delete ──────────────────────────────────────────────────────────────────
|
|
236
|
+
/**
|
|
237
|
+
* Delete a caller: removes its config entry, prefixed env vars, and key dir.
|
|
238
|
+
* The `default` caller cannot be deleted.
|
|
239
|
+
*/
|
|
240
|
+
export function deleteCaller(alias) {
|
|
241
|
+
if (alias === 'default') {
|
|
242
|
+
throw new Error('Cannot delete the "default" caller');
|
|
243
|
+
}
|
|
244
|
+
const config = loadRemoteConfig();
|
|
245
|
+
if (!(alias in config.callers)) {
|
|
246
|
+
throw new Error(`Caller "${alias}" not found`);
|
|
247
|
+
}
|
|
248
|
+
// Clean up prefixed env vars referenced by this caller's env mapping.
|
|
249
|
+
const caller = config.callers[alias];
|
|
250
|
+
if (caller.env) {
|
|
251
|
+
const envUpdates = {};
|
|
252
|
+
for (const mapping of Object.values(caller.env)) {
|
|
253
|
+
const envMatch = /^\$\{(.+)\}$/.exec(mapping);
|
|
254
|
+
if (envMatch)
|
|
255
|
+
envUpdates[envMatch[1]] = ''; // empty string = delete
|
|
256
|
+
}
|
|
257
|
+
if (Object.keys(envUpdates).length > 0)
|
|
258
|
+
setEnvVars(envUpdates);
|
|
259
|
+
}
|
|
260
|
+
const { [alias]: _removed, ...remainingCallers } = config.callers;
|
|
261
|
+
config.callers = remainingCallers;
|
|
262
|
+
saveRemoteConfig(config);
|
|
263
|
+
// Remove the caller's key directory.
|
|
264
|
+
const keysDir = path.join(getCallerKeysDir(), alias);
|
|
265
|
+
if (fs.existsSync(keysDir)) {
|
|
266
|
+
fs.rmSync(keysDir, { recursive: true, force: true });
|
|
267
|
+
}
|
|
268
|
+
log.info(`Deleted caller "${alias}"`);
|
|
269
|
+
}
|
|
270
|
+
//# sourceMappingURL=caller-bootstrap.js.map
|
|
@@ -18,11 +18,20 @@ export declare abstract class BaseIngestor extends EventEmitter {
|
|
|
18
18
|
protected readonly ingestorType: 'websocket' | 'webhook' | 'poll';
|
|
19
19
|
/** Resolved secrets for the parent connection. */
|
|
20
20
|
protected readonly secrets: Record<string, string>;
|
|
21
|
+
/** Optional instance identifier for multi-instance support.
|
|
22
|
+
* When set, this ingestor is one of N instances for the same connection
|
|
23
|
+
* (e.g., watching different Trello boards or Reddit subreddits). */
|
|
24
|
+
protected readonly instanceId?: string | undefined;
|
|
21
25
|
protected state: IngestorState;
|
|
22
26
|
protected buffer: RingBuffer<IngestedEvent>;
|
|
23
27
|
protected counter: number;
|
|
24
28
|
protected lastEventAt: string | null;
|
|
25
29
|
protected errorMessage?: string;
|
|
30
|
+
/** Caller alias that owns this ingestor. Set by IngestorManager after creation. */
|
|
31
|
+
callerAlias: string;
|
|
32
|
+
/** Per-instance epoch for event ID generation.
|
|
33
|
+
* Each instance claims a unique epoch so IDs never collide across restarts. */
|
|
34
|
+
private readonly bootEpoch;
|
|
26
35
|
/** Recently seen idempotency keys for deduplication. */
|
|
27
36
|
private readonly seenKeys;
|
|
28
37
|
constructor(
|
|
@@ -33,11 +42,18 @@ export declare abstract class BaseIngestor extends EventEmitter {
|
|
|
33
42
|
/** Resolved secrets for the parent connection. */
|
|
34
43
|
secrets: Record<string, string>,
|
|
35
44
|
/** Buffer capacity (defaults to DEFAULT_BUFFER_SIZE). */
|
|
36
|
-
bufferSize?: number
|
|
45
|
+
bufferSize?: number,
|
|
46
|
+
/** Optional instance identifier for multi-instance support.
|
|
47
|
+
* When set, this ingestor is one of N instances for the same connection
|
|
48
|
+
* (e.g., watching different Trello boards or Reddit subreddits). */
|
|
49
|
+
instanceId?: string | undefined);
|
|
37
50
|
/** Start the ingestor (connect WebSocket, begin polling, etc.). */
|
|
38
51
|
abstract start(): Promise<void>;
|
|
39
|
-
/** Stop the ingestor cleanly (close connections, clear timers).
|
|
40
|
-
|
|
52
|
+
/** Stop the ingestor cleanly (close connections, clear timers).
|
|
53
|
+
* @param permanent When true, indicates a permanent shutdown (server exit
|
|
54
|
+
* or instance deletion). Webhook ingestors use this to
|
|
55
|
+
* unregister webhooks from the external service. */
|
|
56
|
+
abstract stop(permanent?: boolean): Promise<void>;
|
|
41
57
|
/**
|
|
42
58
|
* Push a new event into the ring buffer.
|
|
43
59
|
* Called by subclasses when they receive data from an external service.
|
|
@@ -18,25 +18,35 @@ const log = createLogger('ingestor');
|
|
|
18
18
|
* When exceeded, the oldest half is pruned. */
|
|
19
19
|
const MAX_SEEN_KEYS = 2000;
|
|
20
20
|
/**
|
|
21
|
-
* Epoch-based event IDs that are monotonically increasing across
|
|
21
|
+
* Epoch-based event IDs that are monotonically increasing across restarts.
|
|
22
22
|
* Format: `bootEpochSeconds * 1_000_000 + counter`.
|
|
23
23
|
*
|
|
24
24
|
* Uses seconds (not milliseconds) so the product stays within Number.MAX_SAFE_INTEGER.
|
|
25
25
|
* This prevents clients with a stale `after_id` cursor from missing events
|
|
26
26
|
* after a reboot — new IDs will always be higher than pre-reboot IDs
|
|
27
|
-
* (assuming <1M events per
|
|
27
|
+
* (assuming <1M events per instance and >1s between restarts).
|
|
28
|
+
*
|
|
29
|
+
* Uses a monotonically increasing module-level epoch so that in-process restarts
|
|
30
|
+
* (e.g., LocalProxy.reinitialize()) also produce strictly increasing IDs.
|
|
31
|
+
* Each new BaseIngestor instance claims the next available epoch.
|
|
28
32
|
*/
|
|
29
|
-
|
|
33
|
+
let nextEpoch = Math.floor(Date.now() / 1000);
|
|
30
34
|
const ID_MULTIPLIER = 1_000_000;
|
|
31
35
|
export class BaseIngestor extends EventEmitter {
|
|
32
36
|
connectionAlias;
|
|
33
37
|
ingestorType;
|
|
34
38
|
secrets;
|
|
39
|
+
instanceId;
|
|
35
40
|
state = 'stopped';
|
|
36
41
|
buffer;
|
|
37
42
|
counter = 0;
|
|
38
43
|
lastEventAt = null;
|
|
39
44
|
errorMessage;
|
|
45
|
+
/** Caller alias that owns this ingestor. Set by IngestorManager after creation. */
|
|
46
|
+
callerAlias = 'unknown';
|
|
47
|
+
/** Per-instance epoch for event ID generation.
|
|
48
|
+
* Each instance claims a unique epoch so IDs never collide across restarts. */
|
|
49
|
+
bootEpoch;
|
|
40
50
|
/** Recently seen idempotency keys for deduplication. */
|
|
41
51
|
seenKeys = new Set();
|
|
42
52
|
constructor(
|
|
@@ -47,12 +57,20 @@ export class BaseIngestor extends EventEmitter {
|
|
|
47
57
|
/** Resolved secrets for the parent connection. */
|
|
48
58
|
secrets,
|
|
49
59
|
/** Buffer capacity (defaults to DEFAULT_BUFFER_SIZE). */
|
|
50
|
-
bufferSize = DEFAULT_BUFFER_SIZE
|
|
60
|
+
bufferSize = DEFAULT_BUFFER_SIZE,
|
|
61
|
+
/** Optional instance identifier for multi-instance support.
|
|
62
|
+
* When set, this ingestor is one of N instances for the same connection
|
|
63
|
+
* (e.g., watching different Trello boards or Reddit subreddits). */
|
|
64
|
+
instanceId) {
|
|
51
65
|
super();
|
|
52
66
|
this.connectionAlias = connectionAlias;
|
|
53
67
|
this.ingestorType = ingestorType;
|
|
54
68
|
this.secrets = secrets;
|
|
69
|
+
this.instanceId = instanceId;
|
|
55
70
|
this.buffer = new RingBuffer(bufferSize);
|
|
71
|
+
// Claim a unique epoch for this instance — ensures event IDs are strictly
|
|
72
|
+
// increasing even when ingestors are restarted in-process.
|
|
73
|
+
this.bootEpoch = nextEpoch++;
|
|
56
74
|
}
|
|
57
75
|
/**
|
|
58
76
|
* Push a new event into the ring buffer.
|
|
@@ -71,14 +89,16 @@ export class BaseIngestor extends EventEmitter {
|
|
|
71
89
|
return;
|
|
72
90
|
}
|
|
73
91
|
const now = new Date();
|
|
74
|
-
const id =
|
|
92
|
+
const id = this.bootEpoch * ID_MULTIPLIER + this.counter++;
|
|
75
93
|
const key = idempotencyKey ?? `${this.connectionAlias}:${crypto.randomUUID()}`;
|
|
76
94
|
const event = {
|
|
77
95
|
id,
|
|
78
96
|
idempotencyKey: key,
|
|
79
97
|
receivedAt: now.toISOString(),
|
|
80
98
|
receivedAtMs: now.getTime(),
|
|
99
|
+
callerAlias: this.callerAlias,
|
|
81
100
|
source: this.connectionAlias,
|
|
101
|
+
...(this.instanceId !== undefined && { instanceId: this.instanceId }),
|
|
82
102
|
eventType,
|
|
83
103
|
data,
|
|
84
104
|
};
|
|
@@ -90,7 +110,7 @@ export class BaseIngestor extends EventEmitter {
|
|
|
90
110
|
this.pruneSeenKeys();
|
|
91
111
|
}
|
|
92
112
|
log.info(`${this.connectionAlias} event #${event.id}: ${eventType}`);
|
|
93
|
-
log.debug(`${this.connectionAlias} event #${event.id} payload:`, JSON.stringify(data
|
|
113
|
+
log.debug(`${this.connectionAlias} event #${event.id} payload size:`, JSON.stringify(data).length);
|
|
94
114
|
this.emit('event', event);
|
|
95
115
|
}
|
|
96
116
|
/**
|
|
@@ -120,6 +140,7 @@ export class BaseIngestor extends EventEmitter {
|
|
|
120
140
|
getStatus() {
|
|
121
141
|
return {
|
|
122
142
|
connection: this.connectionAlias,
|
|
143
|
+
...(this.instanceId !== undefined && { instanceId: this.instanceId }),
|
|
123
144
|
type: this.ingestorType,
|
|
124
145
|
state: this.state,
|
|
125
146
|
bufferedEvents: this.buffer.size,
|
|
@@ -37,9 +37,9 @@ export declare class DiscordGatewayIngestor extends BaseIngestor {
|
|
|
37
37
|
private readonly guildIds;
|
|
38
38
|
private readonly channelIds;
|
|
39
39
|
private readonly userIds;
|
|
40
|
-
constructor(connectionAlias: string, secrets: Record<string, string>, wsConfig: WebSocketIngestorConfig, bufferSize?: number);
|
|
40
|
+
constructor(connectionAlias: string, secrets: Record<string, string>, wsConfig: WebSocketIngestorConfig, bufferSize?: number, instanceId?: string);
|
|
41
41
|
start(): Promise<void>;
|
|
42
|
-
stop(): Promise<void>;
|
|
42
|
+
stop(_permanent?: boolean): Promise<void>;
|
|
43
43
|
private connect;
|
|
44
44
|
private handlePayload;
|
|
45
45
|
private handleHello;
|
|
@@ -41,8 +41,8 @@ export class DiscordGatewayIngestor extends BaseIngestor {
|
|
|
41
41
|
guildIds;
|
|
42
42
|
channelIds;
|
|
43
43
|
userIds;
|
|
44
|
-
constructor(connectionAlias, secrets, wsConfig, bufferSize) {
|
|
45
|
-
super(connectionAlias, 'websocket', secrets, bufferSize);
|
|
44
|
+
constructor(connectionAlias, secrets, wsConfig, bufferSize, instanceId) {
|
|
45
|
+
super(connectionAlias, 'websocket', secrets, bufferSize, instanceId);
|
|
46
46
|
this.wsConfig = wsConfig;
|
|
47
47
|
this.gatewayUrl = wsConfig.gatewayUrl;
|
|
48
48
|
this.intents = wsConfig.intents ?? DEFAULT_INTENTS;
|
|
@@ -56,12 +56,35 @@ export class DiscordGatewayIngestor extends BaseIngestor {
|
|
|
56
56
|
this.connect(this.gatewayUrl);
|
|
57
57
|
return Promise.resolve();
|
|
58
58
|
}
|
|
59
|
-
stop() {
|
|
59
|
+
stop(_permanent) {
|
|
60
60
|
this.state = 'stopped';
|
|
61
61
|
this.clearAllTimers();
|
|
62
62
|
if (this.ws) {
|
|
63
|
-
this.ws
|
|
63
|
+
const ws = this.ws;
|
|
64
64
|
this.ws = null;
|
|
65
|
+
// Wait for the WebSocket to fully close before resolving.
|
|
66
|
+
// This prevents a race where the old connection is still alive when a new
|
|
67
|
+
// ingestor starts. For Discord, this avoids the old gateway session
|
|
68
|
+
// lingering while a new IDENTIFY is sent (which Discord may reject
|
|
69
|
+
// if it sees two sessions for the same bot token).
|
|
70
|
+
return new Promise((resolve) => {
|
|
71
|
+
const onClose = () => {
|
|
72
|
+
ws.removeEventListener('close', onClose);
|
|
73
|
+
resolve();
|
|
74
|
+
};
|
|
75
|
+
if (ws.readyState === WebSocket.CLOSED) {
|
|
76
|
+
resolve();
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
ws.addEventListener('close', onClose);
|
|
80
|
+
ws.close(1000, 'Shutting down');
|
|
81
|
+
// Safety timeout — don't block forever if close event never fires
|
|
82
|
+
setTimeout(() => {
|
|
83
|
+
ws.removeEventListener('close', onClose);
|
|
84
|
+
resolve();
|
|
85
|
+
}, 5000);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
65
88
|
}
|
|
66
89
|
return Promise.resolve();
|
|
67
90
|
}
|
|
@@ -331,11 +354,11 @@ export class DiscordGatewayIngestor extends BaseIngestor {
|
|
|
331
354
|
}
|
|
332
355
|
}
|
|
333
356
|
// ── Self-registration ────────────────────────────────────────────────────
|
|
334
|
-
registerIngestorFactory('websocket:discord', (connectionAlias, config, secrets, bufferSize) => {
|
|
357
|
+
registerIngestorFactory('websocket:discord', (connectionAlias, config, secrets, bufferSize, instanceId) => {
|
|
335
358
|
if (!config.websocket) {
|
|
336
359
|
log.error(`Missing websocket config for ${connectionAlias}`);
|
|
337
360
|
return null;
|
|
338
361
|
}
|
|
339
|
-
return new DiscordGatewayIngestor(connectionAlias, secrets, config.websocket, bufferSize);
|
|
362
|
+
return new DiscordGatewayIngestor(connectionAlias, secrets, config.websocket, bufferSize, instanceId);
|
|
340
363
|
});
|
|
341
364
|
//# sourceMappingURL=discord-gateway.js.map
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared E2E test setup for connection ingestor tests.
|
|
3
|
+
*
|
|
4
|
+
* Provides helpers to:
|
|
5
|
+
* - Load .env.e2e environment variables
|
|
6
|
+
* - Build a RemoteServerConfig with in-memory keys and requested connections
|
|
7
|
+
* - Boot an Express server on a random port
|
|
8
|
+
* - Generate valid webhook signatures (GitHub, Trello)
|
|
9
|
+
* - Wait for ingestor state transitions and event arrival
|
|
10
|
+
*/
|
|
11
|
+
import type { Server } from 'node:http';
|
|
12
|
+
import { IngestorManager } from '../manager.js';
|
|
13
|
+
import type { RemoteServerConfig } from '../../../shared/config.js';
|
|
14
|
+
import type { IngestedEvent, IngestorState } from '../types.js';
|
|
15
|
+
export declare function loadE2EEnv(): void;
|
|
16
|
+
/**
|
|
17
|
+
* Check that all required env vars are set.
|
|
18
|
+
* Returns the missing var names (empty array = all present).
|
|
19
|
+
*/
|
|
20
|
+
export declare function checkEnvVars(vars: string[]): string[];
|
|
21
|
+
/**
|
|
22
|
+
* Build a RemoteServerConfig for E2E tests.
|
|
23
|
+
*
|
|
24
|
+
* Creates a single caller ("e2e-client") with the requested connections.
|
|
25
|
+
* Env vars are injected into caller.env so secret resolution works against
|
|
26
|
+
* process.env (already loaded by loadE2EEnv).
|
|
27
|
+
*/
|
|
28
|
+
export declare function buildE2EConfig(connections: string[], opts?: {
|
|
29
|
+
env?: Record<string, string>;
|
|
30
|
+
ingestorOverrides?: Record<string, Record<string, unknown>>;
|
|
31
|
+
}): RemoteServerConfig;
|
|
32
|
+
export interface E2EServer {
|
|
33
|
+
server: Server;
|
|
34
|
+
baseUrl: string;
|
|
35
|
+
ingestorManager: IngestorManager;
|
|
36
|
+
/** Gracefully shut down server and stop all ingestors. */
|
|
37
|
+
teardown: () => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Boot an Express server with the given config on a random port.
|
|
41
|
+
* Starts all ingestors and returns handles for testing.
|
|
42
|
+
*/
|
|
43
|
+
export declare function bootServer(config: RemoteServerConfig): Promise<E2EServer>;
|
|
44
|
+
/** Generate a valid GitHub HMAC-SHA256 signature for a raw body. */
|
|
45
|
+
export declare function signGitHubPayload(rawBody: string | Buffer, secret: string): string;
|
|
46
|
+
/** Generate a valid Trello HMAC-SHA1 base64 signature for body + callbackUrl. */
|
|
47
|
+
export declare function signTrelloPayload(rawBody: string | Buffer, callbackUrl: string, secret: string): string;
|
|
48
|
+
/**
|
|
49
|
+
* Wait for an ingestor to reach a specific state.
|
|
50
|
+
* Polls every 250ms until the timeout is reached.
|
|
51
|
+
*/
|
|
52
|
+
export declare function waitForIngestorState(manager: IngestorManager, callerAlias: string, connection: string, targetState: IngestorState, timeoutMs?: number): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Poll until at least one event appears for a connection.
|
|
55
|
+
* Returns the events array once non-empty.
|
|
56
|
+
*/
|
|
57
|
+
export declare function pollUntilEvent(manager: IngestorManager, callerAlias: string, connection: string, timeoutMs?: number): Promise<IngestedEvent[]>;
|
|
58
|
+
/** Common shape every IngestedEvent must satisfy (for use with toMatchObject). */
|
|
59
|
+
export declare const INGESTED_EVENT_SHAPE: {
|
|
60
|
+
id: any;
|
|
61
|
+
idempotencyKey: any;
|
|
62
|
+
receivedAt: any;
|
|
63
|
+
receivedAtMs: any;
|
|
64
|
+
callerAlias: string;
|
|
65
|
+
source: any;
|
|
66
|
+
eventType: any;
|
|
67
|
+
data: any;
|
|
68
|
+
};
|
|
69
|
+
//# sourceMappingURL=setup.d.ts.map
|