@wolpertingerlabs/drawlatch 1.0.0-alpha.4 → 1.0.0-alpha.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONNECTIONS.md +9 -1
- package/README.md +393 -465
- package/bin/drawlatch.js +1149 -47
- package/dist/auth/auth.d.ts +10 -0
- package/dist/auth/auth.js +146 -0
- package/dist/auth/env-writer.d.ts +6 -0
- package/dist/auth/env-writer.js +52 -0
- package/dist/auth/password.d.ts +8 -0
- package/dist/auth/password.js +28 -0
- package/dist/auth/sessions.d.ts +13 -0
- package/dist/auth/sessions.js +85 -0
- package/dist/cli/generate-keys.d.ts +4 -4
- package/dist/cli/generate-keys.js +30 -25
- package/dist/connections/{anthropic.json → ai/anthropic.json} +2 -0
- package/dist/connections/{devin.json → ai/devin.json} +2 -0
- package/dist/connections/ai/exa.json +24 -0
- package/dist/connections/{google-ai.json → ai/google-ai.json} +2 -0
- package/dist/connections/{openai.json → ai/openai.json} +2 -0
- package/dist/connections/{openrouter.json → ai/openrouter.json} +2 -0
- package/dist/connections/developer-tools/datadog.json +27 -0
- package/dist/connections/{github.json → developer-tools/github.json} +59 -3
- package/dist/connections/{hex.json → developer-tools/hex.json} +2 -0
- package/dist/connections/{linear.json → developer-tools/linear.json} +2 -0
- package/dist/connections/{lichess.json → gaming/lichess.json} +2 -0
- package/dist/connections/messaging/agentmail.json +21 -0
- package/dist/connections/{discord-bot.json → messaging/discord-bot.json} +2 -0
- package/dist/connections/{discord-oauth.json → messaging/discord-oauth.json} +2 -0
- package/dist/connections/{slack.json → messaging/slack.json} +2 -0
- package/dist/connections/{telegram.json → messaging/telegram.json} +2 -0
- package/dist/connections/{google.json → productivity/google.json} +2 -0
- package/dist/connections/{notion.json → productivity/notion.json} +2 -0
- package/dist/connections/{stripe.json → productivity/stripe.json} +2 -0
- package/dist/connections/{trello.json → productivity/trello.json} +2 -0
- package/dist/connections/{bluesky.json → social-media/bluesky.json} +2 -0
- package/dist/connections/{mastodon.json → social-media/mastodon.json} +2 -0
- package/dist/connections/{reddit.json → social-media/reddit.json} +2 -0
- package/dist/connections/{twitch.json → social-media/twitch.json} +2 -0
- package/dist/connections/{x.json → social-media/x.json} +2 -0
- package/dist/mcp/server.js +268 -33
- package/dist/remote/admin-mutations.d.ts +43 -0
- package/dist/remote/admin-mutations.js +321 -0
- package/dist/remote/admin-types.d.ts +153 -0
- package/dist/remote/admin-types.js +11 -0
- package/dist/remote/admin.d.ts +37 -0
- package/dist/remote/admin.js +317 -0
- package/dist/remote/caller-bootstrap.d.ts +121 -0
- package/dist/remote/caller-bootstrap.js +270 -0
- package/dist/remote/ingestors/base-ingestor.d.ts +5 -0
- package/dist/remote/ingestors/base-ingestor.js +17 -4
- package/dist/remote/ingestors/discord/discord-gateway.js +24 -1
- package/dist/remote/ingestors/e2e/setup.d.ts +69 -0
- package/dist/remote/ingestors/e2e/setup.js +147 -0
- package/dist/remote/ingestors/manager.d.ts +35 -1
- package/dist/remote/ingestors/manager.js +160 -22
- package/dist/remote/ingestors/poll/poll-ingestor.d.ts +2 -0
- package/dist/remote/ingestors/poll/poll-ingestor.js +21 -0
- package/dist/remote/ingestors/slack/socket-mode.js +23 -1
- package/dist/remote/ingestors/types.d.ts +8 -0
- package/dist/remote/ingestors/webhook/github-webhook-ingestor.d.ts +16 -0
- package/dist/remote/ingestors/webhook/github-webhook-ingestor.js +49 -3
- package/dist/remote/ingestors/webhook/webhook-lifecycle-manager.js +9 -9
- package/dist/remote/server.d.ts +25 -33
- package/dist/remote/server.js +595 -735
- package/dist/remote/tool-dispatch.d.ts +84 -0
- package/dist/remote/tool-dispatch.js +910 -0
- package/dist/remote/triggers/rule-engine.d.ts +40 -0
- package/dist/remote/triggers/rule-engine.js +228 -0
- package/dist/remote/triggers/types.d.ts +69 -0
- package/dist/remote/triggers/types.js +10 -0
- package/dist/remote/tunnel-state.d.ts +14 -0
- package/dist/remote/tunnel-state.js +20 -0
- package/dist/remote/tunnel.d.ts +13 -0
- package/dist/remote/tunnel.js +33 -0
- package/dist/shared/config.d.ts +61 -27
- package/dist/shared/config.js +41 -33
- package/dist/shared/connections.d.ts +12 -5
- package/dist/shared/connections.js +52 -14
- package/dist/shared/crypto/index.d.ts +1 -0
- package/dist/shared/crypto/index.js +1 -0
- package/dist/shared/crypto/key-manager.d.ts +81 -0
- package/dist/shared/crypto/key-manager.js +174 -0
- package/dist/shared/env-utils.d.ts +42 -0
- package/dist/shared/env-utils.js +150 -0
- package/dist/shared/migrations.d.ts +40 -0
- package/dist/shared/migrations.js +122 -0
- package/dist/shared/protocol/caller-bundle-crypto.d.ts +37 -0
- package/dist/shared/protocol/caller-bundle-crypto.js +62 -0
- package/dist/shared/protocol/caller-bundle.d.ts +75 -0
- package/dist/shared/protocol/caller-bundle.js +24 -0
- package/dist/shared/protocol/handshake.js +14 -1
- package/dist/shared/protocol/index.d.ts +2 -0
- package/dist/shared/protocol/index.js +2 -0
- package/dist/shared/protocol/sync-client.d.ts +52 -0
- package/dist/shared/protocol/sync-client.js +99 -0
- package/dist/shared/protocol/sync.d.ts +71 -0
- package/dist/shared/protocol/sync.js +176 -0
- package/frontend/dist/assets/index-BdCSPSZK.js +282 -0
- package/frontend/dist/assets/index-BmK26bY2.css +1 -0
- package/frontend/dist/index.html +15 -0
- package/package.json +63 -11
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { resolveCallerRoutes, resolveRoutes, resolveSecrets, } from '../../shared/config.js';
|
|
17
17
|
import { createLogger } from '../../shared/logger.js';
|
|
18
|
+
import { TriggerRuleEngine } from '../triggers/rule-engine.js';
|
|
18
19
|
const log = createLogger('ingestor');
|
|
19
20
|
import { createIngestor } from './registry.js';
|
|
20
21
|
import { WebhookIngestor } from './webhook/base-webhook-ingestor.js';
|
|
@@ -48,8 +49,22 @@ export class IngestorManager {
|
|
|
48
49
|
config;
|
|
49
50
|
/** Active ingestor instances, keyed by `callerAlias:connectionAlias:instanceId`. */
|
|
50
51
|
ingestors = new Map();
|
|
51
|
-
|
|
52
|
+
/** Trigger rule engines per caller. Created during startAll() for callers with triggerRules. */
|
|
53
|
+
triggerEngines = new Map();
|
|
54
|
+
/** Global event listeners (e.g. SSE streams). Called for every event from every ingestor. */
|
|
55
|
+
eventListeners = new Set();
|
|
56
|
+
/**
|
|
57
|
+
* Optional config loader for hot-reload support. When provided, `startOne()`
|
|
58
|
+
* uses it to get fresh config from disk instead of the constructor snapshot.
|
|
59
|
+
*/
|
|
60
|
+
configLoader;
|
|
61
|
+
constructor(config, configLoader) {
|
|
52
62
|
this.config = config;
|
|
63
|
+
this.configLoader = configLoader;
|
|
64
|
+
}
|
|
65
|
+
/** Return fresh config if a loader is available, otherwise the constructor snapshot. */
|
|
66
|
+
getConfig() {
|
|
67
|
+
return this.configLoader ? this.configLoader() : this.config;
|
|
53
68
|
}
|
|
54
69
|
/**
|
|
55
70
|
* Start ingestors for all callers whose connections have an `ingestor` config.
|
|
@@ -63,7 +78,7 @@ export class IngestorManager {
|
|
|
63
78
|
// Resolve routes for this caller (raw + resolved)
|
|
64
79
|
const rawRoutes = resolveCallerRoutes(this.config, callerAlias);
|
|
65
80
|
const callerEnvResolved = resolveSecrets(callerConfig.env ?? {});
|
|
66
|
-
const resolvedRoutes = resolveRoutes(rawRoutes, callerEnvResolved);
|
|
81
|
+
const resolvedRoutes = resolveRoutes(rawRoutes, callerEnvResolved, callerAlias);
|
|
67
82
|
for (let i = 0; i < rawRoutes.length; i++) {
|
|
68
83
|
const rawRoute = rawRoutes[i];
|
|
69
84
|
const resolvedRoute = resolvedRoutes[i];
|
|
@@ -104,6 +119,8 @@ export class IngestorManager {
|
|
|
104
119
|
if (count > 0) {
|
|
105
120
|
log.info(`${count} ingestor(s) started`);
|
|
106
121
|
}
|
|
122
|
+
// Wire up trigger rule engines for callers that have triggerRules
|
|
123
|
+
this.initTriggerEngines();
|
|
107
124
|
}
|
|
108
125
|
/**
|
|
109
126
|
* Internal: create, register, and start a single ingestor instance.
|
|
@@ -124,7 +141,14 @@ export class IngestorManager {
|
|
|
124
141
|
}
|
|
125
142
|
const ingestor = createIngestor(connectionAlias, effectiveConfig, instanceSecrets, overrides?.bufferSize, instanceId);
|
|
126
143
|
if (ingestor) {
|
|
144
|
+
const { caller } = parseKey(key);
|
|
145
|
+
ingestor.callerAlias = caller;
|
|
127
146
|
this.ingestors.set(key, ingestor);
|
|
147
|
+
// Forward events to global listeners (SSE streams, etc.)
|
|
148
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any -- BaseIngestor extends EventEmitter; .on() is inherited
|
|
149
|
+
ingestor.on('event', (event) => {
|
|
150
|
+
this.notifyEventListeners(event);
|
|
151
|
+
});
|
|
128
152
|
log.info(`Starting ${effectiveConfig.type} ingestor for ${key}`);
|
|
129
153
|
try {
|
|
130
154
|
await ingestor.start();
|
|
@@ -134,6 +158,35 @@ export class IngestorManager {
|
|
|
134
158
|
}
|
|
135
159
|
}
|
|
136
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* Initialize trigger rule engines for callers with triggerRules config.
|
|
163
|
+
* Subscribes to 'event' emissions from matching ingestors and dispatches
|
|
164
|
+
* to Claude Code remote triggers.
|
|
165
|
+
*/
|
|
166
|
+
initTriggerEngines() {
|
|
167
|
+
const config = this.getConfig();
|
|
168
|
+
for (const [callerAlias, callerConfig] of Object.entries(config.callers)) {
|
|
169
|
+
if (!callerConfig.triggerRules || callerConfig.triggerRules.length === 0)
|
|
170
|
+
continue;
|
|
171
|
+
// Resolve caller-level secrets for API key access
|
|
172
|
+
const callerEnvResolved = resolveSecrets(callerConfig.env ?? {});
|
|
173
|
+
const engine = new TriggerRuleEngine(callerConfig.triggerRules, callerEnvResolved);
|
|
174
|
+
if (engine.activeRuleCount === 0)
|
|
175
|
+
continue;
|
|
176
|
+
this.triggerEngines.set(callerAlias, engine);
|
|
177
|
+
// Subscribe the engine to all ingestors belonging to this caller
|
|
178
|
+
for (const [key, ingestor] of this.ingestors) {
|
|
179
|
+
const { caller } = parseKey(key);
|
|
180
|
+
if (caller !== callerAlias)
|
|
181
|
+
continue;
|
|
182
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any -- BaseIngestor extends EventEmitter; .on() is inherited
|
|
183
|
+
ingestor.on('event', (event) => {
|
|
184
|
+
engine.handleEvent(event);
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
log.info(`Trigger rule engine for ${callerAlias}: ${engine.activeRuleCount} active rule(s)`);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
137
190
|
/**
|
|
138
191
|
* Stop all running ingestors. Called during graceful shutdown.
|
|
139
192
|
*/
|
|
@@ -204,6 +257,41 @@ export class IngestorManager {
|
|
|
204
257
|
}
|
|
205
258
|
return statuses;
|
|
206
259
|
}
|
|
260
|
+
/**
|
|
261
|
+
* Get status of every ingestor across all callers, augmented with the
|
|
262
|
+
* owning caller alias. Used by the read-only /admin API to render a global
|
|
263
|
+
* dashboard view without iterating per-caller.
|
|
264
|
+
*/
|
|
265
|
+
getAllStatuses() {
|
|
266
|
+
const out = [];
|
|
267
|
+
for (const [key, ingestor] of this.ingestors) {
|
|
268
|
+
const { caller } = parseKey(key);
|
|
269
|
+
out.push({ ...ingestor.getStatus(), callerAlias: caller });
|
|
270
|
+
}
|
|
271
|
+
return out;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Subscribe to all events from all ingestors (current and future).
|
|
275
|
+
* Used by the SSE /events/stream endpoint to fan out events to CLI watchers.
|
|
276
|
+
*/
|
|
277
|
+
onEvent(listener) {
|
|
278
|
+
this.eventListeners.add(listener);
|
|
279
|
+
}
|
|
280
|
+
/** Unsubscribe a global event listener. */
|
|
281
|
+
offEvent(listener) {
|
|
282
|
+
this.eventListeners.delete(listener);
|
|
283
|
+
}
|
|
284
|
+
/** Forward an ingestor event to all global listeners. */
|
|
285
|
+
notifyEventListeners(event) {
|
|
286
|
+
for (const listener of this.eventListeners) {
|
|
287
|
+
try {
|
|
288
|
+
listener(event);
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
// Don't let a broken listener crash ingestor event processing
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
207
295
|
/**
|
|
208
296
|
* Find all webhook ingestor instances that match a given webhook path.
|
|
209
297
|
* Returns all matching instances across all callers (for fan-out dispatch).
|
|
@@ -226,25 +314,41 @@ export class IngestorManager {
|
|
|
226
314
|
* When omitted, starts the default instance (or all instances if listenerInstances is defined).
|
|
227
315
|
*/
|
|
228
316
|
async startOne(callerAlias, connectionAlias, instanceId) {
|
|
229
|
-
|
|
317
|
+
// Get fresh config (from disk in production, or constructor snapshot in tests)
|
|
318
|
+
// so we pick up changes made by tool handlers (e.g. set_connection_enabled,
|
|
319
|
+
// set_listener_params, set_secrets) without requiring a server restart.
|
|
320
|
+
const config = this.getConfig();
|
|
321
|
+
const callerConfig = config.callers[callerAlias];
|
|
230
322
|
if (!callerConfig) {
|
|
231
|
-
return {
|
|
323
|
+
return {
|
|
324
|
+
success: false,
|
|
325
|
+
connection: connectionAlias,
|
|
326
|
+
error: `Unknown caller: ${callerAlias}`,
|
|
327
|
+
};
|
|
232
328
|
}
|
|
233
329
|
const connectionIndex = callerConfig.connections.indexOf(connectionAlias);
|
|
234
330
|
if (connectionIndex === -1) {
|
|
235
|
-
return {
|
|
331
|
+
return {
|
|
332
|
+
success: false,
|
|
333
|
+
connection: connectionAlias,
|
|
334
|
+
error: `Caller does not have connection: ${connectionAlias}`,
|
|
335
|
+
};
|
|
236
336
|
}
|
|
237
|
-
const rawRoutes = resolveCallerRoutes(
|
|
337
|
+
const rawRoutes = resolveCallerRoutes(config, callerAlias);
|
|
238
338
|
const callerEnvResolved = resolveSecrets(callerConfig.env ?? {});
|
|
239
|
-
const resolvedRoutes = resolveRoutes(rawRoutes, callerEnvResolved);
|
|
339
|
+
const resolvedRoutes = resolveRoutes(rawRoutes, callerEnvResolved, callerAlias);
|
|
240
340
|
const rawRoute = rawRoutes[connectionIndex];
|
|
241
341
|
const resolvedRoute = resolvedRoutes[connectionIndex];
|
|
242
342
|
if (!rawRoute.ingestor) {
|
|
243
|
-
return {
|
|
343
|
+
return {
|
|
344
|
+
success: false,
|
|
345
|
+
connection: connectionAlias,
|
|
346
|
+
error: 'This connection does not have an ingestor.',
|
|
347
|
+
};
|
|
244
348
|
}
|
|
245
349
|
// If a specific instanceId is given, start just that one
|
|
246
350
|
if (instanceId) {
|
|
247
|
-
return this.startOneInstance(callerAlias, connectionAlias, instanceId, rawRoute, resolvedRoute);
|
|
351
|
+
return this.startOneInstance(callerAlias, connectionAlias, instanceId, rawRoute, resolvedRoute, config);
|
|
248
352
|
}
|
|
249
353
|
// If listenerInstances is defined, start all instances
|
|
250
354
|
const instances = callerConfig.listenerInstances?.[connectionAlias];
|
|
@@ -253,30 +357,32 @@ export class IngestorManager {
|
|
|
253
357
|
for (const [instId, instOverrides] of Object.entries(instances)) {
|
|
254
358
|
if (instOverrides.disabled)
|
|
255
359
|
continue;
|
|
256
|
-
results.push(await this.startOneInstance(callerAlias, connectionAlias, instId, rawRoute, resolvedRoute));
|
|
360
|
+
results.push(await this.startOneInstance(callerAlias, connectionAlias, instId, rawRoute, resolvedRoute, config));
|
|
257
361
|
}
|
|
258
362
|
return results;
|
|
259
363
|
}
|
|
260
364
|
// Single default instance
|
|
261
|
-
return this.startOneInstance(callerAlias, connectionAlias, undefined, rawRoute, resolvedRoute);
|
|
365
|
+
return this.startOneInstance(callerAlias, connectionAlias, undefined, rawRoute, resolvedRoute, config);
|
|
262
366
|
}
|
|
263
367
|
/** Internal: start a single specific instance. */
|
|
264
|
-
async startOneInstance(callerAlias, connectionAlias, instanceId, rawRoute, resolvedRoute) {
|
|
368
|
+
async startOneInstance(callerAlias, connectionAlias, instanceId, rawRoute, resolvedRoute, config) {
|
|
265
369
|
const key = makeKey(callerAlias, connectionAlias, instanceId ?? DEFAULT_INSTANCE_ID);
|
|
266
370
|
// If already running, return current status
|
|
267
371
|
const existing = this.ingestors.get(key);
|
|
268
372
|
if (existing) {
|
|
269
373
|
const status = existing.getStatus();
|
|
270
|
-
if (status.state === 'connected' ||
|
|
374
|
+
if (status.state === 'connected' ||
|
|
375
|
+
status.state === 'starting' ||
|
|
376
|
+
status.state === 'reconnecting') {
|
|
271
377
|
return { success: true, connection: connectionAlias, instanceId, state: status.state };
|
|
272
378
|
}
|
|
273
379
|
// Remove stopped instance to recreate
|
|
274
380
|
this.ingestors.delete(key);
|
|
275
381
|
}
|
|
276
|
-
const
|
|
382
|
+
const callerCfg = config.callers[callerAlias];
|
|
277
383
|
const overrides = instanceId
|
|
278
|
-
?
|
|
279
|
-
:
|
|
384
|
+
? callerCfg.listenerInstances?.[connectionAlias]?.[instanceId]
|
|
385
|
+
: callerCfg.ingestorOverrides?.[connectionAlias];
|
|
280
386
|
const effectiveConfig = IngestorManager.mergeIngestorConfig(rawRoute.ingestor, overrides);
|
|
281
387
|
// Apply instance params
|
|
282
388
|
const instanceSecrets = { ...resolvedRoute.secrets };
|
|
@@ -289,13 +395,29 @@ export class IngestorManager {
|
|
|
289
395
|
}
|
|
290
396
|
const ingestor = createIngestor(connectionAlias, effectiveConfig, instanceSecrets, overrides?.bufferSize, instanceId);
|
|
291
397
|
if (!ingestor) {
|
|
292
|
-
return {
|
|
398
|
+
return {
|
|
399
|
+
success: false,
|
|
400
|
+
connection: connectionAlias,
|
|
401
|
+
instanceId,
|
|
402
|
+
error: 'Failed to create ingestor.',
|
|
403
|
+
};
|
|
293
404
|
}
|
|
405
|
+
ingestor.callerAlias = callerAlias;
|
|
294
406
|
this.ingestors.set(key, ingestor);
|
|
407
|
+
// Forward events to global listeners (SSE streams, etc.)
|
|
408
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any -- BaseIngestor extends EventEmitter; .on() is inherited
|
|
409
|
+
ingestor.on('event', (event) => {
|
|
410
|
+
this.notifyEventListeners(event);
|
|
411
|
+
});
|
|
295
412
|
log.info(`Starting ${effectiveConfig.type} ingestor for ${key}`);
|
|
296
413
|
try {
|
|
297
414
|
await ingestor.start();
|
|
298
|
-
return {
|
|
415
|
+
return {
|
|
416
|
+
success: true,
|
|
417
|
+
connection: connectionAlias,
|
|
418
|
+
instanceId,
|
|
419
|
+
state: ingestor.getStatus().state,
|
|
420
|
+
};
|
|
299
421
|
}
|
|
300
422
|
catch (err) {
|
|
301
423
|
log.error(`Failed to start ${key}:`, err);
|
|
@@ -318,9 +440,13 @@ export class IngestorManager {
|
|
|
318
440
|
}
|
|
319
441
|
// Stop all instances for this connection
|
|
320
442
|
const prefix = `${callerAlias}:${connectionAlias}:`;
|
|
321
|
-
const keysToStop = Array.from(this.ingestors.keys()).filter(k => k.startsWith(prefix));
|
|
443
|
+
const keysToStop = Array.from(this.ingestors.keys()).filter((k) => k.startsWith(prefix));
|
|
322
444
|
if (keysToStop.length === 0) {
|
|
323
|
-
return {
|
|
445
|
+
return {
|
|
446
|
+
success: false,
|
|
447
|
+
connection: connectionAlias,
|
|
448
|
+
error: 'No ingestor running for this connection.',
|
|
449
|
+
};
|
|
324
450
|
}
|
|
325
451
|
if (keysToStop.length === 1) {
|
|
326
452
|
const parsed = parseKey(keysToStop[0]);
|
|
@@ -340,7 +466,12 @@ export class IngestorManager {
|
|
|
340
466
|
const key = makeKey(callerAlias, connectionAlias, instanceId ?? DEFAULT_INSTANCE_ID);
|
|
341
467
|
const ingestor = this.ingestors.get(key);
|
|
342
468
|
if (!ingestor) {
|
|
343
|
-
return {
|
|
469
|
+
return {
|
|
470
|
+
success: false,
|
|
471
|
+
connection: connectionAlias,
|
|
472
|
+
instanceId,
|
|
473
|
+
error: 'No ingestor running for this connection.',
|
|
474
|
+
};
|
|
344
475
|
}
|
|
345
476
|
log.info(`Stopping ${key}`);
|
|
346
477
|
try {
|
|
@@ -444,7 +575,7 @@ export class IngestorManager {
|
|
|
444
575
|
* - Otherwise, attaches the value as a generic `_instanceParams` bag on the config.
|
|
445
576
|
*/
|
|
446
577
|
static applyInstanceParams(config, secrets, params, fields) {
|
|
447
|
-
const fieldsByKey = new Map(fields.map(f => [f.key, f]));
|
|
578
|
+
const fieldsByKey = new Map(fields.map((f) => [f.key, f]));
|
|
448
579
|
for (const [paramKey, paramValue] of Object.entries(params)) {
|
|
449
580
|
const field = fieldsByKey.get(paramKey);
|
|
450
581
|
if (!field)
|
|
@@ -461,6 +592,13 @@ export class IngestorManager {
|
|
|
461
592
|
if (typeof paramValue === 'string') {
|
|
462
593
|
secrets[paramKey] = paramValue;
|
|
463
594
|
}
|
|
595
|
+
else if (Array.isArray(paramValue) &&
|
|
596
|
+
paramValue.length > 0 &&
|
|
597
|
+
typeof paramValue[0] === 'string') {
|
|
598
|
+
// For text[] instanceKey fields, inject the first element into secrets
|
|
599
|
+
// for lifecycle URL ${VAR} resolution (e.g., repoFilter → "owner/repo")
|
|
600
|
+
secrets[paramKey] = paramValue[0];
|
|
601
|
+
}
|
|
464
602
|
}
|
|
465
603
|
}
|
|
466
604
|
}
|
|
@@ -31,6 +31,8 @@ export declare class PollIngestor extends BaseIngestor {
|
|
|
31
31
|
private readonly responsePath;
|
|
32
32
|
private readonly eventType;
|
|
33
33
|
private readonly pollHeaders;
|
|
34
|
+
private readonly useEtag;
|
|
35
|
+
private lastEtag;
|
|
34
36
|
/** Resolved headers from the parent connection route (injected by manager). */
|
|
35
37
|
private readonly routeHeaders;
|
|
36
38
|
constructor(connectionAlias: string, secrets: Record<string, string>, pollConfig: PollIngestorConfig,
|
|
@@ -42,6 +42,8 @@ export class PollIngestor extends BaseIngestor {
|
|
|
42
42
|
responsePath;
|
|
43
43
|
eventType;
|
|
44
44
|
pollHeaders;
|
|
45
|
+
useEtag;
|
|
46
|
+
lastEtag = null;
|
|
45
47
|
/** Resolved headers from the parent connection route (injected by manager). */
|
|
46
48
|
routeHeaders;
|
|
47
49
|
constructor(connectionAlias, secrets, pollConfig,
|
|
@@ -56,6 +58,7 @@ export class PollIngestor extends BaseIngestor {
|
|
|
56
58
|
this.deduplicateBy = pollConfig.deduplicateBy;
|
|
57
59
|
this.responsePath = pollConfig.responsePath;
|
|
58
60
|
this.eventType = pollConfig.eventType ?? 'poll';
|
|
61
|
+
this.useEtag = pollConfig.etag ?? false;
|
|
59
62
|
this.routeHeaders = routeHeaders;
|
|
60
63
|
// Resolve ${VAR} placeholders in poll-specific headers
|
|
61
64
|
this.pollHeaders = {};
|
|
@@ -95,6 +98,10 @@ export class PollIngestor extends BaseIngestor {
|
|
|
95
98
|
...this.routeHeaders,
|
|
96
99
|
...this.pollHeaders,
|
|
97
100
|
};
|
|
101
|
+
// Add ETag conditional request header if enabled and we have a cached ETag
|
|
102
|
+
if (this.useEtag && this.lastEtag) {
|
|
103
|
+
headers['If-None-Match'] = this.lastEtag;
|
|
104
|
+
}
|
|
98
105
|
// Build request options
|
|
99
106
|
const fetchOptions = {
|
|
100
107
|
method: this.method,
|
|
@@ -115,9 +122,23 @@ export class PollIngestor extends BaseIngestor {
|
|
|
115
122
|
}
|
|
116
123
|
}
|
|
117
124
|
const response = await fetch(this.url, fetchOptions);
|
|
125
|
+
// Handle ETag 304 Not Modified — no new data, not an error
|
|
126
|
+
if (this.useEtag && response.status === 304) {
|
|
127
|
+
this.consecutiveErrors = 0;
|
|
128
|
+
if (this.state !== 'connected')
|
|
129
|
+
this.state = 'connected';
|
|
130
|
+
log.debug(`${this.connectionAlias}: 304 Not Modified (ETag cache hit)`);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
118
133
|
if (!response.ok) {
|
|
119
134
|
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
|
120
135
|
}
|
|
136
|
+
// Store ETag for subsequent conditional requests
|
|
137
|
+
if (this.useEtag) {
|
|
138
|
+
const etag = response.headers.get('etag');
|
|
139
|
+
if (etag)
|
|
140
|
+
this.lastEtag = etag;
|
|
141
|
+
}
|
|
121
142
|
const responseBody = await response.json();
|
|
122
143
|
// Extract items array from response using responsePath
|
|
123
144
|
const items = this.extractItems(responseBody);
|
|
@@ -47,8 +47,30 @@ export class SlackSocketModeIngestor extends BaseIngestor {
|
|
|
47
47
|
this.state = 'stopped';
|
|
48
48
|
this.clearReconnectTimer();
|
|
49
49
|
if (this.ws) {
|
|
50
|
-
this.ws
|
|
50
|
+
const ws = this.ws;
|
|
51
51
|
this.ws = null;
|
|
52
|
+
// Wait for the WebSocket to fully close before resolving.
|
|
53
|
+
// This prevents a race where the old connection is still alive when a new
|
|
54
|
+
// ingestor starts, causing Slack to distribute events to the dying
|
|
55
|
+
// connection (where nobody processes them).
|
|
56
|
+
return new Promise((resolve) => {
|
|
57
|
+
const onClose = () => {
|
|
58
|
+
ws.removeEventListener('close', onClose);
|
|
59
|
+
resolve();
|
|
60
|
+
};
|
|
61
|
+
if (ws.readyState === WebSocket.CLOSED) {
|
|
62
|
+
resolve();
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
ws.addEventListener('close', onClose);
|
|
66
|
+
ws.close(1000, 'Shutting down');
|
|
67
|
+
// Safety timeout — don't block forever if close event never fires
|
|
68
|
+
setTimeout(() => {
|
|
69
|
+
ws.removeEventListener('close', onClose);
|
|
70
|
+
resolve();
|
|
71
|
+
}, 5000);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
52
74
|
}
|
|
53
75
|
return Promise.resolve();
|
|
54
76
|
}
|
|
@@ -86,6 +86,12 @@ export interface PollIngestorConfig {
|
|
|
86
86
|
* Values may contain ${VAR} placeholders.
|
|
87
87
|
* These are merged UNDER the connection's route headers (route headers take precedence). */
|
|
88
88
|
headers?: Record<string, string>;
|
|
89
|
+
/** Enable ETag-based conditional requests.
|
|
90
|
+
* When true, the poller stores the ETag from each successful response and sends
|
|
91
|
+
* `If-None-Match` on subsequent requests. HTTP 304 responses are treated as a
|
|
92
|
+
* successful no-op (no items, no error). Useful for APIs like GitHub Events
|
|
93
|
+
* where 304s do not count against rate limits. */
|
|
94
|
+
etag?: boolean;
|
|
89
95
|
}
|
|
90
96
|
/** A single event received by an ingestor, stored in the ring buffer. */
|
|
91
97
|
export interface IngestedEvent {
|
|
@@ -110,6 +116,8 @@ export interface IngestedEvent {
|
|
|
110
116
|
receivedAt: string;
|
|
111
117
|
/** Unix timestamp (milliseconds) when the event was received by the ingestor. */
|
|
112
118
|
receivedAtMs: number;
|
|
119
|
+
/** Caller alias that owns this event (e.g., 'default', 'alice'). */
|
|
120
|
+
callerAlias: string;
|
|
113
121
|
/** Source connection alias (e.g., 'discord-bot', 'github'). */
|
|
114
122
|
source: string;
|
|
115
123
|
/** Instance identifier for multi-instance listeners (e.g., "project-board").
|
|
@@ -19,7 +19,23 @@ export declare class GitHubWebhookIngestor extends WebhookIngestor {
|
|
|
19
19
|
* Set via `_repoFilter` on the webhook config (injected by IngestorManager).
|
|
20
20
|
*/
|
|
21
21
|
private readonly repoFilter;
|
|
22
|
+
/**
|
|
23
|
+
* Organization filter for org-level webhook registration.
|
|
24
|
+
* When set, lifecycle URLs target the org API instead of the repo API.
|
|
25
|
+
* Set via `_orgFilter` on the webhook config (injected by IngestorManager).
|
|
26
|
+
*/
|
|
27
|
+
private readonly orgFilter;
|
|
22
28
|
constructor(connectionAlias: string, secrets: Record<string, string>, webhookConfig: WebhookIngestorConfig, bufferSize?: number, instanceId?: string);
|
|
29
|
+
/**
|
|
30
|
+
* Return the model ID for multi-instance webhook lifecycle management.
|
|
31
|
+
* For org-level: the org name. For repo-level: the single repo name.
|
|
32
|
+
*/
|
|
33
|
+
protected getModelId(): string | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* Clone webhookConfig with lifecycle URLs rewritten for org-level endpoints.
|
|
36
|
+
* Replaces `repos/${repoFilter}` with `orgs/${orgFilter}` in all lifecycle URLs.
|
|
37
|
+
*/
|
|
38
|
+
private static withOrgLifecycle;
|
|
23
39
|
/**
|
|
24
40
|
* Filter webhooks by repository for multi-instance support.
|
|
25
41
|
* When repoFilter is set, only events from those repos are accepted.
|
|
@@ -23,11 +23,57 @@ export class GitHubWebhookIngestor extends WebhookIngestor {
|
|
|
23
23
|
* Set via `_repoFilter` on the webhook config (injected by IngestorManager).
|
|
24
24
|
*/
|
|
25
25
|
repoFilter;
|
|
26
|
+
/**
|
|
27
|
+
* Organization filter for org-level webhook registration.
|
|
28
|
+
* When set, lifecycle URLs target the org API instead of the repo API.
|
|
29
|
+
* Set via `_orgFilter` on the webhook config (injected by IngestorManager).
|
|
30
|
+
*/
|
|
31
|
+
orgFilter;
|
|
26
32
|
constructor(connectionAlias, secrets, webhookConfig, bufferSize, instanceId) {
|
|
27
|
-
super(connectionAlias, secrets, webhookConfig, bufferSize, instanceId);
|
|
28
|
-
// Repo filter for multi-instance discrimination
|
|
29
33
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any -- injected by IngestorManager for multi-instance support
|
|
30
|
-
|
|
34
|
+
const orgFilter = webhookConfig._orgFilter;
|
|
35
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
|
|
36
|
+
const repoFilterArr = webhookConfig._repoFilter;
|
|
37
|
+
// When orgFilter is set, swap lifecycle URLs to use org endpoints
|
|
38
|
+
if (orgFilter && webhookConfig.lifecycle) {
|
|
39
|
+
webhookConfig = GitHubWebhookIngestor.withOrgLifecycle(webhookConfig);
|
|
40
|
+
}
|
|
41
|
+
else if (!repoFilterArr?.length && !orgFilter && webhookConfig.lifecycle) {
|
|
42
|
+
// Neither filter set — lifecycle registration can't determine the target
|
|
43
|
+
log.warn(`${connectionAlias}: No repoFilter or orgFilter set — webhook auto-registration disabled. ` +
|
|
44
|
+
'Set repoFilter (owner/repo) or orgFilter (org name) to enable.');
|
|
45
|
+
webhookConfig = { ...webhookConfig, lifecycle: undefined };
|
|
46
|
+
}
|
|
47
|
+
super(connectionAlias, secrets, webhookConfig, bufferSize, instanceId);
|
|
48
|
+
this.repoFilter = repoFilterArr ?? [];
|
|
49
|
+
this.orgFilter = orgFilter;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Return the model ID for multi-instance webhook lifecycle management.
|
|
53
|
+
* For org-level: the org name. For repo-level: the single repo name.
|
|
54
|
+
*/
|
|
55
|
+
getModelId() {
|
|
56
|
+
if (this.orgFilter)
|
|
57
|
+
return this.orgFilter;
|
|
58
|
+
return this.repoFilter.length === 1 ? this.repoFilter[0] : undefined;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Clone webhookConfig with lifecycle URLs rewritten for org-level endpoints.
|
|
62
|
+
* Replaces `repos/${repoFilter}` with `orgs/${orgFilter}` in all lifecycle URLs.
|
|
63
|
+
*/
|
|
64
|
+
static withOrgLifecycle(config) {
|
|
65
|
+
if (!config.lifecycle)
|
|
66
|
+
return config;
|
|
67
|
+
const lc = config.lifecycle;
|
|
68
|
+
const swap = (url) => url.replace('repos/${repoFilter}', 'orgs/${orgFilter}');
|
|
69
|
+
return {
|
|
70
|
+
...config,
|
|
71
|
+
lifecycle: {
|
|
72
|
+
list: lc.list ? { ...lc.list, url: swap(lc.list.url) } : undefined,
|
|
73
|
+
register: lc.register ? { ...lc.register, url: swap(lc.register.url) } : undefined,
|
|
74
|
+
unregister: lc.unregister ? { ...lc.unregister, url: swap(lc.unregister.url) } : undefined,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
31
77
|
}
|
|
32
78
|
/**
|
|
33
79
|
* Filter webhooks by repository for multi-instance support.
|
|
@@ -25,7 +25,7 @@ function resolveDeep(value, secrets) {
|
|
|
25
25
|
return resolvePlaceholders(value, secrets);
|
|
26
26
|
}
|
|
27
27
|
if (Array.isArray(value)) {
|
|
28
|
-
return value.map(item => resolveDeep(item, secrets));
|
|
28
|
+
return value.map((item) => resolveDeep(item, secrets));
|
|
29
29
|
}
|
|
30
30
|
if (value !== null && typeof value === 'object') {
|
|
31
31
|
const result = {};
|
|
@@ -87,34 +87,34 @@ export class WebhookLifecycleManager {
|
|
|
87
87
|
if (existingWebhooks !== null && this.config.list) {
|
|
88
88
|
const listConfig = this.config.list;
|
|
89
89
|
// Find a webhook matching our callback URL (and model ID if applicable)
|
|
90
|
-
const matching = existingWebhooks.find(wh => {
|
|
91
|
-
const rawUrl = wh
|
|
90
|
+
const matching = existingWebhooks.find((wh) => {
|
|
91
|
+
const rawUrl = getByPath(wh, listConfig.callbackUrlField);
|
|
92
92
|
const whCallbackUrl = typeof rawUrl === 'string' ? rawUrl : '';
|
|
93
93
|
const urlMatch = whCallbackUrl === callbackUrl;
|
|
94
94
|
if (!urlMatch)
|
|
95
95
|
return false;
|
|
96
96
|
if (modelId && listConfig.modelIdField) {
|
|
97
|
-
const rawModelId = wh
|
|
97
|
+
const rawModelId = getByPath(wh, listConfig.modelIdField);
|
|
98
98
|
return (typeof rawModelId === 'string' ? rawModelId : '') === modelId;
|
|
99
99
|
}
|
|
100
100
|
return true;
|
|
101
101
|
});
|
|
102
102
|
if (matching) {
|
|
103
|
-
const webhookId = String(matching
|
|
103
|
+
const webhookId = String(getByPath(matching, listConfig.idField));
|
|
104
104
|
log.info(`Found existing webhook (ID: ${webhookId}), reusing`);
|
|
105
105
|
return { registered: true, webhookId, lastAttempt: now };
|
|
106
106
|
}
|
|
107
107
|
// Clean up stale webhooks (matching model but wrong callback URL)
|
|
108
108
|
if (modelId && listConfig.modelIdField && this.config.unregister) {
|
|
109
|
-
const stale = existingWebhooks.filter(wh => {
|
|
110
|
-
const rawModelId = wh
|
|
109
|
+
const stale = existingWebhooks.filter((wh) => {
|
|
110
|
+
const rawModelId = getByPath(wh, listConfig.modelIdField);
|
|
111
111
|
const whModelId = typeof rawModelId === 'string' ? rawModelId : '';
|
|
112
|
-
const rawUrl = wh
|
|
112
|
+
const rawUrl = getByPath(wh, listConfig.callbackUrlField);
|
|
113
113
|
const whCallbackUrl = typeof rawUrl === 'string' ? rawUrl : '';
|
|
114
114
|
return whModelId === modelId && whCallbackUrl !== callbackUrl;
|
|
115
115
|
});
|
|
116
116
|
for (const staleWh of stale) {
|
|
117
|
-
const staleId = String(staleWh
|
|
117
|
+
const staleId = String(getByPath(staleWh, listConfig.idField));
|
|
118
118
|
log.info(`Cleaning up stale webhook (ID: ${staleId})`);
|
|
119
119
|
try {
|
|
120
120
|
await this.unregister(staleId);
|
package/dist/remote/server.d.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* - Maintains an audit log of all operations
|
|
13
13
|
* - Rate-limits requests per session
|
|
14
14
|
*/
|
|
15
|
+
import express from 'express';
|
|
15
16
|
import { type RemoteServerConfig, type ResolvedRoute } from '../shared/config.js';
|
|
16
17
|
import { EncryptedChannel, type PublicKeyBundle } from '../shared/crypto/index.js';
|
|
17
18
|
import { HandshakeResponder, type HandshakeInit } from '../shared/protocol/index.js';
|
|
@@ -43,13 +44,25 @@ export interface PendingHandshake {
|
|
|
43
44
|
init: HandshakeInit;
|
|
44
45
|
createdAt: number;
|
|
45
46
|
}
|
|
46
|
-
|
|
47
|
+
/** Sanitized session projection — never includes channel keys or resolved
|
|
48
|
+
* routes (which carry decrypted secrets). Returned by getSessionsSnapshot()
|
|
49
|
+
* for the read-only /admin API. */
|
|
50
|
+
export interface SessionSnapshot {
|
|
51
|
+
/** First 12 chars of the session ID — enough to disambiguate, doesn't
|
|
52
|
+
* expose enough material to forge or replay encrypted requests. */
|
|
53
|
+
sessionIdShort: string;
|
|
54
|
+
callerAlias: string;
|
|
55
|
+
createdAt: number;
|
|
56
|
+
lastActivity: number;
|
|
57
|
+
requestCount: number;
|
|
58
|
+
windowRequests: number;
|
|
59
|
+
windowStart: number;
|
|
60
|
+
}
|
|
61
|
+
/** Loopback guard — used by /sync/listen, /sync/status, /events/stream, /admin.
|
|
62
|
+
* Hoisted to module scope so the admin router and its tests can reuse it. */
|
|
63
|
+
export declare function requireLoopback(req: express.Request, res: express.Response, next: express.NextFunction): void;
|
|
47
64
|
export { resolvePlaceholders } from '../shared/config.js';
|
|
48
|
-
|
|
49
|
-
* Find the first route whose allowedEndpoints match the given URL.
|
|
50
|
-
* Routes with empty allowedEndpoints match nothing.
|
|
51
|
-
*/
|
|
52
|
-
export declare function matchRoute(url: string, routes: ResolvedRoute[]): ResolvedRoute | null;
|
|
65
|
+
export { toolHandlers, executeProxyRequest, isEndpointAllowed, matchRoute, type ToolContext, type ProxyRequestInput, type ProxyRequestResult, type FileAttachment, } from './tool-dispatch.js';
|
|
53
66
|
export declare function checkRateLimit(session: Pick<Session, 'windowRequests' | 'windowStart'>, limit: number): boolean;
|
|
54
67
|
export declare const SESSION_TTL: number;
|
|
55
68
|
export declare const HANDSHAKE_TTL: number;
|
|
@@ -57,36 +70,13 @@ export declare function cleanupSessions(sessionsMap: Map<string, Pick<Session, '
|
|
|
57
70
|
expiredSessions: string[];
|
|
58
71
|
expiredHandshakes: string[];
|
|
59
72
|
};
|
|
60
|
-
export interface ProxyRequestInput {
|
|
61
|
-
method: string;
|
|
62
|
-
url: string;
|
|
63
|
-
headers?: Record<string, string>;
|
|
64
|
-
body?: unknown;
|
|
65
|
-
}
|
|
66
|
-
export interface ProxyRequestResult {
|
|
67
|
-
status: number;
|
|
68
|
-
statusText: string;
|
|
69
|
-
headers: Record<string, string>;
|
|
70
|
-
body: unknown;
|
|
71
|
-
}
|
|
72
73
|
/**
|
|
73
|
-
*
|
|
74
|
+
* Project the active sessions into a sanitized snapshot for read-only use.
|
|
74
75
|
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
* - callboard's `LocalProxy` class (in-process, no encryption)
|
|
78
|
-
*
|
|
79
|
-
* Pure in the sense that it takes routes as input rather than reading global state.
|
|
80
|
-
* The only side effect is the outbound fetch().
|
|
76
|
+
* Drops `channel` (holds AES keys) and `resolvedRoutes` (carry decrypted
|
|
77
|
+
* secrets). Used by the loopback /admin API; never call res.json(session).
|
|
81
78
|
*/
|
|
82
|
-
export declare function
|
|
83
|
-
/** Context passed to every tool handler, providing caller identity and shared services. */
|
|
84
|
-
export interface ToolContext {
|
|
85
|
-
/** The caller alias for the session making this request. */
|
|
86
|
-
callerAlias: string;
|
|
87
|
-
/** The shared ingestor manager (for poll_events / ingestor_status). */
|
|
88
|
-
ingestorManager: IngestorManager;
|
|
89
|
-
}
|
|
79
|
+
export declare function getSessionsSnapshot(): SessionSnapshot[];
|
|
90
80
|
/** Options for creating the app — allows dependency injection for tests */
|
|
91
81
|
export interface CreateAppOptions {
|
|
92
82
|
/** Override config instead of loading from disk */
|
|
@@ -97,6 +87,8 @@ export interface CreateAppOptions {
|
|
|
97
87
|
authorizedPeers?: AuthorizedPeer[];
|
|
98
88
|
/** Override the ingestor manager instead of creating one from config */
|
|
99
89
|
ingestorManager?: IngestorManager;
|
|
90
|
+
/** Disable per-IP rate limiting (for tests that make many requests from localhost) */
|
|
91
|
+
disableRateLimiting?: boolean;
|
|
100
92
|
}
|
|
101
93
|
export declare function createApp(options?: CreateAppOptions): import("express-serve-static-core").Express;
|
|
102
94
|
export declare function main(): void;
|