@wolpertingerlabs/drawlatch 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONNECTIONS.md +197 -0
- package/INGESTORS.md +790 -0
- package/LICENSE +21 -0
- package/README.md +685 -0
- package/dist/cli/generate-keys.d.ts +15 -0
- package/dist/cli/generate-keys.js +107 -0
- package/dist/connections/anthropic.json +16 -0
- package/dist/connections/bluesky.json +26 -0
- package/dist/connections/devin.json +15 -0
- package/dist/connections/discord-bot.json +24 -0
- package/dist/connections/discord-oauth.json +16 -0
- package/dist/connections/github.json +25 -0
- package/dist/connections/google-ai.json +15 -0
- package/dist/connections/google.json +28 -0
- package/dist/connections/hex.json +14 -0
- package/dist/connections/lichess.json +15 -0
- package/dist/connections/linear.json +29 -0
- package/dist/connections/mastodon.json +25 -0
- package/dist/connections/notion.json +33 -0
- package/dist/connections/openai.json +16 -0
- package/dist/connections/openrouter.json +16 -0
- package/dist/connections/reddit.json +28 -0
- package/dist/connections/slack.json +23 -0
- package/dist/connections/stripe.json +25 -0
- package/dist/connections/telegram.json +26 -0
- package/dist/connections/trello.json +25 -0
- package/dist/connections/twitch.json +28 -0
- package/dist/connections/x.json +27 -0
- package/dist/mcp/server.d.ts +13 -0
- package/dist/mcp/server.js +258 -0
- package/dist/remote/ingestors/base-ingestor.d.ts +65 -0
- package/dist/remote/ingestors/base-ingestor.js +132 -0
- package/dist/remote/ingestors/discord/discord-gateway.d.ts +58 -0
- package/dist/remote/ingestors/discord/discord-gateway.js +341 -0
- package/dist/remote/ingestors/discord/index.d.ts +3 -0
- package/dist/remote/ingestors/discord/index.js +3 -0
- package/dist/remote/ingestors/discord/types.d.ts +56 -0
- package/dist/remote/ingestors/discord/types.js +68 -0
- package/dist/remote/ingestors/index.d.ts +16 -0
- package/dist/remote/ingestors/index.js +20 -0
- package/dist/remote/ingestors/manager.d.ts +65 -0
- package/dist/remote/ingestors/manager.js +201 -0
- package/dist/remote/ingestors/poll/index.d.ts +2 -0
- package/dist/remote/ingestors/poll/index.js +2 -0
- package/dist/remote/ingestors/poll/poll-ingestor.d.ts +78 -0
- package/dist/remote/ingestors/poll/poll-ingestor.js +283 -0
- package/dist/remote/ingestors/registry.d.ts +32 -0
- package/dist/remote/ingestors/registry.js +46 -0
- package/dist/remote/ingestors/ring-buffer.d.ts +33 -0
- package/dist/remote/ingestors/ring-buffer.js +62 -0
- package/dist/remote/ingestors/slack/index.d.ts +3 -0
- package/dist/remote/ingestors/slack/index.js +3 -0
- package/dist/remote/ingestors/slack/socket-mode.d.ts +48 -0
- package/dist/remote/ingestors/slack/socket-mode.js +267 -0
- package/dist/remote/ingestors/slack/types.d.ts +70 -0
- package/dist/remote/ingestors/slack/types.js +72 -0
- package/dist/remote/ingestors/types.d.ts +138 -0
- package/dist/remote/ingestors/types.js +13 -0
- package/dist/remote/ingestors/webhook/base-webhook-ingestor.d.ts +112 -0
- package/dist/remote/ingestors/webhook/base-webhook-ingestor.js +119 -0
- package/dist/remote/ingestors/webhook/github-types.d.ts +45 -0
- package/dist/remote/ingestors/webhook/github-types.js +65 -0
- package/dist/remote/ingestors/webhook/github-webhook-ingestor.d.ts +43 -0
- package/dist/remote/ingestors/webhook/github-webhook-ingestor.js +86 -0
- package/dist/remote/ingestors/webhook/index.d.ts +8 -0
- package/dist/remote/ingestors/webhook/index.js +12 -0
- package/dist/remote/ingestors/webhook/stripe-types.d.ts +57 -0
- package/dist/remote/ingestors/webhook/stripe-types.js +108 -0
- package/dist/remote/ingestors/webhook/stripe-webhook-ingestor.d.ts +47 -0
- package/dist/remote/ingestors/webhook/stripe-webhook-ingestor.js +90 -0
- package/dist/remote/ingestors/webhook/trello-types.d.ts +90 -0
- package/dist/remote/ingestors/webhook/trello-types.js +81 -0
- package/dist/remote/ingestors/webhook/trello-webhook-ingestor.d.ts +60 -0
- package/dist/remote/ingestors/webhook/trello-webhook-ingestor.js +126 -0
- package/dist/remote/server.d.ts +103 -0
- package/dist/remote/server.js +536 -0
- package/dist/shared/config.d.ts +213 -0
- package/dist/shared/config.js +269 -0
- package/dist/shared/connections.d.ts +72 -0
- package/dist/shared/connections.js +103 -0
- package/dist/shared/crypto/channel.d.ts +95 -0
- package/dist/shared/crypto/channel.js +175 -0
- package/dist/shared/crypto/index.d.ts +3 -0
- package/dist/shared/crypto/index.js +3 -0
- package/dist/shared/crypto/keys.d.ts +92 -0
- package/dist/shared/crypto/keys.js +143 -0
- package/dist/shared/logger.d.ts +30 -0
- package/dist/shared/logger.js +74 -0
- package/dist/shared/protocol/handshake.d.ts +116 -0
- package/dist/shared/protocol/handshake.js +214 -0
- package/dist/shared/protocol/index.d.ts +3 -0
- package/dist/shared/protocol/index.js +2 -0
- package/dist/shared/protocol/messages.d.ts +46 -0
- package/dist/shared/protocol/messages.js +8 -0
- package/package.json +105 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IngestorManager — owns and manages the lifecycle of all ingestor instances.
|
|
3
|
+
*
|
|
4
|
+
* Keyed by `callerAlias:connectionAlias`, so each caller gets its own
|
|
5
|
+
* ingestor instance (with its own secrets, buffer, and connection state).
|
|
6
|
+
* Multiple sessions from the same caller share the same ingestor/buffer.
|
|
7
|
+
*
|
|
8
|
+
* The manager is created once when the remote server starts, and provides
|
|
9
|
+
* event retrieval and status methods used by the `poll_events` and
|
|
10
|
+
* `ingestor_status` tool handlers.
|
|
11
|
+
*/
|
|
12
|
+
import { resolveCallerRoutes, resolveRoutes, resolveSecrets, } from '../../shared/config.js';
|
|
13
|
+
import { createLogger } from '../../shared/logger.js';
|
|
14
|
+
const log = createLogger('ingestor');
|
|
15
|
+
import { createIngestor } from './registry.js';
|
|
16
|
+
import { WebhookIngestor } from './webhook/base-webhook-ingestor.js';
|
|
17
|
+
// Import providers so they self-register their factories.
|
|
18
|
+
// Each provider calls registerIngestorFactory() at module load time.
|
|
19
|
+
import './discord/discord-gateway.js';
|
|
20
|
+
import './slack/socket-mode.js';
|
|
21
|
+
import './webhook/github-webhook-ingestor.js';
|
|
22
|
+
import './webhook/stripe-webhook-ingestor.js';
|
|
23
|
+
import './webhook/trello-webhook-ingestor.js';
|
|
24
|
+
import './poll/poll-ingestor.js';
|
|
25
|
+
export class IngestorManager {
|
|
26
|
+
config;
|
|
27
|
+
/** Active ingestor instances, keyed by `callerAlias:connectionAlias`. */
|
|
28
|
+
ingestors = new Map();
|
|
29
|
+
constructor(config) {
|
|
30
|
+
this.config = config;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Start ingestors for all callers whose connections have an `ingestor` config.
|
|
34
|
+
* Called once when the remote server starts listening.
|
|
35
|
+
*/
|
|
36
|
+
async startAll() {
|
|
37
|
+
for (const [callerAlias, callerConfig] of Object.entries(this.config.callers)) {
|
|
38
|
+
// Resolve routes for this caller (raw + resolved)
|
|
39
|
+
const rawRoutes = resolveCallerRoutes(this.config, callerAlias);
|
|
40
|
+
const callerEnvResolved = resolveSecrets(callerConfig.env ?? {});
|
|
41
|
+
const resolvedRoutes = resolveRoutes(rawRoutes, callerEnvResolved);
|
|
42
|
+
for (let i = 0; i < rawRoutes.length; i++) {
|
|
43
|
+
const rawRoute = rawRoutes[i];
|
|
44
|
+
const resolvedRoute = resolvedRoutes[i];
|
|
45
|
+
const connectionAlias = callerConfig.connections[i];
|
|
46
|
+
// Skip connections without an ingestor config
|
|
47
|
+
if (!rawRoute.ingestor)
|
|
48
|
+
continue;
|
|
49
|
+
// Get caller-level overrides for this connection
|
|
50
|
+
const overrides = callerConfig.ingestorOverrides?.[connectionAlias];
|
|
51
|
+
// Skip if explicitly disabled by caller
|
|
52
|
+
if (overrides?.disabled) {
|
|
53
|
+
log.info(`Skipping disabled ingestor for ${callerAlias}:${connectionAlias}`);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
const key = `${callerAlias}:${connectionAlias}`;
|
|
57
|
+
if (this.ingestors.has(key))
|
|
58
|
+
continue;
|
|
59
|
+
// Merge caller overrides into a copy of the template config
|
|
60
|
+
const effectiveConfig = IngestorManager.mergeIngestorConfig(rawRoute.ingestor, overrides);
|
|
61
|
+
// For poll ingestors, attach the resolved route headers so the factory
|
|
62
|
+
// can pass them through for authenticated HTTP requests.
|
|
63
|
+
if (effectiveConfig.type === 'poll') {
|
|
64
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any -- private property consumed by poll factory
|
|
65
|
+
effectiveConfig._resolvedRouteHeaders = resolvedRoute.headers;
|
|
66
|
+
}
|
|
67
|
+
const ingestor = createIngestor(connectionAlias, effectiveConfig, resolvedRoute.secrets, overrides?.bufferSize);
|
|
68
|
+
if (ingestor) {
|
|
69
|
+
this.ingestors.set(key, ingestor);
|
|
70
|
+
log.info(`Starting ${effectiveConfig.type} ingestor for ${key}`);
|
|
71
|
+
try {
|
|
72
|
+
await ingestor.start();
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
log.error(`Failed to start ${key}:`, err);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const count = this.ingestors.size;
|
|
81
|
+
if (count > 0) {
|
|
82
|
+
log.info(`${count} ingestor(s) started`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Stop all running ingestors. Called during graceful shutdown.
|
|
87
|
+
*/
|
|
88
|
+
async stopAll() {
|
|
89
|
+
const stops = Array.from(this.ingestors.entries()).map(async ([key, ingestor]) => {
|
|
90
|
+
log.info(`Stopping ${key}`);
|
|
91
|
+
try {
|
|
92
|
+
await ingestor.stop();
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
log.error(`Error stopping ${key}:`, err);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
await Promise.all(stops);
|
|
99
|
+
this.ingestors.clear();
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Get events for a specific caller and connection.
|
|
103
|
+
* @param callerAlias The caller whose events to retrieve.
|
|
104
|
+
* @param connectionAlias The connection to filter by.
|
|
105
|
+
* @param afterId Return events with id > afterId. Pass -1 for all.
|
|
106
|
+
*/
|
|
107
|
+
getEvents(callerAlias, connectionAlias, afterId = -1) {
|
|
108
|
+
const key = `${callerAlias}:${connectionAlias}`;
|
|
109
|
+
const ingestor = this.ingestors.get(key);
|
|
110
|
+
if (!ingestor)
|
|
111
|
+
return [];
|
|
112
|
+
return ingestor.getEvents(afterId);
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Get events across all ingestors for a caller, sorted chronologically.
|
|
116
|
+
* @param callerAlias The caller whose events to retrieve.
|
|
117
|
+
* @param afterId Return events with id > afterId. Pass -1 for all.
|
|
118
|
+
*/
|
|
119
|
+
getAllEvents(callerAlias, afterId = -1) {
|
|
120
|
+
const events = [];
|
|
121
|
+
const prefix = `${callerAlias}:`;
|
|
122
|
+
for (const [key, ingestor] of this.ingestors) {
|
|
123
|
+
if (key.startsWith(prefix)) {
|
|
124
|
+
events.push(...ingestor.getEvents(afterId));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
// Sort by receivedAt (ISO strings sort lexicographically)
|
|
128
|
+
events.sort((a, b) => a.receivedAt.localeCompare(b.receivedAt));
|
|
129
|
+
return events;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Get status of all ingestors for a caller.
|
|
133
|
+
*/
|
|
134
|
+
getStatuses(callerAlias) {
|
|
135
|
+
const statuses = [];
|
|
136
|
+
const prefix = `${callerAlias}:`;
|
|
137
|
+
for (const [key, ingestor] of this.ingestors) {
|
|
138
|
+
if (key.startsWith(prefix)) {
|
|
139
|
+
statuses.push(ingestor.getStatus());
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return statuses;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Find all webhook ingestor instances that match a given webhook path.
|
|
146
|
+
* Returns all matching instances across all callers (for fan-out dispatch).
|
|
147
|
+
*
|
|
148
|
+
* @param path - The webhook path segment (e.g., 'github' from /webhooks/github).
|
|
149
|
+
*/
|
|
150
|
+
getWebhookIngestors(path) {
|
|
151
|
+
const matches = [];
|
|
152
|
+
for (const ingestor of this.ingestors.values()) {
|
|
153
|
+
if (ingestor instanceof WebhookIngestor && ingestor.webhookPath === path) {
|
|
154
|
+
matches.push(ingestor);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return matches;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Merge caller-level ingestor overrides into a copy of the template config.
|
|
161
|
+
* Override fields replace template values; omitted fields inherit the template defaults.
|
|
162
|
+
*/
|
|
163
|
+
static mergeIngestorConfig(templateConfig, overrides) {
|
|
164
|
+
if (!overrides)
|
|
165
|
+
return templateConfig;
|
|
166
|
+
// Deep-copy to avoid mutating the shared template
|
|
167
|
+
const merged = {
|
|
168
|
+
type: templateConfig.type,
|
|
169
|
+
...(templateConfig.websocket && {
|
|
170
|
+
websocket: { ...templateConfig.websocket },
|
|
171
|
+
}),
|
|
172
|
+
...(templateConfig.webhook && {
|
|
173
|
+
webhook: { ...templateConfig.webhook },
|
|
174
|
+
}),
|
|
175
|
+
...(templateConfig.poll && {
|
|
176
|
+
poll: { ...templateConfig.poll },
|
|
177
|
+
}),
|
|
178
|
+
};
|
|
179
|
+
// Apply WebSocket-specific overrides
|
|
180
|
+
if (merged.websocket) {
|
|
181
|
+
const ws = merged.websocket;
|
|
182
|
+
if (overrides.intents !== undefined)
|
|
183
|
+
ws.intents = overrides.intents;
|
|
184
|
+
if (overrides.eventFilter !== undefined)
|
|
185
|
+
ws.eventFilter = overrides.eventFilter;
|
|
186
|
+
if (overrides.guildIds !== undefined)
|
|
187
|
+
ws.guildIds = overrides.guildIds;
|
|
188
|
+
if (overrides.channelIds !== undefined)
|
|
189
|
+
ws.channelIds = overrides.channelIds;
|
|
190
|
+
if (overrides.userIds !== undefined)
|
|
191
|
+
ws.userIds = overrides.userIds;
|
|
192
|
+
}
|
|
193
|
+
// Apply poll-specific overrides
|
|
194
|
+
if (merged.poll) {
|
|
195
|
+
if (overrides.intervalMs !== undefined)
|
|
196
|
+
merged.poll.intervalMs = overrides.intervalMs;
|
|
197
|
+
}
|
|
198
|
+
return merged;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=manager.js.map
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Polling ingestor — interval-based HTTP poller.
|
|
3
|
+
*
|
|
4
|
+
* Periodically hits an HTTP endpoint using the connection's resolved
|
|
5
|
+
* secrets and headers, extracts individual items from the response,
|
|
6
|
+
* deduplicates by a configurable field, and pushes new items into
|
|
7
|
+
* the ring buffer.
|
|
8
|
+
*
|
|
9
|
+
* Unlike WebSocket ingestors (which maintain persistent connections)
|
|
10
|
+
* or webhook ingestors (which are passive receivers), poll ingestors
|
|
11
|
+
* are active requesters on a timer.
|
|
12
|
+
*
|
|
13
|
+
* Designed as a single concrete class — all service-specific behavior
|
|
14
|
+
* (response shape, event type, deduplication field) is parameterized
|
|
15
|
+
* via `PollIngestorConfig` rather than requiring subclasses.
|
|
16
|
+
*
|
|
17
|
+
* @see https://developers.notion.com/reference (Notion — first implementor)
|
|
18
|
+
* @see https://developers.linear.app/docs/graphql/working-with-the-graphql-api (Linear — first implementor)
|
|
19
|
+
*/
|
|
20
|
+
import { BaseIngestor } from '../base-ingestor.js';
|
|
21
|
+
import type { PollIngestorConfig } from '../types.js';
|
|
22
|
+
export declare class PollIngestor extends BaseIngestor {
|
|
23
|
+
private pollTimer;
|
|
24
|
+
private readonly seenIds;
|
|
25
|
+
private consecutiveErrors;
|
|
26
|
+
private readonly url;
|
|
27
|
+
private readonly intervalMs;
|
|
28
|
+
private readonly method;
|
|
29
|
+
private readonly body;
|
|
30
|
+
private readonly deduplicateBy;
|
|
31
|
+
private readonly responsePath;
|
|
32
|
+
private readonly eventType;
|
|
33
|
+
private readonly pollHeaders;
|
|
34
|
+
/** Resolved headers from the parent connection route (injected by manager). */
|
|
35
|
+
private readonly routeHeaders;
|
|
36
|
+
constructor(connectionAlias: string, secrets: Record<string, string>, pollConfig: PollIngestorConfig,
|
|
37
|
+
/** Pre-resolved headers from the connection's route. */
|
|
38
|
+
routeHeaders: Record<string, string>, bufferSize?: number);
|
|
39
|
+
start(): Promise<void>;
|
|
40
|
+
stop(): Promise<void>;
|
|
41
|
+
private poll;
|
|
42
|
+
/**
|
|
43
|
+
* Extract items from the response using the configured responsePath.
|
|
44
|
+
* E.g., "results" extracts response.results,
|
|
45
|
+
* "data.issues.nodes" extracts response.data.issues.nodes.
|
|
46
|
+
* If no responsePath, returns the response itself (expects a top-level array).
|
|
47
|
+
*/
|
|
48
|
+
private extractItems;
|
|
49
|
+
/**
|
|
50
|
+
* Resolve a dot-separated path on an object.
|
|
51
|
+
* E.g., "data.name" on { data: { name: "t3_abc" } } → "t3_abc".
|
|
52
|
+
* Single-segment paths (e.g., "id") resolve as a simple property lookup.
|
|
53
|
+
*/
|
|
54
|
+
private static resolveNestedPath;
|
|
55
|
+
/**
|
|
56
|
+
* Check if an item should be pushed into the buffer (dedup check).
|
|
57
|
+
* Returns true if the item is new (not previously seen).
|
|
58
|
+
* Supports dot-separated paths in `deduplicateBy` (e.g., "data.name").
|
|
59
|
+
*/
|
|
60
|
+
private shouldPush;
|
|
61
|
+
/**
|
|
62
|
+
* Prune the seen IDs set to prevent unbounded memory growth.
|
|
63
|
+
* Removes the oldest half of entries. Since Set preserves insertion order,
|
|
64
|
+
* we can iterate and delete the first N entries.
|
|
65
|
+
*/
|
|
66
|
+
private pruneSeenIds;
|
|
67
|
+
/**
|
|
68
|
+
* Extract an idempotency key from a poll item using the `deduplicateBy` field.
|
|
69
|
+
* Supports dot-separated paths (e.g., "data.name").
|
|
70
|
+
* Returns `undefined` if no dedup field is configured or the field is absent.
|
|
71
|
+
*/
|
|
72
|
+
private extractItemIdempotencyKey;
|
|
73
|
+
/**
|
|
74
|
+
* Resolve ${VAR} placeholders in a string using a secrets map.
|
|
75
|
+
*/
|
|
76
|
+
static resolvePlaceholders(str: string, secrets: Record<string, string>): string;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=poll-ingestor.d.ts.map
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Polling ingestor — interval-based HTTP poller.
|
|
3
|
+
*
|
|
4
|
+
* Periodically hits an HTTP endpoint using the connection's resolved
|
|
5
|
+
* secrets and headers, extracts individual items from the response,
|
|
6
|
+
* deduplicates by a configurable field, and pushes new items into
|
|
7
|
+
* the ring buffer.
|
|
8
|
+
*
|
|
9
|
+
* Unlike WebSocket ingestors (which maintain persistent connections)
|
|
10
|
+
* or webhook ingestors (which are passive receivers), poll ingestors
|
|
11
|
+
* are active requesters on a timer.
|
|
12
|
+
*
|
|
13
|
+
* Designed as a single concrete class — all service-specific behavior
|
|
14
|
+
* (response shape, event type, deduplication field) is parameterized
|
|
15
|
+
* via `PollIngestorConfig` rather than requiring subclasses.
|
|
16
|
+
*
|
|
17
|
+
* @see https://developers.notion.com/reference (Notion — first implementor)
|
|
18
|
+
* @see https://developers.linear.app/docs/graphql/working-with-the-graphql-api (Linear — first implementor)
|
|
19
|
+
*/
|
|
20
|
+
import { BaseIngestor } from '../base-ingestor.js';
|
|
21
|
+
import { registerIngestorFactory } from '../registry.js';
|
|
22
|
+
import { createLogger } from '../../../shared/logger.js';
|
|
23
|
+
const log = createLogger('poll');
|
|
24
|
+
// ── Constants ─────────────────────────────────────────────────────────
|
|
25
|
+
/** Maximum number of seen IDs to track for deduplication.
|
|
26
|
+
* When exceeded, oldest entries are pruned. */
|
|
27
|
+
const MAX_SEEN_IDS = 10_000;
|
|
28
|
+
/** Minimum allowed poll interval (to prevent accidental API flooding). */
|
|
29
|
+
const MIN_INTERVAL_MS = 5_000; // 5 seconds
|
|
30
|
+
/** Maximum consecutive errors before transitioning to 'error' state. */
|
|
31
|
+
const MAX_CONSECUTIVE_ERRORS = 10;
|
|
32
|
+
// ── Poll Ingestor ─────────────────────────────────────────────────────
|
|
33
|
+
export class PollIngestor extends BaseIngestor {
|
|
34
|
+
pollTimer = null;
|
|
35
|
+
seenIds = new Set();
|
|
36
|
+
consecutiveErrors = 0;
|
|
37
|
+
url;
|
|
38
|
+
intervalMs;
|
|
39
|
+
method;
|
|
40
|
+
body;
|
|
41
|
+
deduplicateBy;
|
|
42
|
+
responsePath;
|
|
43
|
+
eventType;
|
|
44
|
+
pollHeaders;
|
|
45
|
+
/** Resolved headers from the parent connection route (injected by manager). */
|
|
46
|
+
routeHeaders;
|
|
47
|
+
constructor(connectionAlias, secrets, pollConfig,
|
|
48
|
+
/** Pre-resolved headers from the connection's route. */
|
|
49
|
+
routeHeaders, bufferSize) {
|
|
50
|
+
super(connectionAlias, 'poll', secrets, bufferSize);
|
|
51
|
+
// Resolve ${VAR} placeholders in URL
|
|
52
|
+
this.url = PollIngestor.resolvePlaceholders(pollConfig.url, secrets);
|
|
53
|
+
this.intervalMs = Math.max(pollConfig.intervalMs, MIN_INTERVAL_MS);
|
|
54
|
+
this.method = (pollConfig.method ?? 'GET').toUpperCase();
|
|
55
|
+
this.body = pollConfig.body;
|
|
56
|
+
this.deduplicateBy = pollConfig.deduplicateBy;
|
|
57
|
+
this.responsePath = pollConfig.responsePath;
|
|
58
|
+
this.eventType = pollConfig.eventType ?? 'poll';
|
|
59
|
+
this.routeHeaders = routeHeaders;
|
|
60
|
+
// Resolve ${VAR} placeholders in poll-specific headers
|
|
61
|
+
this.pollHeaders = {};
|
|
62
|
+
if (pollConfig.headers) {
|
|
63
|
+
for (const [k, v] of Object.entries(pollConfig.headers)) {
|
|
64
|
+
this.pollHeaders[k] = PollIngestor.resolvePlaceholders(v, secrets);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
// ── Lifecycle ──────────────────────────────────────────────────────
|
|
69
|
+
async start() {
|
|
70
|
+
this.state = 'starting';
|
|
71
|
+
log.info(`Starting poll ingestor for ${this.connectionAlias} ` +
|
|
72
|
+
`(${this.method} ${this.url}, every ${this.intervalMs}ms)`);
|
|
73
|
+
// Do an initial poll immediately
|
|
74
|
+
await this.poll();
|
|
75
|
+
// Then set up the recurring interval
|
|
76
|
+
this.pollTimer = setInterval(() => {
|
|
77
|
+
void this.poll();
|
|
78
|
+
}, this.intervalMs);
|
|
79
|
+
// If the initial poll succeeded, state is already 'connected'.
|
|
80
|
+
// If it failed, state is 'reconnecting' — the timer will retry.
|
|
81
|
+
}
|
|
82
|
+
stop() {
|
|
83
|
+
this.state = 'stopped';
|
|
84
|
+
if (this.pollTimer) {
|
|
85
|
+
clearInterval(this.pollTimer);
|
|
86
|
+
this.pollTimer = null;
|
|
87
|
+
}
|
|
88
|
+
return Promise.resolve();
|
|
89
|
+
}
|
|
90
|
+
// ── Core poll logic ──────────────────────────────────────────────
|
|
91
|
+
async poll() {
|
|
92
|
+
try {
|
|
93
|
+
// Build headers: route headers (from connection template) merged with poll-specific headers
|
|
94
|
+
const headers = {
|
|
95
|
+
...this.routeHeaders,
|
|
96
|
+
...this.pollHeaders,
|
|
97
|
+
};
|
|
98
|
+
// Build request options
|
|
99
|
+
const fetchOptions = {
|
|
100
|
+
method: this.method,
|
|
101
|
+
headers,
|
|
102
|
+
};
|
|
103
|
+
// Add body for POST/PUT/PATCH
|
|
104
|
+
if (this.body !== undefined && this.method !== 'GET' && this.method !== 'HEAD') {
|
|
105
|
+
if (typeof this.body === 'string') {
|
|
106
|
+
fetchOptions.body = PollIngestor.resolvePlaceholders(this.body, this.secrets);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const serialized = JSON.stringify(this.body);
|
|
110
|
+
fetchOptions.body = PollIngestor.resolvePlaceholders(serialized, this.secrets);
|
|
111
|
+
// Ensure Content-Type is set for JSON bodies
|
|
112
|
+
if (!headers['Content-Type'] && !headers['content-type']) {
|
|
113
|
+
headers['Content-Type'] = 'application/json';
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const response = await fetch(this.url, fetchOptions);
|
|
118
|
+
if (!response.ok) {
|
|
119
|
+
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
|
120
|
+
}
|
|
121
|
+
const responseBody = await response.json();
|
|
122
|
+
// Extract items array from response using responsePath
|
|
123
|
+
const items = this.extractItems(responseBody);
|
|
124
|
+
if (!Array.isArray(items)) {
|
|
125
|
+
throw new Error(`Expected array at responsePath "${this.responsePath ?? '(root)'}", got ${typeof items}`);
|
|
126
|
+
}
|
|
127
|
+
// Process each item
|
|
128
|
+
let newItemCount = 0;
|
|
129
|
+
for (const item of items) {
|
|
130
|
+
if (this.shouldPush(item)) {
|
|
131
|
+
const idempotencyKey = this.extractItemIdempotencyKey(item);
|
|
132
|
+
this.pushEvent(this.eventType, item, idempotencyKey);
|
|
133
|
+
newItemCount++;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
// Success — reset error state
|
|
137
|
+
this.consecutiveErrors = 0;
|
|
138
|
+
if (this.state !== 'connected') {
|
|
139
|
+
this.state = 'connected';
|
|
140
|
+
}
|
|
141
|
+
if (newItemCount > 0) {
|
|
142
|
+
log.info(`${this.connectionAlias}: ${newItemCount} new item(s) from ${items.length} total`);
|
|
143
|
+
}
|
|
144
|
+
log.debug(`${this.connectionAlias}: poll complete — ${items.length} items, ${newItemCount} new`);
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
this.consecutiveErrors++;
|
|
148
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
149
|
+
this.errorMessage = message;
|
|
150
|
+
if (this.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
|
151
|
+
this.state = 'error';
|
|
152
|
+
log.error(`${this.connectionAlias}: ${MAX_CONSECUTIVE_ERRORS} consecutive errors, giving up: ${message}`);
|
|
153
|
+
// Stop the timer on permanent error
|
|
154
|
+
if (this.pollTimer) {
|
|
155
|
+
clearInterval(this.pollTimer);
|
|
156
|
+
this.pollTimer = null;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
this.state = 'reconnecting';
|
|
161
|
+
log.warn(`${this.connectionAlias}: poll failed (attempt ${this.consecutiveErrors}/${MAX_CONSECUTIVE_ERRORS}): ${message}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// ── Response parsing ──────────────────────────────────────────────
|
|
166
|
+
/**
|
|
167
|
+
* Extract items from the response using the configured responsePath.
|
|
168
|
+
* E.g., "results" extracts response.results,
|
|
169
|
+
* "data.issues.nodes" extracts response.data.issues.nodes.
|
|
170
|
+
* If no responsePath, returns the response itself (expects a top-level array).
|
|
171
|
+
*/
|
|
172
|
+
extractItems(responseBody) {
|
|
173
|
+
if (!this.responsePath)
|
|
174
|
+
return responseBody;
|
|
175
|
+
const parts = this.responsePath.split('.');
|
|
176
|
+
let current = responseBody;
|
|
177
|
+
for (const part of parts) {
|
|
178
|
+
if (current === null || current === undefined || typeof current !== 'object') {
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
current = current[part];
|
|
182
|
+
}
|
|
183
|
+
return current;
|
|
184
|
+
}
|
|
185
|
+
// ── Deduplication ──────────────────────────────────────────────────
|
|
186
|
+
/**
|
|
187
|
+
* Resolve a dot-separated path on an object.
|
|
188
|
+
* E.g., "data.name" on { data: { name: "t3_abc" } } → "t3_abc".
|
|
189
|
+
* Single-segment paths (e.g., "id") resolve as a simple property lookup.
|
|
190
|
+
*/
|
|
191
|
+
static resolveNestedPath(obj, dotPath) {
|
|
192
|
+
if (obj === null || obj === undefined || typeof obj !== 'object')
|
|
193
|
+
return undefined;
|
|
194
|
+
const parts = dotPath.split('.');
|
|
195
|
+
let current = obj;
|
|
196
|
+
for (const part of parts) {
|
|
197
|
+
if (current === null || current === undefined || typeof current !== 'object') {
|
|
198
|
+
return undefined;
|
|
199
|
+
}
|
|
200
|
+
current = current[part];
|
|
201
|
+
}
|
|
202
|
+
return current;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Check if an item should be pushed into the buffer (dedup check).
|
|
206
|
+
* Returns true if the item is new (not previously seen).
|
|
207
|
+
* Supports dot-separated paths in `deduplicateBy` (e.g., "data.name").
|
|
208
|
+
*/
|
|
209
|
+
shouldPush(item) {
|
|
210
|
+
if (!this.deduplicateBy)
|
|
211
|
+
return true;
|
|
212
|
+
const idValue = PollIngestor.resolveNestedPath(item, this.deduplicateBy);
|
|
213
|
+
if (idValue === undefined || idValue === null)
|
|
214
|
+
return true;
|
|
215
|
+
// Ensure safe stringification: only accept primitives for dedup IDs
|
|
216
|
+
const id = typeof idValue === 'string' || typeof idValue === 'number' || typeof idValue === 'boolean'
|
|
217
|
+
? String(idValue)
|
|
218
|
+
: JSON.stringify(idValue);
|
|
219
|
+
if (this.seenIds.has(id))
|
|
220
|
+
return false;
|
|
221
|
+
// Add to seen set, prune if over limit
|
|
222
|
+
this.seenIds.add(id);
|
|
223
|
+
if (this.seenIds.size > MAX_SEEN_IDS) {
|
|
224
|
+
this.pruneSeenIds();
|
|
225
|
+
}
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Prune the seen IDs set to prevent unbounded memory growth.
|
|
230
|
+
* Removes the oldest half of entries. Since Set preserves insertion order,
|
|
231
|
+
* we can iterate and delete the first N entries.
|
|
232
|
+
*/
|
|
233
|
+
pruneSeenIds() {
|
|
234
|
+
const pruneCount = Math.floor(this.seenIds.size / 2);
|
|
235
|
+
let removed = 0;
|
|
236
|
+
for (const id of this.seenIds) {
|
|
237
|
+
if (removed >= pruneCount)
|
|
238
|
+
break;
|
|
239
|
+
this.seenIds.delete(id);
|
|
240
|
+
removed++;
|
|
241
|
+
}
|
|
242
|
+
log.debug(`${this.connectionAlias}: pruned ${removed} seen IDs (${this.seenIds.size} remaining)`);
|
|
243
|
+
}
|
|
244
|
+
// ── Idempotency ─────────────────────────────────────────────────
|
|
245
|
+
/**
|
|
246
|
+
* Extract an idempotency key from a poll item using the `deduplicateBy` field.
|
|
247
|
+
* Supports dot-separated paths (e.g., "data.name").
|
|
248
|
+
* Returns `undefined` if no dedup field is configured or the field is absent.
|
|
249
|
+
*/
|
|
250
|
+
extractItemIdempotencyKey(item) {
|
|
251
|
+
if (!this.deduplicateBy)
|
|
252
|
+
return undefined;
|
|
253
|
+
const idValue = PollIngestor.resolveNestedPath(item, this.deduplicateBy);
|
|
254
|
+
if (idValue === undefined || idValue === null)
|
|
255
|
+
return undefined;
|
|
256
|
+
const id = typeof idValue === 'string' || typeof idValue === 'number' || typeof idValue === 'boolean'
|
|
257
|
+
? String(idValue)
|
|
258
|
+
: JSON.stringify(idValue);
|
|
259
|
+
return `poll:${this.connectionAlias}:${id}`;
|
|
260
|
+
}
|
|
261
|
+
// ── Helpers ────────────────────────────────────────────────────────
|
|
262
|
+
/**
|
|
263
|
+
* Resolve ${VAR} placeholders in a string using a secrets map.
|
|
264
|
+
*/
|
|
265
|
+
static resolvePlaceholders(str, secrets) {
|
|
266
|
+
return str.replace(/\$\{(\w+)\}/g, (match, name) => {
|
|
267
|
+
if (name in secrets)
|
|
268
|
+
return secrets[name];
|
|
269
|
+
return match;
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
// ── Self-registration ──────────────────────────────────────────────────
|
|
274
|
+
registerIngestorFactory('poll', (connectionAlias, config, secrets, bufferSize) => {
|
|
275
|
+
if (!config.poll) {
|
|
276
|
+
log.error(`Missing poll config for ${connectionAlias}`);
|
|
277
|
+
return null;
|
|
278
|
+
}
|
|
279
|
+
return new PollIngestor(connectionAlias, secrets, config.poll,
|
|
280
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unnecessary-condition -- route headers injected by manager via private property; may be absent
|
|
281
|
+
config._resolvedRouteHeaders ?? {}, bufferSize);
|
|
282
|
+
});
|
|
283
|
+
//# sourceMappingURL=poll-ingestor.js.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ingestor factory registry.
|
|
3
|
+
*
|
|
4
|
+
* Decouples the IngestorManager from specific ingestor implementations.
|
|
5
|
+
* Each provider (Discord, Slack, webhook, poll) registers its own factory
|
|
6
|
+
* function at module load time; the manager calls `createIngestor()` without
|
|
7
|
+
* knowing which concrete classes exist.
|
|
8
|
+
*/
|
|
9
|
+
import type { BaseIngestor } from './base-ingestor.js';
|
|
10
|
+
import type { IngestorConfig } from './types.js';
|
|
11
|
+
/** Signature for a factory that creates an ingestor from its config. */
|
|
12
|
+
export type IngestorFactory = (connectionAlias: string, config: IngestorConfig, secrets: Record<string, string>, bufferSize?: number) => BaseIngestor | null;
|
|
13
|
+
/**
|
|
14
|
+
* Register a factory for a given ingestor key.
|
|
15
|
+
*
|
|
16
|
+
* Convention for keys:
|
|
17
|
+
* - WebSocket protocols: `websocket:<protocol>` (e.g., `websocket:discord`, `websocket:slack`)
|
|
18
|
+
* - Webhook protocols: `webhook:<protocol>` (e.g., `webhook:generic` for GitHub, `webhook:stripe`)
|
|
19
|
+
* - Other types: the type name directly (e.g., `poll`)
|
|
20
|
+
*/
|
|
21
|
+
export declare function registerIngestorFactory(key: string, factory: IngestorFactory): void;
|
|
22
|
+
/**
|
|
23
|
+
* Create an ingestor instance using the registered factory for its config type.
|
|
24
|
+
*
|
|
25
|
+
* For WebSocket ingestors, the key is `websocket:<protocol>`.
|
|
26
|
+
* For Webhook ingestors, the key is `webhook:<protocol>` (default: `webhook:generic`).
|
|
27
|
+
* For other types, the key is just the type name.
|
|
28
|
+
*
|
|
29
|
+
* Returns `null` if no factory is registered or if the factory declines to create.
|
|
30
|
+
*/
|
|
31
|
+
export declare function createIngestor(connectionAlias: string, config: IngestorConfig, secrets: Record<string, string>, bufferSize?: number): BaseIngestor | null;
|
|
32
|
+
//# sourceMappingURL=registry.d.ts.map
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ingestor factory registry.
|
|
3
|
+
*
|
|
4
|
+
* Decouples the IngestorManager from specific ingestor implementations.
|
|
5
|
+
* Each provider (Discord, Slack, webhook, poll) registers its own factory
|
|
6
|
+
* function at module load time; the manager calls `createIngestor()` without
|
|
7
|
+
* knowing which concrete classes exist.
|
|
8
|
+
*/
|
|
9
|
+
import { createLogger } from '../../shared/logger.js';
|
|
10
|
+
const log = createLogger('ingestor');
|
|
11
|
+
/** Registered factories keyed by type string (e.g., 'websocket:discord'). */
|
|
12
|
+
const factories = new Map();
|
|
13
|
+
/**
|
|
14
|
+
* Register a factory for a given ingestor key.
|
|
15
|
+
*
|
|
16
|
+
* Convention for keys:
|
|
17
|
+
* - WebSocket protocols: `websocket:<protocol>` (e.g., `websocket:discord`, `websocket:slack`)
|
|
18
|
+
* - Webhook protocols: `webhook:<protocol>` (e.g., `webhook:generic` for GitHub, `webhook:stripe`)
|
|
19
|
+
* - Other types: the type name directly (e.g., `poll`)
|
|
20
|
+
*/
|
|
21
|
+
export function registerIngestorFactory(key, factory) {
|
|
22
|
+
factories.set(key, factory);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Create an ingestor instance using the registered factory for its config type.
|
|
26
|
+
*
|
|
27
|
+
* For WebSocket ingestors, the key is `websocket:<protocol>`.
|
|
28
|
+
* For Webhook ingestors, the key is `webhook:<protocol>` (default: `webhook:generic`).
|
|
29
|
+
* For other types, the key is just the type name.
|
|
30
|
+
*
|
|
31
|
+
* Returns `null` if no factory is registered or if the factory declines to create.
|
|
32
|
+
*/
|
|
33
|
+
export function createIngestor(connectionAlias, config, secrets, bufferSize) {
|
|
34
|
+
const key = config.type === 'websocket'
|
|
35
|
+
? `websocket:${config.websocket?.protocol ?? 'generic'}`
|
|
36
|
+
: config.type === 'webhook'
|
|
37
|
+
? `webhook:${config.webhook?.protocol ?? 'generic'}`
|
|
38
|
+
: config.type;
|
|
39
|
+
const factory = factories.get(key);
|
|
40
|
+
if (!factory) {
|
|
41
|
+
log.error(`No factory registered for "${key}" (${connectionAlias})`);
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
return factory(connectionAlias, config, secrets, bufferSize);
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=registry.js.map
|