@hogsend/engine 0.24.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hogsend/engine",
3
- "version": "0.24.0",
3
+ "version": "0.25.0",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -40,14 +40,14 @@
40
40
  "svix": "^1.95.1",
41
41
  "winston": "^3.19.0",
42
42
  "zod": "^4.4.3",
43
- "@hogsend/core": "^0.24.0",
44
- "@hogsend/db": "^0.24.0",
45
- "@hogsend/email": "^0.24.0",
46
- "@hogsend/plugin-posthog": "^0.24.0",
47
- "@hogsend/plugin-resend": "^0.24.0"
43
+ "@hogsend/core": "^0.25.0",
44
+ "@hogsend/db": "^0.25.0",
45
+ "@hogsend/email": "^0.25.0",
46
+ "@hogsend/plugin-posthog": "^0.25.0",
47
+ "@hogsend/plugin-resend": "^0.25.0"
48
48
  },
49
49
  "optionalDependencies": {
50
- "@hogsend/plugin-postmark": "^0.24.0"
50
+ "@hogsend/plugin-postmark": "^0.25.0"
51
51
  },
52
52
  "devDependencies": {
53
53
  "@types/node": "^22.15.3",
@@ -0,0 +1,52 @@
1
+ import type { DefinedConnectorAction } from "./define-action.js";
2
+
3
+ /**
4
+ * The process-wide connector-action registry, set once by `createHogsendClient`.
5
+ * Mirrors {@link ConnectorRegistry}: keyed by `${connectorId}:${name}`,
6
+ * last-writer-wins, with a lazy empty fallback so a self-booting context (the
7
+ * standalone {@link sendConnectorAction}) resolves cleanly even before any
8
+ * container ran (it just finds no actions).
9
+ */
10
+ export class ConnectorActionRegistry {
11
+ private readonly byKey = new Map<string, DefinedConnectorAction>();
12
+
13
+ constructor(actions: DefinedConnectorAction[] = []) {
14
+ for (const action of actions) this.register(action);
15
+ }
16
+
17
+ register(action: DefinedConnectorAction): void {
18
+ this.byKey.set(`${action.connectorId}:${action.name}`, action);
19
+ }
20
+
21
+ get(connectorId: string, name: string): DefinedConnectorAction | undefined {
22
+ return this.byKey.get(`${connectorId}:${name}`);
23
+ }
24
+
25
+ getAll(): DefinedConnectorAction[] {
26
+ return [...this.byKey.values()];
27
+ }
28
+
29
+ count(): number {
30
+ return this.byKey.size;
31
+ }
32
+ }
33
+
34
+ let installed: ConnectorActionRegistry | undefined;
35
+ let fallback: ConnectorActionRegistry | undefined;
36
+
37
+ export function setConnectorActionRegistry(
38
+ registry: ConnectorActionRegistry,
39
+ ): void {
40
+ installed = registry;
41
+ }
42
+
43
+ export function getConnectorActionRegistry(): ConnectorActionRegistry {
44
+ if (installed) return installed;
45
+ if (!fallback) fallback = new ConnectorActionRegistry();
46
+ return fallback;
47
+ }
48
+
49
+ /** Reset the installed registry — only for test cleanup. */
50
+ export function resetConnectorActionRegistry(): void {
51
+ installed = undefined;
52
+ }
@@ -0,0 +1,55 @@
1
+ import type { Database } from "@hogsend/db";
2
+ import type { Logger } from "../lib/logger.js";
3
+
4
+ /**
5
+ * Connector OUTBOUND ACTIONS — the journey-callable, socket-free face of a
6
+ * connector. Distinct from event fan-out (the durable `emitOutbound` →
7
+ * destination spine): an action is an IMPERATIVE call a journey/workflow makes
8
+ * ("post this to #general", "@mention these members", "DM this user"). On
9
+ * Discord these are pure bot-REST calls needing only the bot token — so they run
10
+ * on ANY replica and are INDEPENDENT of the inbound gateway socket (a deployment
11
+ * with the gateway off can still send).
12
+ *
13
+ * A platform plugin contributes actions (e.g. `discordActions`); the consumer
14
+ * registers them via `createHogsendClient({ connectorActions })`. A journey
15
+ * invokes one through the standalone {@link sendConnectorAction} export (NOT on
16
+ * `ctx` — features are standalone imports, mirroring `sendEmail()`).
17
+ */
18
+ /** A contact resolved for an outbound action — a platform-neutral projection. */
19
+ export interface ResolvedActionContact {
20
+ id: string;
21
+ email: string | null;
22
+ discordId: string | null;
23
+ externalId: string | null;
24
+ properties: Record<string, unknown>;
25
+ }
26
+
27
+ export interface ConnectorActionCtx {
28
+ db: Database;
29
+ logger: Logger;
30
+ /**
31
+ * Resolve a contact by email, external id, or a platform id (e.g. a Discord
32
+ * snowflake). The engine owns the contacts schema, so a plugin action resolves
33
+ * a recipient WITHOUT coupling to `@hogsend/db`. Null when no live contact
34
+ * matches.
35
+ */
36
+ resolveContact(ref: string): Promise<ResolvedActionContact | null>;
37
+ }
38
+
39
+ export interface DefinedConnectorAction<A = unknown, R = unknown> {
40
+ /** The connector this action belongs to (e.g. "discord"). Keys the registry. */
41
+ connectorId: string;
42
+ /** Action name, unique within the connector (e.g. "sendChannelMessage"). */
43
+ name: string;
44
+ /** Optional human description (Studio enumeration / docs). */
45
+ description?: string;
46
+ /** Perform the outbound action. Single-object-in, result-out. */
47
+ run(args: A, ctx: ConnectorActionCtx): Promise<R>;
48
+ }
49
+
50
+ /** Identity/validating authoring helper (mirrors `defineConnector`/`defineDestination`). */
51
+ export function defineConnectorAction<A, R>(
52
+ def: DefinedConnectorAction<A, R>,
53
+ ): DefinedConnectorAction<A, R> {
54
+ return def;
55
+ }
@@ -0,0 +1,291 @@
1
+ import type { HogsendClient } from "../container.js";
2
+ import {
3
+ type ConnectorHeartbeatHandle,
4
+ startConnectorHeartbeat,
5
+ } from "../lib/connector-heartbeat.js";
6
+ import { ingestEvent } from "../lib/ingestion.js";
7
+ import {
8
+ acquireLeaderLease,
9
+ newLeaseToken,
10
+ releaseLeaderLease,
11
+ renewLeaderLease,
12
+ } from "../lib/leader-lease.js";
13
+ import type { Logger } from "../lib/logger.js";
14
+ import type { DefinedConnector } from "./define-connector.js";
15
+
16
+ /**
17
+ * The connector-agnostic INBOUND RUNTIME seam. A `gateway`-transport connector
18
+ * (Discord today, Slack tomorrow) needs a long-lived process holding a socket to
19
+ * the platform. Rather than a hand-wired standalone service per consumer, the
20
+ * engine boots that runtime INLINE inside the process every consumer already
21
+ * runs (the Hatchet worker by default), gated by a Redis LEADER LEASE so exactly
22
+ * ONE replica ever holds the socket — one bot token permits one live session.
23
+ *
24
+ * A platform plugin contributes a {@link ConnectorRuntimeFactory} (supplied to
25
+ * `createWorker({ connectorRuntimes })`); the engine owns everything
26
+ * platform-neutral: lease election, the in-process dispatch→transform→ingest
27
+ * sink (no HTTP hop, no shared secret), the owned heartbeat, and shutdown
28
+ * ordering. A second connector reuses this verbatim — it only writes a
29
+ * `defineConnector` + a runtime factory, and touches zero engine code.
30
+ */
31
+
32
+ /** The minimal long-lived runtime a platform plugin supplies (e.g. a discord.js socket). */
33
+ export interface ConnectorRuntime {
34
+ /** Open the socket. Rejects loudly on a fatal config error (bad token/intents). */
35
+ start(): Promise<void>;
36
+ /** Close the socket and clear timers (awaited on demotion / shutdown). */
37
+ stop(): Promise<void>;
38
+ /** Platform metadata to fold into the heartbeat (e.g. `{ intents }`). */
39
+ getMetadata(): Record<string, unknown>;
40
+ }
41
+
42
+ /**
43
+ * What the engine injects into a runtime factory. `ingest` is the in-process
44
+ * sink: hand it a raw platform dispatch and the engine runs the connector's own
45
+ * `transform` then `ingestEvent` — the EXACT pair the HTTP ingress route runs,
46
+ * minus the network hop. `onMetadata` folds a late-observed field (e.g. the
47
+ * guild id seen at GUILD_CREATE) into the live heartbeat.
48
+ */
49
+ export interface ConnectorRuntimeDeps {
50
+ ingest(
51
+ dispatchType: string,
52
+ data: unknown,
53
+ ): Promise<{ ok: boolean; status: number }>;
54
+ onMetadata(patch: Record<string, unknown>): void;
55
+ logger: Logger;
56
+ }
57
+
58
+ /**
59
+ * Build a runtime for a connector, or return `null` when it is not configured to
60
+ * run here (e.g. the platform's bot token env is unset) — the engine then simply
61
+ * skips it without holding a lease.
62
+ */
63
+ export type ConnectorRuntimeFactory = (
64
+ deps: ConnectorRuntimeDeps,
65
+ ) => ConnectorRuntime | null;
66
+
67
+ export interface ConnectorRuntimesHandle {
68
+ /** Release every lease + delete heartbeats BEFORE stopping sockets. */
69
+ stop(): Promise<void>;
70
+ }
71
+
72
+ const LEASE_TTL_MS = 30_000;
73
+ const RENEW_MS = 10_000;
74
+ const ELECT_MS = 5_000;
75
+
76
+ /** Build the in-process dispatch→transform→ingest sink for one connector. */
77
+ function makeIngest(client: HogsendClient, connector: DefinedConnector) {
78
+ return async (
79
+ dispatchType: string,
80
+ data: unknown,
81
+ ): Promise<{ ok: boolean; status: number }> => {
82
+ try {
83
+ // Reconstruct the EXACT envelope the HTTP ingress route receives
84
+ // (`{ __t, d }`) so the connector transform is byte-identical whether the
85
+ // dispatch arrived over HTTP or in-process.
86
+ const payload = { __t: dispatchType, d: data };
87
+ const event = await connector.transform(payload, {
88
+ db: client.db,
89
+ logger: client.logger,
90
+ transport: "gateway",
91
+ });
92
+ if (!event) return { ok: true, status: 200 };
93
+ await ingestEvent({
94
+ db: client.db,
95
+ registry: client.registry,
96
+ hatchet: client.hatchet,
97
+ logger: client.logger,
98
+ event,
99
+ // Pass the active analytics provider so a Discord-keyed contact merge
100
+ // stitches the analytics person too (the HTTP route omits this; in-proc
101
+ // can do better since it already holds the container).
102
+ analytics: client.analytics,
103
+ });
104
+ return { ok: true, status: 200 };
105
+ } catch (err) {
106
+ client.logger.warn("Connector runtime in-process ingest failed", {
107
+ connectorId: connector.meta.id,
108
+ dispatchType,
109
+ error: err instanceof Error ? err.message : String(err),
110
+ });
111
+ return { ok: false, status: 500 };
112
+ }
113
+ };
114
+ }
115
+
116
+ /**
117
+ * Per-connector lease controller: a single timer loop that races for the lease,
118
+ * starts the runtime + heartbeat on a win, renews while it leads, and demotes
119
+ * (stop socket + heartbeat) the moment a renew is lost — then re-enters the
120
+ * race, giving bounded automatic failover within the TTL with no two-holder
121
+ * overlap.
122
+ */
123
+ function startController(
124
+ client: HogsendClient,
125
+ connector: DefinedConnector,
126
+ factory: ConnectorRuntimeFactory,
127
+ ): ConnectorRuntimesHandle | null {
128
+ const { logger } = client;
129
+ const connectorId = connector.meta.id;
130
+ const leaseKey = `hogsend:connector-runtime:${connectorId}:leader`;
131
+
132
+ let heartbeat: ConnectorHeartbeatHandle | undefined;
133
+
134
+ const runtime = factory({
135
+ ingest: makeIngest(client, connector),
136
+ onMetadata: (patch) => heartbeat?.state.setMetadata(patch),
137
+ logger,
138
+ });
139
+ // `null` ⇒ not configured to run here (e.g. no bot token). Skip cleanly — no
140
+ // lease held, no heartbeat written, dashboard stays Offline (truthfully).
141
+ if (!runtime) {
142
+ logger.debug("Connector runtime not configured; skipping", { connectorId });
143
+ return null;
144
+ }
145
+ // Non-null alias the hoisted closures below capture (a const keeps TS's
146
+ // post-guard narrowing; the bare `runtime` widens back to `| null` inside
147
+ // them).
148
+ const rt = runtime;
149
+
150
+ let leading = false;
151
+ let token = "";
152
+ let stopped = false;
153
+ let timer: ReturnType<typeof setTimeout> | undefined;
154
+
155
+ /** Drop leadership: heartbeat key deleted FIRST (immediate Offline), then socket. */
156
+ async function demote(): Promise<void> {
157
+ leading = false;
158
+ await heartbeat?.stop();
159
+ heartbeat = undefined;
160
+ try {
161
+ await rt.stop();
162
+ } catch (err) {
163
+ logger.warn("Connector runtime stop failed during demotion", {
164
+ connectorId,
165
+ error: err instanceof Error ? err.message : String(err),
166
+ });
167
+ }
168
+ }
169
+
170
+ async function tick(): Promise<void> {
171
+ if (stopped) return;
172
+ try {
173
+ if (!leading) {
174
+ token = newLeaseToken();
175
+ const won = await acquireLeaderLease({
176
+ key: leaseKey,
177
+ token,
178
+ ttlMs: LEASE_TTL_MS,
179
+ });
180
+ if (won) {
181
+ leading = true;
182
+ heartbeat = startConnectorHeartbeat(connectorId, logger);
183
+ heartbeat.state.setMetadata(rt.getMetadata());
184
+ logger.info("Connector runtime acquired lease; opening socket", {
185
+ connectorId,
186
+ });
187
+ try {
188
+ await rt.start();
189
+ } catch (err) {
190
+ // A fatal start (bad token / disallowed intents) — release the lease
191
+ // so another replica (or a fixed redeploy) can try, and surface it.
192
+ logger.error("Connector runtime failed to start; releasing lease", {
193
+ connectorId,
194
+ error: err instanceof Error ? err.message : String(err),
195
+ });
196
+ await heartbeat.stop();
197
+ heartbeat = undefined;
198
+ leading = false;
199
+ await releaseLeaderLease({ key: leaseKey, token });
200
+ }
201
+ }
202
+ } else {
203
+ const renewed = await renewLeaderLease({
204
+ key: leaseKey,
205
+ token,
206
+ ttlMs: LEASE_TTL_MS,
207
+ });
208
+ if (!renewed) {
209
+ logger.warn("Connector runtime lost lease; demoting", {
210
+ connectorId,
211
+ });
212
+ await demote();
213
+ }
214
+ }
215
+ } catch (err) {
216
+ logger.warn("Connector runtime election tick failed", {
217
+ connectorId,
218
+ error: err instanceof Error ? err.message : String(err),
219
+ });
220
+ } finally {
221
+ if (!stopped) {
222
+ timer = setTimeout(() => void tick(), leading ? RENEW_MS : ELECT_MS);
223
+ timer.unref?.();
224
+ }
225
+ }
226
+ }
227
+
228
+ void tick();
229
+
230
+ return {
231
+ async stop() {
232
+ stopped = true;
233
+ if (timer) clearTimeout(timer);
234
+ if (leading) {
235
+ await demote();
236
+ await releaseLeaderLease({ key: leaseKey, token });
237
+ }
238
+ },
239
+ };
240
+ }
241
+
242
+ export interface StartConnectorRuntimesArgs {
243
+ client: HogsendClient;
244
+ /** Platform runtime factories keyed by connector id (from createWorker opts). */
245
+ factories: Record<string, ConnectorRuntimeFactory>;
246
+ }
247
+
248
+ /**
249
+ * Boot the inline runtimes for every registered `gateway`-transport connector
250
+ * that has a supplied factory. Fire-and-forget per connector (each runs its own
251
+ * lease loop); returns a handle whose `stop()` releases all leases + deletes
252
+ * heartbeats before stopping sockets (graceful, heartbeat-first ordering).
253
+ *
254
+ * No-ops cleanly when there are no gateway connectors, no factories, or a
255
+ * factory declines (returns null) — so a deploy with nothing configured carries
256
+ * zero cost.
257
+ */
258
+ export function startConnectorRuntimes(
259
+ args: StartConnectorRuntimesArgs,
260
+ ): ConnectorRuntimesHandle {
261
+ const { client, factories } = args;
262
+ const gateways = client.connectorRegistry.getByTransport("gateway");
263
+ const controllers: ConnectorRuntimesHandle[] = [];
264
+
265
+ for (const connector of gateways) {
266
+ const factory = factories[connector.meta.id];
267
+ if (!factory) {
268
+ client.logger.debug(
269
+ "Gateway connector has no runtime factory; skipping",
270
+ {
271
+ connectorId: connector.meta.id,
272
+ },
273
+ );
274
+ continue;
275
+ }
276
+ const controller = startController(client, connector, factory);
277
+ if (controller) controllers.push(controller);
278
+ }
279
+
280
+ if (controllers.length > 0) {
281
+ client.logger.info("Connector runtimes started", {
282
+ count: controllers.length,
283
+ });
284
+ }
285
+
286
+ return {
287
+ async stop() {
288
+ await Promise.allSettled(controllers.map((c) => c.stop()));
289
+ },
290
+ };
291
+ }
package/src/container.ts CHANGED
@@ -20,6 +20,11 @@ import {
20
20
  buildBucketRegistry,
21
21
  collectBucketReactionJourneys,
22
22
  } from "./buckets/registry.js";
23
+ import {
24
+ ConnectorActionRegistry,
25
+ setConnectorActionRegistry,
26
+ } from "./connectors/action-registry-singleton.js";
27
+ import type { DefinedConnectorAction } from "./connectors/define-action.js";
23
28
  import type { DefinedConnector } from "./connectors/define-connector.js";
24
29
  import { connectorsFromEnv } from "./connectors/presets/index.js";
25
30
  import {
@@ -161,6 +166,13 @@ export interface HogsendClient {
161
166
  * `/v1/connectors/:id/*` routes read `get(id).handlers`.
162
167
  */
163
168
  connectorRegistry: ConnectorRegistry;
169
+ /**
170
+ * The connector OUTBOUND ACTION registry, keyed by `${connectorId}:${name}`.
171
+ * Holds the journey-callable imperative actions (Discord post / broadcast /
172
+ * mention / DM) the standalone `sendConnectorAction()` resolves. Socket-free —
173
+ * independent of any inbound gateway runtime. Empty when none are wired.
174
+ */
175
+ connectorActionRegistry: ConnectorActionRegistry;
164
176
  hatchet: HatchetClient;
165
177
  /**
166
178
  * The client repo's migration journal (`migrations/meta/_journal.json`),
@@ -281,6 +293,14 @@ export interface HogsendClientOptions {
281
293
  * connectors. Defaults to none.
282
294
  */
283
295
  connectors?: DefinedConnector[];
296
+ /**
297
+ * Connector OUTBOUND ACTIONS (e.g. `discordActions` from `@hogsend/plugin-discord`)
298
+ * — journey-callable imperative actions registered into the
299
+ * {@link ConnectorActionRegistry} and invoked via the standalone
300
+ * `sendConnectorAction()`. Socket-free; independent of any inbound gateway
301
+ * runtime. Defaults to none.
302
+ */
303
+ connectorActions?: DefinedConnectorAction[];
284
304
  /**
285
305
  * @deprecated pass `connectors` instead. Back-compat array of
286
306
  * `defineWebhookSource()` sources; converted to webhook-transport connectors.
@@ -679,6 +699,15 @@ export function createHogsendClient(
679
699
  `Connector registry loaded: ${connectorRegistry.count()} connectors`,
680
700
  );
681
701
 
702
+ // Build + install the connector ACTION registry (outbound imperative actions),
703
+ // the action sibling of the connector registry above. Runs in BOTH the API and
704
+ // worker (both call createHogsendClient), so `sendConnectorAction()` resolves
705
+ // in either process.
706
+ const connectorActionRegistry = new ConnectorActionRegistry(
707
+ opts.connectorActions ?? [],
708
+ );
709
+ setConnectorActionRegistry(connectorActionRegistry);
710
+
682
711
  // Optional: auto-seed a PostHog DESTINATION on the outbound spine so the email
683
712
  // lifecycle fans out to PostHog durably. Default OFF (ENABLE_POSTHOG_DESTINATION)
684
713
  // to avoid double-emit alongside the fire-and-forget capture path. Idempotent +
@@ -725,6 +754,7 @@ export function createHogsendClient(
725
754
  bucketRegistry,
726
755
  listRegistry,
727
756
  connectorRegistry,
757
+ connectorActionRegistry,
728
758
  hatchet: opts.overrides?.hatchet ?? hatchet,
729
759
  clientJournal: opts.clientJournal ?? { entries: [] },
730
760
  defaults,
package/src/env.ts CHANGED
@@ -207,6 +207,19 @@ export const env = createEnv({
207
207
  // (the route 401s), so a gateway worker cannot relay without it. MUST be
208
208
  // high-entropy — generate with `openssl rand -base64 32`.
209
209
  CONNECTOR_INGRESS_SECRET: z.string().min(32).optional(),
210
+ // --- Connector runtimes (inline gateway sockets) ---
211
+ // The long-lived inbound socket for gateway-transport connectors (Discord)
212
+ // runs INLINE inside the host process below, gated by a Redis leader lease so
213
+ // exactly ONE replica holds it. Auto-on: a registered gateway connector +
214
+ // its bot token present is enough. Enum (not z.coerce.boolean) so an explicit
215
+ // "false" actually disables it (z.coerce.boolean treats "false" as true).
216
+ ENABLE_CONNECTOR_RUNTIMES: z.enum(["true", "false"]).default("true"),
217
+ // Which already-deployed process hosts the inline runtime. "worker" (default)
218
+ // is the committed home; "standalone" defers to the advanced discord-worker
219
+ // entry; "api" is reserved (host it yourself via startConnectorRuntimes).
220
+ CONNECTOR_RUNTIME_HOST: z
221
+ .enum(["worker", "api", "standalone"])
222
+ .default("worker"),
210
223
  // --- Outbound destination presets (Phase 3) ---
211
224
  // Which `defineDestination()` PRESETS are registered into the process
212
225
  // destination registry the delivery task resolves by `endpoint.kind`. csv of
package/src/index.ts CHANGED
@@ -91,6 +91,18 @@ export {
91
91
  resetBucketRegistry,
92
92
  setBucketRegistry,
93
93
  } from "./buckets/registry-singleton.js";
94
+ export {
95
+ ConnectorActionRegistry,
96
+ getConnectorActionRegistry,
97
+ resetConnectorActionRegistry,
98
+ setConnectorActionRegistry,
99
+ } from "./connectors/action-registry-singleton.js";
100
+ export {
101
+ type ConnectorActionCtx,
102
+ type DefinedConnectorAction,
103
+ defineConnectorAction,
104
+ type ResolvedActionContact,
105
+ } from "./connectors/define-action.js";
94
106
  // --- Inbound connectors: unified authoring layer ---
95
107
  export {
96
108
  type ConnectorCtx,
@@ -115,6 +127,13 @@ export {
115
127
  resetConnectorRegistry,
116
128
  setConnectorRegistry,
117
129
  } from "./connectors/registry-singleton.js";
130
+ export {
131
+ type ConnectorRuntime,
132
+ type ConnectorRuntimeDeps,
133
+ type ConnectorRuntimeFactory,
134
+ type ConnectorRuntimesHandle,
135
+ startConnectorRuntimes,
136
+ } from "./connectors/runtime.js";
118
137
  export {
119
138
  createHogsendClient,
120
139
  type HogsendClient,
@@ -201,6 +220,18 @@ export {
201
220
  type BucketTransitionSource,
202
221
  emitBucketTransition,
203
222
  } from "./lib/bucket-emit.js";
223
+ // --- Connector outbound actions (journey-callable, socket-free) ---
224
+ export {
225
+ type SendConnectorActionArgs,
226
+ sendConnectorAction,
227
+ } from "./lib/connector-actions.js";
228
+ // --- Connector-runtime liveness heartbeat (connector-neutral) ---
229
+ export {
230
+ type ConnectorHeartbeat,
231
+ type ConnectorHeartbeatHandle,
232
+ getConnectorHeartbeat,
233
+ startConnectorHeartbeat,
234
+ } from "./lib/connector-heartbeat.js";
204
235
  // --- Single-use link codes (native connector /link → /verify identify loop) ---
205
236
  export {
206
237
  type CreateLinkCodeResult,
@@ -289,6 +320,13 @@ export {
289
320
  type IngestResult,
290
321
  ingestEvent,
291
322
  } from "./lib/ingestion.js";
323
+ // --- Leader lease (connector-runtime singleton election) ---
324
+ export {
325
+ acquireLeaderLease,
326
+ newLeaseToken,
327
+ releaseLeaderLease,
328
+ renewLeaderLease,
329
+ } from "./lib/leader-lease.js";
292
330
  // --- Logging ---
293
331
  export { createLogger, type Logger } from "./lib/logger.js";
294
332
  export { createTrackedMailer } from "./lib/mailer.js";
@@ -0,0 +1,90 @@
1
+ import { contacts, type Database } from "@hogsend/db";
2
+ import { and, eq, isNull, or } from "drizzle-orm";
3
+ import { getConnectorActionRegistry } from "../connectors/action-registry-singleton.js";
4
+ import type { ResolvedActionContact } from "../connectors/define-action.js";
5
+ import { env } from "../env.js";
6
+ import { getDb } from "./db.js";
7
+ import { createLogger } from "./logger.js";
8
+
9
+ /**
10
+ * Resolve a contact for an outbound action by email, external id, or a platform
11
+ * id (e.g. a Discord snowflake). Matches text columns only (NOT the uuid `id`,
12
+ * which would force an invalid-uuid cast error for an email ref). First live
13
+ * match wins.
14
+ */
15
+ async function resolveContact(
16
+ db: Database,
17
+ ref: string,
18
+ ): Promise<ResolvedActionContact | null> {
19
+ if (!ref) return null;
20
+ const rows = await db
21
+ .select({
22
+ id: contacts.id,
23
+ email: contacts.email,
24
+ discordId: contacts.discordId,
25
+ externalId: contacts.externalId,
26
+ properties: contacts.properties,
27
+ })
28
+ .from(contacts)
29
+ .where(
30
+ and(
31
+ isNull(contacts.deletedAt),
32
+ or(
33
+ eq(contacts.email, ref),
34
+ eq(contacts.externalId, ref),
35
+ eq(contacts.discordId, ref),
36
+ ),
37
+ ),
38
+ )
39
+ .limit(1);
40
+ const row = rows[0];
41
+ if (!row) return null;
42
+ return {
43
+ id: row.id,
44
+ email: row.email ?? null,
45
+ discordId: row.discordId ?? null,
46
+ externalId: row.externalId ?? null,
47
+ properties: (row.properties ?? {}) as Record<string, unknown>,
48
+ };
49
+ }
50
+
51
+ export interface SendConnectorActionArgs {
52
+ /** The connector the action belongs to (e.g. "discord"). */
53
+ connectorId: string;
54
+ /** The action name (e.g. "sendChannelMessage"). */
55
+ action: string;
56
+ /** The action's own args object (shape defined by the action). */
57
+ args?: unknown;
58
+ }
59
+
60
+ /**
61
+ * Invoke a registered connector outbound action from a journey/workflow. The
62
+ * standalone, socket-free counterpart to `sendEmail()` — single-object-in,
63
+ * result-out, NOT on `JourneyContext` (features are standalone imports). Throws
64
+ * when the action isn't registered (wire it via
65
+ * `createHogsendClient({ connectorActions })`).
66
+ *
67
+ * Independent of any inbound gateway runtime: a deployment with the gateway off
68
+ * (or "Worker Offline") can still send — Discord actions are bot-REST needing
69
+ * only the bot token.
70
+ */
71
+ export async function sendConnectorAction(
72
+ input: SendConnectorActionArgs,
73
+ ): Promise<unknown> {
74
+ const action = getConnectorActionRegistry().get(
75
+ input.connectorId,
76
+ input.action,
77
+ );
78
+ if (!action) {
79
+ throw new Error(
80
+ `no connector action "${input.connectorId}:${input.action}" is registered ` +
81
+ "(pass it via createHogsendClient({ connectorActions }))",
82
+ );
83
+ }
84
+ const db = getDb();
85
+ return action.run(input.args, {
86
+ db,
87
+ logger: createLogger(env.LOG_LEVEL),
88
+ resolveContact: (ref: string) => resolveContact(db, ref),
89
+ });
90
+ }
@@ -0,0 +1,177 @@
1
+ import type { Logger } from "./logger.js";
2
+ import { getRedis } from "./redis.js";
3
+
4
+ /**
5
+ * Connector-runtime liveness heartbeat — the connector-neutral generalization of
6
+ * {@link ./discord-gateway-heartbeat.ts}. A long-lived inbound runtime (the
7
+ * leased gateway socket) writes a TTL'd Redis key on an interval; readers (the
8
+ * admin `/connectors` projection Studio reads) treat a fresh key as "this
9
+ * connector's runtime is alive". Because ONLY the lease-holder writes it (see
10
+ * connectors/runtime.ts), a fresh key means "this deployment's elected leader
11
+ * owns the socket" — liveness is OWNED, not merely observed, so a stray process
12
+ * can no longer light the dashboard green.
13
+ *
14
+ * The payload carries an opaque `metadata` blob (e.g. `{ guildId, intents }` for
15
+ * Discord) folded in by the runtime — so the heartbeat stays platform-neutral
16
+ * while still surfacing the bits Studio shows. Everything is best-effort: a
17
+ * missing/unreachable Redis never crashes the runtime and reads back as "down".
18
+ *
19
+ * The legacy Discord key (`hogsend:discord-gateway:heartbeat`, still written by
20
+ * the standalone `discord-worker.ts` hatch via {@link startDiscordGatewayHeartbeat})
21
+ * is honoured as a READ fallback for `connectorId === "discord"`, so a
22
+ * mid-rollout deploy where the old standalone worker is still the writer keeps
23
+ * showing green until the inline runtime takes over.
24
+ */
25
+ const TTL_SECONDS = 30;
26
+ const REFRESH_MS = 10_000;
27
+
28
+ const heartbeatKey = (connectorId: string) =>
29
+ `hogsend:connector-runtime:${connectorId}:heartbeat`;
30
+
31
+ /** The legacy standalone-Discord key — read-only fallback for one minor. */
32
+ const LEGACY_DISCORD_KEY = "hogsend:discord-gateway:heartbeat";
33
+
34
+ export interface ConnectorHeartbeat {
35
+ /** True when a fresh runtime heartbeat is present in Redis. */
36
+ alive: boolean;
37
+ /** ISO timestamp the runtime last wrote, when alive. */
38
+ lastSeenAt?: string;
39
+ /** Opaque platform metadata the runtime folded in (e.g. guildId, intents). */
40
+ metadata?: Record<string, unknown>;
41
+ }
42
+
43
+ /** The JSON shape persisted under {@link heartbeatKey}. */
44
+ interface HeartbeatPayload {
45
+ lastSeenAt: string;
46
+ metadata?: Record<string, unknown>;
47
+ }
48
+
49
+ export interface ConnectorHeartbeatState {
50
+ /**
51
+ * Merge a metadata patch (read-merge-write) and flush immediately, so Studio
52
+ * reflects a late-observed field (e.g. the guild id seen at GUILD_CREATE)
53
+ * without waiting for the next refresh tick.
54
+ */
55
+ setMetadata(patch: Record<string, unknown>): void;
56
+ }
57
+
58
+ export interface ConnectorHeartbeatHandle {
59
+ state: ConnectorHeartbeatState;
60
+ /** Clear the timer and delete the key for an immediate "down" on demotion. */
61
+ stop(): Promise<void>;
62
+ }
63
+
64
+ /**
65
+ * Begin writing a connector-runtime heartbeat. Writes once immediately, then
66
+ * refreshes every {@link REFRESH_MS} with a {@link TTL_SECONDS} expiry — so an
67
+ * ungraceful death (or a lost lease) is reflected as "down" within the TTL.
68
+ */
69
+ export function startConnectorHeartbeat(
70
+ connectorId: string,
71
+ logger: Logger,
72
+ ): ConnectorHeartbeatHandle {
73
+ let warned = false;
74
+ let metadata: Record<string, unknown> = {};
75
+ const key = heartbeatKey(connectorId);
76
+
77
+ const write = async () => {
78
+ const payload: HeartbeatPayload = {
79
+ lastSeenAt: new Date().toISOString(),
80
+ ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
81
+ };
82
+ try {
83
+ await getRedis().set(key, JSON.stringify(payload), "EX", TTL_SECONDS);
84
+ } catch (err) {
85
+ if (!warned) {
86
+ warned = true;
87
+ logger.debug("Connector heartbeat write failed (Redis unreachable?)", {
88
+ connectorId,
89
+ error: err instanceof Error ? err.message : String(err),
90
+ });
91
+ }
92
+ }
93
+ };
94
+
95
+ void write();
96
+ const timer = setInterval(() => void write(), REFRESH_MS);
97
+ // Never hold the process open for the heartbeat alone.
98
+ timer.unref?.();
99
+
100
+ return {
101
+ state: {
102
+ setMetadata(patch: Record<string, unknown>) {
103
+ metadata = { ...metadata, ...patch };
104
+ void write();
105
+ },
106
+ },
107
+ async stop() {
108
+ clearInterval(timer);
109
+ try {
110
+ await getRedis().del(key);
111
+ } catch {
112
+ // Best-effort — the TTL expires it anyway.
113
+ }
114
+ },
115
+ };
116
+ }
117
+
118
+ /** Normalize a stored payload string into a {@link ConnectorHeartbeat}. */
119
+ function parsePayload(raw: string): ConnectorHeartbeat {
120
+ try {
121
+ const parsed = JSON.parse(raw) as HeartbeatPayload;
122
+ return {
123
+ alive: true,
124
+ lastSeenAt: parsed.lastSeenAt,
125
+ ...(parsed.metadata ? { metadata: parsed.metadata } : {}),
126
+ };
127
+ } catch {
128
+ // Legacy plain-string value (a bare ISO timestamp) — alive, no metadata.
129
+ return { alive: true, lastSeenAt: raw };
130
+ }
131
+ }
132
+
133
+ /**
134
+ * Normalize the LEGACY Discord heartbeat (`{ lastSeenAt, guildId?, intents? }`)
135
+ * into the connector-neutral shape so the admin projection reads one schema.
136
+ */
137
+ function parseLegacyDiscord(raw: string): ConnectorHeartbeat {
138
+ try {
139
+ const parsed = JSON.parse(raw) as {
140
+ lastSeenAt: string;
141
+ guildId?: string;
142
+ intents?: number;
143
+ };
144
+ const metadata: Record<string, unknown> = {};
145
+ if (parsed.guildId) metadata.guildId = parsed.guildId;
146
+ if (typeof parsed.intents === "number") metadata.intents = parsed.intents;
147
+ return {
148
+ alive: true,
149
+ lastSeenAt: parsed.lastSeenAt,
150
+ ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
151
+ };
152
+ } catch {
153
+ return { alive: true, lastSeenAt: raw };
154
+ }
155
+ }
156
+
157
+ /**
158
+ * Read a connector-runtime heartbeat. Resolves to `{ alive: false }` when the
159
+ * key is missing or Redis is unreachable. For `connectorId === "discord"`, falls
160
+ * back to the legacy standalone-worker key so a mid-rollout deploy stays green.
161
+ */
162
+ export async function getConnectorHeartbeat(
163
+ connectorId: string,
164
+ ): Promise<ConnectorHeartbeat> {
165
+ try {
166
+ const redis = getRedis();
167
+ const raw = await redis.get(heartbeatKey(connectorId));
168
+ if (raw) return parsePayload(raw);
169
+ if (connectorId === "discord") {
170
+ const legacy = await redis.get(LEGACY_DISCORD_KEY);
171
+ if (legacy) return parseLegacyDiscord(legacy);
172
+ }
173
+ return { alive: false };
174
+ } catch {
175
+ return { alive: false };
176
+ }
177
+ }
@@ -0,0 +1,102 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import type { Redis } from "ioredis";
3
+ import { getRedis } from "./redis.js";
4
+
5
+ /**
6
+ * A connector-neutral distributed leader lease over Redis. The connector runtime
7
+ * needs EXACTLY ONE process (across N replicated Hatchet workers) to hold a given
8
+ * platform socket at a time — one bot token permits one live Gateway session — so
9
+ * the runtime races for a lease keyed by connector id and only the winner opens
10
+ * the socket. Losers idle and re-race, giving bounded automatic failover within
11
+ * the TTL with no two-holders overlap.
12
+ *
13
+ * The lease is a `SET key token NX PX ttl` (atomic acquire-if-absent with an
14
+ * expiry), renewed by a Lua compare-then-PEXPIRE and released by a Lua
15
+ * compare-then-DEL. The compare on the caller's unique `token` is the fence: a
16
+ * process that has LOST the lease (expired, taken over) can neither renew nor
17
+ * release it, so it can never stomp the new holder. Built on the shared engine
18
+ * Redis singleton ({@link getRedis}); every op accepts an explicit client for
19
+ * tests.
20
+ *
21
+ * Everything is best-effort against Redis faults: acquire/renew return `false`
22
+ * (caller stays/loses leader, never opens a second socket), release swallows.
23
+ * Fail-safe means "no lease ⇒ no socket", never "two sockets".
24
+ */
25
+
26
+ /** Renew the lease ONLY if we still own it, then extend its expiry. */
27
+ const RENEW_LUA =
28
+ 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
29
+
30
+ /** Delete the lease ONLY if we still own it (never delete someone else's). */
31
+ const RELEASE_LUA =
32
+ 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
33
+
34
+ /** A process-unique fencing token to stamp on an acquired lease. */
35
+ export function newLeaseToken(): string {
36
+ return randomUUID();
37
+ }
38
+
39
+ /**
40
+ * Try to acquire the lease. Returns `true` only when this call set the key (it
41
+ * was absent). A `false` means another holder owns it OR Redis is unreachable —
42
+ * either way the caller must NOT open the socket.
43
+ */
44
+ export async function acquireLeaderLease(args: {
45
+ key: string;
46
+ token: string;
47
+ ttlMs: number;
48
+ redis?: Redis;
49
+ }): Promise<boolean> {
50
+ const redis = args.redis ?? getRedis();
51
+ try {
52
+ const res = await redis.set(args.key, args.token, "PX", args.ttlMs, "NX");
53
+ return res === "OK";
54
+ } catch {
55
+ return false;
56
+ }
57
+ }
58
+
59
+ /**
60
+ * Renew our hold on the lease (compare-and-extend). Returns `false` if we no
61
+ * longer own it (expired / taken over) or Redis is unreachable — the caller must
62
+ * then self-demote and stop the socket.
63
+ */
64
+ export async function renewLeaderLease(args: {
65
+ key: string;
66
+ token: string;
67
+ ttlMs: number;
68
+ redis?: Redis;
69
+ }): Promise<boolean> {
70
+ const redis = args.redis ?? getRedis();
71
+ try {
72
+ const res = await redis.eval(
73
+ RENEW_LUA,
74
+ 1,
75
+ args.key,
76
+ args.token,
77
+ String(args.ttlMs),
78
+ );
79
+ return res === 1;
80
+ } catch {
81
+ return false;
82
+ }
83
+ }
84
+
85
+ /**
86
+ * Release the lease if (and only if) we still own it. Best-effort: a failure is
87
+ * swallowed because the TTL expires the key anyway. Returns `true` when our own
88
+ * key was deleted.
89
+ */
90
+ export async function releaseLeaderLease(args: {
91
+ key: string;
92
+ token: string;
93
+ redis?: Redis;
94
+ }): Promise<boolean> {
95
+ const redis = args.redis ?? getRedis();
96
+ try {
97
+ const res = await redis.eval(RELEASE_LUA, 1, args.key, args.token);
98
+ return res === 1;
99
+ } catch {
100
+ return false;
101
+ }
102
+ }
@@ -4,8 +4,8 @@ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
4
4
  import { and, count, isNotNull, isNull } from "drizzle-orm";
5
5
  import type { AppEnv } from "../../app.js";
6
6
  import { getDestinationRegistry } from "../../destinations/registry-singleton.js";
7
+ import { getConnectorHeartbeat } from "../../lib/connector-heartbeat.js";
7
8
  import { signConnectorState } from "../../lib/connector-state.js";
8
- import { getDiscordGatewayHeartbeat } from "../../lib/discord-gateway-heartbeat.js";
9
9
  import {
10
10
  getDerivedCredential,
11
11
  getProviderCredential,
@@ -281,14 +281,25 @@ export const adminConnectorsRouter = new OpenAPIHono<AppEnv>()
281
281
  const derived = (await getDerivedCredential(db, id).catch(
282
282
  () => null,
283
283
  )) as Record<string, unknown> | null;
284
- const heartbeat = await getDiscordGatewayHeartbeat();
284
+ const heartbeat = await getConnectorHeartbeat(id);
285
+ // guildId/intents ride in the connector-neutral heartbeat metadata
286
+ // blob (folded in by the inline runtime; the legacy Discord key is
287
+ // normalized into the same shape on read).
288
+ const heartbeatGuildId =
289
+ typeof heartbeat.metadata?.guildId === "string"
290
+ ? heartbeat.metadata.guildId
291
+ : null;
292
+ const heartbeatIntents =
293
+ typeof heartbeat.metadata?.intents === "number"
294
+ ? heartbeat.metadata.intents
295
+ : null;
285
296
 
286
297
  const derivedGuildId =
287
298
  typeof derived?.discordGuildId === "string"
288
299
  ? derived.discordGuildId
289
300
  : null;
290
301
  // Prefer the live worker-observed guild; fall back to the stored one.
291
- const guildId = heartbeat.guildId ?? derivedGuildId;
302
+ const guildId = heartbeatGuildId ?? derivedGuildId;
292
303
  // Prefer the LIVE worker-reported intents (the derived credential never
293
304
  // carries discordIntents — install writes only the guild id); fall back
294
305
  // to the derived value for forward-compat if it ever IS written.
@@ -296,7 +307,7 @@ export const adminConnectorsRouter = new OpenAPIHono<AppEnv>()
296
307
  typeof derived?.discordIntents === "number"
297
308
  ? derived.discordIntents
298
309
  : null;
299
- const intents = heartbeat.intents ?? derivedIntents;
310
+ const intents = heartbeatIntents ?? derivedIntents;
300
311
  // Tri-state: a guild id (either source) confirms the bot is in a
301
312
  // server; otherwise unknown (null), NOT a false "not installed".
302
313
  const botInstalled: boolean | null = guildId ? true : null;
@@ -354,10 +365,14 @@ export const adminConnectorsRouter = new OpenAPIHono<AppEnv>()
354
365
  const derived = (await getDerivedCredential(db, "discord").catch(
355
366
  () => null,
356
367
  )) as Record<string, unknown> | null;
357
- const heartbeat = await getDiscordGatewayHeartbeat();
368
+ const heartbeat = await getConnectorHeartbeat("discord");
358
369
  // Prefer the live worker-observed guild; fall back to the stored one.
370
+ const heartbeatGuildId =
371
+ typeof heartbeat.metadata?.guildId === "string"
372
+ ? heartbeat.metadata.guildId
373
+ : null;
359
374
  const guildId =
360
- heartbeat.guildId ??
375
+ heartbeatGuildId ??
361
376
  (typeof derived?.discordGuildId === "string"
362
377
  ? derived.discordGuildId
363
378
  : null);
package/src/worker.ts CHANGED
@@ -3,6 +3,10 @@ import {
3
3
  selectBucketReactionTasks,
4
4
  selectBucketTasks,
5
5
  } from "./buckets/registry.js";
6
+ import {
7
+ type ConnectorRuntimeFactory,
8
+ startConnectorRuntimes,
9
+ } from "./connectors/runtime.js";
6
10
  import type { HogsendClient } from "./container.js";
7
11
  import type { DefinedJourney } from "./journeys/define-journey.js";
8
12
  import { parseEnabledFilter, selectJourneyTasks } from "./journeys/registry.js";
@@ -39,6 +43,14 @@ export interface CreateWorkerOptions {
39
43
  enabledBuckets?: string;
40
44
  /** Extra client tasks registered alongside the built-in workflows. */
41
45
  extraWorkflows?: unknown[];
46
+ /**
47
+ * Inbound connector-runtime factories keyed by connector id (e.g.
48
+ * `{ discord: createDiscordRuntime }`). When `CONNECTOR_RUNTIME_HOST=worker`
49
+ * (default) and `ENABLE_CONNECTOR_RUNTIMES` is on, the worker elects a single
50
+ * leader replica per gateway connector and holds its socket inline — no
51
+ * separate service, no ingress secret.
52
+ */
53
+ connectorRuntimes?: Record<string, ConnectorRuntimeFactory>;
42
54
  }
43
55
 
44
56
  export interface Worker {
@@ -99,11 +111,14 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
99
111
  // lifecycle. `_worker` is captured for stop().
100
112
  let _worker: Awaited<ReturnType<typeof hatchet.worker>> | undefined;
101
113
  let _stopHeartbeat: (() => Promise<void>) | undefined;
114
+ let _stopRuntimes: (() => Promise<void>) | undefined;
102
115
 
103
116
  async function stop(): Promise<void> {
104
- // Delete the heartbeat first (an immediate "worker down" signal) before the
105
- // Redis connection is torn down below.
117
+ // Delete the worker heartbeat first (an immediate "worker down" signal) and
118
+ // release any connector-runtime leases + their heartbeats BEFORE the Redis
119
+ // connection is torn down below.
106
120
  await _stopHeartbeat?.();
121
+ await _stopRuntimes?.();
107
122
  await Promise.allSettled([
108
123
  _worker?.stop(),
109
124
  // Shut down the injected analytics instance (same object the worker's
@@ -142,6 +157,26 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
142
157
  // (read via GET /v1/health). Best-effort; never blocks the listener.
143
158
  _stopHeartbeat = startWorkerHeartbeat(container.logger);
144
159
 
160
+ // Inline connector runtimes (gateway sockets) — the worker is the default
161
+ // host. Each gateway connector with a supplied factory races a Redis leader
162
+ // lease so exactly one replica holds its socket; the in-process sink feeds
163
+ // transform→ingest with no HTTP hop. Auto-on; a factory that declines (e.g.
164
+ // no bot token) simply no-ops. Started AFTER the heartbeat and BEFORE the
165
+ // blocking `_worker.start()` below (anything after it is dead code at
166
+ // runtime until shutdown).
167
+ if (
168
+ container.env.ENABLE_CONNECTOR_RUNTIMES === "true" &&
169
+ container.env.CONNECTOR_RUNTIME_HOST === "worker" &&
170
+ opts.connectorRuntimes &&
171
+ Object.keys(opts.connectorRuntimes).length > 0
172
+ ) {
173
+ const runtimes = startConnectorRuntimes({
174
+ client: container,
175
+ factories: opts.connectorRuntimes,
176
+ });
177
+ _stopRuntimes = () => runtimes.stop();
178
+ }
179
+
145
180
  // Boot-time backfill / criteria-change re-eval (Section 6.6 B): diff each
146
181
  // enabled bucket's criteriaHash against bucket_configs and trigger a
147
182
  // backfill/re-eval run where it differs. Kicked off BEFORE the listener