@automagik/omni 2.260906.2 → 2.260906.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/drizzle/0057_connector_lifecycle_contract.sql +40 -0
- package/db/drizzle/meta/_journal.json +7 -0
- package/dist/commands/webhooks.d.ts.map +1 -1
- package/dist/index.js +223 -8
- package/dist/sdk/client.d.ts +21 -0
- package/dist/sdk/client.d.ts.map +1 -1
- package/dist/sdk/index.d.ts +1 -1
- package/dist/sdk/index.d.ts.map +1 -1
- package/dist/sdk/index.js +7 -0
- package/dist/sdk/types.generated.d.ts +376 -10
- package/dist/sdk/types.generated.d.ts.map +1 -1
- package/dist/server/index.js +342 -7
- package/package.json +1 -1
package/dist/server/index.js
CHANGED
|
@@ -22567,7 +22567,24 @@ var init_registry = __esm(() => {
|
|
|
22567
22567
|
component: exports_external.string(),
|
|
22568
22568
|
reason: exports_external.string(),
|
|
22569
22569
|
severity: exports_external.enum(["warning", "critical"])
|
|
22570
|
-
}), { description: "System health degradation detected" })
|
|
22570
|
+
}), { description: "System health degradation detected" }),
|
|
22571
|
+
connectorStalled: createEventSchema("system.connector.stalled", exports_external.object({
|
|
22572
|
+
sourceId: exports_external.string(),
|
|
22573
|
+
sourceName: exports_external.string(),
|
|
22574
|
+
expectedIntervalSeconds: exports_external.number(),
|
|
22575
|
+
lastReceivedAt: exports_external.number().nullable(),
|
|
22576
|
+
lastHeartbeatAt: exports_external.number().nullable(),
|
|
22577
|
+
silentForSeconds: exports_external.number(),
|
|
22578
|
+
stalledAt: exports_external.number()
|
|
22579
|
+
}), { description: "A supervised connector went silent beyond its declared cadence" }),
|
|
22580
|
+
connectorRecovered: createEventSchema("system.connector.recovered", exports_external.object({
|
|
22581
|
+
sourceId: exports_external.string(),
|
|
22582
|
+
sourceName: exports_external.string(),
|
|
22583
|
+
expectedIntervalSeconds: exports_external.number(),
|
|
22584
|
+
stalledForSeconds: exports_external.number(),
|
|
22585
|
+
recoveredBy: exports_external.enum(["event", "heartbeat", "rearmed"]),
|
|
22586
|
+
recoveredAt: exports_external.number()
|
|
22587
|
+
}), { description: "A stalled connector signalled again (event, heartbeat, or re-declared cadence)" })
|
|
22571
22588
|
};
|
|
22572
22589
|
CustomEventSchemas = {
|
|
22573
22590
|
chatUnreadUpdated: createEventSchema("custom.chat.unread-updated", exports_external.object({
|
|
@@ -38946,6 +38963,113 @@ var init_automations = __esm(() => {
|
|
|
38946
38963
|
init_sweeper();
|
|
38947
38964
|
});
|
|
38948
38965
|
|
|
38966
|
+
// ../core/src/connectors/liveness.ts
|
|
38967
|
+
function latestConnectorSignal(row) {
|
|
38968
|
+
let best = { at: row.createdAt, kind: "rearmed" };
|
|
38969
|
+
const candidates = [
|
|
38970
|
+
{ at: row.livenessArmedAt, kind: "rearmed" },
|
|
38971
|
+
{ at: row.lastReceivedAt, kind: "event" },
|
|
38972
|
+
{ at: row.lastHeartbeatAt, kind: "heartbeat" }
|
|
38973
|
+
];
|
|
38974
|
+
for (const candidate of candidates) {
|
|
38975
|
+
if (candidate.at && candidate.at.getTime() >= best.at.getTime()) {
|
|
38976
|
+
best = { at: candidate.at, kind: candidate.kind };
|
|
38977
|
+
}
|
|
38978
|
+
}
|
|
38979
|
+
return best;
|
|
38980
|
+
}
|
|
38981
|
+
async function sweepConnectorLiveness(deps) {
|
|
38982
|
+
const clock = deps.now ?? (() => new Date);
|
|
38983
|
+
const now = clock();
|
|
38984
|
+
const rows = await deps.repo.findSupervised();
|
|
38985
|
+
const stats = { scanned: rows.length, stalled: 0, recovered: 0, errors: 0 };
|
|
38986
|
+
for (const row of rows) {
|
|
38987
|
+
try {
|
|
38988
|
+
await processRow2(row, now, deps, stats);
|
|
38989
|
+
} catch (err2) {
|
|
38990
|
+
stats.errors += 1;
|
|
38991
|
+
deps.logger.error("connector liveness: failed to process source", {
|
|
38992
|
+
sourceId: row.id,
|
|
38993
|
+
sourceName: row.name,
|
|
38994
|
+
error: err2 instanceof Error ? err2.message : String(err2)
|
|
38995
|
+
});
|
|
38996
|
+
}
|
|
38997
|
+
}
|
|
38998
|
+
if (stats.stalled > 0 || stats.recovered > 0) {
|
|
38999
|
+
deps.logger.info("connector liveness sweep tick", { ...stats });
|
|
39000
|
+
}
|
|
39001
|
+
return stats;
|
|
39002
|
+
}
|
|
39003
|
+
async function processRow2(row, now, deps, stats) {
|
|
39004
|
+
const signal = latestConnectorSignal(row);
|
|
39005
|
+
const silentMs = Math.max(0, now.getTime() - signal.at.getTime());
|
|
39006
|
+
const overdue = silentMs > row.expectedIntervalSeconds * 1000;
|
|
39007
|
+
const silentForSeconds = Math.round(silentMs / 1000);
|
|
39008
|
+
if (overdue) {
|
|
39009
|
+
if (row.livenessStatus === "stalled")
|
|
39010
|
+
return;
|
|
39011
|
+
if (!await deps.repo.markStalled(row.id, now))
|
|
39012
|
+
return;
|
|
39013
|
+
const payload = {
|
|
39014
|
+
sourceId: row.id,
|
|
39015
|
+
sourceName: row.name,
|
|
39016
|
+
expectedIntervalSeconds: row.expectedIntervalSeconds,
|
|
39017
|
+
lastReceivedAt: row.lastReceivedAt?.getTime() ?? null,
|
|
39018
|
+
lastHeartbeatAt: row.lastHeartbeatAt?.getTime() ?? null,
|
|
39019
|
+
silentForSeconds,
|
|
39020
|
+
stalledAt: now.getTime()
|
|
39021
|
+
};
|
|
39022
|
+
const eventId = await publishTransition(deps, "system.connector.stalled", { ...payload }, row);
|
|
39023
|
+
stats.stalled += 1;
|
|
39024
|
+
await runHook(deps.logger, "onStalled", row, () => deps.onStalled?.(row, payload, eventId));
|
|
39025
|
+
return;
|
|
39026
|
+
}
|
|
39027
|
+
if (row.livenessStatus === "stalled") {
|
|
39028
|
+
if (!await deps.repo.markRecovered(row.id, now))
|
|
39029
|
+
return;
|
|
39030
|
+
const stalledSince = row.stalledAt ?? signal.at;
|
|
39031
|
+
const payload = {
|
|
39032
|
+
sourceId: row.id,
|
|
39033
|
+
sourceName: row.name,
|
|
39034
|
+
expectedIntervalSeconds: row.expectedIntervalSeconds,
|
|
39035
|
+
stalledForSeconds: Math.max(0, Math.round((now.getTime() - stalledSince.getTime()) / 1000)),
|
|
39036
|
+
recoveredBy: signal.kind,
|
|
39037
|
+
recoveredAt: now.getTime()
|
|
39038
|
+
};
|
|
39039
|
+
await publishTransition(deps, "system.connector.recovered", { ...payload }, row);
|
|
39040
|
+
stats.recovered += 1;
|
|
39041
|
+
await runHook(deps.logger, "onRecovered", row, () => deps.onRecovered?.(row, payload));
|
|
39042
|
+
}
|
|
39043
|
+
}
|
|
39044
|
+
async function publishTransition(deps, type, payload, row) {
|
|
39045
|
+
if (!deps.eventBus) {
|
|
39046
|
+
deps.logger.warn("connector liveness: no event bus, transition persisted without event", {
|
|
39047
|
+
type,
|
|
39048
|
+
sourceName: row.name
|
|
39049
|
+
});
|
|
39050
|
+
return null;
|
|
39051
|
+
}
|
|
39052
|
+
const result = await deps.eventBus.publishGeneric(type, payload, {
|
|
39053
|
+
source: "connector-liveness",
|
|
39054
|
+
...row.tenantId ? { tenantId: row.tenantId } : {}
|
|
39055
|
+
});
|
|
39056
|
+
return result.id;
|
|
39057
|
+
}
|
|
39058
|
+
async function runHook(logger5, name, row, hook) {
|
|
39059
|
+
try {
|
|
39060
|
+
await hook();
|
|
39061
|
+
} catch (err2) {
|
|
39062
|
+
logger5.error(`connector liveness: ${name} hook failed`, {
|
|
39063
|
+
sourceId: row.id,
|
|
39064
|
+
sourceName: row.name,
|
|
39065
|
+
error: err2 instanceof Error ? err2.message : String(err2)
|
|
39066
|
+
});
|
|
39067
|
+
}
|
|
39068
|
+
}
|
|
39069
|
+
|
|
39070
|
+
// ../core/src/connectors/index.ts
|
|
39071
|
+
var init_connectors = () => {};
|
|
39072
|
+
|
|
38949
39073
|
// ../core/src/sessions/reset.ts
|
|
38950
39074
|
class InMemorySessionActivityStore {
|
|
38951
39075
|
activities = new Map;
|
|
@@ -43678,6 +43802,7 @@ __export(exports_src, {
|
|
|
43678
43802
|
tenantLabelBucketCount: () => tenantLabelBucketCount,
|
|
43679
43803
|
tenantEventsProcessed: () => tenantEventsProcessed,
|
|
43680
43804
|
sweepFollowUps: () => sweepFollowUps,
|
|
43805
|
+
sweepConnectorLiveness: () => sweepConnectorLiveness,
|
|
43681
43806
|
substituteTemplateObject: () => substituteTemplateObject,
|
|
43682
43807
|
substituteTemplate: () => substituteTemplate,
|
|
43683
43808
|
stampsEqual: () => stampsEqual,
|
|
@@ -43740,6 +43865,7 @@ __export(exports_src, {
|
|
|
43740
43865
|
natsConnectionStatus: () => natsConnectionStatus,
|
|
43741
43866
|
matchesPattern: () => matchesPattern,
|
|
43742
43867
|
markdownToWhatsApp: () => markdownToWhatsApp,
|
|
43868
|
+
latestConnectorSignal: () => latestConnectorSignal,
|
|
43743
43869
|
jsonEquals: () => jsonEquals,
|
|
43744
43870
|
isValidUuid: () => isValidUuid,
|
|
43745
43871
|
isValidJsonSchema: () => isValidJsonSchema,
|
|
@@ -44065,6 +44191,7 @@ var init_src = __esm(() => {
|
|
|
44065
44191
|
init_observability();
|
|
44066
44192
|
init_secrets();
|
|
44067
44193
|
init_automations();
|
|
44194
|
+
init_connectors();
|
|
44068
44195
|
init_reset();
|
|
44069
44196
|
init_providers();
|
|
44070
44197
|
init_cache();
|
|
@@ -172164,6 +172291,9 @@ __export(exports_schema, {
|
|
|
172164
172291
|
conversations: () => conversations,
|
|
172165
172292
|
contentTypes: () => contentTypes,
|
|
172166
172293
|
consumerOffsets: () => consumerOffsets,
|
|
172294
|
+
connectorWindowSemanticsValues: () => connectorWindowSemanticsValues,
|
|
172295
|
+
connectorMutationPolicies: () => connectorMutationPolicies,
|
|
172296
|
+
connectorLivenessStatuses: () => connectorLivenessStatuses,
|
|
172167
172297
|
conditionOperators: () => conditionOperators,
|
|
172168
172298
|
closeContactOutcomes: () => closeContactOutcomes,
|
|
172169
172299
|
closeContactLogs: () => closeContactLogs,
|
|
@@ -172212,7 +172342,7 @@ __export(exports_schema, {
|
|
|
172212
172342
|
accessRules: () => accessRules,
|
|
172213
172343
|
accessModes: () => accessModes
|
|
172214
172344
|
});
|
|
172215
|
-
var channelTypes, agentTypes, agentSystems, agentEntityTypes, debounceMode, splitDelayMode, supersedeMode, replyFilterMode, agentSessionStrategies, ruleTypes, accessModes, settingValueTypes, apiKeyStatuses, apiKeyProfiles, eventTypes, contentTypes, chatTypes, messageSources, messageTypes, messageStatuses, deliveryStatuses, scheduledMessageDeliveryModes, scheduledMessageStatuses, jobStatuses, providerSchemas, agentProviders, agents, agentRoutes, agentSessions, apiKeys, apiKeyAuditLogs, apiKeysRelations, apiKeyAuditLogsRelations, instances, whatsappTemplates, whatsappFlowKeys, scheduledMessages, persons, platformIdentities, conversations, chats, chatParticipants, omniGroups, messages2, omniEvents, handoffLogs, closeContactOutcomes, closeContactLogs, accessRules, globalSettings, settingChangeHistory, batchJobs, syncJobTypes, syncJobs, mediaContent, chatIdMappings, pluginStorage, agentProvidersRelations, agentsRelations, instancesRelations, syncJobsRelations, personsRelations, platformIdentitiesRelations, conversationsRelations, chatsRelations, chatParticipantsRelations, messagesRelations, omniEventsRelations, accessRulesRelations, globalSettingsRelations, settingChangeHistoryRelations, batchJobsRelations, mediaContentRelations, chatIdMappingsRelations, deadLetterStatuses, deadLetterEvents, payloadStorageConfig, payloadStages, eventPayloads, webhookSignatureAlgorithms, webhookSources, eventSchemas, conditionOperators, actionTypes, automationDebounceModes, automations2, automationLogStatuses, automationLogs, consumerOffsets, automationsRelations, automationLogsRelations, triggerLogs, triggerLogsRelations, agentRoutesRelations, agentTaskStatuses, agentTasks, agentTasksRelations, turnStatuses, turnActions, turns, turnsRelations, followUpDisarmReasons, chatFollowUpState, chatFollowUpStateRelations, processedEvents, genieHosts, tenantStatuses, principalTypes, principalStatuses, tenantRoles, membershipStatuses, credentialClasses, authCredentialStatuses, platformApiKeyStatuses, tenants, principals, tenantMemberships, tenantRolePolicies, platformApiKeys, tenantKeyLineage, authCredentials, tenantAuditLogs, platformAuditLogs, platformProviderCatalog, tenantProviderConfig, platformSettings, tenantSettings, platformSettingChangeHistory, tenantSettingChangeHistory, platformPluginStorage, tenantPluginStorage, platformPayloadStorageConfig, tenantPayloadStorageOverrides, tenantMigrationLedger, tenantMigrationLedgerHistory;
|
|
172345
|
+
var channelTypes, agentTypes, agentSystems, agentEntityTypes, debounceMode, splitDelayMode, supersedeMode, replyFilterMode, agentSessionStrategies, ruleTypes, accessModes, settingValueTypes, apiKeyStatuses, apiKeyProfiles, eventTypes, contentTypes, chatTypes, messageSources, messageTypes, messageStatuses, deliveryStatuses, scheduledMessageDeliveryModes, scheduledMessageStatuses, jobStatuses, providerSchemas, agentProviders, agents, agentRoutes, agentSessions, apiKeys, apiKeyAuditLogs, apiKeysRelations, apiKeyAuditLogsRelations, instances, whatsappTemplates, whatsappFlowKeys, scheduledMessages, persons, platformIdentities, conversations, chats, chatParticipants, omniGroups, messages2, omniEvents, handoffLogs, closeContactOutcomes, closeContactLogs, accessRules, globalSettings, settingChangeHistory, batchJobs, syncJobTypes, syncJobs, mediaContent, chatIdMappings, pluginStorage, agentProvidersRelations, agentsRelations, instancesRelations, syncJobsRelations, personsRelations, platformIdentitiesRelations, conversationsRelations, chatsRelations, chatParticipantsRelations, messagesRelations, omniEventsRelations, accessRulesRelations, globalSettingsRelations, settingChangeHistoryRelations, batchJobsRelations, mediaContentRelations, chatIdMappingsRelations, deadLetterStatuses, deadLetterEvents, payloadStorageConfig, payloadStages, eventPayloads, webhookSignatureAlgorithms, connectorLivenessStatuses, connectorWindowSemanticsValues, connectorMutationPolicies, webhookSources, eventSchemas, conditionOperators, actionTypes, automationDebounceModes, automations2, automationLogStatuses, automationLogs, consumerOffsets, automationsRelations, automationLogsRelations, triggerLogs, triggerLogsRelations, agentRoutesRelations, agentTaskStatuses, agentTasks, agentTasksRelations, turnStatuses, turnActions, turns, turnsRelations, followUpDisarmReasons, chatFollowUpState, chatFollowUpStateRelations, processedEvents, genieHosts, tenantStatuses, principalTypes, principalStatuses, tenantRoles, membershipStatuses, credentialClasses, authCredentialStatuses, platformApiKeyStatuses, tenants, principals, tenantMemberships, tenantRolePolicies, platformApiKeys, tenantKeyLineage, authCredentials, tenantAuditLogs, platformAuditLogs, platformProviderCatalog, tenantProviderConfig, platformSettings, tenantSettings, platformSettingChangeHistory, tenantSettingChangeHistory, platformPluginStorage, tenantPluginStorage, platformPayloadStorageConfig, tenantPayloadStorageOverrides, tenantMigrationLedger, tenantMigrationLedgerHistory;
|
|
172216
172346
|
var init_schema2 = __esm(() => {
|
|
172217
172347
|
init_events();
|
|
172218
172348
|
init_types5();
|
|
@@ -173425,6 +173555,9 @@ var init_schema2 = __esm(() => {
|
|
|
173425
173555
|
eventStageIdx: index("event_payloads_event_stage_idx").on(table3.eventId, table3.stage)
|
|
173426
173556
|
}));
|
|
173427
173557
|
webhookSignatureAlgorithms = ["hmac-sha256", "hmac-sha1", "token-match"];
|
|
173558
|
+
connectorLivenessStatuses = ["healthy", "stalled"];
|
|
173559
|
+
connectorWindowSemanticsValues = ["future_only", "includes_in_progress"];
|
|
173560
|
+
connectorMutationPolicies = ["same_id", "new_id"];
|
|
173428
173561
|
webhookSources = pgTable("webhook_sources", {
|
|
173429
173562
|
id: uuid("id").primaryKey().defaultRandom(),
|
|
173430
173563
|
name: varchar("name", { length: 100 }).notNull().unique(),
|
|
@@ -173436,6 +173569,14 @@ var init_schema2 = __esm(() => {
|
|
|
173436
173569
|
enabled: boolean2("enabled").notNull().default(true),
|
|
173437
173570
|
lastReceivedAt: timestamp("last_received_at", { withTimezone: true }),
|
|
173438
173571
|
totalReceived: integer("total_received").notNull().default(0),
|
|
173572
|
+
expectedIntervalSeconds: integer("expected_interval_seconds"),
|
|
173573
|
+
lastHeartbeatAt: timestamp("last_heartbeat_at", { withTimezone: true }),
|
|
173574
|
+
heartbeatCount: integer("heartbeat_count").notNull().default(0),
|
|
173575
|
+
livenessStatus: varchar("liveness_status", { length: 20 }).$type(),
|
|
173576
|
+
livenessArmedAt: timestamp("liveness_armed_at", { withTimezone: true }),
|
|
173577
|
+
stalledAt: timestamp("stalled_at", { withTimezone: true }),
|
|
173578
|
+
windowSemantics: varchar("window_semantics", { length: 40 }).$type(),
|
|
173579
|
+
mutationPolicy: varchar("mutation_policy", { length: 20 }).$type(),
|
|
173439
173580
|
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
|
173440
173581
|
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
|
173441
173582
|
tenantId: uuid("tenant_id").references(() => tenants.id, { onDelete: "restrict" })
|
|
@@ -173443,7 +173584,8 @@ var init_schema2 = __esm(() => {
|
|
|
173443
173584
|
tenantIdx: index("webhook_sources_tenant_idx").on(table3.tenantId),
|
|
173444
173585
|
tenantNameUq: uniqueIndex("webhook_sources_tenant_name_uq").on(table3.tenantId, table3.name).where(sql`${table3.tenantId} IS NOT NULL`),
|
|
173445
173586
|
nameIdx: uniqueIndex("webhook_sources_name_idx").on(table3.name),
|
|
173446
|
-
enabledIdx: index("webhook_sources_enabled_idx").on(table3.enabled)
|
|
173587
|
+
enabledIdx: index("webhook_sources_enabled_idx").on(table3.enabled),
|
|
173588
|
+
supervisedIdx: index("webhook_sources_supervised_idx").on(table3.enabled).where(sql`${table3.expectedIntervalSeconds} IS NOT NULL`)
|
|
173447
173589
|
}));
|
|
173448
173590
|
eventSchemas = pgTable("event_schemas", {
|
|
173449
173591
|
id: uuid("id").primaryKey().defaultRandom(),
|
|
@@ -177895,6 +178037,9 @@ __export(exports_src3, {
|
|
|
177895
178037
|
conversations: () => conversations,
|
|
177896
178038
|
contentTypes: () => contentTypes,
|
|
177897
178039
|
consumerOffsets: () => consumerOffsets,
|
|
178040
|
+
connectorWindowSemanticsValues: () => connectorWindowSemanticsValues,
|
|
178041
|
+
connectorMutationPolicies: () => connectorMutationPolicies,
|
|
178042
|
+
connectorLivenessStatuses: () => connectorLivenessStatuses,
|
|
177898
178043
|
conditionOperators: () => conditionOperators,
|
|
177899
178044
|
closeDb: () => closeDb,
|
|
177900
178045
|
closeContactOutcomes: () => closeContactOutcomes,
|
|
@@ -250680,7 +250825,7 @@ var init_sentry_scrub = __esm(() => {
|
|
|
250680
250825
|
var require_package7 = __commonJS((exports, module) => {
|
|
250681
250826
|
module.exports = {
|
|
250682
250827
|
name: "@omni/api",
|
|
250683
|
-
version: "2.260906.
|
|
250828
|
+
version: "2.260906.3",
|
|
250684
250829
|
private: true,
|
|
250685
250830
|
type: "module",
|
|
250686
250831
|
exports: {
|
|
@@ -302543,6 +302688,7 @@ var init_route_ownership = __esm(() => {
|
|
|
302543
302688
|
"POST /api/v2/voice/leave",
|
|
302544
302689
|
"POST /api/v2/webhook-sources",
|
|
302545
302690
|
"POST /api/v2/webhooks/:source",
|
|
302691
|
+
"POST /api/v2/webhooks/:source/heartbeat",
|
|
302546
302692
|
"PUT /api/v2/agent-state/:agentId/:chatId",
|
|
302547
302693
|
"PUT /api/v2/follow-up/agents/:id",
|
|
302548
302694
|
"PUT /api/v2/follow-up/chats/:id",
|
|
@@ -358067,6 +358213,9 @@ class DeadLetterService {
|
|
|
358067
358213
|
}
|
|
358068
358214
|
async create(options) {
|
|
358069
358215
|
const insertData = prepareDeadLetterInsert(options);
|
|
358216
|
+
if (options.autoRetry === false) {
|
|
358217
|
+
insertData.nextAutoRetryAt = null;
|
|
358218
|
+
}
|
|
358070
358219
|
const [result] = await this.db.insert(deadLetterEvents).values(insertData).returning();
|
|
358071
358220
|
if (!result) {
|
|
358072
358221
|
throw new Error("Failed to create dead letter entry");
|
|
@@ -363201,7 +363350,10 @@ class WebhookService {
|
|
|
363201
363350
|
if (data.signatureSecret && !data.signatureConfig) {
|
|
363202
363351
|
throw new ValidationError2("signatureSecret cannot be set without a signatureConfig");
|
|
363203
363352
|
}
|
|
363204
|
-
|
|
363353
|
+
let values2 = data.signatureSecret ? { ...data, signatureSecret: sealCredentialField(this.tenantId, data.signatureSecret) } : data;
|
|
363354
|
+
if (values2.expectedIntervalSeconds != null) {
|
|
363355
|
+
values2 = { ...values2, livenessArmedAt: new Date, livenessStatus: "healthy" };
|
|
363356
|
+
}
|
|
363205
363357
|
const [created] = await this.db.insert(webhookSources).values(values2).returning();
|
|
363206
363358
|
if (!created) {
|
|
363207
363359
|
throw new Error("Failed to create webhook source");
|
|
@@ -363227,12 +363379,28 @@ class WebhookService {
|
|
|
363227
363379
|
patch.signatureSecret = null;
|
|
363228
363380
|
}
|
|
363229
363381
|
}
|
|
363382
|
+
await this.applyLivenessArming(id, data, patch);
|
|
363230
363383
|
const [updated] = await this.db.update(webhookSources).set({ ...patch, updatedAt: new Date }).where(eq(webhookSources.id, id)).returning();
|
|
363231
363384
|
if (!updated) {
|
|
363232
363385
|
throw new NotFoundError("WebhookSource", id);
|
|
363233
363386
|
}
|
|
363234
363387
|
return updated;
|
|
363235
363388
|
}
|
|
363389
|
+
async applyLivenessArming(id, data, patch) {
|
|
363390
|
+
if (data.expectedIntervalSeconds === undefined)
|
|
363391
|
+
return;
|
|
363392
|
+
if (data.expectedIntervalSeconds === null) {
|
|
363393
|
+
patch.livenessStatus = null;
|
|
363394
|
+
patch.livenessArmedAt = null;
|
|
363395
|
+
patch.stalledAt = null;
|
|
363396
|
+
return;
|
|
363397
|
+
}
|
|
363398
|
+
patch.livenessArmedAt = new Date;
|
|
363399
|
+
const existing = await this.getById(id);
|
|
363400
|
+
if (existing.livenessStatus === null) {
|
|
363401
|
+
patch.livenessStatus = "healthy";
|
|
363402
|
+
}
|
|
363403
|
+
}
|
|
363236
363404
|
async delete(id) {
|
|
363237
363405
|
const result = await this.db.delete(webhookSources).where(eq(webhookSources.id, id)).returning();
|
|
363238
363406
|
if (!result.length) {
|
|
@@ -363360,6 +363528,101 @@ class WebhookService {
|
|
|
363360
363528
|
}
|
|
363361
363529
|
return { eventId, published: false };
|
|
363362
363530
|
}
|
|
363531
|
+
async heartbeat(sourceName) {
|
|
363532
|
+
const source = await this.getByName(sourceName);
|
|
363533
|
+
if (!source) {
|
|
363534
|
+
throw new NotFoundError("WebhookSource", sourceName);
|
|
363535
|
+
}
|
|
363536
|
+
if (!source.enabled) {
|
|
363537
|
+
throw new OmniError({
|
|
363538
|
+
code: ERROR_CODES.FORBIDDEN,
|
|
363539
|
+
message: `Webhook source '${sourceName}' is disabled`,
|
|
363540
|
+
context: { sourceName }
|
|
363541
|
+
});
|
|
363542
|
+
}
|
|
363543
|
+
const now = new Date;
|
|
363544
|
+
await this.db.update(webhookSources).set({
|
|
363545
|
+
lastHeartbeatAt: now,
|
|
363546
|
+
heartbeatCount: sql`${webhookSources.heartbeatCount} + 1`,
|
|
363547
|
+
updatedAt: now
|
|
363548
|
+
}).where(eq(webhookSources.id, source.id));
|
|
363549
|
+
return {
|
|
363550
|
+
ok: true,
|
|
363551
|
+
source: sourceName,
|
|
363552
|
+
heartbeatAt: now.toISOString(),
|
|
363553
|
+
livenessStatus: source.livenessStatus,
|
|
363554
|
+
expectedIntervalSeconds: source.expectedIntervalSeconds
|
|
363555
|
+
};
|
|
363556
|
+
}
|
|
363557
|
+
async sweepLiveness(options = {}) {
|
|
363558
|
+
const { deadLetters } = options;
|
|
363559
|
+
const hooks2 = deadLetters ? {
|
|
363560
|
+
onStalled: (row, payload, publishedEventId) => fileStallDeadLetter(deadLetters, row, payload, publishedEventId),
|
|
363561
|
+
onRecovered: (row) => resolveStallDeadLetters(deadLetters, row)
|
|
363562
|
+
} : {};
|
|
363563
|
+
return sweepConnectorLiveness({
|
|
363564
|
+
repo: this.livenessRepo(),
|
|
363565
|
+
eventBus: this.eventBus,
|
|
363566
|
+
logger: log111,
|
|
363567
|
+
...hooks2
|
|
363568
|
+
});
|
|
363569
|
+
}
|
|
363570
|
+
livenessRepo() {
|
|
363571
|
+
return {
|
|
363572
|
+
findSupervised: async () => {
|
|
363573
|
+
const rows = await this.db.select().from(webhookSources).where(and2(eq(webhookSources.enabled, true), isNotNull(webhookSources.expectedIntervalSeconds)));
|
|
363574
|
+
return rows.filter((row) => row.expectedIntervalSeconds !== null).map((row) => ({
|
|
363575
|
+
id: row.id,
|
|
363576
|
+
name: row.name,
|
|
363577
|
+
expectedIntervalSeconds: row.expectedIntervalSeconds,
|
|
363578
|
+
lastReceivedAt: row.lastReceivedAt,
|
|
363579
|
+
lastHeartbeatAt: row.lastHeartbeatAt,
|
|
363580
|
+
livenessArmedAt: row.livenessArmedAt,
|
|
363581
|
+
livenessStatus: row.livenessStatus,
|
|
363582
|
+
stalledAt: row.stalledAt,
|
|
363583
|
+
createdAt: row.createdAt,
|
|
363584
|
+
tenantId: row.tenantId
|
|
363585
|
+
}));
|
|
363586
|
+
},
|
|
363587
|
+
markStalled: async (id, at) => {
|
|
363588
|
+
const res = await this.db.update(webhookSources).set({ livenessStatus: "stalled", stalledAt: at, updatedAt: at }).where(and2(eq(webhookSources.id, id), sql`${webhookSources.livenessStatus} IS DISTINCT FROM 'stalled'`)).returning({ id: webhookSources.id });
|
|
363589
|
+
return res.length > 0;
|
|
363590
|
+
},
|
|
363591
|
+
markRecovered: async (id, at) => {
|
|
363592
|
+
const res = await this.db.update(webhookSources).set({ livenessStatus: "healthy", stalledAt: null, updatedAt: at }).where(and2(eq(webhookSources.id, id), eq(webhookSources.livenessStatus, "stalled"))).returning({ id: webhookSources.id });
|
|
363593
|
+
return res.length > 0;
|
|
363594
|
+
}
|
|
363595
|
+
};
|
|
363596
|
+
}
|
|
363597
|
+
}
|
|
363598
|
+
async function fileStallDeadLetter(deadLetters, row, payload, publishedEventId) {
|
|
363599
|
+
const event = {
|
|
363600
|
+
id: publishedEventId ?? generateId(),
|
|
363601
|
+
type: "system.connector.stalled",
|
|
363602
|
+
payload: { ...payload },
|
|
363603
|
+
metadata: {
|
|
363604
|
+
correlationId: generateId(),
|
|
363605
|
+
source: "connector-liveness",
|
|
363606
|
+
...row.tenantId ? { tenantId: row.tenantId } : {}
|
|
363607
|
+
},
|
|
363608
|
+
timestamp: payload.stalledAt
|
|
363609
|
+
};
|
|
363610
|
+
await deadLetters.create({
|
|
363611
|
+
event,
|
|
363612
|
+
subject: "system.connector.stalled.internal.global",
|
|
363613
|
+
error: new Error(`Connector '${row.name}' declared >=1 event or heartbeat per ${row.expectedIntervalSeconds}s ` + `but has been silent for ${payload.silentForSeconds}s`),
|
|
363614
|
+
retryCount: 0,
|
|
363615
|
+
autoRetry: false
|
|
363616
|
+
});
|
|
363617
|
+
}
|
|
363618
|
+
async function resolveStallDeadLetters(deadLetters, row) {
|
|
363619
|
+
const { items } = await deadLetters.list({ status: ["pending"], eventType: ["system.connector.stalled"] });
|
|
363620
|
+
for (const entry of items) {
|
|
363621
|
+
const stored = entry.payload;
|
|
363622
|
+
if (stored.payload?.sourceId === row.id) {
|
|
363623
|
+
await deadLetters.resolve(entry.id, "connector-liveness: recovered");
|
|
363624
|
+
}
|
|
363625
|
+
}
|
|
363363
363626
|
}
|
|
363364
363627
|
var log111;
|
|
363365
363628
|
var init_webhooks = __esm(() => {
|
|
@@ -369789,6 +370052,7 @@ function registerWebhookSchemas(registry4) {
|
|
|
369789
370052
|
registry4.register("CreateWebhookSourceRequest", CreateWebhookSourceSchema);
|
|
369790
370053
|
registry4.register("TriggerEventRequest", TriggerEventSchema);
|
|
369791
370054
|
registry4.register("WebhookReceiveResponse", WebhookReceiveResponseSchema);
|
|
370055
|
+
registry4.register("WebhookHeartbeatResponse", WebhookHeartbeatResponseSchema);
|
|
369792
370056
|
registry4.registerPath({
|
|
369793
370057
|
method: "get",
|
|
369794
370058
|
path: "/webhook-sources",
|
|
@@ -369887,6 +370151,25 @@ function registerWebhookSchemas(registry4) {
|
|
|
369887
370151
|
400: { description: "Body is not a JSON object", content: { "application/json": { schema: ErrorSchema } } }
|
|
369888
370152
|
}
|
|
369889
370153
|
});
|
|
370154
|
+
registry4.registerPath({
|
|
370155
|
+
method: "post",
|
|
370156
|
+
path: "/webhooks/{source}/heartbeat",
|
|
370157
|
+
operationId: "heartbeatWebhookSource",
|
|
370158
|
+
tags: ["Webhooks"],
|
|
370159
|
+
summary: "Connector heartbeat",
|
|
370160
|
+
description: 'Record a connector heartbeat: "I ran, zero events found". Resets the liveness window of a supervised ' + "source (one declaring expectedIntervalSeconds) so silence-beyond-window detection can tell quiet from " + "dead. Creates NO journal event \u2014 only the stalled/recovered transitions are journaled. No request body.",
|
|
370161
|
+
request: {
|
|
370162
|
+
params: exports_external.object({ source: exports_external.string().openapi({ description: "Source name" }) })
|
|
370163
|
+
},
|
|
370164
|
+
responses: {
|
|
370165
|
+
200: {
|
|
370166
|
+
description: "Heartbeat recorded",
|
|
370167
|
+
content: { "application/json": { schema: WebhookHeartbeatResponseSchema } }
|
|
370168
|
+
},
|
|
370169
|
+
403: { description: "Source disabled", content: { "application/json": { schema: ErrorSchema } } },
|
|
370170
|
+
404: { description: "Source not found", content: { "application/json": { schema: ErrorSchema } } }
|
|
370171
|
+
}
|
|
370172
|
+
});
|
|
369890
370173
|
registry4.registerPath({
|
|
369891
370174
|
method: "post",
|
|
369892
370175
|
path: "/webhooks/ingress/{source}",
|
|
@@ -369925,7 +370208,7 @@ function registerWebhookSchemas(registry4) {
|
|
|
369925
370208
|
}
|
|
369926
370209
|
});
|
|
369927
370210
|
}
|
|
369928
|
-
var WebhookSignatureConfigSchema, WebhookEventTypeMappingSchema, WebhookSourceSchema, CreateWebhookSourceSchema, TriggerEventSchema, WebhookReceiveResponseSchema;
|
|
370211
|
+
var WebhookSignatureConfigSchema, WebhookEventTypeMappingSchema, WebhookSourceSchema, CreateWebhookSourceSchema, WebhookHeartbeatResponseSchema, TriggerEventSchema, WebhookReceiveResponseSchema;
|
|
369929
370212
|
var init_webhooks2 = __esm(() => {
|
|
369930
370213
|
init_src5();
|
|
369931
370214
|
init_zod_openapi();
|
|
@@ -369963,6 +370246,16 @@ var init_webhooks2 = __esm(() => {
|
|
|
369963
370246
|
}),
|
|
369964
370247
|
hasSignatureSecret: exports_external.boolean().openapi({ description: "Whether a signature secret is stored (secret is write-only)" }),
|
|
369965
370248
|
enabled: exports_external.boolean().openapi({ description: "Whether enabled" }),
|
|
370249
|
+
lastReceivedAt: exports_external.string().datetime().nullable().openapi({ description: "When the last webhook was received" }),
|
|
370250
|
+
totalReceived: exports_external.number().int().openapi({ description: "Total webhooks received" }),
|
|
370251
|
+
expectedIntervalSeconds: exports_external.number().int().nullable().openapi({ description: "Declared cadence: >=1 event or heartbeat per N seconds. Null = unsupervised" }),
|
|
370252
|
+
lastHeartbeatAt: exports_external.string().datetime().nullable().openapi({ description: 'Last heartbeat ("ran, zero events")' }),
|
|
370253
|
+
heartbeatCount: exports_external.number().int().openapi({ description: "Total heartbeats received" }),
|
|
370254
|
+
livenessStatus: exports_external.enum(connectorLivenessStatuses).nullable().openapi({ description: "Liveness state; null = unsupervised. Transitions emit system.connector.* events" }),
|
|
370255
|
+
livenessArmedAt: exports_external.string().datetime().nullable().openapi({ description: "When the cadence was (re)declared" }),
|
|
370256
|
+
stalledAt: exports_external.string().datetime().nullable().openapi({ description: "When the current stall began" }),
|
|
370257
|
+
windowSemantics: exports_external.enum(connectorWindowSemanticsValues).nullable().openapi({ description: "Declared window semantics; null = undeclared" }),
|
|
370258
|
+
mutationPolicy: exports_external.enum(connectorMutationPolicies).nullable().openapi({ description: "Declared upstream-mutation re-emit policy; null = undeclared" }),
|
|
369966
370259
|
createdAt: exports_external.string().datetime().openapi({ description: "Creation timestamp" }),
|
|
369967
370260
|
updatedAt: exports_external.string().datetime().openapi({ description: "Last update timestamp" })
|
|
369968
370261
|
});
|
|
@@ -369979,7 +370272,23 @@ var init_webhooks2 = __esm(() => {
|
|
|
369979
370272
|
eventTypeMapping: WebhookEventTypeMappingSchema.nullable().optional().openapi({
|
|
369980
370273
|
description: "Semantic event-type extraction (e.g. header X-GitHub-Event: push emits custom.{source}.push). " + "Null or absent keeps the legacy collapsed custom.webhook.{source} type for every delivery."
|
|
369981
370274
|
}),
|
|
369982
|
-
enabled: exports_external.boolean().default(true).openapi({ description: "Whether enabled" })
|
|
370275
|
+
enabled: exports_external.boolean().default(true).openapi({ description: "Whether enabled" }),
|
|
370276
|
+
expectedIntervalSeconds: exports_external.number().int().min(1).max(2592000).nullable().optional().openapi({
|
|
370277
|
+
description: "Declared cadence: the connector promises >=1 event or heartbeat per N seconds (1s-30d). Declaring it " + "arms liveness supervision (silence beyond the window emits system.connector.stalled and marks the " + "source unhealthy); null disarms it."
|
|
370278
|
+
}),
|
|
370279
|
+
windowSemantics: exports_external.enum(connectorWindowSemanticsValues).nullable().optional().openapi({
|
|
370280
|
+
description: "Declared time-window semantics: 'future_only' (only not-yet-started items) or 'includes_in_progress'. " + "Informational contract for consumers; null = undeclared."
|
|
370281
|
+
}),
|
|
370282
|
+
mutationPolicy: exports_external.enum(connectorMutationPolicies).nullable().optional().openapi({
|
|
370283
|
+
description: "How the source re-emits a changed upstream item: 'same_id' (reschedule case \u2014 consumers must key on " + "id+content) or 'new_id'. Feeds the idempotency key template choice; null = undeclared."
|
|
370284
|
+
})
|
|
370285
|
+
});
|
|
370286
|
+
WebhookHeartbeatResponseSchema = exports_external.object({
|
|
370287
|
+
ok: exports_external.literal(true).openapi({ description: "Heartbeat recorded" }),
|
|
370288
|
+
source: exports_external.string().openapi({ description: "Webhook source name" }),
|
|
370289
|
+
heartbeatAt: exports_external.string().datetime().openapi({ description: "When the heartbeat was recorded" }),
|
|
370290
|
+
livenessStatus: exports_external.enum(connectorLivenessStatuses).nullable().openapi({ description: "Status before this heartbeat (a stalled source recovers on the next sweep tick)" }),
|
|
370291
|
+
expectedIntervalSeconds: exports_external.number().int().nullable().openapi({ description: "Declared cadence, if any" })
|
|
369983
370292
|
});
|
|
369984
370293
|
TriggerEventSchema = exports_external.object({
|
|
369985
370294
|
eventType: exports_external.string().min(1).openapi({ description: "Event type (must start with custom.)" }),
|
|
@@ -382522,6 +382831,12 @@ var init_webhooks3 = __esm(() => {
|
|
|
382522
382831
|
});
|
|
382523
382832
|
return c.json(result);
|
|
382524
382833
|
});
|
|
382834
|
+
webhooksRoutes.post("/webhooks/:source/heartbeat", async (c) => {
|
|
382835
|
+
const sourceName = c.req.param("source");
|
|
382836
|
+
const services = c.get("services");
|
|
382837
|
+
const result = await services.webhooks.heartbeat(sourceName);
|
|
382838
|
+
return c.json(result);
|
|
382839
|
+
});
|
|
382525
382840
|
triggerEventSchema = exports_external.object({
|
|
382526
382841
|
eventType: exports_external.string().min(1).refine((t) => t.startsWith("custom."), { message: 'Event type must start with "custom."' }).describe("Event type (must be custom.*)"),
|
|
382527
382842
|
payload: exports_external.record(exports_external.string(), exports_external.unknown()).describe("Event payload"),
|
|
@@ -386672,6 +386987,26 @@ function setupScheduler(services, channelRegistry2) {
|
|
|
386672
386987
|
});
|
|
386673
386988
|
}
|
|
386674
386989
|
});
|
|
386990
|
+
scheduler.register({
|
|
386991
|
+
name: "connector-liveness-sweeper",
|
|
386992
|
+
cron: "*/30 * * * * *",
|
|
386993
|
+
runOnStart: false,
|
|
386994
|
+
handler: async () => {
|
|
386995
|
+
await withCronMonitor("connector-liveness-sweeper", "*/30 * * * * *", 1, 1, async () => {
|
|
386996
|
+
const startTime3 = Date.now();
|
|
386997
|
+
try {
|
|
386998
|
+
const stats = await services.webhooks.sweepLiveness({ deadLetters: services.deadLetters });
|
|
386999
|
+
recordScheduledJob("connector-liveness-sweeper", "success", (Date.now() - startTime3) / 1000);
|
|
387000
|
+
if (stats.stalled > 0 || stats.recovered > 0) {
|
|
387001
|
+
log130.info("Connector liveness sweep tick", { ...stats });
|
|
387002
|
+
}
|
|
387003
|
+
} catch (err2) {
|
|
387004
|
+
recordScheduledJob("connector-liveness-sweeper", "failure", (Date.now() - startTime3) / 1000);
|
|
387005
|
+
throw err2;
|
|
387006
|
+
}
|
|
387007
|
+
});
|
|
387008
|
+
}
|
|
387009
|
+
});
|
|
386675
387010
|
if (channelRegistry2) {
|
|
386676
387011
|
const scheduledMessages2 = createScheduledMessageSweeper(services, channelRegistry2);
|
|
386677
387012
|
scheduler.register({
|