@shipfox/api-server 5.0.0 → 6.0.0

package/src/modules.ts

@@ -1,115 +1,168 @@
 import {annotationsModule} from '@shipfox/annotations';
-import {agentModule} from '@shipfox/api-agent';
-import {authModule} from '@shipfox/api-auth';
+import {annotationsInterModuleContract} from '@shipfox/annotations-dto/inter-module';
+import {createAgentModule} from '@shipfox/api-agent';
+import {agentInterModuleContract} from '@shipfox/api-agent-dto/inter-module';
+import {createAuthModule} from '@shipfox/api-auth';
+import {config as authConfig} from '@shipfox/api-auth/config';
+import {authInterModuleContract} from '@shipfox/api-auth-dto/inter-module';
 import {createDefinitionsModule} from '@shipfox/api-definitions';
+import {definitionsInterModuleContract} from '@shipfox/api-definitions-dto/inter-module';
 import {dispatcherModule} from '@shipfox/api-dispatcher';
-import {
-  buildAgentToolCatalogs,
-  buildAgentToolSelectionCatalogs,
-  createIntegrationsContext,
-  createWorkspaceConnectionSnapshotLoader,
-  getIntegrationConnectionById,
-} from '@shipfox/api-integration-core';
-import {logsModule} from '@shipfox/api-logs';
+import {emailChallengesModule} from '@shipfox/api-email-challenges';
+import {createIntegrationsContext, type WebhookDeliverySource} from '@shipfox/api-integration-core';
+import {integrationsInterModuleContract} from '@shipfox/api-integration-core-dto';
+import {createLogsModule} from '@shipfox/api-logs';
 import {createProjectsModule} from '@shipfox/api-projects';
-import {runnersModule} from '@shipfox/api-runners';
-import {deleteSecrets, getSecret, secretsModule, setSecrets} from '@shipfox/api-secrets';
-import {triggersModule} from '@shipfox/api-triggers';
-import {
-  loadRunningLeasedStep,
-  setAgentToolMaterializationServices,
-  setSourceControl,
-  workflowsModule,
-} from '@shipfox/api-workflows';
+import {projectsInterModuleContract} from '@shipfox/api-projects-dto';
+import {createRunnersModule} from '@shipfox/api-runners';
+import {runnersInterModuleContract} from '@shipfox/api-runners-dto/inter-module';
+import {createSecretsModule} from '@shipfox/api-secrets';
+import {secretsInterModuleContract} from '@shipfox/api-secrets-dto/inter-module';
+import {createTriggersModule} from '@shipfox/api-triggers';
+import {createWorkflowsModule} from '@shipfox/api-workflows';
+import {workflowsInterModuleContract} from '@shipfox/api-workflows-dto/inter-module';
 import {workspacesModule} from '@shipfox/api-workspaces';
+import {workspacesInterModuleContract} from '@shipfox/api-workspaces-dto/inter-module';
+import {durationToSeconds} from '@shipfox/node-jwt';
 import type {ShipfoxModule} from '@shipfox/node-module';
+import {
+  createInMemoryInterModuleTransport,
+  registerInterModulePresentations,
+} from '@shipfox/node-module/inter-module';
 
-export async function defaultModules(): Promise<ShipfoxModule[]> {
+export interface DefaultModulesOptions {
+  webhookDeliverySource?: WebhookDeliverySource | undefined;
+}
+
+export async function defaultModules(
+  options: DefaultModulesOptions = {},
+): Promise<ShipfoxModule[]> {
+  const interModuleTransport = createInMemoryInterModuleTransport();
+  const workflowsClient = interModuleTransport.createClient(workflowsInterModuleContract);
+  const authClient = interModuleTransport.createClient(authInterModuleContract);
+  const agentClient = interModuleTransport.createClient(agentInterModuleContract);
+  const runnersClient = interModuleTransport.createClient(runnersInterModuleContract);
+  const projectsClient = interModuleTransport.createClient(projectsInterModuleContract);
+  const definitionsClient = interModuleTransport.createClient(definitionsInterModuleContract);
+  const annotationsClient = interModuleTransport.createClient(annotationsInterModuleContract);
+  const secretsClient = interModuleTransport.createClient(secretsInterModuleContract);
+  const workspacesClient = interModuleTransport.createClient(workspacesInterModuleContract);
+  const integrationsClient = interModuleTransport.createClient(integrationsInterModuleContract);
   const integrations = await createIntegrationsContext({
+    workspaces: workspacesClient,
     secrets: {
-      deleteSecrets,
+      deleteSecrets: async (params) => (await secretsClient.deleteSecrets(params)).deleted,
       linear: {
-        getSecret: (params) =>
-          getSecret({
-            ...params,
-            namespace: `system/integrations/linear/${params.namespace}`,
-          }),
-        setSecrets: (params) =>
-          setSecrets({
-            ...params,
-            namespace: `system/integrations/linear/${params.namespace}`,
-          }),
-        deleteSecrets: (params) =>
-          deleteSecrets({
-            ...params,
-            namespace: `system/integrations/linear/${params.namespace}`,
-          }),
+        getSecret: async (params) =>
+          (
+            await secretsClient.getSecret({
+              ...params,
+              namespace: `system/integrations/linear/${params.namespace}`,
+            })
+          ).value,
+        setSecrets: async (params) => {
+          const {editedBy, ...secretParams} = params;
+          await secretsClient.setSecrets({
+            ...secretParams,
+            namespace: `system/integrations/linear/${secretParams.namespace}`,
+            ...(editedBy === undefined ? {} : {editedBy}),
+          });
+        },
+        deleteSecrets: async (params) =>
+          (
+            await secretsClient.deleteSecrets({
+              ...params,
+              namespace: `system/integrations/linear/${params.namespace}`,
+            })
+          ).deleted,
       },
       jira: {
-        getSecret: (params) =>
-          getSecret({...params, namespace: `system/integrations/jira/${params.namespace}`}),
-        setSecrets: (params) =>
-          setSecrets({...params, namespace: `system/integrations/jira/${params.namespace}`}),
-        deleteSecrets: (params) =>
-          deleteSecrets({...params, namespace: `system/integrations/jira/${params.namespace}`}),
+        getSecret: async (params) =>
+          (
+            await secretsClient.getSecret({
+              ...params,
+              namespace: `system/integrations/jira/${params.namespace}`,
+            })
+          ).value,
+        setSecrets: async (params) => {
+          const {editedBy, ...secretParams} = params;
+          await secretsClient.setSecrets({
+            ...secretParams,
+            namespace: `system/integrations/jira/${secretParams.namespace}`,
+            ...(editedBy === undefined ? {} : {editedBy}),
+          });
+        },
+        deleteSecrets: async (params) =>
+          (
+            await secretsClient.deleteSecrets({
+              ...params,
+              namespace: `system/integrations/jira/${params.namespace}`,
+            })
+          ).deleted,
       },
       slack: {
-        getSecret: (params) =>
-          getSecret({
-            ...params,
-            namespace: `system/integrations/slack/${params.namespace}`,
-          }),
-        setSecrets: (params) =>
-          setSecrets({
-            ...params,
-            namespace: `system/integrations/slack/${params.namespace}`,
-          }),
-        deleteSecrets: (params) =>
-          deleteSecrets({
-            ...params,
-            namespace: `system/integrations/slack/${params.namespace}`,
-          }),
+        getSecret: async (params) =>
+          (
+            await secretsClient.getSecret({
+              ...params,
+              namespace: `system/integrations/slack/${params.namespace}`,
+            })
+          ).value,
+        setSecrets: async (params) => {
+          const {editedBy, ...secretParams} = params;
+          await secretsClient.setSecrets({
+            ...secretParams,
+            namespace: `system/integrations/slack/${secretParams.namespace}`,
+            ...(editedBy === undefined ? {} : {editedBy}),
+          });
+        },
+        deleteSecrets: async (params) =>
+          (
+            await secretsClient.deleteSecrets({
+              ...params,
+              namespace: `system/integrations/slack/${params.namespace}`,
+            })
+          ).deleted,
       },
     },
-    agentTools: {loadLeasedAgentStep: loadRunningLeasedStep},
-  });
-  const [agentToolSelectionCatalogs, agentToolCatalogs] = await Promise.all([
-    buildAgentToolSelectionCatalogs(integrations.registry),
-    buildAgentToolCatalogs(integrations.registry),
-  ]);
-  const loadWorkspaceConnectionSnapshot = createWorkspaceConnectionSnapshotLoader(
-    integrations.registry,
-  );
-
-  // The checkout-token route resolves intents and mints credentials through the
-  // source-control service; wire it into the workflows module before serving.
-  setSourceControl(integrations.sourceControl);
-  setAgentToolMaterializationServices({
-    catalogs: agentToolCatalogs,
-    loadWorkspaceConnectionSnapshot,
-    getIntegrationConnectionById,
+    agentTools: {workflows: workflowsClient},
+    webhookDeliverySource: options.webhookDeliverySource,
   });
-  const projectsModule = createProjectsModule({sourceControl: integrations.sourceControl});
+  const projectsModule = createProjectsModule({integrations: integrationsClient});
   const definitionsModule = createDefinitionsModule({
-    sourceControl: integrations.sourceControl,
-    agentToolSelectionCatalogs,
-    loadWorkspaceConnectionSnapshot,
-    getIntegrationConnectionById,
+    projects: projectsClient,
+    integrations: integrationsClient,
   });
 
-  return [
-    authModule,
+  const modules = [
+    emailChallengesModule,
+    createAuthModule({workspaces: workspacesClient}),
     workspacesModule,
-    secretsModule,
-    agentModule,
+    createSecretsModule(projectsClient),
+    createAgentModule({secrets: secretsClient}),
     integrations.module,
     projectsModule,
     definitionsModule,
-    workflowsModule,
+    createWorkflowsModule({
+      annotations: annotationsClient,
+      agent: agentClient,
+      definitions: definitionsClient,
+      auth: authClient,
+      projects: projectsClient,
+      runners: runnersClient,
+      secrets: secretsClient,
+      integrations: integrationsClient,
+    }),
     annotationsModule,
-    runnersModule,
-    logsModule,
-    triggersModule,
+    createRunnersModule({auth: authClient}),
+    createLogsModule({
+      workflows: workflowsClient,
+      jobLeaseTokenTtlSeconds: durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN),
+    }),
+    createTriggersModule({workflows: workflowsClient}),
     dispatcherModule,
   ];
+  registerInterModulePresentations({transport: interModuleTransport, modules});
+  interModuleTransport.seal();
+  return modules;
 }

package/src/server.test.ts

@@ -1,4 +1,6 @@
 import type {
+  ModuleService,
+  ModuleServicesHandle,
   ModuleWorker,
   ModuleWorkersHandle,
   ShipfoxModule,
@@ -8,6 +10,7 @@ import {createServer, runServer, type ServerHandle} from './server.js';
 
 const mocks = vi.hoisted(() => {
   const workersHandle = {stop: vi.fn()};
+  const servicesHandle = {stop: vi.fn()};
   return {
     captureException: vi.fn(),
     closeApp: vi.fn(),
@@ -23,13 +26,13 @@ const mocks = vi.hoisted(() => {
     logger: {error: vi.fn(), info: vi.fn()},
     parseApiTrustProxy: vi.fn(),
     registerModuleMetrics: vi.fn(),
-    resetPublishers: vi.fn(),
-    resetSubscribers: vi.fn(),
     runModuleStartupTasks: vi.fn(),
     shutdownServiceMetrics: vi.fn(),
+    startModuleServices: vi.fn(),
     startModuleWorkers: vi.fn(),
     startServiceMetrics: vi.fn(),
     workersHandle,
+    servicesHandle,
     apiConfig: {
       API_PORT: undefined as number | undefined,
       API_TRUST_PROXY: 'false',
@@ -51,9 +54,8 @@ vi.mock('@shipfox/node-module', () => ({
   aggregateLoginMethods: mocks.aggregateLoginMethods,
   initializeModules: mocks.initializeModules,
   registerModuleMetrics: mocks.registerModuleMetrics,
-  resetPublishers: mocks.resetPublishers,
-  resetSubscribers: mocks.resetSubscribers,
   runModuleStartupTasks: mocks.runModuleStartupTasks,
+  startModuleServices: mocks.startModuleServices,
   startModuleWorkers: mocks.startModuleWorkers,
 }));
 vi.mock('@shipfox/node-opentelemetry', () => ({
@@ -87,12 +89,28 @@ function worker(taskQueue = 'runtime-queue'): ModuleWorker {
   };
 }
 
+function service(name = 'runtime-service'): ModuleService {
+  return {name, shutdownTimeoutMs: 10_000, start: vi.fn()};
+}
+
 function lastStartModuleWorkersOptions(): StartModuleWorkersOptions {
   const options = mocks.startModuleWorkers.mock.calls.at(-1)?.[0];
   if (!options) throw new Error('startModuleWorkers was not called');
   return options as StartModuleWorkersOptions;
 }
 
+function lastStartModuleServicesOptions(): {
+  services: ModuleService[];
+  onServiceFailure?: (error: unknown, service: ModuleService) => void | Promise<void>;
+} {
+  const options = mocks.startModuleServices.mock.calls.at(-1)?.[0];
+  if (!options) throw new Error('startModuleServices was not called');
+  return options as {
+    services: ModuleService[];
+    onServiceFailure?: (error: unknown, service: ModuleService) => void | Promise<void>;
+  };
+}
+
 function resetMocks(): void {
   vi.useRealTimers();
   mocks.captureException.mockReset();
@@ -110,13 +128,13 @@ function resetMocks(): void {
   mocks.logger.info.mockReset();
   mocks.parseApiTrustProxy.mockReset();
   mocks.registerModuleMetrics.mockReset();
-  mocks.resetPublishers.mockReset();
-  mocks.resetSubscribers.mockReset();
   mocks.runModuleStartupTasks.mockReset();
   mocks.shutdownServiceMetrics.mockReset();
+  mocks.startModuleServices.mockReset();
   mocks.startModuleWorkers.mockReset();
   mocks.startServiceMetrics.mockReset();
   mocks.workersHandle.stop.mockReset();
+  mocks.servicesHandle.stop.mockReset();
   mocks.apiConfig.API_PORT = undefined;
   mocks.apiConfig.API_TRUST_PROXY = 'false';
   mocks.apiConfig.E2E_ENABLED = false;
@@ -132,13 +150,16 @@ function resetMocks(): void {
     routes: [],
     e2eRoutes: [],
     workers: [worker()],
+    services: [service()],
   });
   mocks.listen.mockResolvedValue('http://127.0.0.1:3000');
   mocks.parseApiTrustProxy.mockReturnValue(false);
   mocks.runModuleStartupTasks.mockResolvedValue(undefined);
   mocks.shutdownServiceMetrics.mockResolvedValue(undefined);
   mocks.startModuleWorkers.mockResolvedValue(mocks.workersHandle);
+  mocks.startModuleServices.mockResolvedValue(mocks.servicesHandle);
   mocks.workersHandle.stop.mockResolvedValue(undefined);
+  mocks.servicesHandle.stop.mockResolvedValue(undefined);
 }
 
 const servers: ServerHandle[] = [];
@@ -173,8 +194,14 @@ describe('createServer', () => {
     expect(mocks.aggregateLoginMethods).toHaveBeenCalledWith({modules});
     expect(mocks.createPostgresClient).toHaveBeenCalledOnce();
     expect(mocks.initializeModules).toHaveBeenCalledWith({modules});
-    expect(mocks.registerModuleMetrics).toHaveBeenCalledWith({modules});
-    expect(mocks.runModuleStartupTasks).toHaveBeenCalledWith({modules});
+    expect(mocks.registerModuleMetrics).toHaveBeenCalledWith({
+      modules,
+      context: {outboxRegistry: undefined},
+    });
+    expect(mocks.runModuleStartupTasks).toHaveBeenCalledWith({
+      modules,
+      context: {outboxRegistry: undefined},
+    });
     expect(mocks.createApp).toHaveBeenCalledWith({
       auth: [],
       routes: [],
@@ -201,6 +228,7 @@ describe('createServer', () => {
     await server.start();
 
     expect(mocks.startModuleWorkers).toHaveBeenCalledOnce();
+    expect(mocks.startModuleServices).toHaveBeenCalledOnce();
     expect(mocks.listen).toHaveBeenCalledOnce();
   });
 
@@ -230,6 +258,35 @@ describe('createServer', () => {
     );
   });
 
+  it('starts module services after workers and before listening for HTTP requests', async () => {
+    const server = await createTestServer({modules: [module()]});
+    let resolveServices: ((handle: ModuleServicesHandle) => void) | undefined;
+    const servicesStarted = new Promise<void>((resolve) => {
+      mocks.startModuleServices.mockImplementationOnce(
+        () =>
+          new Promise<ModuleServicesHandle>((innerResolve) => {
+            resolveServices = innerResolve;
+            resolve();
+          }),
+      );
+    });
+
+    const start = server.start();
+    await servicesStarted;
+
+    expect(mocks.listen).not.toHaveBeenCalled();
+    expect(mocks.startModuleWorkers.mock.invocationCallOrder[0]).toBeLessThan(
+      mocks.startModuleServices.mock.invocationCallOrder[0] ?? 0,
+    );
+
+    resolveServices?.(mocks.servicesHandle);
+    await start;
+
+    expect(mocks.startModuleServices.mock.invocationCallOrder[0]).toBeLessThan(
+      mocks.listen.mock.invocationCallOrder[0] ?? 0,
+    );
+  });
+
   it('passes API_PORT to the HTTP listener when configured', async () => {
     mocks.apiConfig.API_PORT = 55_291;
     const server = await createTestServer({modules: [module()]});
@@ -250,6 +307,20 @@ describe('createServer', () => {
     expect(mocks.listen).not.toHaveBeenCalled();
   });
 
+  it('does not listen when module service startup fails', async () => {
+    const failure = new Error('service configuration invalid');
+    mocks.startModuleServices.mockRejectedValueOnce(failure);
+    const server = await createTestServer({modules: [module()]});
+
+    const result = server.start();
+
+    await expect(result).rejects.toBe(failure);
+    expect(mocks.listen).not.toHaveBeenCalled();
+
+    await server.stop();
+    expect(mocks.workersHandle.stop).toHaveBeenCalledOnce();
+  });
+
   it('releases service metrics and Postgres when boot fails', async () => {
     const failure = new Error('migration failed');
     mocks.initializeModules.mockRejectedValueOnce(failure);
@@ -260,8 +331,6 @@ describe('createServer', () => {
     expect(mocks.shutdownServiceMetrics.mock.invocationCallOrder[0]).toBeLessThan(
       mocks.closePostgresClient.mock.invocationCallOrder[0] ?? 0,
     );
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
   });
 
   it('rejects a second server while another server owns process resources', async () => {
@@ -287,8 +356,6 @@ describe('createServer', () => {
 
     await expect(result).rejects.toBe(failure);
     expect(mocks.closePostgresClient).toHaveBeenCalledOnce();
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
     expect(mocks.logger.error).toHaveBeenCalledWith(
       {err: cleanupFailure, bootError: failure},
       'Failed to clean up API server boot',
@@ -305,11 +372,10 @@ describe('createServer', () => {
     await secondStop;
 
     expect(mocks.closeApp).toHaveBeenCalledOnce();
+    expect(mocks.servicesHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.workersHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.shutdownServiceMetrics).toHaveBeenCalledOnce();
     expect(mocks.closePostgresClient).toHaveBeenCalledOnce();
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
     expect(mocks.closeErrorMonitoring).toHaveBeenCalledWith(2_000);
   });
 
@@ -319,11 +385,10 @@ describe('createServer', () => {
     await server.stop();
 
     expect(mocks.closeApp).toHaveBeenCalledOnce();
+    expect(mocks.servicesHandle.stop).not.toHaveBeenCalled();
     expect(mocks.workersHandle.stop).not.toHaveBeenCalled();
     expect(mocks.shutdownServiceMetrics).toHaveBeenCalledOnce();
     expect(mocks.closePostgresClient).toHaveBeenCalledOnce();
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
   });
 
   it('waits for worker startup before stopping', async () => {
@@ -351,11 +416,10 @@ describe('createServer', () => {
 
     expect(mocks.listen).not.toHaveBeenCalled();
     expect(mocks.closeApp).toHaveBeenCalledOnce();
+    expect(mocks.servicesHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.workersHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.shutdownServiceMetrics).toHaveBeenCalledOnce();
     expect(mocks.closePostgresClient).toHaveBeenCalledOnce();
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
     expect(mocks.closeErrorMonitoring).toHaveBeenCalledWith(2_000);
   });
 
@@ -366,6 +430,9 @@ describe('createServer', () => {
     await server.stop();
 
     expect(mocks.closeApp.mock.invocationCallOrder[0]).toBeLessThan(
+      mocks.servicesHandle.stop.mock.invocationCallOrder[0] ?? 0,
+    );
+    expect(mocks.servicesHandle.stop.mock.invocationCallOrder[0]).toBeLessThan(
       mocks.workersHandle.stop.mock.invocationCallOrder[0] ?? 0,
     );
     expect(mocks.workersHandle.stop.mock.invocationCallOrder[0]).toBeLessThan(
@@ -375,9 +442,6 @@ describe('createServer', () => {
       mocks.closePostgresClient.mock.invocationCallOrder[0] ?? 0,
     );
     expect(mocks.closePostgresClient.mock.invocationCallOrder[0]).toBeLessThan(
-      mocks.resetPublishers.mock.invocationCallOrder[0] ?? 0,
-    );
-    expect(mocks.resetSubscribers.mock.invocationCallOrder[0]).toBeLessThan(
       mocks.closeErrorMonitoring.mock.invocationCallOrder[0] ?? 0,
     );
   });
@@ -389,22 +453,15 @@ describe('createServer', () => {
     mocks.workersHandle.stop.mockRejectedValueOnce(new Error('stop workers failed'));
     mocks.shutdownServiceMetrics.mockRejectedValueOnce(new Error('shutdown metrics failed'));
     mocks.closePostgresClient.mockRejectedValueOnce(new Error('close Postgres failed'));
-    mocks.resetPublishers.mockImplementationOnce(() => {
-      throw new Error('reset publishers failed');
-    });
-    mocks.resetSubscribers.mockImplementationOnce(() => {
-      throw new Error('reset subscribers failed');
-    });
     mocks.closeErrorMonitoring.mockRejectedValueOnce(new Error('close monitoring failed'));
 
     const result = server.stop();
 
     await expect(result).rejects.toBeInstanceOf(AggregateError);
+    expect(mocks.servicesHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.workersHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.shutdownServiceMetrics).toHaveBeenCalledOnce();
     expect(mocks.closePostgresClient).toHaveBeenCalledOnce();
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
     expect(mocks.closeErrorMonitoring).toHaveBeenCalledWith(2_000);
   });
 
@@ -443,6 +500,27 @@ describe('createServer', () => {
     );
   });
 
+  it('reports service failures after closing HTTP and error monitoring', async () => {
+    const onServiceFailure = vi.fn();
+    const server = await createTestServer({modules: [module()], onServiceFailure});
+    await server.start();
+    const failure = new Error('poller failed');
+    const failedService = service();
+
+    await lastStartModuleServicesOptions().onServiceFailure?.(failure, failedService);
+
+    expect(mocks.logger.error).toHaveBeenCalledWith(
+      {err: failure, service: 'runtime-service'},
+      'Module service stopped unexpectedly',
+    );
+    expect(mocks.captureException).toHaveBeenCalledWith(failure);
+    expect(mocks.closeApp).toHaveBeenCalledOnce();
+    expect(mocks.closeErrorMonitoring).toHaveBeenCalledWith(2_000);
+    expect(mocks.closeErrorMonitoring.mock.invocationCallOrder[0]).toBeLessThan(
+      onServiceFailure.mock.invocationCallOrder[0] ?? 0,
+    );
+  });
+
   it('reports a timed-out HTTP close before invoking the worker failure hook', async () => {
     vi.useFakeTimers();
     const onWorkerFailure = vi.fn();
@@ -459,7 +537,7 @@ describe('createServer', () => {
 
     expect(mocks.logger.error).toHaveBeenCalledWith(
       {timeoutMs: 10_000},
-      'Timed out closing HTTP server after worker failure',
+      'Timed out closing HTTP server after module runtime failure',
     );
     expect(onWorkerFailure).toHaveBeenCalledWith(
       failure,
@@ -478,7 +556,7 @@ describe('createServer', () => {
 
     expect(mocks.logger.error).toHaveBeenCalledWith(
       {err: closeFailure},
-      'Failed to close HTTP server after worker failure',
+      'Failed to close HTTP server after module runtime failure',
     );
     expect(mocks.closeErrorMonitoring).toHaveBeenCalledWith(2_000);
     expect(mocks.closeErrorMonitoring.mock.invocationCallOrder[0]).toBeLessThan(
@@ -519,11 +597,10 @@ describe('runServer', () => {
     await expect(result).rejects.toBe(failure);
     expect(onStartupFailure).toHaveBeenCalledWith(failure);
     expect(mocks.closeApp).toHaveBeenCalledOnce();
+    expect(mocks.servicesHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.workersHandle.stop).toHaveBeenCalledOnce();
     expect(mocks.shutdownServiceMetrics).toHaveBeenCalledOnce();
     expect(mocks.closePostgresClient).toHaveBeenCalledOnce();
-    expect(mocks.resetPublishers).toHaveBeenCalledOnce();
-    expect(mocks.resetSubscribers).toHaveBeenCalledOnce();
     expect(mocks.closeErrorMonitoring).toHaveBeenCalledWith(2_000);
     expect(onStartupFailure.mock.invocationCallOrder[0]).toBeLessThan(
       mocks.closeErrorMonitoring.mock.invocationCallOrder[0] ?? 0,
@@ -557,10 +634,25 @@ describe('runServer', () => {
   it('exits unsuccessfully when a module worker fails', async () => {
     vi.spyOn(process, 'once').mockImplementation((() => process) as never);
     const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
-    await runServer({modules: [module()]});
+    const handle = await runServer({modules: [module()]});
 
     await lastStartModuleWorkersOptions().onWorkerFailure?.(new Error('poller failed'), worker());
 
     expect(exitSpy).toHaveBeenCalledWith(1);
+    await handle.stop();
+  });
+
+  it('exits unsuccessfully when a module service fails', async () => {
+    vi.spyOn(process, 'once').mockImplementation((() => process) as never);
+    const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
+    const handle = await runServer({modules: [module()]});
+
+    await lastStartModuleServicesOptions().onServiceFailure?.(
+      new Error('poller failed'),
+      service(),
+    );
+
+    expect(exitSpy).toHaveBeenCalledWith(1);
+    await handle.stop();
   });
 });