@empyria/restate 0.4.12 → 0.4.14

package/AGENTS.md

@@ -74,6 +74,11 @@ Attempted to assign to readonly property`). Use `mock.module('@restatedev/restat
 any mocking) if you need to restore it — reading it back off the live `clients` binding later
 may observe an already-mocked value. See `test/Admin.test.js`.
 
+`setupRestate`'s shutdown decision is `shouldDeleteOnShutdown` (unit tested); `keepDeploymentOnShutdown` and
+`forceRegistration` exist because invocations are pinned to their deployment: Restate 1.7 keeps the deployment ID when the
+same address registers again, but only `force: true` updates its handler list. Check both against a live server when
+touching them.
+
 `setupRestate` (in `lib/Admin.js`) is not unit tested: it binds real HTTP/2 + health-check
 ports, installs process-wide `SIGTERM`/`SIGINT` handlers, and its returned `forceClose` calls
 `process.exit()` — none of which are safe to exercise in-process in a test run.

package/README.md

@@ -118,13 +118,25 @@ endpoint, then create jobs with either:
   starts with the application.** The job has the stable `id` you give it, so calling it on every start never adds a second
   job: the same request changes nothing, and a changed request (another schedule, target or payload) replaces the pending
   run under the same ID. Call it once the endpoint has registered, e.g. from `setupRestate`'s `registrationCallbackFn`.
+  It also repairs a job whose chain broke: each run schedules the next, so a pending run that was killed, cancelled or
+  lost with a deleted deployment would stop the job for good. When the same request finds its pending run overdue by
+  more than `OVERDUE_GRACE_MS` (60 s), `ensure` schedules a fresh one; a replaced run that still arrives does nothing,
+  because a run only works when it is the job's current one.
 
 Cron expressions are `croner` patterns (five fields, or six with seconds for tests). The target handler is sent the
 `payload` as its input, or no input when there is none.
 
 ### An endpoint that does not register itself
 
-`setupRestate` serves, registers the deployment, and deletes it again on SIGINT/SIGTERM. When something else owns
+`setupRestate` serves, registers the deployment, and deletes it again on SIGINT/SIGTERM. Invocations in flight are
+pinned to their deployment, so deleting it strands them: a restarted process cannot finish them and they can only be
+killed. For a process restarted at the same address, pass `keepDeploymentOnShutdown: true` and `forceRegistration: true`:
+Restate keeps the deployment ID when the same address registers again (checked against Restate 1.7), running
+invocations resume on the new process, and `force` updates the services and handlers in place (a plain
+re-registration keeps the old handler list). Only compatible changes are safe this way: an invocation of a removed or
+renamed handler cannot resume.
+
+When something else owns
 registration (a supervisor that registers a process's address once and restarts the process), use
 `serveEndpoint({port, services})` instead: it only serves. Invocations already running on a deployment are pinned to its
 address, so the deployment must survive the process stopping and come back at the same address.

package/lib/Admin.js

@@ -581,7 +581,15 @@ export function defineObject(def) {
 /**
  * Starts the HTTP/2 Restate endpoint, a health-check HTTP server, registers
  * the deployment with the admin API, and wires SIGTERM/SIGINT for graceful
- * shutdown (deregistering the deployment before exiting).
+ * shutdown (deregistering the deployment before exiting, unless `keepDeploymentOnShutdown`).
+ *
+ * Invocations in flight are pinned to their deployment. Deleting it on shutdown means a restarted process cannot
+ * finish them, and they cannot even be cancelled (only killed). For a process that is restarted at the same address
+ * (a dev loop, a service that is redeployed in place), pass `keepDeploymentOnShutdown: true` and
+ * `forceRegistration: true`: Restate keeps the deployment ID when the same address registers again, so running
+ * invocations resume on the new process, and `force` updates the deployment's services and handlers in place (a plain
+ * re-registration keeps the old handler list). Only compatible changes are safe that way: an invocation of a handler
+ * that was removed or renamed cannot resume.
  * @param {Object} params
  * @param {string} params.restateAdminURL
  * @param {string} params.host
@@ -590,6 +598,9 @@ export function defineObject(def) {
  * @param {Parameters<typeof restate.createEndpointHandler>[0]['services']} [params.services]
  * @param {() => void} [params.shutdownCallbackFn]
  * @param {(deploymentId: string) => void} [params.registrationCallbackFn]
+ * @param {boolean} [params.keepDeploymentOnShutdown=false] Leave the deployment registered on SIGINT/SIGTERM.
+ * @param {boolean} [params.forceRegistration=false] Register with `force: true`, updating an existing deployment at
+ *   the same address in place.
  * @returns {Promise<RestateServer>}
  */
 export async function setupRestate({
@@ -600,6 +611,8 @@ export async function setupRestate({
 	services = [],
 	shutdownCallbackFn,
 	registrationCallbackFn,
+	keepDeploymentOnShutdown = false,
+	forceRegistration = false,
 }) {
 	const handler = restate.createEndpointHandler({ services })
 
@@ -647,8 +660,12 @@ export async function setupRestate({
 		isHealthy = false
 
 		try {
-			if (deploymentId && deploymentId !== 'NA') {
+			if (shouldDeleteOnShutdown({ deploymentId, keepDeploymentOnShutdown })) {
 				await deleteDeployment(restateAdminURL, deploymentId)
+			} else if (keepDeploymentOnShutdown && deploymentId) {
+				console.log(
+					`Keeping deployment ${deploymentId}: invocations in flight resume when it comes back`,
+				)
 			}
 		} catch (err) {
 			console.error(err)
@@ -688,7 +705,11 @@ export async function setupRestate({
 
 			for (let attempt = 1; attempt <= attempts; attempt++) {
 				try {
-					const result = await registerDeployment({ restateAdminURL, serviceURL })
+					const result = await registerDeployment({
+						restateAdminURL,
+						serviceURL,
+						force: forceRegistration,
+					})
 
 					try {
 						registrationCallbackFn?.(result.id)
@@ -751,15 +772,27 @@ export async function setupRestate({
 }
 
 /**
- * POSTs this process's HTTP endpoint to the Restate admin API so its services are discoverable.
- * @param {{restateAdminURL: string, serviceURL: string}} params
+ * Whether `setupRestate`'s shutdown removes the deployment: only a real one (registered, not `'NA'`), and not when the
+ * caller keeps it for the next start.
+ * @param {{deploymentId: string|undefined, keepDeploymentOnShutdown?: boolean}} params
+ * @returns {boolean}
+ */
+export function shouldDeleteOnShutdown({ deploymentId, keepDeploymentOnShutdown = false }) {
+	return Boolean(deploymentId) && deploymentId !== 'NA' && !keepDeploymentOnShutdown
+}
+
+/**
+ * POSTs this process's HTTP endpoint to the Restate admin API so its services are discoverable. Registering an
+ * address that is already registered returns the same deployment; with `force` its services and handlers are
+ * updated in place (without it, Restate keeps the ones it knew).
+ * @param {{restateAdminURL: string, serviceURL: string, force?: boolean}} params
  * @returns {Promise<RegisterDeploymentResponse>}
  */
-export async function registerDeployment({ restateAdminURL, serviceURL }) {
+export async function registerDeployment({ restateAdminURL, serviceURL, force = false }) {
 	const uri = serviceURL
 
 	/** @type {RegisterDeploymentRequest} */
-	const body = { uri, force: null }
+	const body = { uri, force: force ? true : null }
 
 	console.log(`Registering: ${uri} at: ${restateAdminURL}/deployments`)
 
@@ -872,7 +905,8 @@ export function createRestateAdmin({ restateAdminURL, restateURL }) {
 		pollInvocation: ({ invocationId, intervalMs }) =>
 			pollInvocation({ restateURL, invocationId, intervalMs }),
 
-		getInvocationOutput: ({ invocationId }) => getInvocationOutput({ restateURL, invocationId }),
+		getInvocationOutput: ({ invocationId }) =>
+			getInvocationOutput({ restateURL, invocationId }),
 
 		sendMessage: ({ name, message, key, payload, idempotencyKey }) =>
 			sendMessage({ restateURL, name, message, key, payload, idempotencyKey }),

package/lib/Cron.js

@@ -27,6 +27,15 @@ export const RETRY_FOREVER = {
 	exponentiationFactor: 2,
 }
 
+/**
+ * How late a job's pending run may be before `ensure` treats it as lost and schedules a fresh one. A run is normally
+ * late only by the time the services were down; a run that is late by more than this, when the application starts and
+ * calls `ensure`, has most likely been killed, cancelled or lost with a deleted deployment, and nothing else would ever
+ * schedule the next one. Replacing a run that is merely slow is harmless: `execute` ignores a run that is no longer
+ * the job's current one.
+ */
+export const OVERDUE_GRACE_MS = 60_000
+
 export const cronJobInitiator = restate.service({
 	name: 'CronJobInitiator',
 	handlers: {
@@ -56,7 +65,8 @@ export const cronJob = restate.object({
 		 * code that starts with the application and cannot know whether the job was created before.
 		 *
 		 * - no job yet: schedules it, as `initiate` does;
-		 * - the same request already scheduled: changes nothing and returns the current state;
+		 * - the same request already scheduled: changes nothing and returns the current state, unless its pending run
+		 *   is overdue by more than {@link OVERDUE_GRACE_MS} (killed, cancelled or lost): then it schedules a fresh one;
 		 * - a different request (another schedule, target or payload): cancels the pending run and
 		 *   schedules the new request under the same job ID.
 		 */
@@ -64,7 +74,11 @@ export const cronJob = restate.object({
 			const wanted = normalizeRequest(request)
 			const jobState = await ctx.get(JOB_STATE_NAME)
 
-			if (jobState && sameRequest(jobState.request, wanted)) return jobState
+			if (jobState && sameRequest(jobState.request, wanted)) {
+				const dueAt = jobState.next_execution_at ?? Date.parse(jobState.next_execution_time)
+				const now = await ctx.date.now()
+				if (!(Number.isFinite(dueAt) && now > dueAt + OVERDUE_GRACE_MS)) return jobState
+			}
 			if (jobState) ctx.cancel(jobState.next_execution_id)
 
 			return await scheduleNextExecution(ctx, wanted)
@@ -74,6 +88,15 @@ export const cronJob = restate.object({
 			if (!jobState) {
 				throw new TerminalError('Job not found.')
 			}
+			// Only the job's current run works and schedules the next one. A run that `ensure` replaced (or that a
+			// changed schedule superseded) and that still arrives does nothing, so a job never runs as two chains.
+			const self = ctx.request?.().id
+			if (self && jobState.next_execution_id && self !== jobState.next_execution_id) {
+				ctx.console?.log?.(
+					`CronJob ${ctx.key}: run ${self} is not the current one (${jobState.next_execution_id}), skipped`,
+				)
+				return
+			}
 
 			// execute the task
 			const { service, method, key, payload } = jobState.request
@@ -156,6 +179,8 @@ const scheduleNextExecution = async (ctx, request) => {
 	const jobState = {
 		request,
 		next_execution_time: next.toString(),
+		// The same instant as a number, so `ensure` can tell an overdue run without parsing a date string.
+		next_execution_at: next.getTime(),
 		next_execution_id: await handle.invocationId,
 	}
 	ctx.set(JOB_STATE_NAME, jobState)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@empyria/restate",
-	"version": "0.4.12",
+	"version": "0.4.14",
 	"description": "Restate.dev helpers for the Empyria nanoservice framework",
 	"license": "MIT",
 	"author": "Imre Fazekas <imre.fazekas@icloud.com>",

package/test/Admin.test.js

@@ -20,6 +20,7 @@ import {
 	sendMessageWithDiscovery,
 	sendMessageAsyncWithDiscovery,
 	registerDeployment,
+	shouldDeleteOnShutdown,
 	sendMessageAsynSafe,
 	defineWorkflow,
 	defineService,
@@ -521,6 +522,42 @@ describe('registerDeployment', () => {
 			registerDeployment({ restateAdminURL: 'http://admin', serviceURL: 'http://svc' }),
 		).rejects.toThrow(/bad uri/)
 	})
+
+	test('sends force: null by default and force: true when asked, to the deployments endpoint', async () => {
+		const sent = []
+		globalThis.fetch = mock(async (url, init) => {
+			sent.push({ url, body: JSON.parse(init.body) })
+			return jsonResponse({ id: 'dep-1', services: [] })
+		})
+		await registerDeployment({ restateAdminURL: 'http://admin', serviceURL: 'http://svc:9080' })
+		await registerDeployment({
+			restateAdminURL: 'http://admin',
+			serviceURL: 'http://svc:9080',
+			force: true,
+		})
+		expect(sent).toEqual([
+			{ url: 'http://admin/deployments', body: { uri: 'http://svc:9080', force: null } },
+			{ url: 'http://admin/deployments', body: { uri: 'http://svc:9080', force: true } },
+		])
+	})
+})
+
+describe('shouldDeleteOnShutdown', () => {
+	test('deletes a registered deployment by default', () => {
+		expect(shouldDeleteOnShutdown({ deploymentId: 'dep-1' })).toBe(true)
+	})
+
+	test('keeps it when asked, so invocations in flight can resume after a restart', () => {
+		expect(
+			shouldDeleteOnShutdown({ deploymentId: 'dep-1', keepDeploymentOnShutdown: true }),
+		).toBe(false)
+	})
+
+	test('never deletes without a real deployment', () => {
+		for (const deploymentId of [undefined, '', 'NA']) {
+			expect(shouldDeleteOnShutdown({ deploymentId })).toBe(false)
+		}
+	})
 })
 
 // `clients` is a live ESM namespace binding (read-only — Object.assign/property

package/test/Cron.test.js

@@ -5,6 +5,7 @@ import {
 	ensureCronJob,
 	setJobStateName,
 	JOB_STATE_NAME,
+	OVERDUE_GRACE_MS,
 } from '../lib/Cron.js'
 import { TerminalError } from '@restatedev/restate-sdk'
 
@@ -34,7 +35,7 @@ describe('cronJobInitiator.create', () => {
 
 // Every cronJob handler that schedules a next run needs the same "date/objectSendClient/set"
 // wiring; this factory keeps the mocked context consistent across tests.
-function makeSchedulingCtx({ existing } = {}) {
+function makeSchedulingCtx({ existing, now = Date.now(), invocationId } = {}) {
 	const sets = []
 	const cancelled = []
 	return {
@@ -42,7 +43,8 @@ function makeSchedulingCtx({ existing } = {}) {
 		cancelled,
 		cancel: (invocationId) => cancelled.push(invocationId),
 		get: async () => existing,
-		date: { now: async () => Date.now() },
+		date: { now: async () => now },
+		...(invocationId ? { request: () => ({ id: invocationId }) } : {}),
 		key: 'job-1',
 		objectSendClient: () => ({
 			execute: () => ({ invocationId: Promise.resolve('exec-inv-1') }),
@@ -109,6 +111,37 @@ describe('cronJob.execute', () => {
 		expect(ctx.sets.length).toBe(1)
 	})
 
+	test("a run that is not the job's current one does nothing and schedules nothing", async () => {
+		const sent = []
+		const ctx = makeSchedulingCtx({
+			invocationId: 'inv-replaced',
+			existing: {
+				request: { service: 'Svc', method: 'do', cronExpression: '* * * * *' },
+				next_execution_id: 'inv-current',
+			},
+		})
+		ctx.genericSend = (opts) => sent.push(opts)
+		await cronJob.object.execute(ctx)
+		expect(sent).toEqual([])
+		expect(ctx.sets).toEqual([])
+	})
+
+	test('the current run works and schedules the next one', async () => {
+		const sent = []
+		const ctx = makeSchedulingCtx({
+			invocationId: 'inv-current',
+			existing: {
+				request: { service: 'Svc', method: 'do', cronExpression: '* * * * *' },
+				next_execution_id: 'inv-current',
+			},
+		})
+		ctx.genericSend = (opts) => sent.push(opts)
+		await cronJob.object.execute(ctx)
+		expect(sent.length).toBe(1)
+		expect(ctx.sets.length).toBe(1)
+		expect(ctx.sets[0][1].next_execution_at).toBeNumber()
+	})
+
 	test('throws when there is no job state to execute', async () => {
 		const ctx = makeSchedulingCtx({ existing: undefined })
 		await expect(cronJob.object.execute(ctx)).rejects.toThrow(TerminalError)
@@ -179,6 +212,42 @@ describe('cronJob.ensure', () => {
 		expect(ctx.cancelled).toEqual([])
 	})
 
+	test('the same request with a pending run that is long overdue (lost) schedules a fresh run', async () => {
+		const now = Date.parse('2026-10-05T08:00:00Z')
+		const existing = {
+			request,
+			next_execution_time: new Date(now - OVERDUE_GRACE_MS - 1000).toString(),
+			next_execution_at: now - OVERDUE_GRACE_MS - 1000,
+			next_execution_id: 'inv-killed',
+		}
+		const ctx = makeSchedulingCtx({ existing, now })
+		const state = await cronJob.object.ensure(ctx, request)
+		expect(state.next_execution_id).toBe('exec-inv-1')
+		expect(state.next_execution_at).toBeGreaterThan(now)
+		expect(ctx.cancelled).toEqual(['inv-killed'])
+	})
+
+	test('a state written before next_execution_at existed is judged by its date string', async () => {
+		const now = Date.parse('2026-10-05T08:00:00Z')
+		const existing = {
+			request,
+			next_execution_time: new Date(now - 10 * 60_000).toString(),
+			next_execution_id: 'inv-killed',
+		}
+		const ctx = makeSchedulingCtx({ existing, now })
+		expect((await cronJob.object.ensure(ctx, request)).next_execution_id).toBe('exec-inv-1')
+	})
+
+	test('a pending run that is due soon, or only a little late, is left alone', async () => {
+		const now = Date.parse('2026-10-05T08:00:00Z')
+		for (const dueAt of [now + 30_000, now - OVERDUE_GRACE_MS + 1000]) {
+			const existing = { request, next_execution_at: dueAt, next_execution_id: 'inv-pending' }
+			const ctx = makeSchedulingCtx({ existing, now })
+			expect(await cronJob.object.ensure(ctx, request)).toBe(existing)
+			expect(ctx.cancelled).toEqual([])
+		}
+	})
+
 	test('unknown fields and field order do not make a request "different"', async () => {
 		const existing = { request, next_execution_time: 'T', next_execution_id: 'inv-old' }
 		const ctx = makeSchedulingCtx({ existing })