@platformatic/runtime 3.63.0 → 3.64.0

package/index.d.ts

@@ -9,7 +9,7 @@ import { EventEmitter } from 'node:events'
 import { Logger } from 'pino'
 import { PlatformaticRuntimeConfig } from './config.js'
 
-export type RuntimeConfiguration = Promise<Configuration<PlatformaticRuntimeConfig>>
+export type RuntimeConfiguration = Configuration<PlatformaticRuntimeConfig>
 
 export type ApplicationCommandContext = {
   colorette: typeof colorette
@@ -72,6 +72,8 @@ export namespace errors {
   ) => FastifyError
   export const ExtensionHealthRoutesUnavailableError: () => FastifyError
   export const LastProfileTimeoutError: (id: string) => FastifyError
+  export const SchedulerJobNotFoundError: (name: string) => FastifyError
+  export const DuplicateSchedulerJobError: (name: string) => FastifyError
 }
 
 export namespace symbols {
@@ -145,6 +147,45 @@ export interface RuntimeSharedContextUpdateOptions extends SharedContextUpdateOp
   context?: object
 }
 
+export interface SchedulerJobBase {
+  name: string
+  cron: string
+  paused: boolean
+  maxRetries: number
+  lastExecutedAt?: string | null
+  lastStatus?: 'success' | 'failed' | null
+  nextRunAt?: string | null
+}
+
+export interface ConfiguredSchedulerJob extends SchedulerJobBase {
+  source: 'config'
+  callbackUrl: string
+  method: string
+  headers?: Record<string, string>
+  body?: string | Record<string, unknown>
+}
+
+export interface ApplicationSchedulerJob extends SchedulerJobBase {
+  source: 'application'
+  applicationId: string
+  scheduleId: string
+  tasks: string[]
+}
+
+export type SchedulerJob = ConfiguredSchedulerJob | ApplicationSchedulerJob
+
+export interface SchedulerRunResult {
+  name: string
+  success: boolean
+  executedAt: string
+}
+
+export interface ApplicationSchedule {
+  id: string
+  cron: string
+  tasks: string[]
+}
+
 export interface RuntimeMetadata {
   pid: number
   cwd: string
@@ -175,6 +216,10 @@ export declare class ManagementClient {
   getApplicationOpenapiSchema (id: string): Promise<unknown>
   getApplicationGraphqlSchema (id: string): Promise<unknown>
   getMetrics (format?: string): Promise<{ metrics: unknown }>
+  getSchedulerJobs (): Promise<SchedulerJob[]>
+  pauseSchedulerJob (name: string): Promise<SchedulerJob>
+  resumeSchedulerJob (name: string): Promise<SchedulerJob>
+  runSchedulerJob (name: string): Promise<SchedulerRunResult>
   startApplication (id: string): Promise<void>
   stopApplication (id: string): Promise<void>
   restartApplication (id: string): Promise<void>
@@ -286,6 +331,11 @@ export declare class Runtime extends EventEmitter {
   getRuntimeStatus (): string
   getRuntimeMetadata (): Promise<RuntimeMetadata>
   getRuntimeEnv (): Record<string, string>
+  // `includeMeta: true` returns the full merged Configuration object (which
+  // carries the kMetadata symbol key: `{ root, path, env, module }`), not a
+  // plain Record; the argument-invariant overload below covers every other
+  // call.
+  getRuntimeConfig (includeMeta: true): RuntimeConfiguration
   getRuntimeConfig (includeMeta?: boolean): Record<string, unknown>
   getApplicationsIds (): string[]
   /**
@@ -339,9 +389,23 @@ export declare class Runtime extends EventEmitter {
   restartApplication (id: string): Promise<void>
   addApplications (applications: unknown[], start?: boolean): Promise<ApplicationDetails[]>
   removeApplications (applications: string[], silent?: boolean): Promise<ApplicationDetails[]>
+  startApplicationProfiling (id: string, options: Record<string, unknown> & { allWorkers: true }, ensureStarted?: boolean): Promise<{ workers: number[] }>
   startApplicationProfiling (id: string, options?: Record<string, unknown>, ensureStarted?: boolean): Promise<void>
+  stopApplicationProfiling (id: string, options: Record<string, unknown> & { allWorkers: true }, ensureStarted?: boolean): Promise<Array<{ workerIndex: number, profile: Buffer }>>
+  stopApplicationProfiling (id: string, options: Record<string, unknown> & { includeSampleCount: true }, ensureStarted?: boolean): Promise<{ profile: Buffer, sampleCount: number }>
   stopApplicationProfiling (id: string, options?: Record<string, unknown>, ensureStarted?: boolean): Promise<Buffer>
-  getApplicationLastProfile (id: string, options?: Record<string, unknown>, ensureStarted?: boolean): Promise<{ profile: Buffer, timestamp: number | null, preserved: boolean }>
+  getApplicationLastProfile (id: string, options?: Record<string, unknown>, ensureStarted?: boolean): Promise<{ profile: Buffer, timestamp: number | null, sampleCount: number | null, preserved: boolean }>
+  // Patches one application's resolved configuration through the same
+  // per-application JSON-Patch mechanism the runtime already applies during
+  // create()/start().
+  setApplicationConfigPatch (applicationId: string, patch: Array<Record<string, unknown>>): void
+  removeApplicationConfigPatch (applicationId: string): void
+  getSchedulerJobs (): SchedulerJob[]
+  pauseSchedulerJob (name: string): Promise<SchedulerJob>
+  resumeSchedulerJob (name: string): Promise<SchedulerJob>
+  runSchedulerJob (name: string): Promise<SchedulerRunResult>
+  getApplicationScheduledTasks (id: string): Promise<ApplicationSchedule[]>
+  runApplicationScheduledTasks (id: string, scheduleId: string, scheduledTime: number): Promise<unknown>
 }
 
 export function wrapInRuntimeConfig (
@@ -357,13 +421,27 @@ export declare function loadConfiguration (
   context?: ConfigurationOptions
 ): Promise<RuntimeConfiguration>
 
+// create() additionally accepts `setupSignals`, a Runtime-only option (it is
+// never read outside this package) that controls whether SIGUSR2/close-with-grace
+// handlers are installed.
+export interface RuntimeCreateContext extends ConfigurationOptions<PlatformaticRuntimeConfig> {
+  setupSignals?: boolean
+}
+
 export function create (
   root: string,
   source?: string | PlatformaticRuntimeConfig,
-  context?: ConfigurationOptions
+  context?: RuntimeCreateContext
 ): Promise<Runtime>
 
-export declare function prepareApplication (config: RuntimeConfiguration, application: object): object
+// The runtime's actual per-application preparation (worker defaults, paths,
+// telemetry hooks, capability resolution) also takes the resolved worker
+// count as a third argument, and is itself async.
+export declare function prepareApplication (
+  config: RuntimeConfiguration,
+  application: object,
+  workers?: PlatformaticRuntimeConfig['workers']
+): Promise<object>
 
 export declare function transform (
   config: RuntimeConfiguration,

package/lib/errors.js

@@ -140,6 +140,17 @@ export const MessagingError = createError(
   'Cannot send a message to application "%s": %s'
 )
 
+export const SchedulerJobNotFoundError = createError(
+  `${ERROR_PREFIX}_SCHEDULER_JOB_NOT_FOUND`,
+  'Scheduler "%s" not found',
+  404
+)
+
+export const DuplicateSchedulerJobError = createError(
+  `${ERROR_PREFIX}_DUPLICATE_SCHEDULER_JOB`,
+  'Scheduler "%s" is already registered'
+)
+
 export const MissingPprofCapture = createError(
   `${ERROR_PREFIX}_MISSING_PPROF_CAPTURE`,
   'Please install @platformatic/wattpm-pprof-capture'

package/lib/management-api.js

@@ -79,6 +79,23 @@ export async function managementApiPlugin (app, opts) {
     await runtime.restart(applications)
   })
 
+  app.get('/scheduler', async () => {
+    const jobs = await runtime.getSchedulerJobs()
+    return { jobs }
+  })
+
+  app.post('/scheduler/:name/pause', async request => {
+    return runtime.pauseSchedulerJob(request.params.name)
+  })
+
+  app.post('/scheduler/:name/resume', async request => {
+    return runtime.resumeSchedulerJob(request.params.name)
+  })
+
+  app.post('/scheduler/:name/run', async request => {
+    return runtime.runSchedulerJob(request.params.name)
+  })
+
   app.get('/applications', async () => {
     return runtime.getApplications()
   })
@@ -203,8 +220,8 @@ export async function managementApiPlugin (app, opts) {
     app.log.debug('start profiling', { id })
 
     const options = request.body || {}
-    await runtime.startApplicationProfiling(id, options)
-    reply.code(200).send({})
+    const result = await runtime.startApplicationProfiling(id, options)
+    reply.code(200).send(result ?? {})
   })
 
   app.post('/applications/:id/pprof/stop', async (request, reply) => {
@@ -213,6 +230,19 @@ export async function managementApiPlugin (app, opts) {
 
     const options = request.body || {}
     const profileData = await runtime.stopApplicationProfiling(id, options)
+
+    // With allWorkers there is one profile per worker: return them as JSON
+    // with base64 encoded payloads since a single binary body cannot carry
+    // multiple profiles.
+    if (options.allWorkers && Array.isArray(profileData)) {
+      const profiles = profileData.map(({ workerIndex, profile }) => ({
+        workerIndex,
+        profile: Buffer.from(profile).toString('base64')
+      }))
+
+      return { profiles }
+    }
+
     reply.type('application/octet-stream').code(200).send(profileData)
   })
 

package/lib/management-handlers.js

@@ -19,6 +19,10 @@ const allOperations = [
   'getApplicationOpenapiSchema',
   'getApplicationGraphqlSchema',
   'getMetrics',
+  'getSchedulerJobs',
+  'pauseSchedulerJob',
+  'resumeSchedulerJob',
+  'runSchedulerJob',
   'startApplication',
   'stopApplication',
   'restartApplication',
@@ -96,6 +100,10 @@ export function createManagementHandlers (runtime, allowedOperations) {
     return runtime.getMetrics(format)
   })
 
+  register('getSchedulerJobs', async () => {
+    return runtime.getSchedulerJobs()
+  })
+
   // Write operations
   register('startApplication', async ({ id }) => {
     await runtime.startApplication(id)
@@ -113,6 +121,18 @@ export function createManagementHandlers (runtime, allowedOperations) {
     return runtime.restart(applications)
   })
 
+  register('pauseSchedulerJob', async ({ name }) => {
+    return runtime.pauseSchedulerJob(name)
+  })
+
+  register('resumeSchedulerJob', async ({ name }) => {
+    return runtime.resumeSchedulerJob(name)
+  })
+
+  register('runSchedulerJob', async ({ name }) => {
+    return runtime.runSchedulerJob(name)
+  })
+
   register('addApplications', async ({ applications, start }) => {
     const config = runtime.getRuntimeConfig(true)
 

package/lib/runtime.js

@@ -93,6 +93,7 @@ const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
 const kHeapCheckCounter = Symbol('plt.runtime.worker.heapCheckCounter')
 const kLastHeapStats = Symbol('plt.runtime.worker.lastHeapStats')
 const kProfilingELUGates = Symbol('plt.runtime.worker.profilingELUGates')
+const kWorkerScheduledTasks = Symbol('plt.runtime.worker.scheduledTasks')
 const kHealthITCTimeoutMs = 5000
 const kProfilingELUHysteresis = 0.1
 const kLastProfileTimeoutMs = 10000
@@ -116,6 +117,10 @@ const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by th
 
 const MAX_LISTENERS_COUNT = 100
 
+function hasWorkerIndex (applicationId) {
+  return /^.+:\d+$/.test(applicationId)
+}
+
 function parseOrigins (origins) {
   if (!origins) return undefined
 
@@ -352,8 +357,8 @@ export class Runtime extends EventEmitter {
     await this.addApplications(this.#config.applications)
     await this.#setDispatcher(config.undici)
 
-    if (config.scheduler && !this.#context.build) {
-      this.#scheduler = startScheduler(config.scheduler, this.#dispatcher, logger)
+    if (!this.#context.build) {
+      this.#scheduler = startScheduler(config.scheduler ?? [], this.#dispatcher, logger)
     }
 
     this.#updateStatus('init')
@@ -708,6 +713,7 @@ export class Runtime extends EventEmitter {
 
     for (const application of applications) {
       this.#dynamicWorkersScaler?.remove(application)
+      await this.#scheduler?.removeApplicationJobs(application)
       this.#applications.delete(application)
       this.#applicationRestartCounts.delete(application)
     }
@@ -816,6 +822,8 @@ export class Runtime extends EventEmitter {
       await this.#startWorker(config, applicationConfig, workers, id, i, silent)
     }
 
+    await this.#registerApplicationSchedulerJobs(id)
+
     this.emitAndNotify('application:started', id)
     await this.#dynamicWorkersScaler?.applyPendingUpdate(id)
   }
@@ -829,6 +837,7 @@ export class Runtime extends EventEmitter {
     const workersCount = workersIds.length
 
     this.emitAndNotify('application:stopping', id)
+    await this.#scheduler?.stopApplicationJobs(id)
 
     if (typeof workersCount === 'number') {
       const stopInvocations = []
@@ -882,6 +891,7 @@ export class Runtime extends EventEmitter {
         await this.#replaceWorker(config, applicationConfig, workersCount, id, workerIndex, worker, true)
       }
 
+      await this.#registerApplicationSchedulerJobs(id)
       this.#incrementApplicationRestartCount(id)
       this.emitAndNotify('application:restarted', id)
     } finally {
@@ -907,17 +917,68 @@ export class Runtime extends EventEmitter {
   }
 
   async startApplicationProfiling (id, options = {}, ensureStarted = true) {
-    const service = await this.#getApplicationById(id, ensureStarted)
     this.#validatePprofCapturePreload()
 
-    return sendViaITC(service, 'startProfiling', options)
+    const { allWorkers, ...profilingOptions } = options
+
+    if (!allWorkers || hasWorkerIndex(id)) {
+      const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
+      return sendViaITC(service, 'startProfiling', profilingOptions)
+    }
+
+    const started = []
+    const alreadyProfiling = []
+    let firstError
+
+    for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
+      try {
+        await sendViaITC(worker, 'startProfiling', profilingOptions)
+        started.push(workerIndex)
+      } catch (error) {
+        // A worker which is already being profiled is considered covered, but
+        // if no other worker could be started the error is still reported.
+        if (error.code === 'PLT_PPROF_PROFILING_ALREADY_STARTED') {
+          alreadyProfiling.push(workerIndex)
+        }
+
+        firstError ??= error
+      }
+    }
+
+    if (started.length === 0 && firstError) {
+      throw firstError
+    }
+
+    return { workers: started.concat(alreadyProfiling).sort((a, b) => a - b) }
   }
 
   async stopApplicationProfiling (id, options = {}, ensureStarted = true) {
-    const service = await this.#getApplicationById(id, ensureStarted)
     this.#validatePprofCapturePreload()
 
-    return sendViaITC(service, 'stopProfiling', options)
+    const { allWorkers, ...profilingOptions } = options
+
+    if (!allWorkers || hasWorkerIndex(id)) {
+      const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
+      return sendViaITC(service, 'stopProfiling', profilingOptions)
+    }
+
+    const profiles = []
+    let firstError
+
+    for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
+      try {
+        const profile = await sendViaITC(worker, 'stopProfiling', profilingOptions)
+        profiles.push({ workerIndex, profile })
+      } catch (error) {
+        firstError ??= error
+      }
+    }
+
+    if (profiles.length === 0 && firstError) {
+      throw firstError
+    }
+
+    return profiles
   }
 
   async getApplicationLastProfile (id, options = {}, ensureStarted = true) {
@@ -935,7 +996,11 @@ export class Runtime extends EventEmitter {
       // surface as an unhandled rejection.
       const pull = (async () => {
         const service = await this.#getApplicationById(id, ensureStarted)
-        const result = await sendViaITC(service, 'getLastProfile', { ...options, includeTimestamp: true })
+        const result = await sendViaITC(service, 'getLastProfile', {
+          ...options,
+          includeTimestamp: true,
+          includeSampleCount: true
+        })
         return { service, result }
       })()
       pull.catch(() => {})
@@ -947,7 +1012,9 @@ export class Runtime extends EventEmitter {
 
         // An older capture module which does not support includeTimestamp
         // returns the raw profile.
-        const value = result instanceof Uint8Array ? { profile: result, timestamp: null } : result
+        const value = result instanceof Uint8Array
+          ? { profile: result, timestamp: null, sampleCount: null }
+          : { sampleCount: null, ...result }
 
         // A strictly newer live window supersedes the preserved overload
         // profile: prune it so the preserved copy naturally expires once the
@@ -979,7 +1046,12 @@ export class Runtime extends EventEmitter {
     if (preserved) {
       // The preserved flag lets consumers distinguish post-mortem evidence
       // from a live window and judge it together with the timestamp.
-      return { profile: preserved.profile, timestamp: preserved.timestamp, preserved: true }
+      return {
+        profile: preserved.profile,
+        timestamp: preserved.timestamp,
+        sampleCount: preserved.sampleCount,
+        preserved: true
+      }
     }
 
     throw error
@@ -1888,17 +1960,27 @@ export class Runtime extends EventEmitter {
   }
 
   async getApplicationMeta (id) {
-    const application = await this.#getApplicationById(id)
+    const hasWorkerId = /^.+:\d+$/.test(id)
+    const attempts = hasWorkerId ? 1 : Math.max(1, this.#workers.getKeys(id).length)
 
-    try {
-      return await sendViaITC(application, 'getApplicationMeta')
-    } catch (e) {
-      // The application exports no meta, return an empty object
-      if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
-        return {}
-      }
+    for (let attempt = 0; attempt < attempts; attempt++) {
+      const application = await this.#getApplicationById(id)
 
-      throw e
+      try {
+        return await sendViaITC(application, 'getApplicationMeta')
+      } catch (e) {
+        // The application exports no meta, return an empty object
+        if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
+          return {}
+        }
+
+        // A parallel restart can stop the selected worker while metadata is
+        // being retrieved. Retry another worker unless one was requested
+        // explicitly or every worker available at the start has been tried.
+        if (e.code !== 'PLT_RUNTIME_APPLICATION_WORKER_EXIT' || attempt === attempts - 1) {
+          throw e
+        }
+      }
     }
   }
 
@@ -1973,6 +2055,79 @@ export class Runtime extends EventEmitter {
     return sendViaITC(application, 'getApplicationGraphQLSchema')
   }
 
+  async getApplicationScheduledTasks (id) {
+    const application = await this.#getApplicationById(id, true)
+
+    return sendViaITC(application, 'getApplicationScheduledTasks')
+  }
+
+  async runApplicationScheduledTasks (id, scheduleId, scheduledTime) {
+    const application = await this.#getApplicationById(id, true)
+
+    return sendViaITC(application, 'runApplicationScheduledTasks', { scheduleId, scheduledTime })
+  }
+
+  getSchedulerJobs () {
+    return this.#scheduler?.getJobs() ?? []
+  }
+
+  pauseSchedulerJob (name) {
+    return this.#schedulerOrThrow().pauseJob(name)
+  }
+
+  resumeSchedulerJob (name) {
+    return this.#schedulerOrThrow().resumeJob(name)
+  }
+
+  runSchedulerJob (name) {
+    return this.#schedulerOrThrow().runJob(name)
+  }
+
+  #schedulerOrThrow () {
+    if (!this.#scheduler) {
+      throw new Error('The scheduler is not configured')
+    }
+
+    return this.#scheduler
+  }
+
+  async #registerApplicationSchedulerJobs (id) {
+    if (!this.#scheduler) {
+      return
+    }
+
+    const pausedJobs = new Set(
+      this.#scheduler
+        .getJobs()
+        .filter(job => job.applicationId === id && job.paused)
+        .map(job => job.name)
+    )
+
+    await this.#scheduler.removeApplicationJobs(id)
+
+    const workerId = this.#workers.getKeys(id)[0]
+    const schedules = this.#workers.get(workerId)?.[kWorkerScheduledTasks] ?? []
+    for (const schedule of schedules) {
+      const name = `${id}:${schedule.id}`
+      this.#scheduler.addJob(
+        {
+          name,
+          cron: schedule.cron,
+          source: 'application',
+          applicationId: id,
+          scheduleId: schedule.id,
+          tasks: schedule.tasks,
+          maxRetries: 3
+        },
+        ({ scheduledTime }) => this.runApplicationScheduledTasks(id, schedule.id, scheduledTime)
+      )
+
+      if (pausedJobs.has(name)) {
+        await this.#scheduler.pauseJob(name)
+      }
+    }
+  }
+
   async getWorkers (includeRaw = false) {
     const status = {}
 
@@ -2414,7 +2569,7 @@ export class Runtime extends EventEmitter {
     // The event only carries metadata: the profile can be retrieved on demand
     // via getApplicationLastProfile. We use emit instead of emitAndNotify since
     // other workers are not interested in this event.
-    worker[kITC].on('profile:captured', ({ type, timestamp }) => {
+    worker[kITC].on('profile:captured', ({ type, timestamp, sampleCount }) => {
       // A strictly newer completed window supersedes the preserved overload
       // profile: once the worker is past the overload and producing windows
       // again, its old evidence must not be served anymore.
@@ -2430,7 +2585,8 @@ export class Runtime extends EventEmitter {
         application: applicationId,
         worker: index,
         type,
-        timestamp
+        timestamp,
+        sampleCount: sampleCount ?? null
       })
     })
 
@@ -2492,8 +2648,12 @@ export class Runtime extends EventEmitter {
     // the worker can be retrieved (see getApplicationLastProfile) even while
     // the worker event loop is blocked, and it survives the worker being
     // replaced by the health checks.
-    worker[kITC].on('profile:overload', ({ type, timestamp, profile }) => {
-      this.#lastOverloadProfiles.set(`${workerId}:${type}`, { profile, timestamp })
+    worker[kITC].on('profile:overload', ({ type, timestamp, profile, sampleCount }) => {
+      this.#lastOverloadProfiles.set(`${workerId}:${type}`, {
+        profile,
+        timestamp,
+        sampleCount: sampleCount ?? null
+      })
     })
 
     // Preserved overload profiles outlive their worker only for a grace
@@ -3000,22 +3160,24 @@ export class Runtime extends EventEmitter {
     this.emitAndNotify('application:worker:starting', eventPayload)
 
     try {
-      let workerUrl
+      let workerStartResult
       if (config.startTimeout > 0) {
-        workerUrl = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
+        workerStartResult = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
 
-        if (workerUrl === kTimeout) {
+        if (workerStartResult === kTimeout) {
           this.emitAndNotify('application:worker:startTimeout', eventPayload)
           this.logger.error(`The ${label} failed to start in ${config.startTimeout}ms. Forcefully killing the thread.`)
           worker.terminate()
           throw new ApplicationStartTimeoutError(id, config.startTimeout)
         }
       } else {
-        workerUrl = await sendViaITC(worker, 'start')
+        workerStartResult = await sendViaITC(worker, 'start')
       }
 
       await this.#avoidOutOfOrderThreadLogs()
 
+      const { url: workerUrl, scheduledTasks } = workerStartResult
+      worker[kWorkerScheduledTasks] = scheduledTasks
       if (workerUrl) {
         this.#url = workerUrl
 
@@ -3338,7 +3500,17 @@ export class Runtime extends EventEmitter {
       this.#workers.set(newWorkerId, newWorker)
 
       // Add the worker to the mesh
-      await this.#startWorker(configForNewWorker, applicationConfig, workersCount, applicationId, newIndex, false, 0, newWorker, true)
+      await this.#startWorker(
+        configForNewWorker,
+        applicationConfig,
+        workersCount,
+        applicationId,
+        newIndex,
+        false,
+        0,
+        newWorker,
+        true
+      )
 
       // Make sure the runtime hasn't been stopped in the meanwhile
       if (this.#status !== 'started') {
@@ -3388,6 +3560,37 @@ export class Runtime extends EventEmitter {
     return this.#getWorkerByIdOrNext(applicationId, workerId, ensureStarted, mustExist)
   }
 
+  // Profiling start and stop must address the same worker: resolve an id
+  // without an explicit worker index to the first worker deterministically,
+  // as the round-robin used by #getApplicationById would rotate to a
+  // different worker between the two calls.
+  async #getApplicationWorkerForProfiling (applicationId, ensureStarted) {
+    if (hasWorkerIndex(applicationId)) {
+      return this.#getApplicationById(applicationId, ensureStarted)
+    }
+
+    if (!this.#applications.has(applicationId)) {
+      throw new ApplicationNotFoundError(applicationId, this.getApplicationsIds().join(', '))
+    }
+
+    const [firstWorker] = this.#workers.getKeys(applicationId)
+    return this.#getWorkerByIdOrNext(applicationId, firstWorker?.split(':')[1], ensureStarted)
+  }
+
+  async #getApplicationWorkersForProfiling (applicationId, ensureStarted) {
+    if (!this.#applications.has(applicationId)) {
+      throw new ApplicationNotFoundError(applicationId, this.getApplicationsIds().join(', '))
+    }
+
+    const workers = []
+    for (const key of this.#workers.getKeys(applicationId)) {
+      const workerIndex = parseInt(key.split(':')[1], 10)
+      workers.push({ workerIndex, worker: await this.#getWorkerByIdOrNext(applicationId, workerIndex, ensureStarted) })
+    }
+
+    return workers
+  }
+
   // This method can work in two modes: when workerId is provided, it will return the specific worker
   // otherwise it will return the next available worker for the application.
   async #getWorkerByIdOrNext (applicationId, workerId, ensureStarted = false, mustExist = true) {

package/lib/scaling-algorithm.js

@@ -83,7 +83,7 @@ export class ScalingAlgorithm {
 
         const newWorkersCount = appMinWorkers - workersCount
         totalWorkersCount += newWorkersCount
-        totalAvailableMemory += newWorkersCount * avgHeapUsed
+        totalAvailableMemory -= newWorkersCount * avgHeapUsed
         continue
       }
 
@@ -96,7 +96,7 @@ export class ScalingAlgorithm {
 
         const removedWorkersCount = workersCount - appMaxWorkers
         totalWorkersCount -= removedWorkersCount
-        totalAvailableMemory -= removedWorkersCount * avgHeapUsed
+        totalAvailableMemory += removedWorkersCount * avgHeapUsed
         continue
       }
 
@@ -111,7 +111,7 @@ export class ScalingAlgorithm {
 
           const removedWorkersCount = 1
           totalWorkersCount -= removedWorkersCount
-          totalAvailableMemory -= removedWorkersCount * avgHeapUsed
+          totalAvailableMemory += removedWorkersCount * avgHeapUsed
         }
       }
     }

package/lib/scheduler.js

@@ -1,68 +1,176 @@
 import { CronJob, validateCronExpression } from 'cron'
 import { setTimeout } from 'node:timers/promises'
 import { request } from 'undici'
+import { DuplicateSchedulerJobError, SchedulerJobNotFoundError } from './errors.js'
 
 export class SchedulerService {
   constructor (schedulerConfig, dispatcher, logger) {
     this.logger = logger
-    this.jobsConfig = []
-    this.cronJobs = []
     this.dispatcher = dispatcher
-    this.validateCronSchedulers(schedulerConfig)
-  }
+    this.jobs = new Map()
+    this.started = false
 
-  validateCronSchedulers (schedulerConfig) {
     for (const config of schedulerConfig) {
-      // Skip disabled schedulers
-      if (config.enabled === false) {
-        continue
-      }
+      this.addJob({ ...config, source: 'config' })
+    }
+  }
 
-      // Validate cron expression
-      const validation = validateCronExpression(config.cron)
-      if (!validation.valid) {
-        throw new Error(`Invalid cron expression "${config.cron}" for scheduler "${config.name}"`)
-      }
+  addJob (config, executor) {
+    // Skip disabled schedulers
+    if (config.enabled === false) {
+      return null
+    }
 
-      // Set defaults for optional fields
-      const job = {
-        ...config,
-        headers: config.headers || {},
-        body: config.body || {},
-        maxRetries: config.maxRetries || 3
-      }
-      this.jobsConfig.push(job)
+    // Validate cron expression
+    const validation = validateCronExpression(config.cron)
+    if (!validation.valid) {
+      throw new Error(`Invalid cron expression "${config.cron}" for scheduler "${config.name}"`)
+    }
+
+    // Set defaults for optional fields
+    const job = {
+      ...config,
+      headers: config.headers || {},
+      body: config.body || {},
+      maxRetries: config.maxRetries ?? 3,
+      source: config.source ?? 'application',
+      paused: false,
+      lastExecutedAt: null,
+      lastStatus: null,
+      cronJob: null,
+      executor
     }
+
+    if (this.jobs.has(job.name)) {
+      throw new DuplicateSchedulerJobError(job.name)
+    }
+
+    this.jobs.set(job.name, job)
+
+    if (this.started) {
+      this.#startJob(job)
+    }
+
+    return this.#serializeJob(job)
   }
 
   start () {
-    for (const job of this.jobsConfig) {
-      this.logger.info(`Configuring scheduler "${job.name}" with cron "${job.cron}"`)
-      const cronJob = CronJob.from({
-        cronTime: job.cron,
-        onTick: async () => {
-          this.logger.info(`Executing scheduler "${job.name}"`)
-          // This cannot throw, the try/catch is inside
-          await this.executeCallback(job)
-        },
-        start: true,
-        timeZone: 'UTC',
-        waitForCompletion: true
-      })
-
-      this.cronJobs.push(cronJob)
+    this.started = true
+
+    for (const job of this.jobs.values()) {
+      this.#startJob(job)
     }
   }
 
   async stop () {
-    for (const job of this.cronJobs) {
-      await job.stop()
+    this.started = false
+
+    for (const job of this.jobs.values()) {
+      await this.#stopJob(job)
+    }
+  }
+
+  getJobs () {
+    return Array.from(this.jobs.values(), job => this.#serializeJob(job))
+  }
+
+  // Stops the local cron trigger for a job without removing its configuration.
+  // Used when an external coordinator (e.g. ICC) takes over the scheduling, so
+  // that multiple instances of the same runtime do not execute the job concurrently.
+  async pauseJob (name) {
+    const job = this.#getJobOrThrow(name)
+
+    if (job.paused) {
+      return this.#serializeJob(job)
+    }
+
+    job.paused = true
+    await this.#stopJob(job)
+    this.logger.info(`Scheduler "${name}" paused`)
+    return this.#serializeJob(job)
+  }
+
+  async resumeJob (name) {
+    const job = this.#getJobOrThrow(name)
+
+    if (!job.paused) {
+      return this.#serializeJob(job)
+    }
+
+    job.paused = false
+
+    if (this.started) {
+      this.#startJob(job)
+    }
+
+    this.logger.info(`Scheduler "${name}" resumed`)
+    return this.#serializeJob(job)
+  }
+
+  async removeApplicationJobs (applicationId) {
+    for (const [name, job] of this.jobs) {
+      if (job.applicationId === applicationId) {
+        await this.#stopJob(job)
+        this.jobs.delete(name)
+      }
+    }
+  }
+
+  async stopApplicationJobs (applicationId) {
+    for (const job of this.jobs.values()) {
+      if (job.applicationId === applicationId) {
+        await this.#stopJob(job)
+      }
+    }
+  }
+
+  // Executes a job immediately, regardless of its cron schedule or paused state.
+  // Used by an external coordinator to trigger the execution in this instance.
+  async runJob (name) {
+    const job = this.#getJobOrThrow(name)
+    return this.executeCallback(job)
+  }
+
+  #getJobOrThrow (name) {
+    const job = this.jobs.get(name)
+    if (!job) {
+      throw new SchedulerJobNotFoundError(name)
+    }
+    return job
+  }
+
+  #startJob (job) {
+    if (job.cronJob || job.paused) {
+      return
+    }
+
+    this.logger.info(`Configuring scheduler "${job.name}" with cron "${job.cron}"`)
+    job.cronJob = CronJob.from({
+      cronTime: job.cron,
+      onTick: async () => {
+        this.logger.info(`Executing scheduler "${job.name}"`)
+        // This cannot throw, the try/catch is inside
+        await this.executeCallback(job)
+      },
+      start: true,
+      timeZone: 'UTC',
+      waitForCompletion: true
+    })
+  }
+
+  async #stopJob (job) {
+    if (!job.cronJob) {
+      return
     }
+
+    await job.cronJob.stop()
+    job.cronJob = null
   }
 
   async executeCallback (scheduler) {
     let attempt = 0
     let success = false
+    const scheduledTime = Date.now()
 
     while (!success && attempt < scheduler.maxRetries) {
       try {
@@ -74,29 +182,32 @@ export class SchedulerService {
           )
           await setTimeout(delay)
         }
-        const headers = {
-          'x-retry-attempt': attempt + 1,
-          ...scheduler.headers
-        }
-
-        const bodyString = typeof scheduler.body === 'string' ? scheduler.body : JSON.stringify(scheduler.body)
-        const response = await request(scheduler.callbackUrl, {
-          method: scheduler.method,
-          headers,
-          body: bodyString,
-          dispatcher: this.dispatcher
-        })
-
-        // Consumes the body, but we are not interested in the body content,
-        // we don't save it anywere, so we just dump it
-        await response.body.dump()
-
-        if (response.statusCode >= 200 && response.statusCode < 300) {
-          this.logger.info(`Scheduler "${scheduler.name}" executed successfully`)
-          success = true
+        if (scheduler.executor) {
+          await scheduler.executor({ scheduledTime })
         } else {
-          throw new Error(`HTTP error ${response.statusCode}`)
+          const headers = {
+            'x-retry-attempt': attempt + 1,
+            ...scheduler.headers
+          }
+
+          const bodyString = typeof scheduler.body === 'string' ? scheduler.body : JSON.stringify(scheduler.body)
+          const response = await request(scheduler.callbackUrl, {
+            method: scheduler.method,
+            headers,
+            body: bodyString,
+            dispatcher: this.dispatcher
+          })
+
+          // Consume the body to release the connection.
+          await response.body.dump()
+
+          if (response.statusCode < 200 || response.statusCode >= 300) {
+            throw new Error(`HTTP error ${response.statusCode}`)
+          }
         }
+
+        this.logger.info(`Scheduler "${scheduler.name}" executed successfully`)
+        success = true
       } catch (error) {
         this.logger.error(
           `Error executing scheduler "${scheduler.name}" (attempt ${attempt + 1}/${scheduler.maxRetries}):`,
@@ -106,9 +217,30 @@ export class SchedulerService {
       }
     }
 
+    scheduler.lastExecutedAt = new Date().toISOString()
+    scheduler.lastStatus = success ? 'success' : 'failed'
+
     if (!success) {
       this.logger.error(`Scheduler "${scheduler.name}" failed after ${scheduler.maxRetries} attempts`)
     }
+
+    return { name: scheduler.name, success, executedAt: scheduler.lastExecutedAt }
+  }
+
+  #serializeJob (job) {
+    const { cronJob, executor, ...serializable } = job
+    let nextRunAt = null
+
+    if (cronJob) {
+      try {
+        nextRunAt = cronJob.nextDate().toJSDate().toISOString()
+        /* c8 ignore next 3 */
+      } catch {
+        nextRunAt = null
+      }
+    }
+
+    return { ...serializable, nextRunAt }
   }
 }
 

package/lib/worker/itc.js

@@ -205,7 +205,16 @@ export async function setupITC (controller, application, dispatcher, sharedConte
         }
 
         dispatcher.replaceServer(await controller.capability.getDispatchTarget())
-        return application.entrypoint ? controller.capability.getUrl() : null
+
+        const scheduledTasks =
+          typeof controller.capability.getScheduledTasks === 'function'
+            ? await controller.capability.getScheduledTasks()
+            : []
+
+        return {
+          url: application.entrypoint ? controller.capability.getUrl() : null,
+          scheduledTasks
+        }
       },
 
       async stop ({ force, dependents }) {
@@ -305,6 +314,22 @@ export async function setupITC (controller, application, dispatcher, sharedConte
         }
       },
 
+      async getApplicationScheduledTasks () {
+        if (typeof controller.capability.getScheduledTasks !== 'function') {
+          return []
+        }
+
+        return controller.capability.getScheduledTasks()
+      },
+
+      async runApplicationScheduledTasks ({ scheduleId, scheduledTime }) {
+        if (typeof controller.capability.runScheduledTasks !== 'function') {
+          throw new Error(`Application "${application.id}" does not support scheduled task execution`)
+        }
+
+        return controller.capability.runScheduledTasks(scheduleId, scheduledTime)
+      },
+
       async getApplicationMeta () {
         try {
           return await controller.capability.getMeta()

package/lib/worker/management.js

@@ -70,6 +70,22 @@ export class ManagementClient {
     return this.#send('getMetrics', { format })
   }
 
+  getSchedulerJobs () {
+    return this.#send('getSchedulerJobs')
+  }
+
+  pauseSchedulerJob (name) {
+    return this.#send('pauseSchedulerJob', { name })
+  }
+
+  resumeSchedulerJob (name) {
+    return this.#send('resumeSchedulerJob', { name })
+  }
+
+  runSchedulerJob (name) {
+    return this.#send('runSchedulerJob', { name })
+  }
+
   startApplication (id) {
     return this.#send('startApplication', { id })
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "3.63.0",
+  "version": "3.64.0",
   "description": "Platformatic runtime",
   "main": "index.js",
   "type": "module",
@@ -39,14 +39,14 @@
     "typescript": "^5.9.3",
     "undici-oidc-interceptor": "^0.9.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/composer": "3.63.0",
-    "@platformatic/db": "3.63.0",
-    "@platformatic/gateway": "3.63.0",
-    "@platformatic/service": "3.63.0",
-    "@platformatic/sql-graphql": "3.63.0",
-    "@platformatic/sql-mapper": "3.63.0",
-    "@platformatic/node": "3.63.0",
-    "@platformatic/wattpm-pprof-capture": "3.63.0"
+    "@platformatic/db": "3.64.0",
+    "@platformatic/composer": "3.64.0",
+    "@platformatic/gateway": "3.64.0",
+    "@platformatic/service": "3.64.0",
+    "@platformatic/node": "3.64.0",
+    "@platformatic/sql-graphql": "3.64.0",
+    "@platformatic/wattpm-pprof-capture": "3.64.0",
+    "@platformatic/sql-mapper": "3.64.0"
   },
   "dependencies": {
     "@fastify/accepts": "^5.0.0",
@@ -77,13 +77,13 @@
     "undici": "^7.27.2",
     "undici-thread-interceptor": "^1.3.1",
     "ws": "^8.16.0",
-    "@platformatic/basic": "3.63.0",
-    "@platformatic/generators": "3.63.0",
-    "@platformatic/foundation": "3.63.0",
-    "@platformatic/globals": "3.63.0",
-    "@platformatic/metrics": "3.63.0",
-    "@platformatic/telemetry": "3.63.0",
-    "@platformatic/itc": "3.63.0"
+    "@platformatic/foundation": "3.64.0",
+    "@platformatic/globals": "3.64.0",
+    "@platformatic/generators": "3.64.0",
+    "@platformatic/itc": "3.64.0",
+    "@platformatic/basic": "3.64.0",
+    "@platformatic/metrics": "3.64.0",
+    "@platformatic/telemetry": "3.64.0"
   },
   "engines": {
     "node": ">=22.19.0"

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.63.0.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.64.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Platformatic Runtime Config",
   "type": "object",