@platformatic/runtime 3.63.0 → 3.65.0

package/lib/runtime.js

@@ -5,6 +5,7 @@ import {
   executeInParallel,
   executeWithTimeout,
   features,
+  kEnvFileFallbackKeys,
   kMetadata,
   kTimeout,
   parseMemorySize
@@ -93,6 +94,7 @@ const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
 const kHeapCheckCounter = Symbol('plt.runtime.worker.heapCheckCounter')
 const kLastHeapStats = Symbol('plt.runtime.worker.lastHeapStats')
 const kProfilingELUGates = Symbol('plt.runtime.worker.profilingELUGates')
+const kWorkerScheduledTasks = Symbol('plt.runtime.worker.scheduledTasks')
 const kHealthITCTimeoutMs = 5000
 const kProfilingELUHysteresis = 0.1
 const kLastProfileTimeoutMs = 10000
@@ -116,6 +118,10 @@ const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by th
 
 const MAX_LISTENERS_COUNT = 100
 
+function hasWorkerIndex (applicationId) {
+  return /^.+:\d+$/.test(applicationId)
+}
+
 function parseOrigins (origins) {
   if (!origins) return undefined
 
@@ -141,6 +147,24 @@ function formatMetricValue (value) {
   return `${value}`
 }
 
+// Resolves the setup function of a runtime extension out of its module namespace.
+//
+// The canonical form is a default export, but transpilers and bundlers emit
+// faux ESM modules, in which the real exports are properties of `module.exports`
+// and therefore only reachable via an additional `default` hop. The setup
+// function can also be exported as a named `setup` export, at both levels.
+function resolveExtensionSetup (imported) {
+  const candidates = [imported?.default, imported?.setup, imported?.default?.default, imported?.default?.setup]
+
+  for (const candidate of candidates) {
+    if (typeof candidate === 'function') {
+      return candidate
+    }
+  }
+
+  return null
+}
+
 // Always run operations in parallel to avoid deadlocks when services have dependencies
 const DEFAULT_CONCURRENCY = availableParallelism() * 2
 const MAX_BOOTSTRAP_ATTEMPTS = 5
@@ -352,8 +376,8 @@ export class Runtime extends EventEmitter {
     await this.addApplications(this.#config.applications)
     await this.#setDispatcher(config.undici)
 
-    if (config.scheduler && !this.#context.build) {
-      this.#scheduler = startScheduler(config.scheduler, this.#dispatcher, logger)
+    if (!this.#context.build) {
+      this.#scheduler = startScheduler(config.scheduler ?? [], this.#dispatcher, logger)
     }
 
     this.#updateStatus('init')
@@ -708,6 +732,7 @@ export class Runtime extends EventEmitter {
 
     for (const application of applications) {
       this.#dynamicWorkersScaler?.remove(application)
+      await this.#scheduler?.removeApplicationJobs(application)
       this.#applications.delete(application)
       this.#applicationRestartCounts.delete(application)
     }
@@ -816,6 +841,8 @@ export class Runtime extends EventEmitter {
       await this.#startWorker(config, applicationConfig, workers, id, i, silent)
     }
 
+    await this.#registerApplicationSchedulerJobs(id)
+
     this.emitAndNotify('application:started', id)
     await this.#dynamicWorkersScaler?.applyPendingUpdate(id)
   }
@@ -829,6 +856,7 @@ export class Runtime extends EventEmitter {
     const workersCount = workersIds.length
 
     this.emitAndNotify('application:stopping', id)
+    await this.#scheduler?.stopApplicationJobs(id)
 
     if (typeof workersCount === 'number') {
       const stopInvocations = []
@@ -882,6 +910,7 @@ export class Runtime extends EventEmitter {
         await this.#replaceWorker(config, applicationConfig, workersCount, id, workerIndex, worker, true)
       }
 
+      await this.#registerApplicationSchedulerJobs(id)
       this.#incrementApplicationRestartCount(id)
       this.emitAndNotify('application:restarted', id)
     } finally {
@@ -907,17 +936,68 @@ export class Runtime extends EventEmitter {
   }
 
   async startApplicationProfiling (id, options = {}, ensureStarted = true) {
-    const service = await this.#getApplicationById(id, ensureStarted)
     this.#validatePprofCapturePreload()
 
-    return sendViaITC(service, 'startProfiling', options)
+    const { allWorkers, ...profilingOptions } = options
+
+    if (!allWorkers || hasWorkerIndex(id)) {
+      const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
+      return sendViaITC(service, 'startProfiling', profilingOptions)
+    }
+
+    const started = []
+    const alreadyProfiling = []
+    let firstError
+
+    for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
+      try {
+        await sendViaITC(worker, 'startProfiling', profilingOptions)
+        started.push(workerIndex)
+      } catch (error) {
+        // A worker which is already being profiled is considered covered, but
+        // if no other worker could be started the error is still reported.
+        if (error.code === 'PLT_PPROF_PROFILING_ALREADY_STARTED') {
+          alreadyProfiling.push(workerIndex)
+        }
+
+        firstError ??= error
+      }
+    }
+
+    if (started.length === 0 && firstError) {
+      throw firstError
+    }
+
+    return { workers: started.concat(alreadyProfiling).sort((a, b) => a - b) }
   }
 
   async stopApplicationProfiling (id, options = {}, ensureStarted = true) {
-    const service = await this.#getApplicationById(id, ensureStarted)
     this.#validatePprofCapturePreload()
 
-    return sendViaITC(service, 'stopProfiling', options)
+    const { allWorkers, ...profilingOptions } = options
+
+    if (!allWorkers || hasWorkerIndex(id)) {
+      const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
+      return sendViaITC(service, 'stopProfiling', profilingOptions)
+    }
+
+    const profiles = []
+    let firstError
+
+    for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
+      try {
+        const profile = await sendViaITC(worker, 'stopProfiling', profilingOptions)
+        profiles.push({ workerIndex, profile })
+      } catch (error) {
+        firstError ??= error
+      }
+    }
+
+    if (profiles.length === 0 && firstError) {
+      throw firstError
+    }
+
+    return profiles
   }
 
   async getApplicationLastProfile (id, options = {}, ensureStarted = true) {
@@ -929,39 +1009,45 @@ export class Runtime extends EventEmitter {
 
     try {
       // Bound the whole retrieval with a single timeout budget: resolving the
-      // worker round-trips to it when ensureStarted is set, and the profile
-      // pull does too — both hang if the worker event loop is blocked. Attach
+      // workers round-trips to them when ensureStarted is set, and the profile
+      // pulls do too — both hang if a worker event loop is blocked. Attach
       // a noop handler so that a late settlement after the timeout does not
       // surface as an unhandled rejection.
-      const pull = (async () => {
-        const service = await this.#getApplicationById(id, ensureStarted)
-        const result = await sendViaITC(service, 'getLastProfile', { ...options, includeTimestamp: true })
-        return { service, result }
-      })()
+      const pull = this.#pullLastProfiles(id, options, ensureStarted)
       pull.catch(() => {})
 
       const outcome = await executeWithTimeout(pull, timeout, kTimeout)
 
       if (outcome !== kTimeout) {
-        const { service, result } = outcome
-
-        // An older capture module which does not support includeTimestamp
-        // returns the raw profile.
-        const value = result instanceof Uint8Array ? { profile: result, timestamp: null } : result
-
-        // A strictly newer live window supersedes the preserved overload
-        // profile: prune it so the preserved copy naturally expires once the
-        // worker is healthy again and its profiles are being consumed.
-        if (value.timestamp != null) {
-          const key = `${service[kApplicationId]}:${service[kWorkerId]}:${type}`
-          const preserved = this.#lastOverloadProfiles.get(key)
+        let best = null
+
+        for (const { service, result } of outcome) {
+          // An older capture module which does not support includeTimestamp
+          // returns the raw profile.
+          const value = result instanceof Uint8Array
+            ? { profile: result, timestamp: null, sampleCount: null }
+            : { sampleCount: null, ...result }
+
+          // A strictly newer live window supersedes the preserved overload
+          // profile: prune it so the preserved copy naturally expires once the
+          // worker is healthy again and its profiles are being consumed.
+          if (value.timestamp != null) {
+            const key = `${service[kApplicationId]}:${service[kWorkerId]}:${type}`
+            const preserved = this.#lastOverloadProfiles.get(key)
+
+            if (preserved && preserved.timestamp < value.timestamp) {
+              this.#lastOverloadProfiles.delete(key)
+            }
+          }
 
-          if (preserved && preserved.timestamp < value.timestamp) {
-            this.#lastOverloadProfiles.delete(key)
+          // For an application-level id the newest window across the workers
+          // wins, mirroring the preserved overload profile fallback below.
+          if (!best || (value.timestamp != null && (best.timestamp == null || value.timestamp > best.timestamp))) {
+            best = value
           }
         }
 
-        return { ...value, preserved: false }
+        return { ...best, preserved: false }
       }
 
       // The worker event loop is not responding (e.g. it is hard-blocked).
@@ -979,12 +1065,73 @@ export class Runtime extends EventEmitter {
     if (preserved) {
       // The preserved flag lets consumers distinguish post-mortem evidence
       // from a live window and judge it together with the timestamp.
-      return { profile: preserved.profile, timestamp: preserved.timestamp, preserved: true }
+      return {
+        profile: preserved.profile,
+        timestamp: preserved.timestamp,
+        sampleCount: preserved.sampleCount,
+        preserved: true
+      }
     }
 
     throw error
   }
 
+  // Pulls the last profile from the addressed worker, or from every worker of
+  // the application when no explicit worker index is given: the application
+  // "last profile" is the newest window among its workers, so one arbitrary
+  // worker cannot answer for all of them. Per-worker failures with fallback
+  // codes are ignored as long as at least one worker yields a profile.
+  async #pullLastProfiles (id, options, ensureStarted) {
+    const pullOptions = { ...options, includeTimestamp: true, includeSampleCount: true }
+
+    const pullWorker = async service => {
+      const result = await sendViaITC(service, 'getLastProfile', pullOptions)
+      return { service, result }
+    }
+
+    if (/^.+:\d+$/.test(id)) {
+      return [await pullWorker(await this.#getApplicationById(id, ensureStarted))]
+    }
+
+    if (!this.#applications.has(id)) {
+      throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
+    }
+
+    const keys = this.#workers.getKeys(id)
+
+    // No worker is currently registered: resolve the id as usual so that the
+    // canonical error is raised.
+    if (keys.length === 0) {
+      return [await pullWorker(await this.#getApplicationById(id, ensureStarted))]
+    }
+
+    const settled = await Promise.allSettled(
+      keys.map(async key => {
+        const service = await this.#getWorkerByIdOrNext(id, key.split(':')[1], ensureStarted)
+        return pullWorker(service)
+      })
+    )
+
+    const profiles = []
+    let firstError
+
+    for (const outcome of settled) {
+      if (outcome.status === 'fulfilled') {
+        profiles.push(outcome.value)
+      } else if (!kLastProfileFallbackCodes.has(outcome.reason?.code)) {
+        throw outcome.reason
+      } else {
+        firstError ??= outcome.reason
+      }
+    }
+
+    if (profiles.length === 0) {
+      throw firstError
+    }
+
+    return profiles
+  }
+
   // The final profile of an overload pause is pushed by the worker and
   // preserved in the main thread (see the profile:overload listener), so that
   // the evidence of what saturated a worker survives the worker being blocked
@@ -1888,17 +2035,27 @@ export class Runtime extends EventEmitter {
   }
 
   async getApplicationMeta (id) {
-    const application = await this.#getApplicationById(id)
+    const hasWorkerId = /^.+:\d+$/.test(id)
+    const attempts = hasWorkerId ? 1 : Math.max(1, this.#workers.getKeys(id).length)
 
-    try {
-      return await sendViaITC(application, 'getApplicationMeta')
-    } catch (e) {
-      // The application exports no meta, return an empty object
-      if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
-        return {}
-      }
+    for (let attempt = 0; attempt < attempts; attempt++) {
+      const application = await this.#getApplicationById(id)
 
-      throw e
+      try {
+        return await sendViaITC(application, 'getApplicationMeta')
+      } catch (e) {
+        // The application exports no meta, return an empty object
+        if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
+          return {}
+        }
+
+        // A parallel restart can stop the selected worker while metadata is
+        // being retrieved. Retry another worker unless one was requested
+        // explicitly or every worker available at the start has been tried.
+        if (e.code !== 'PLT_RUNTIME_APPLICATION_WORKER_EXIT' || attempt === attempts - 1) {
+          throw e
+        }
+      }
     }
   }
 
@@ -1973,6 +2130,79 @@ export class Runtime extends EventEmitter {
     return sendViaITC(application, 'getApplicationGraphQLSchema')
   }
 
+  async getApplicationScheduledTasks (id) {
+    const application = await this.#getApplicationById(id, true)
+
+    return sendViaITC(application, 'getApplicationScheduledTasks')
+  }
+
+  async runApplicationScheduledTasks (id, scheduleId, scheduledTime) {
+    const application = await this.#getApplicationById(id, true)
+
+    return sendViaITC(application, 'runApplicationScheduledTasks', { scheduleId, scheduledTime })
+  }
+
+  getSchedulerJobs () {
+    return this.#scheduler?.getJobs() ?? []
+  }
+
+  pauseSchedulerJob (name) {
+    return this.#schedulerOrThrow().pauseJob(name)
+  }
+
+  resumeSchedulerJob (name) {
+    return this.#schedulerOrThrow().resumeJob(name)
+  }
+
+  runSchedulerJob (name) {
+    return this.#schedulerOrThrow().runJob(name)
+  }
+
+  #schedulerOrThrow () {
+    if (!this.#scheduler) {
+      throw new Error('The scheduler is not configured')
+    }
+
+    return this.#scheduler
+  }
+
+  async #registerApplicationSchedulerJobs (id) {
+    if (!this.#scheduler) {
+      return
+    }
+
+    const pausedJobs = new Set(
+      this.#scheduler
+        .getJobs()
+        .filter(job => job.applicationId === id && job.paused)
+        .map(job => job.name)
+    )
+
+    await this.#scheduler.removeApplicationJobs(id)
+
+    const workerId = this.#workers.getKeys(id)[0]
+    const schedules = this.#workers.get(workerId)?.[kWorkerScheduledTasks] ?? []
+    for (const schedule of schedules) {
+      const name = `${id}:${schedule.id}`
+      this.#scheduler.addJob(
+        {
+          name,
+          cron: schedule.cron,
+          source: 'application',
+          applicationId: id,
+          scheduleId: schedule.id,
+          tasks: schedule.tasks,
+          maxRetries: 3
+        },
+        ({ scheduledTime }) => this.runApplicationScheduledTasks(id, schedule.id, scheduledTime)
+      )
+
+      if (pausedJobs.has(name)) {
+        await this.#scheduler.pauseJob(name)
+      }
+    }
+  }
+
   async getWorkers (includeRaw = false) {
     const status = {}
 
@@ -2278,7 +2508,10 @@ export class Runtime extends EventEmitter {
           codeRangeSizeMb
         },
         inspectorOptions,
-        dirname: this.#root
+        dirname: this.#root,
+        // Keys of the worker environment which only come from an env file of the runtime:
+        // the env file of the application is allowed to override those.
+        envFileFallbackKeys: this.#env[kEnvFileFallbackKeys] ?? []
       },
       argv: applicationConfig.arguments,
       execArgv,
@@ -2414,7 +2647,7 @@ export class Runtime extends EventEmitter {
     // The event only carries metadata: the profile can be retrieved on demand
     // via getApplicationLastProfile. We use emit instead of emitAndNotify since
     // other workers are not interested in this event.
-    worker[kITC].on('profile:captured', ({ type, timestamp }) => {
+    worker[kITC].on('profile:captured', ({ type, timestamp, sampleCount }) => {
       // A strictly newer completed window supersedes the preserved overload
       // profile: once the worker is past the overload and producing windows
       // again, its old evidence must not be served anymore.
@@ -2430,7 +2663,8 @@ export class Runtime extends EventEmitter {
         application: applicationId,
         worker: index,
         type,
-        timestamp
+        timestamp,
+        sampleCount: sampleCount ?? null
       })
     })
 
@@ -2492,8 +2726,12 @@ export class Runtime extends EventEmitter {
     // the worker can be retrieved (see getApplicationLastProfile) even while
     // the worker event loop is blocked, and it survives the worker being
     // replaced by the health checks.
-    worker[kITC].on('profile:overload', ({ type, timestamp, profile }) => {
-      this.#lastOverloadProfiles.set(`${workerId}:${type}`, { profile, timestamp })
+    worker[kITC].on('profile:overload', ({ type, timestamp, profile, sampleCount }) => {
+      this.#lastOverloadProfiles.set(`${workerId}:${type}`, {
+        profile,
+        timestamp,
+        sampleCount: sampleCount ?? null
+      })
     })
 
     // Preserved overload profiles outlive their worker only for a grace
@@ -3000,22 +3238,24 @@ export class Runtime extends EventEmitter {
     this.emitAndNotify('application:worker:starting', eventPayload)
 
     try {
-      let workerUrl
+      let workerStartResult
       if (config.startTimeout > 0) {
-        workerUrl = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
+        workerStartResult = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
 
-        if (workerUrl === kTimeout) {
+        if (workerStartResult === kTimeout) {
           this.emitAndNotify('application:worker:startTimeout', eventPayload)
           this.logger.error(`The ${label} failed to start in ${config.startTimeout}ms. Forcefully killing the thread.`)
           worker.terminate()
           throw new ApplicationStartTimeoutError(id, config.startTimeout)
         }
       } else {
-        workerUrl = await sendViaITC(worker, 'start')
+        workerStartResult = await sendViaITC(worker, 'start')
       }
 
       await this.#avoidOutOfOrderThreadLogs()
 
+      const { url: workerUrl, scheduledTasks } = workerStartResult
+      worker[kWorkerScheduledTasks] = scheduledTasks
       if (workerUrl) {
         this.#url = workerUrl
 
@@ -3338,7 +3578,17 @@ export class Runtime extends EventEmitter {
       this.#workers.set(newWorkerId, newWorker)
 
       // Add the worker to the mesh
-      await this.#startWorker(configForNewWorker, applicationConfig, workersCount, applicationId, newIndex, false, 0, newWorker, true)
+      await this.#startWorker(
+        configForNewWorker,
+        applicationConfig,
+        workersCount,
+        applicationId,
+        newIndex,
+        false,
+        0,
+        newWorker,
+        true
+      )
 
       // Make sure the runtime hasn't been stopped in the meanwhile
       if (this.#status !== 'started') {
@@ -3388,6 +3638,37 @@ export class Runtime extends EventEmitter {
     return this.#getWorkerByIdOrNext(applicationId, workerId, ensureStarted, mustExist)
   }
 
+  // Profiling start and stop must address the same worker: resolve an id
+  // without an explicit worker index to the first worker deterministically,
+  // as the round-robin used by #getApplicationById would rotate to a
+  // different worker between the two calls.
+  async #getApplicationWorkerForProfiling (applicationId, ensureStarted) {
+    if (hasWorkerIndex(applicationId)) {
+      return this.#getApplicationById(applicationId, ensureStarted)
+    }
+
+    if (!this.#applications.has(applicationId)) {
+      throw new ApplicationNotFoundError(applicationId, this.getApplicationsIds().join(', '))
+    }
+
+    const [firstWorker] = this.#workers.getKeys(applicationId)
+    return this.#getWorkerByIdOrNext(applicationId, firstWorker?.split(':')[1], ensureStarted)
+  }
+
+  async #getApplicationWorkersForProfiling (applicationId, ensureStarted) {
+    if (!this.#applications.has(applicationId)) {
+      throw new ApplicationNotFoundError(applicationId, this.getApplicationsIds().join(', '))
+    }
+
+    const workers = []
+    for (const key of this.#workers.getKeys(applicationId)) {
+      const workerIndex = parseInt(key.split(':')[1], 10)
+      workers.push({ workerIndex, worker: await this.#getWorkerByIdOrNext(applicationId, workerIndex, ensureStarted) })
+    }
+
+    return workers
+  }
+
   // This method can work in two modes: when workerId is provided, it will return the specific worker
   // otherwise it will return the next available worker for the application.
   async #getWorkerByIdOrNext (applicationId, workerId, ensureStarted = false, mustExist = true) {
@@ -4073,13 +4354,15 @@ export class Runtime extends EventEmitter {
     for (const extension of extensions) {
       const { path, options } = typeof extension === 'string' ? { path: extension } : extension
 
-      let setup
+      let imported
       try {
-        setup = (await import(pathToFileURL(path))).default
+        imported = await import(pathToFileURL(path))
       } catch (e) {
         throw new FailedToLoadExtensionError(path, e.message, { cause: e })
       }
 
+      const setup = resolveExtensionSetup(imported)
+
       if (typeof setup !== 'function') {
         throw new InvalidExtensionError(path)
       }

package/lib/scaling-algorithm.js

@@ -83,7 +83,7 @@ export class ScalingAlgorithm {
 
         const newWorkersCount = appMinWorkers - workersCount
         totalWorkersCount += newWorkersCount
-        totalAvailableMemory += newWorkersCount * avgHeapUsed
+        totalAvailableMemory -= newWorkersCount * avgHeapUsed
         continue
       }
 
@@ -96,7 +96,7 @@ export class ScalingAlgorithm {
 
         const removedWorkersCount = workersCount - appMaxWorkers
         totalWorkersCount -= removedWorkersCount
-        totalAvailableMemory -= removedWorkersCount * avgHeapUsed
+        totalAvailableMemory += removedWorkersCount * avgHeapUsed
         continue
       }
 
@@ -111,7 +111,7 @@ export class ScalingAlgorithm {
 
           const removedWorkersCount = 1
           totalWorkersCount -= removedWorkersCount
-          totalAvailableMemory -= removedWorkersCount * avgHeapUsed
+          totalAvailableMemory += removedWorkersCount * avgHeapUsed
         }
       }
     }