@outputai/core 0.15.1-next.e254388.0 → 0.15.1-next.fd6bb62.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.15.1-next.e254388.0",
+  "version": "0.15.1-next.fd6bb62.0",
   "description": "The core module of the output framework",
   "type": "module",
   "engines": {

package/src/worker/configs.js

@@ -22,13 +22,31 @@ const envVarSchema = z.object( {
   // Worker concurrency — tune these via env vars to adjust for your workload.
   // Each step (API, LLM, etc.) call is one activity. Lower this to reduce memory pressure.
   TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 40 ) ),
-  // Workflows are lightweight state machines — this can be high.
-  TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTIONS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 200 ) ),
-  // LRU cache for sticky workflow execution. Lower values free memory faster after surges.
-  TEMPORAL_MAX_CACHED_WORKFLOWS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 1000 ) ),
+  /*
+  Workflow tasks that may run at once. These are short state transitions, so this is rarely the
+  throughput limit - activity concurrency above is. Left unset so @temporalio/worker applies its
+  own default of 40; the startup 'Worker sizing' log reports it as workflowTaskSlots.
+  https://typescript.temporal.io/api/interfaces/worker.WorkerOptions#maxconcurrentworkflowtaskexecutions
+  */
+  TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTIONS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().optional() ),
+  /*
+  LRU cache for sticky workflow execution, holding each workflow's state until it completes or is
+  evicted. Left unset so @temporalio/worker sizes it from the isolate's heap limit, which a flat
+  value here cannot track. That formula assumes roughly 600 cached workflows per GB, so set this
+  where payloads are heavier - the startup 'Worker sizing' log reports what it resolved to.
+  https://typescript.temporal.io/api/interfaces/worker.WorkerOptions#maxcachedworkflows
+  */
+  TEMPORAL_MAX_CACHED_WORKFLOWS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().optional() ),
   // How aggressively the worker pulls tasks from Temporal.
   TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_POLLS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 5 ) ),
   TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_POLLS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 5 ) ),
+  /*
+  Threads running workflow sandboxes. Each is a V8 isolate and NODE_OPTIONS heap caps apply per
+  isolate, so this multiplies the heap the process can commit - the startup 'Worker sizing'
+  log reports the total. Left unset so @temporalio/worker picks: 1 with reuseV8Context, 2 without.
+  https://typescript.temporal.io/api/interfaces/worker.WorkerOptions#workflowthreadpoolsize
+  */
+  TEMPORAL_WORKFLOW_THREAD_POOL_SIZE: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().optional() ),
   // JSON-encoded Temporal Worker tuner options.
   TEMPORAL_WORKER_TUNER: workerTunerEnvSchema,
   // Activity configs
@@ -64,6 +82,7 @@ export const maxConcurrentWorkflowTaskExecutions = envVars.TEMPORAL_MAX_CONCURRE
 export const maxCachedWorkflows = envVars.TEMPORAL_MAX_CACHED_WORKFLOWS;
 export const maxConcurrentActivityTaskPolls = envVars.TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_POLLS;
 export const maxConcurrentWorkflowTaskPolls = envVars.TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_POLLS;
+export const workflowThreadPoolSize = envVars.TEMPORAL_WORKFLOW_THREAD_POOL_SIZE;
 export const workerTuner = envVars.TEMPORAL_WORKER_TUNER;
 export const namespace = envVars.TEMPORAL_NAMESPACE;
 export const taskQueue = envVars.OUTPUT_CATALOG_ID;

package/src/worker/index.js

@@ -1,5 +1,5 @@
 import { Worker, NativeConnection } from '@temporalio/worker';
-import * as configs from './configs.js';
+import { address, apiKey, namespace, taskQueue, grpcProxy, hookFlushTimeoutMs } from './configs.js';
 import { loadActivities } from './loader/activities.js';
 import { loadWorkflows } from './loader/workflows.js';
 import { loadHooks } from './loader/hooks.js';
@@ -15,7 +15,7 @@ import { CatalogPublisher } from './catalog_workflow/catalog_publisher.js';
 import { mainEventBus } from '#bus';
 import { flushPendingHooks } from '#hooks/pending_hooks';
 import { BusEventType } from '#consts';
-import { setupTelemetry } from './telemetry.js';
+import { setupTelemetry, logIsolateSizing } from './telemetry.js';
 import { TemporalConnectionMonitor } from './connection_monitor.js';
 import { bindGlobalFunctions } from './global_functions.js';
 import { setupClientConfig } from '#temporal/client';
@@ -24,28 +24,12 @@ import { setupTemporalLogger } from './temporal_logger.js';
 import { WorkerRunner } from './worker_runner.js';
 import { shutdownServices } from './shutdown.js';
 import { init as initCredentials } from '#credentials/init';
+import { buildWorkerOptions } from './worker_options.js';
 
 import './log_hooks.js';
 
 const log = createChildLogger( 'Worker' );
 
-const {
-  address,
-  apiKey,
-  namespace,
-  taskQueue,
-  grpcProxy,
-  maxConcurrentWorkflowTaskExecutions,
-  maxConcurrentActivityTaskExecutions,
-  maxCachedWorkflows,
-  maxConcurrentActivityTaskPolls,
-  maxConcurrentWorkflowTaskPolls,
-  shutdownForceTime,
-  shutdownGraceTime,
-  hookFlushTimeoutMs,
-  workerTuner
-} = configs;
-
 const services = {
   connection: null,
   connectionMonitor: null,
@@ -113,34 +97,19 @@ const execute = async () => {
   await run( () => services.catalogPublisher.run() );
 
   log.info( 'Creating Temporal worker...' );
-  const worker = await run( () => {
-    if ( workerTuner ) {
-      log.info( 'Using worker tuner options', { ...workerTuner } );
-    }
-
-    return Worker.create( {
-      connection,
-      namespace,
-      taskQueue,
-      workflowsPath,
-      activities,
-      sinks,
-      interceptors: initInterceptors( { activities, workflows } ),
-      // tuner isn't compatible with concurrent task executions configs
-      ...( workerTuner ? {
-        tuner: workerTuner
-      } : {
-        maxConcurrentWorkflowTaskExecutions,
-        maxConcurrentActivityTaskExecutions
-      } ),
-      maxCachedWorkflows,
-      maxConcurrentActivityTaskPolls,
-      maxConcurrentWorkflowTaskPolls,
-      bundlerOptions: { webpackConfigHook },
-      ...( shutdownForceTime !== undefined && { shutdownForceTime } ),
-      ...( shutdownGraceTime !== undefined && { shutdownGraceTime } )
-    } );
-  } );
+  const worker = await run( () => Worker.create( {
+    connection,
+    namespace,
+    taskQueue,
+    workflowsPath,
+    activities,
+    sinks,
+    interceptors: initInterceptors( { activities, workflows } ),
+    bundlerOptions: { webpackConfigHook },
+    ...buildWorkerOptions()
+  } ) );
+
+  run( () => logIsolateSizing( { worker } ) );
 
   log.info( 'Setting up telemetry...' );
   run( () => setupTelemetry( { worker } ) );

package/src/worker/telemetry.js

@@ -1,21 +1,141 @@
+import { totalmem } from 'node:os';
+import { getHeapStatistics } from 'node:v8';
 import { serializeError } from '#helpers/error_serializer';
 import { createChildLogger } from '#logger';
 import { workerTelemetryIntervalMs } from './configs.js';
 
 const log = createChildLogger( 'Telemetry' );
 
+/**
+ * Resolve the memory limit to compare the worker against: the container's limit, or the
+ * machine's total memory when there is none.
+ *
+ * `process.constrainedMemory()` returns a sentinel near 2^64 rather than 0 when the process is
+ * unconstrained, so values at or above the machine's memory are treated as no limit.
+ *
+ * @returns {number} Bytes.
+ */
+const resolveMemoryLimit = () => {
+  const constrainedMemory = process.constrainedMemory();
+  const machineMemory = totalmem();
+  return constrainedMemory > 0 && constrainedMemory < machineMemory ? constrainedMemory : machineMemory;
+};
+
+/**
+ * Log the worker's isolate and cache sizing. Warn when the projected
+ * heap exceeds the memory limit, or when workflow task slots exceed
+ * the cache.
+ *
+ * The worker runs one isolate on the main thread and one per workflow
+ * thread. A NODE_OPTIONS heap cap applies to each isolate, not to the
+ * process, so on a 4 GB container `--max-old-space-size-percentage=85`
+ * allows each isolate ~3.4 GB and two isolates can commit well over
+ * the container's memory.
+ *
+ * The cache and the task slots are sized independently:
+ * `@temporalio/worker` derives `maxCachedWorkflows` from the heap limit
+ * but fixes `workflowTaskSlots` at 40. On a small container the slots
+ * can exceed the cache, which causes tasks to evict each other's
+ * workflows and forces history replays.
+ *
+ * Logged fields are read from `worker.options`, so each is the resolved
+ * value rather than the requested one:
+ *
+ * - `workflowThreadPoolSize`: threads running workflow sandboxes.
+ *   Defaults to 1 with `reuseV8Context`, 2 without. Ignored under
+ *   `debugMode`.
+ * - `reuseV8Context`: whether workflows share one VM context per thread
+ *   or each get a new one. Enabled by default. Upstream estimates
+ *   roughly 600 cached workflows per GB with it and 250 without.
+ * - `isolateCount`: `1 + workflowThreadPoolSize`, the multiplier on the
+ *   per-isolate heap cap. 1 under `debugMode`, which runs workflows in
+ *   the main isolate and spawns no workflow threads.
+ * - `heapSizeLimit`: the main isolate's V8 heap ceiling, as resolved
+ *   from NODE_OPTIONS.
+ * - `projectedCommittedHeap`: `isolateCount * heapSizeLimit`, the heap
+ *   the process can commit if every isolate fills. Only the main
+ *   thread's limit is readable, so this assumes the workflow threads
+ *   have the same one.
+ * - `memoryLimit`: the limit the projection is compared against. The
+ *   container's limit, or the machine's total memory when unconstrained.
+ * - `maxCachedWorkflows`: sticky cache size. Derived from
+ *   `heapSizeLimit` when TEMPORAL_MAX_CACHED_WORKFLOWS is unset.
+ * - `workflowTaskSlots`: workflow tasks that may run at once, from the
+ *   tuner. Undefined with a resource-based tuner, which sizes slots at
+ *   runtime.
+ *
+ * @param {object} params Parameters.
+ * @param {import('@temporalio/worker').Worker} params.worker The created Temporal worker.
+ */
+export const logIsolateSizing = ( { worker } ) => {
+  const { workflowThreadPoolSize, reuseV8Context, maxCachedWorkflows, tuner, debugMode } = worker.options;
+  // debugMode runs workflows in the main isolate and spawns no workflow threads.
+  const isolateCount = debugMode ? 1 : 1 + workflowThreadPoolSize;
+  const heapSizeLimit = getHeapStatistics().heap_size_limit;
+  const memoryLimit = resolveMemoryLimit();
+  const workflowTaskSlots = tuner?.workflowTaskSlotSupplier?.numSlots;
+  const projectedCommittedHeap = isolateCount * heapSizeLimit;
+
+  log.info( 'Worker sizing', {
+    workflowThreadPoolSize,
+    reuseV8Context,
+    isolateCount,
+    heapSizeLimit,
+    projectedCommittedHeap,
+    memoryLimit,
+    maxCachedWorkflows,
+    workflowTaskSlots
+  } );
+
+  if ( projectedCommittedHeap > memoryLimit ) {
+    log.warn( 'Worker isolates may commit more heap than the host allows', { isolateCount, heapSizeLimit, projectedCommittedHeap, memoryLimit } );
+  }
+
+  if ( workflowTaskSlots > maxCachedWorkflows ) {
+    log.warn( 'Worker runs more workflow tasks than it can cache, forcing replays', { workflowTaskSlots, maxCachedWorkflows } );
+  }
+};
+
+/**
+ * Start an interval that logs worker status and memory every
+ * OUTPUT_WORKER_TELEMETRY_INTERVAL_MS. Disabled when it is 0.
+ *
+ * Each record holds the Temporal worker's `status` and a `memory`
+ * block. The memory fields mix process-wide and main-isolate scopes:
+ *
+ * - `availableMemory`: memory the process can still allocate, within
+ *   the container limit.
+ * - `memoryLimit`: the container's limit, or the machine's total memory
+ *   when unconstrained.
+ * - `memoryUsage`: raw `process.memoryUsage()`. `rss` is process-wide;
+ *   `heapTotal` and `heapUsed` cover only the main isolate.
+ * - `nonMainHeap`: `memoryUsage.rss - memoryUsage.heapUsed`, the
+ *   resident memory outside the main isolate's live objects: workflow
+ *   thread heaps, Rust core allocations, buffers, and heap pages V8 has
+ *   committed but not filled. An upper bound on the workflow thread
+ *   heap, not a measurement of it. Cached workflow memory is counted
+ *   here, so it grows with the cache while `heapUsed` stays flat.
+ * - `heapSizeLimit`: the main isolate's V8 heap ceiling. Constant, but
+ *   included in each sample so records are self-contained.
+ *
+ * @param {object} params Parameters.
+ * @param {import('@temporalio/worker').Worker} params.worker The created Temporal worker.
+ */
 export const setupTelemetry = ( { worker } ) => {
   if ( workerTelemetryIntervalMs <= 0 ) {
     return;
   }
   setInterval( () => {
     try {
+      const memoryUsage = process.memoryUsage();
       log.info( 'Worker', {
         status: worker.getStatus(),
         memory: {
           availableMemory: process.availableMemory(),
-          constrainedMemory: process.constrainedMemory(),
-          memoryUsage: process.memoryUsage()
+          memoryLimit: resolveMemoryLimit(),
+          memoryUsage,
+          nonMainHeap: memoryUsage.rss - memoryUsage.heapUsed,
+          heapSizeLimit: getHeapStatistics().heap_size_limit
         }
       } );
     } catch ( error ) {

package/src/worker/worker_options.js

@@ -0,0 +1,43 @@
+import * as configs from './configs.js';
+import { createChildLogger } from '#logger';
+
+const log = createChildLogger( 'Worker' );
+
+const {
+  maxConcurrentWorkflowTaskExecutions,
+  maxConcurrentActivityTaskExecutions,
+  maxCachedWorkflows,
+  maxConcurrentActivityTaskPolls,
+  maxConcurrentWorkflowTaskPolls,
+  workflowThreadPoolSize,
+  shutdownForceTime,
+  shutdownGraceTime,
+  workerTuner
+} = configs;
+
+/**
+ * Builds the concurrency, cache and shutdown options passed to `Worker.create`.
+ * Unset values are omitted so @temporalio/worker applies its own defaults.
+ * @returns {object} partial Temporal worker options
+ */
+export const buildWorkerOptions = () => {
+  if ( workerTuner ) {
+    log.info( 'Using worker tuner options', { ...workerTuner } );
+  }
+  return {
+    // tuner isn't compatible with concurrent task executions configs
+    ...( workerTuner ? {
+      tuner: workerTuner
+    } : {
+      ...( maxConcurrentWorkflowTaskExecutions !== undefined && { maxConcurrentWorkflowTaskExecutions } ),
+      maxConcurrentActivityTaskExecutions
+    } ),
+    ...( maxCachedWorkflows !== undefined && { maxCachedWorkflows } ),
+    maxConcurrentActivityTaskPolls,
+    maxConcurrentWorkflowTaskPolls,
+    // Omitted when unset so @temporalio/worker applies its own default, which depends on reuseV8Context.
+    ...( workflowThreadPoolSize !== undefined && { workflowThreadPoolSize } ),
+    ...( shutdownForceTime !== undefined && { shutdownForceTime } ),
+    ...( shutdownGraceTime !== undefined && { shutdownGraceTime } )
+  };
+};