@highstate/backend 0.18.0 → 0.20.0

package/src/shared/resolvers/effective-output-type.ts

@@ -0,0 +1,162 @@
+import type { ComponentModel, EntityModel, InstanceInput, InstanceModel } from "@highstate/contract"
+
+export type InstanceTypeContext = {
+  instance: InstanceModel
+  component: ComponentModel
+  entities: Readonly<Record<string, EntityModel>>
+}
+
+export type ResolveEffectiveOutputTypeOptions = {
+  input: InstanceInput
+  fallbackType: string
+  getInstanceContext: (instanceId: string) => InstanceTypeContext | undefined
+  visited?: Set<string>
+}
+
+/**
+ * Resolves the effective entity type of a connected output.
+ *
+ * The function starts from the connected output declaration and then applies
+ * two additional transformations used by the platform:
+ *
+ * 1. `fromInput` forwarding.
+ * If the output references `fromInput`, it follows that input edge and reuses
+ * the upstream effective type when the forwarding source is deterministic.
+ *
+ * 2. Input `path` traversal.
+ * If the edge provides a `path`, the type is narrowed by following entity
+ * inclusions segment-by-segment.
+ *
+ * The function is intentionally conservative.
+ * If a forwarding source is ambiguous (missing, multiple, hub/injection-backed,
+ * unknown node/entity, or cyclic), it falls back to the declared type.
+ *
+ * @param options.input The connected instance output reference being consumed.
+ * @param options.fallbackType The type to use when resolution cannot continue safely.
+ * @param options.getInstanceContext A lookup that returns instance/component/entities for an instance ID.
+ * @param options.visited A recursion guard set used to prevent forwarding cycles.
+ * @returns The effective resolved output entity type.
+ */
+export function resolveEffectiveOutputType(options: ResolveEffectiveOutputTypeOptions): string {
+  const visited = options.visited ?? new Set<string>()
+
+  const producer = options.getInstanceContext(options.input.instanceId)
+  if (!producer) {
+    return resolveTypeByPathOrFallbackInclusion({
+      rootType: options.fallbackType,
+      path: options.input.path,
+      entities: undefined,
+      fallbackType: options.fallbackType,
+    })
+  }
+
+  const outputSpec = producer.component.outputs[options.input.output]
+  if (!outputSpec) {
+    return resolveTypeByPathOrFallbackInclusion({
+      rootType: options.fallbackType,
+      path: options.input.path,
+      entities: producer.entities,
+      fallbackType: options.fallbackType,
+    })
+  }
+
+  let effectiveType: string = outputSpec.type
+
+  if (outputSpec.fromInput) {
+    const visitedKey = `${producer.instance.id}:${options.input.output}`
+    if (!visited.has(visitedKey)) {
+      visited.add(visitedKey)
+
+      const forwardedInputName = outputSpec.fromInput
+      const forwardedFallback = producer.component.inputs[forwardedInputName]?.type ?? effectiveType
+      const hasHubInputs = (producer.instance.hubInputs?.[forwardedInputName]?.length ?? 0) > 0
+      const hasInjectionInputs = (producer.instance.injectionInputs?.length ?? 0) > 0
+      const directInputs = producer.instance.inputs?.[forwardedInputName] ?? []
+
+      if (!hasHubInputs && !hasInjectionInputs && directInputs.length === 1) {
+        const forwardedInput = directInputs[0]
+        if (forwardedInput) {
+          effectiveType = resolveEffectiveOutputType({
+            input: forwardedInput,
+            fallbackType: forwardedFallback,
+            getInstanceContext: options.getInstanceContext,
+            visited,
+          })
+        } else {
+          effectiveType = forwardedFallback
+        }
+      } else {
+        effectiveType = forwardedFallback
+      }
+    }
+  }
+
+  return resolveTypeByPathOrFallbackInclusion({
+    rootType: effectiveType,
+    path: options.input.path,
+    entities: producer.entities,
+    fallbackType: options.fallbackType,
+  })
+}
+
+function resolveTypeByPathOrFallbackInclusion(options: {
+  rootType: string
+  path: string | undefined
+  entities: Readonly<Record<string, EntityModel>> | undefined
+  fallbackType: string
+}): string {
+  const { rootType, path, entities, fallbackType } = options
+
+  if (!path || !entities) {
+    return resolveTypeByImplicitInclusion(rootType, fallbackType, entities)
+  }
+
+  let currentType = rootType
+  const segments = path.split(".")
+
+  for (const segment of segments) {
+    const entity = entities[currentType]
+    if (!entity) {
+      return rootType
+    }
+
+    const inclusion = entity.inclusions?.find(inc => inc.field === segment)
+    if (!inclusion) {
+      return rootType
+    }
+
+    currentType = inclusion.type
+  }
+
+  return currentType
+}
+
+function resolveTypeByImplicitInclusion(
+  rootType: string,
+  fallbackType: string,
+  entities: Readonly<Record<string, EntityModel>> | undefined,
+): string {
+  if (!entities) {
+    return rootType
+  }
+
+  if (rootType === fallbackType) {
+    return rootType
+  }
+
+  const rootEntity = entities[rootType]
+  if (!rootEntity) {
+    return rootType
+  }
+
+  if (rootEntity.extensions?.includes(fallbackType)) {
+    return rootType
+  }
+
+  const inclusion = rootEntity.inclusions?.find(inc => inc.type === fallbackType)
+  if (inclusion) {
+    return fallbackType
+  }
+
+  return rootType
+}

package/src/shared/resolvers/index.ts

@@ -1,3 +1,4 @@
+export * from "./effective-output-type"
 export * from "./graph-resolver"
 export * from "./input"
 export * from "./input-hash"

package/src/shared/resolvers/input.ts

@@ -1,12 +1,15 @@
 import {
   type ComponentModel,
+  type EntityModel,
   type HubInput,
   type HubModel,
   type InstanceInput,
   type InstanceModel,
+  inputKey,
   isUnitModel,
 } from "@highstate/contract"
 import { fromEntries, mapValues } from "remeda"
+import { resolveEffectiveOutputType } from "./effective-output-type"
 import { GraphResolver } from "./graph-resolver"
 
 export type InputResolverNode =
@@ -14,10 +17,12 @@ export type InputResolverNode =
       kind: "instance"
       instance: InstanceModel
       component: ComponentModel
+      entities: Readonly<Record<string, EntityModel>>
     }
   | {
       kind: "hub"
       hub: HubModel
+      entities: Readonly<Record<string, EntityModel>>
     }
 
 export type ResolvedInstanceInput = {
@@ -30,6 +35,7 @@ export type InputResolverOutput =
       kind: "instance"
       instance: InstanceModel
       component: ComponentModel
+      entities: Readonly<Record<string, EntityModel>>
       resolvedInputs: Record<string, ResolvedInstanceInput[]>
       resolvedOutputs: Record<string, InstanceInput[]> | undefined
       resolvedInjectionInputs: ResolvedInstanceInput[]
@@ -37,6 +43,7 @@ export type InputResolverOutput =
     }
   | {
       kind: "hub"
+      hub: HubModel
       resolvedInputs: ResolvedInstanceInput[]
     }
 
@@ -78,6 +85,25 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
     return dependencies
   }
 
+  private resolveOutputTypeForInput(input: InstanceInput, fallbackType: string): string {
+    return resolveEffectiveOutputType({
+      input,
+      fallbackType,
+      getInstanceContext: instanceId => {
+        const output = this.outputs.get(`instance:${instanceId}`)
+        if (!output || output.kind !== "instance") {
+          return undefined
+        }
+
+        return {
+          instance: output.instance,
+          component: output.component,
+          entities: output.entities,
+        }
+      },
+    })
+  }
+
   processNode(node: InputResolverNode): InputResolverOutput {
     const getHubOutput = (input: HubInput) => {
       const output = this.outputs.get(`hub:${input.hubId}`)
@@ -118,7 +144,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
       const hubResult: Map<string, ResolvedInstanceInput> = new Map()
 
       const addHubResult = (input: ResolvedInstanceInput) => {
-        hubResult.set(`${input.input.instanceId}:${input.input.output}`, input)
+        hubResult.set(inputKey(input.input), input)
       }
 
       for (const input of node.hub.inputs ?? []) {
@@ -130,7 +156,10 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
           continue
         }
 
-        addHubResult({ input, type: componentInput.type })
+        addHubResult({
+          input,
+          type: this.resolveOutputTypeForInput(input, componentInput.type),
+        })
       }
 
       for (const injectionInput of node.hub.injectionInputs ?? []) {
@@ -143,6 +172,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
 
       return {
         kind: "hub",
+        hub: node.hub,
         resolvedInputs: Array.from(hubResult.values()),
       }
     }
@@ -160,6 +190,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
         kind: "instance",
         instance: node.instance,
         component: node.component,
+        entities: node.entities,
         resolvedInputs: mapValues(node.instance.resolvedInputs, (inputs, inputName) => {
           const componentInput = node.component.inputs[inputName]
           if (!componentInput) {
@@ -189,7 +220,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
         resolvedInputsMap.set(inputName, inputs)
       }
 
-      inputs.set(`${input.input.instanceId}:${input.input.output}`, input)
+      inputs.set(inputKey(input.input), input)
     }
 
     const addInstanceInput = (inputName: string, input: InstanceInput) => {
@@ -211,17 +242,37 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
       }
 
       if (isUnitModel(component)) {
-        addInstanceResult(inputName, { input, type: componentInput.type })
+        addInstanceResult(inputName, {
+          input,
+          type: this.resolveOutputTypeForInput(input, componentInput.type),
+        })
         return
       }
 
       if (resolvedOutputs) {
         for (const output of resolvedOutputs) {
-          addInstanceResult(inputName, { input: output, type: componentInput.type })
+          addInstanceResult(inputName, {
+            input: {
+              ...output,
+              // keep explicit path from the edge while preserving already-resolved output path
+              path: input.path ?? output.path,
+            },
+            type: this.resolveOutputTypeForInput(
+              {
+                instanceId: input.instanceId,
+                output: input.output,
+                path: input.path ?? output.path,
+              },
+              componentInput.type,
+            ),
+          })
         }
       } else {
         // if the instance is not evaluated, we a forced to use the input as is
-        addInstanceResult(inputName, { input, type: componentInput.type })
+        addInstanceResult(inputName, {
+          input,
+          type: this.resolveOutputTypeForInput(input, componentInput.type),
+        })
       }
     }
 
@@ -237,7 +288,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
     for (const injectionInput of node.instance.injectionInputs ?? []) {
       const { resolvedInputs } = getHubOutput(injectionInput)
       for (const input of resolvedInputs) {
-        injectionInputs.set(`${input.input.instanceId}:${input.input.output}`, input)
+        injectionInputs.set(inputKey(input.input), input)
       }
     }
 
@@ -248,7 +299,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
       for (const hubInput of hubInputs) {
         const { resolvedInputs } = getHubOutput(hubInput)
         for (const input of resolvedInputs) {
-          allInputs.set(`${input.input.instanceId}:${input.input.output}`, input)
+          allInputs.set(inputKey(input.input), input)
         }
       }
 
@@ -256,7 +307,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
         if (input.type === componentInput.type) {
           addInstanceInput(inputName, input.input)
 
-          const key = `${input.input.instanceId}:${input.input.output}`
+          const key = inputKey(input.input)
           if (injectionInputs.has(key)) {
             matchedInjectionInputs.set(key, input)
           }
@@ -275,6 +326,7 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
       kind: "instance",
       instance: node.instance,
       component: node.component,
+      entities: node.entities,
       resolvedInputs,
       resolvedOutputs: node.instance.resolvedOutputs,
       resolvedInjectionInputs: Array.from(injectionInputs.values()),

package/src/shared/utils/index.ts

@@ -1,3 +1,4 @@
 export * from "./async-batcher"
 export * from "./hash"
 export * from "./promise-tracker"
+export * from "./stable-json"

package/src/shared/utils/stable-json.ts

@@ -0,0 +1,41 @@
+export function stableJsonStringify(value: unknown): string {
+  if (value === null) {
+    return "null"
+  }
+
+  if (Array.isArray(value)) {
+    return `[${value.map(stableJsonStringify).join(",")}]`
+  }
+
+  switch (typeof value) {
+    case "string":
+      return JSON.stringify(value)
+    case "number": {
+      if (!Number.isFinite(value)) {
+        throw new Error("Snapshot content contains a non-finite number")
+      }
+
+      return JSON.stringify(value)
+    }
+    case "boolean":
+      return value ? "true" : "false"
+    case "object": {
+      const record = value as Record<string, unknown>
+      const keys = Object.keys(record).sort()
+      const parts: string[] = []
+
+      for (const key of keys) {
+        const item = record[key]
+        if (item === undefined) {
+          throw new Error("Snapshot content contains undefined")
+        }
+
+        parts.push(`${JSON.stringify(key)}:${stableJsonStringify(item)}`)
+      }
+
+      return `{${parts.join(",")}}`
+    }
+    default:
+      throw new Error(`Snapshot content contains non-JSON value of type "${typeof value}"`)
+  }
+}

package/src/terminal/manager.ts

@@ -1,5 +1,6 @@
 import type { Logger } from "pino"
 import type { ProjectUnlockService } from "../business"
+import type { ObjectRefIndexService } from "../business/object-ref-index"
 import type { DatabaseManager, Terminal, TerminalSession } from "../database"
 import type { PubSubManager } from "../pubsub"
 import type { TerminalSessionOutput } from "../shared/models/project/terminal"
@@ -35,6 +36,7 @@ export class TerminalManager {
     private readonly database: DatabaseManager,
     private readonly pubsubManager: PubSubManager,
     private readonly projectUnlockService: ProjectUnlockService,
+    private readonly objectRefIndexService: ObjectRefIndexService,
     private readonly logger: Logger,
   ) {
     this.projectUnlockService.registerUnlockTask(
@@ -113,6 +115,8 @@ export class TerminalManager {
       },
     })
 
+    await this.objectRefIndexService.track(projectId, [session.id])
+
     const output = toTerminalSessionOutput(terminal, session)
 
     this.logger.info({ msg: "terminal session created", id: output.id })
@@ -319,6 +323,7 @@ export class TerminalManager {
     database: DatabaseManager,
     pubsubManager: PubSubManager,
     projectUnlockService: ProjectUnlockService,
+    objectRefIndexService: ObjectRefIndexService,
     logger: Logger,
   ): TerminalManager {
     return new TerminalManager(
@@ -326,6 +331,7 @@ export class TerminalManager {
       database,
       pubsubManager,
       projectUnlockService,
+      objectRefIndexService,
       logger.child({ service: "TerminalManager" }),
     )
   }

package/src/worker/manager.ts

@@ -2,6 +2,7 @@ import type { Logger } from "pino"
 import type { ApiKeyService, ProjectUnlockService } from "../business"
 import type { DatabaseManager, Worker, WorkerVersion } from "../database"
 import type { PubSubManager } from "../pubsub"
+import type { WorkerVersionStatus } from "../shared"
 import type { WorkerBackend } from "./abstractions"
 import { PassThrough } from "node:stream"
 import { z } from "zod"
@@ -9,6 +10,9 @@ import { type AsyncBatcher, createAsyncBatcher } from "../shared"
 
 export const workerManagerConfig = z.object({
   HIGHSTATE_WORKER_API_PATH: z.string().default("/var/run/highstate.sock"),
+  HIGHSTATE_WORKER_START_MAX_ATTEMPTS: z.coerce.number().int().positive().default(5),
+  HIGHSTATE_WORKER_RESTART_BACKOFF_BASE_MS: z.coerce.number().int().nonnegative().default(1000),
+  HIGHSTATE_WORKER_RESTART_BACKOFF_MAX_MS: z.coerce.number().int().nonnegative().default(30000),
 })
 
 type RunningWorkerInfo = {
@@ -22,7 +26,40 @@ type RunningWorkerInfo = {
   lineBuffer?: string
 }
 
-export const maxWorkerStartAttempts = 5
+function getWorkerRestartBackoffMs(failedAttempts: number, baseMs: number, maxMs: number): number {
+  if (failedAttempts <= 0) {
+    return 0
+  }
+
+  const exponent = Math.max(0, failedAttempts - 1)
+  const uncapped = baseMs * 2 ** exponent
+
+  return Math.min(maxMs, uncapped)
+}
+
+async function waitForAbortableDelay(delayMs: number, signal: AbortSignal): Promise<boolean> {
+  if (delayMs <= 0) {
+    return true
+  }
+
+  if (signal.aborted) {
+    return false
+  }
+
+  return await new Promise(resolve => {
+    const onAbort = () => {
+      clearTimeout(timer)
+      resolve(false)
+    }
+
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", onAbort)
+      resolve(true)
+    }, delayMs)
+
+    signal.addEventListener("abort", onAbort, { once: true })
+  })
+}
 
 export class WorkerManager {
   constructor(
@@ -50,6 +87,17 @@ export class WorkerManager {
 
   private readonly runningWorkers = new Map<string, RunningWorkerInfo>()
 
+  private async publishWorkerVersionStatus(
+    projectId: string,
+    workerVersionId: string,
+    status: WorkerVersionStatus,
+  ): Promise<void> {
+    await this.pubsubManager.publish(["worker-version-status", projectId], {
+      workerVersionId,
+      status,
+    })
+  }
+
   private async startWorkerVersion(
     projectId: string,
     workerVersion: WorkerVersion & { worker: Worker },
@@ -70,6 +118,7 @@ export class WorkerManager {
     // calculate attempt number
     const previousFailedAttempts = existingInfo?.failedAttempts ?? 0
     const failedAttempts = restart ? previousFailedAttempts + 1 : 0
+    const maxWorkerStartAttempts = this.config.HIGHSTATE_WORKER_START_MAX_ATTEMPTS
 
     // check if max attempts reached
     if (failedAttempts >= maxWorkerStartAttempts) {
@@ -95,6 +144,8 @@ export class WorkerManager {
         },
       })
 
+      await this.publishWorkerVersionStatus(projectId, workerVersion.id, "error")
+
       // clean up from running workers map
       if (existingInfo) {
         existingInfo.logBatcher && void existingInfo.logBatcher.flush()
@@ -104,6 +155,45 @@ export class WorkerManager {
       return
     }
 
+    if (restart && existingInfo) {
+      const restartBackoffMs = getWorkerRestartBackoffMs(
+        failedAttempts,
+        this.config.HIGHSTATE_WORKER_RESTART_BACKOFF_BASE_MS,
+        this.config.HIGHSTATE_WORKER_RESTART_BACKOFF_MAX_MS,
+      )
+
+      if (restartBackoffMs > 0) {
+        this.logger.debug(
+          { projectId, workerVersionId: workerVersion.id, restartBackoffMs, failedAttempts },
+          `delaying worker version "%s" restart for %s ms after attempt %s`,
+          workerVersion.id,
+          restartBackoffMs,
+          failedAttempts,
+        )
+
+        await this.writeWorkerLog(
+          projectId,
+          workerVersion.id,
+          `worker restart scheduled in ${restartBackoffMs}ms`,
+        )
+
+        const canContinue = await waitForAbortableDelay(
+          restartBackoffMs,
+          existingInfo.abortController.signal,
+        )
+
+        if (!canContinue) {
+          return
+        }
+
+        const currentInfo = this.runningWorkers.get(workerVersion.id)
+
+        if (currentInfo !== existingInfo || currentInfo.abortController.signal.aborted) {
+          return
+        }
+      }
+    }
+
     // regenerate API token
     const apiKey = await this.apiKeyService.regenerateToken(projectId, workerVersion.apiKeyId)
     const stdout = new PassThrough()
@@ -184,6 +274,8 @@ export class WorkerManager {
       },
     })
 
+    await this.publishWorkerVersionStatus(projectId, workerVersion.id, "starting")
+
     void this.workerBackend
       .run({
         projectId,
@@ -272,6 +364,8 @@ export class WorkerManager {
       },
     })
 
+    await this.publishWorkerVersionStatus(projectId, workerVersionId, "running")
+
     this.logger.debug(
       { projectId, workerVersionId },
       `worker version "%s" is now running in project "%s"`,
@@ -311,6 +405,8 @@ export class WorkerManager {
       },
     })
 
+    await this.publishWorkerVersionStatus(info.projectId, workerVersionId, "stopped")
+
     this.logger.debug(
       { projectId: info.projectId, workerVersionId },
       `stopped worker version "%s"`,