@highstate/backend 0.9.15 → 0.9.18

package/src/shared/models/project/worker.ts

@@ -0,0 +1,107 @@
+import { z } from "zod"
+import { objectMetaSchema } from "@highstate/contract"
+
+export const workerStatus = z.enum([
+  // transient statuses
+  "running",
+  "starting",
+])
+
+export const MAX_WORKER_START_ATTEMPTS = 5
+
+export const workerSchema = z.object({
+  id: z.uuidv7(),
+
+  meta: objectMetaSchema.pick({
+    title: true,
+    description: true,
+    icon: true,
+    iconColor: true,
+    createdAt: true,
+    updatedAt: true,
+  }),
+
+  /**
+   * The status of the worker.
+   */
+  status: workerStatus,
+
+  /**
+   * The number of failed attempts to start the worker. When a worker fails to start,
+   * this count is incremented.
+   * If the worker fails to start more than 5 times in a row, it is
+   * considered unhealthy and will not be started again
+   *
+   * This count is reset when the worker is successfully started and connected.
+   */
+  failedStartAttempts: z.number(),
+
+  /**
+   * The fully qualified base docker image without tag and digest.
+   *
+   * It serves as the identity of the worker and can be shared across workers.
+   *
+   * Multiple workers sharing the same identity also share the same service accoun
+   * and have equal access to all its resources.
+   *
+   * Example: `ghcr.io/highstate-io/kubernetes-worker`
+   */
+  identity: z.string(),
+
+  /**
+   * The fully qualified docker image name for this worker.
+   *
+   * Example: `ghcr.io/highstate-io/kubernetes-worker:v1.0.0@sha256:abc123...`
+   */
+  image: z.string(),
+
+  /**
+   * The ID of the service account owned by this worker.
+   */
+  serviceAccountId: z.uuidv7(),
+
+  /**
+   * The ID of the API key used to authenticate this worker with the system.
+   *
+   * Will be generated on the first run of the worker and will be rotated every time the worker is restarted.
+   */
+  apiKeyId: z.uuidv7(),
+})
+
+export const workerUnitRegistrationSchema = z.object({
+  id: z.uuidv7(),
+
+  meta: objectMetaSchema.pick({ createdAt: true, updatedAt: true }),
+
+  workerId: z.uuidv7(),
+  instanceId: z.string(),
+
+  /**
+   * The fully qualified base docker image with tag and digest.
+   */
+  image: z.string(),
+
+  /**
+   * The parameters to pass to the worker.
+   */
+  params: z.record(z.string(), z.unknown()),
+})
+
+export const unitWorkerSchema = z.object({
+  name: z.string(),
+  image: z.string(),
+  params: z.record(z.string(), z.unknown()),
+})
+
+export type Worker = z.infer<typeof workerSchema>
+export type WorkerUnitRegistration = z.infer<typeof workerUnitRegistrationSchema>
+export type UnitWorker = z.infer<typeof unitWorkerSchema>
+
+export function getWorkerIdentity(image: string): string {
+  const parts = image.split(":")
+  if (parts.length < 2) {
+    throw new Error(`Invalid image format: ${image}`)
+  }
+
+  return parts.slice(0, -1).join(":") // Remove the tag and digest
+}

package/src/shared/resolvers/graph-resolver.ts

@@ -2,6 +2,7 @@ import type { Logger } from "pino"
 import { unique } from "remeda"
 
 export type ResolverOutputHandler<TOutput> = (id: string, value: TOutput) => void
+export type DependentSetHandler = (id: string, dependentSet: Set<string> | undefined) => void
 
 export abstract class GraphResolver<TNode, TOutput> {
   private readonly workset: Set<string> = new Set()
@@ -13,6 +14,7 @@ export abstract class GraphResolver<TNode, TOutput> {
     private readonly nodes: ReadonlyMap<string, TNode>,
     protected readonly logger: Logger,
     private readonly outputHandler?: ResolverOutputHandler<TOutput>,
+    private readonly dependentSetHandler?: DependentSetHandler,
   ) {}
 
   addToWorkset(nodeId: string): void {
@@ -32,6 +34,15 @@ export abstract class GraphResolver<TNode, TOutput> {
     return this.outputMap
   }
 
+  /**
+   * The map of dependencies for each node.
+   *
+   * The key is the node identifier, and the value is an array of identifiers of the nodes which depend on it.
+   */
+  get dependents(): ReadonlyMap<string, Set<string>> {
+    return this.dependentMap
+  }
+
   requireOutput(nodeId: string): TOutput {
     const output = this.outputMap.get(nodeId)
     if (!output) {
@@ -54,27 +65,17 @@ export abstract class GraphResolver<TNode, TOutput> {
   protected abstract processNode(node: TNode, logger: Logger): TOutput | Promise<TOutput>
 
   /**
-   * Gets all identifiers of the nodes that depend on the given node directly or indirectly.
+   * Gets the identifiers of the nodes that depend on the given node directly.
+   *
+   * Returns an empty array if there are no dependents.
    */
-  getAllDependents(nodeId: string): string[] {
-    const result = new Set<string>()
-    const stack: string[] = [nodeId]
-
-    while (stack.length > 0) {
-      const dependents = this.dependentMap.get(stack.pop()!)
-      if (!dependents) {
-        continue
-      }
-
-      for (const dependentId of dependents) {
-        if (!result.has(dependentId)) {
-          result.add(dependentId)
-          stack.push(dependentId)
-        }
-      }
+  getDependents(nodeId: string): string[] {
+    const dependents = this.dependentMap.get(nodeId)
+    if (!dependents) {
+      return []
     }
 
-    return Array.from(result)
+    return Array.from(dependents)
   }
 
   /**
@@ -175,12 +176,15 @@ export abstract class GraphResolver<TNode, TOutput> {
           return
         }
 
+        const changedDependentMaps = new Set<string>()
+
         // remove all dependent nodes
         const oldDependencies = this.dependencyMap.get(nodeId) ?? []
         for (const depId of oldDependencies) {
           const dependantSet = this.dependentMap.get(depId)
           if (dependantSet) {
             dependantSet.delete(nodeId)
+            changedDependentMaps.add(depId)
             if (dependantSet.size === 0) {
               this.dependentMap.delete(depId)
             }
@@ -196,6 +200,15 @@ export abstract class GraphResolver<TNode, TOutput> {
           }
 
           dependantSet.add(nodeId)
+          changedDependentMaps.add(depId)
+        }
+
+        // if the dependent map has changed, notify the handler
+        if (this.dependentSetHandler) {
+          for (const depId of changedDependentMaps) {
+            const dependents = this.dependentMap.get(depId)
+            this.dependentSetHandler(depId, dependents)
+          }
         }
 
         this.outputMap.set(nodeId, output)
@@ -209,3 +222,33 @@ export abstract class GraphResolver<TNode, TOutput> {
     }
   }
 }
+
+/**
+ * Gets all identifiers of the nodes that depend on the given node directly or indirectly.
+ *
+ * @param dependentMap The map of dependents, received from the graph resolver.
+ * @param nodeId The identifier of the node to get dependents for.
+ */
+export function getAllDependents(
+  dependentMap: ReadonlyMap<string, Iterable<string>>,
+  nodeId: string,
+): string[] {
+  const result = new Set<string>()
+  const stack: string[] = [nodeId]
+
+  while (stack.length > 0) {
+    const dependents = dependentMap.get(stack.pop()!)
+    if (!dependents) {
+      continue
+    }
+
+    for (const dependentId of dependents) {
+      if (!result.has(dependentId)) {
+        result.add(dependentId)
+        stack.push(dependentId)
+      }
+    }
+  }
+
+  return Array.from(result)
+}

package/src/shared/resolvers/index.ts

@@ -0,0 +1,5 @@
+export * from "./graph-resolver"
+export * from "./registry"
+export * from "./input"
+export * from "./input-hash"
+export * from "./validation"

package/src/shared/resolvers/input-hash.ts

@@ -1,7 +1,11 @@
-import type { InstanceState } from "../state"
 import type { ResolvedInstanceInput } from "./input"
+import type { InstanceState } from "../models/project"
 import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
-import { sha256 } from "crypto-hash"
+import { crc32 } from "@aws-crypto/crc32"
+import { encode } from "@msgpack/msgpack"
+import { sha256 } from "@noble/hashes/sha2"
+import { bytesToHex } from "@noble/ciphers/utils"
+import { int32ToBytes } from "../utils"
 import { GraphResolver } from "./graph-resolver"
 
 export type InputHashNode = {
@@ -9,11 +13,12 @@ export type InputHashNode = {
   component: ComponentModel
   resolvedInputs: Record<string, ResolvedInstanceInput[]>
   state: InstanceState | undefined
-  sourceHash: string | undefined
+  sourceHash: number | undefined
 }
 
 export type InputHashOutput = {
-  inputHash: string
+  inputHash: number
+  dependencyOutputHash: string
   outputHash: string
 }
 
@@ -33,17 +38,31 @@ export class InputHashResolver extends GraphResolver<InputHashNode, InputHashOut
     return dependencies
   }
 
-  async processNode({
+  processNode({
     instance,
     component,
     resolvedInputs,
     sourceHash,
     state,
-  }: InputHashNode): Promise<InputHashOutput> {
-    let sink = component.definitionHash + JSON.stringify(instance.args ?? {})
+  }: InputHashNode): InputHashOutput {
+    const inputHashSink: Uint8Array[] = []
 
+    // 1. include the component definition hash
+    inputHashSink.push(int32ToBytes(component.definitionHash))
+
+    // 2. include the input hash nonce if available
+    if (state?.operationStatus?.inputHashNonce) {
+      inputHashSink.push(int32ToBytes(state.operationStatus.inputHashNonce))
+    }
+
+    // 3. include instance args encoded as msgpack
+    if (instance.args) {
+      inputHashSink.push(encode(instance.args))
+    }
+
+    // 4. include the source hash if available
     if (sourceHash) {
-      sink += sourceHash
+      inputHashSink.push(int32ToBytes(sourceHash))
     } else if (isUnitModel(component)) {
       this.logger.warn(
         { instanceId: instance.id },
@@ -55,28 +74,47 @@ export class InputHashResolver extends GraphResolver<InputHashNode, InputHashOut
       //
       .sort(([a], [b]) => a.localeCompare(b))
 
+    const dependencyInstanceIds = new Set<string>()
+
+    // 5. include the sorted resolved inputs
     for (const [inputKey, inputs] of sortedInputs) {
       if (Object.keys(inputs).length === 0) {
         continue
       }
 
-      sink += inputKey
+      // 5.1. include the input key to distinguish different inputs with possibly the same instanceId
+      inputHashSink.push(Buffer.from(inputKey))
 
-      const instanceIds = inputs.map(input => input.input.instanceId)
-      instanceIds.sort()
+      // the instances inside the input should also have stable order
+      const instanceIds = inputs.map(input => input.input.instanceId).sort()
 
       for (const instanceId of instanceIds) {
         const dependency = this.outputs.get(instanceId)
         if (!dependency) continue
 
-        sink += dependency.inputHash
-        sink += dependency.outputHash
+        // 5.2 include both input and output hashes of the dependency
+        inputHashSink.push(int32ToBytes(dependency.inputHash))
+        inputHashSink.push(Buffer.from(dependency.outputHash))
+
+        dependencyInstanceIds.add(instanceId)
       }
     }
 
+    // also calculate the dependency output hash as the combined output hashes of all unique dependencies
+    const dependencyOutputHashSink: Uint8Array[] = []
+    const sortedDependencyInstanceIds = Array.from(dependencyInstanceIds).sort()
+
+    for (const dependencyInstanceId of sortedDependencyInstanceIds) {
+      const dependency = this.outputs.get(dependencyInstanceId)
+      if (!dependency) continue
+
+      dependencyOutputHashSink.push(Buffer.from(dependencyInstanceId))
+    }
+
     return {
-      inputHash: await sha256(sink),
-      outputHash: state?.outputHash ?? "",
+      inputHash: crc32(Buffer.concat(inputHashSink)),
+      dependencyOutputHash: bytesToHex(sha256(Buffer.concat(dependencyOutputHashSink))),
+      outputHash: state?.operationStatus?.outputHash ?? "",
     }
   }
 }

package/src/shared/resolvers/input.ts

@@ -1,8 +1,8 @@
-import type { HubModel } from "../project"
 import {
   isUnitModel,
   type ComponentModel,
   type HubInput,
+  type HubModel,
   type InstanceInput,
   type InstanceModel,
 } from "@highstate/contract"
@@ -44,14 +44,6 @@ export type InputResolverOutput =
  * Resolves the all recursive instance and hub inputs based on its direct inputs and injected inputs.
  */
 export class InputResolver extends GraphResolver<InputResolverNode, InputResolverOutput> {
-  // constructor(
-  //   nodes: ReadonlyMap<string, InputResolverNode>,
-  //   logger: Logger,
-  //   outputHandler?: ResolverOutputHandler<InputResolverOutput>,
-  // ) {
-  //   super(nodes, logger, outputHandler)
-  // }
-
   getNodeDependencies(node: InputResolverNode): string[] {
     const dependencies: string[] = []
 
@@ -284,7 +276,15 @@ export class InputResolver extends GraphResolver<InputResolverNode, InputResolve
   }
 }
 
-export function getResolvedHubInputs(output: InputResolverOutput): ResolvedInstanceInput[] {
+export function getResolvedHubInputs(
+  outputMap: ReadonlyMap<string, InputResolverOutput>,
+  hubId: string,
+): ResolvedInstanceInput[] {
+  const output = outputMap.get(`hub:${hubId}`)
+  if (!output) {
+    return []
+  }
+
   if (output.kind !== "hub") {
     throw new Error("Expected hub node")
   }
@@ -293,8 +293,14 @@ export function getResolvedHubInputs(output: InputResolverOutput): ResolvedInsta
 }
 
 export function getResolvedInstanceInputs(
-  output: InputResolverOutput,
+  outputMap: ReadonlyMap<string, InputResolverOutput>,
+  instanceId: string,
 ): Record<string, ResolvedInstanceInput[]> {
+  const output = outputMap.get(`instance:${instanceId}`)
+  if (!output) {
+    return {}
+  }
+
   if (output.kind !== "instance") {
     throw new Error("Expected instance node")
   }
@@ -303,8 +309,14 @@ export function getResolvedInstanceInputs(
 }
 
 export function getResolvedInjectionInstanceInputs(
-  output: InputResolverOutput,
+  outputMap: ReadonlyMap<string, InputResolverOutput>,
+  instanceId: string,
 ): ResolvedInstanceInput[] {
+  const output = outputMap.get(`instance:${instanceId}`)
+  if (!output) {
+    return []
+  }
+
   if (output.kind !== "instance") {
     throw new Error("Expected instance node")
   }
@@ -313,11 +325,33 @@ export function getResolvedInjectionInstanceInputs(
 }
 
 export function getMatchedInjectionInstanceInputs(
-  output: InputResolverOutput,
+  outputMap: ReadonlyMap<string, InputResolverOutput>,
+  instanceId: string,
 ): ResolvedInstanceInput[] {
+  const output = outputMap.get(`instance:${instanceId}`)
+  if (!output) {
+    return []
+  }
+
   if (output.kind !== "instance") {
     throw new Error("Expected instance node")
   }
 
   return output.matchedInjectionInputs
 }
+
+export function getResolvedInstanceOutputs(
+  outputMap: ReadonlyMap<string, InputResolverOutput>,
+  instanceId: string,
+): Record<string, InstanceInput[]> | undefined {
+  const output = outputMap.get(`instance:${instanceId}`)
+  if (!output) {
+    return undefined
+  }
+
+  if (output.kind !== "instance") {
+    throw new Error("Expected instance node")
+  }
+
+  return output.resolvedOutputs
+}

package/src/shared/resolvers/registry.ts

@@ -1,6 +1,6 @@
 import type { Logger } from "pino"
-import type { GraphResolver, ResolverOutputHandler } from "./graph-resolver"
-import type { InstanceState } from "../state"
+import type { DependentSetHandler, GraphResolver, ResolverOutputHandler } from "./graph-resolver"
+import type { InstanceState } from "../models/project"
 import { StateResolver } from "./state"
 import { InputResolver, type InputResolverNode, type InputResolverOutput } from "./input"
 import { InputHashResolver, type InputHashNode, type InputHashOutput } from "./input-hash"
@@ -26,5 +26,6 @@ export const resolverFactories = {
     nodes: ReadonlyMap<string, unknown>,
     logger: Logger,
     outputHandler?: ResolverOutputHandler<unknown>,
+    dependentSetHandler?: DependentSetHandler,
   ) => GraphResolver<unknown, unknown>
 >

package/src/shared/resolvers/state.ts

@@ -1,9 +1,9 @@
-import type { InstanceState } from "../state"
+import type { InstanceState } from "../models/project"
 import { GraphResolver } from "./graph-resolver"
 
 export class StateResolver extends GraphResolver<InstanceState, void> {
   protected getNodeDependencies(node: InstanceState): string[] {
-    return node.dependencyIds
+    return node.operationStatus?.dependencyIds ?? []
   }
 
   protected processNode(): void {

package/src/shared/resolvers/validation.ts

@@ -1,11 +1,15 @@
-import type { ComponentModel, InstanceModel } from "@highstate/contract"
 import type { ResolvedInstanceInput } from "./input"
+import type { InstanceState } from "../models"
+import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
 import { Ajv } from "ajv"
+import styles from "ansi-styles"
+import { parseArgumentValue } from "../utils"
 import { GraphResolver } from "./graph-resolver"
 
 export type ValidationNode = {
   instance: InstanceModel
   component: ComponentModel
+  state: InstanceState | undefined
   resolvedInputs: Record<string, ResolvedInstanceInput[]>
 }
 
@@ -14,11 +18,7 @@ export type ValidationOutput =
       status: "ok"
     }
   | {
-      status: "invalid-args"
-      errorText: string
-    }
-  | {
-      status: "invalid-inputs" | "missing-inputs"
+      status: "error"
       errorText: string
     }
 
@@ -38,34 +38,65 @@ export class ValidationResolver extends GraphResolver<ValidationNode, Validation
     return dependencies
   }
 
-  processNode({ instance, component, resolvedInputs }: ValidationNode): ValidationOutput {
+  processNode({ instance, component, state, resolvedInputs }: ValidationNode): ValidationOutput {
     const ajv = new Ajv({ strict: false })
 
     this.logger.debug({ instanceId: instance.id }, "validating instance")
 
+    const validationErrors: string[] = []
+
     for (const [name, argument] of Object.entries(component.args)) {
-      if (!argument.required && !instance.args?.[name]) {
-        continue
-      }
+      try {
+        const value = parseArgumentValue(instance.args?.[name])
 
-      if (!ajv.validate(argument.schema, instance.args?.[name])) {
-        this.logger.debug({ instanceId: instance.id, argumentName: name }, "invalid argument")
+        if (!argument.required && value === undefined) {
+          continue
+        }
+
+        if (!ajv.validate(argument.schema, value)) {
+          this.logger.debug({ instanceId: instance.id, argumentName: name }, "invalid argument")
 
-        return {
-          status: "invalid-args",
-          errorText: `invalid argument "${name}": ${ajv.errorsText()}`,
+          validationErrors.push(
+            `Invalid argument ` +
+              `"${styles.blueBright.open}${name}${styles.reset.close}": ` +
+              ajv.errorsText(),
+          )
         }
+      } catch (error) {
+        this.logger.debug(
+          { instanceId: instance.id, argumentName: name, error },
+          "failed to validate argument",
+        )
+
+        validationErrors.push(
+          `Failed to validate argument ` +
+            `"${styles.blueBright.open}${name}${styles.reset.close}": ` +
+            (error instanceof Error ? error.message : String(error)),
+        )
       }
     }
 
-    for (const inputs of Object.values(resolvedInputs)) {
+    if (isUnitModel(component)) {
+      for (const [secret, secretSchema] of Object.entries(component.secrets)) {
+        if (secretSchema.required && !state?.secretNames?.includes(secret)) {
+          validationErrors.push(
+            `Missing required secret ` +
+              `"${styles.blueBright.open}${secret}${styles.reset.close}"`,
+          )
+        }
+      }
+    }
+
+    for (const [key, inputs] of Object.entries(resolvedInputs)) {
       for (const input of inputs) {
         const inputInstance = this.outputs.get(input.input.instanceId)
         if (inputInstance?.status !== "ok") {
-          return {
-            status: "invalid-inputs",
-            errorText: `instance "${input.input.instanceId}" has errors`,
-          }
+          validationErrors.push(
+            `Invalid input ` +
+              `"${styles.blueBright.open}${key}${styles.reset.close}":\n` +
+              `"${styles.blueBright.open}${input.input.instanceId}${styles.reset.close}" ` +
+              `has validation errors`,
+          )
         }
       }
     }
@@ -76,13 +107,39 @@ export class ValidationResolver extends GraphResolver<ValidationNode, Validation
       }
 
       if (!resolvedInputs[name] || !resolvedInputs[name].length) {
-        return {
-          status: "missing-inputs",
-          errorText: `input "${name}" is missing`,
-        }
+        validationErrors.push(
+          `Missing required input ` +
+            `"${styles.blueBright.open}${name}${styles.reset.close}" ` +
+            `of type ` +
+            `"${styles.greenBright.open}${input.type}${styles.reset.close}"`,
+        )
       }
     }
 
-    return { status: "ok" }
+    if (validationErrors.length === 0) {
+      return { status: "ok" }
+    }
+
+    const numberPrefixLength = (validationErrors.length + 1).toString().length + 2 // +2 for the ". " prefix
+
+    const formattedError = validationErrors
+      .map((error, index) => {
+        const lines = error.split("\n")
+        const prefix = `${index + 1}. `
+
+        return lines
+          .map((line, lineIndex) => {
+            const linePrefix = lineIndex === 0 ? prefix : " ".repeat(numberPrefixLength)
+
+            return `${linePrefix}${line}`
+          })
+          .join("\r\n")
+      })
+      .join("\r\n")
+
+    return {
+      status: "error",
+      errorText: formattedError,
+    }
   }
 }

package/src/shared/utils/args.ts

@@ -0,0 +1,25 @@
+import { yamlValueSchema } from "@highstate/contract"
+import { parse } from "yaml"
+
+const yamlResultCache = new WeakMap<object, unknown>()
+
+/**
+ * Parses an argument value which can be wrapped in a YAML structure.
+ *
+ * @param value The value to parse.
+ */
+export function parseArgumentValue(value: unknown): unknown {
+  const yamlResult = yamlValueSchema.safeParse(value)
+  if (!yamlResult.success) {
+    return value
+  }
+
+  const existingResult = yamlResultCache.get(value as object)
+  if (existingResult !== undefined) {
+    return existingResult
+  }
+
+  const result = parse(yamlResult.data.value) as unknown
+  yamlResultCache.set(value as object, result)
+  return result
+}