@highstate/backend 0.9.16 → 0.9.18

package/src/library/worker/main.ts

@@ -1,18 +1,18 @@
-import type { WorkerData, WorkerResponse } from "./protocol"
+import type { WorkerData } from "./protocol"
 import { parentPort, workerData } from "node:worker_threads"
 import { pino } from "pino"
 import { errorToString } from "../../common"
-import { evaluateInstances } from "./evaluator"
+import { evaluateProject } from "./evaluator"
 import { loadComponents } from "./loader.lite"
 
 const data = workerData as WorkerData
 
-const logger = pino({ name: "library-worker", level: data.logLevel })
+const logger = pino({ name: "library-worker" })
 
 try {
   const library = await loadComponents(logger, data.libraryModulePaths)
 
-  const results = evaluateInstances(
+  const result = evaluateProject(
     logger,
     library,
     data.allInstances,
@@ -20,12 +20,12 @@ try {
     data.instanceIds,
   )
 
-  parentPort!.postMessage({ type: "results", results } satisfies WorkerResponse)
+  parentPort!.postMessage(result)
 } catch (error) {
-  logger.error({ error }, "failed to evaluate")
+  logger.error({ error }, "failed to evaluate project")
 
   parentPort!.postMessage({
-    type: "error",
+    success: false,
     error: errorToString(error),
-  } satisfies WorkerResponse)
+  })
 }

package/src/library/worker/protocol.ts

@@ -1,5 +1,4 @@
 import type { InstanceModel } from "@highstate/contract"
-import type { InstanceEvaluationResult } from "../abstractions"
 import type { ResolvedInstanceInput } from "../../shared"
 
 export type WorkerData = {
@@ -10,13 +9,3 @@ export type WorkerData = {
   resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
   instanceIds: string[]
 }
-
-export type WorkerResponse =
-  | {
-      type: "results"
-      results: InstanceEvaluationResult[]
-    }
-  | {
-      type: "error"
-      error: string
-    }

package/src/lock/index.ts

@@ -1,3 +1,4 @@
 export * from "./abstractions"
 export * from "./factory"
 export * from "./manager"
+export * from "./test"

package/src/lock/manager.ts

@@ -3,9 +3,24 @@ import { join } from "remeda"
 
 export type LockKeyMap = {
   /**
-   * Lock for a specific project.
+   * Lock for the project names to ensure uniqueness.
    */
-  project: [projectId: string]
+  "project-name": [name: string]
+
+  /**
+   * Lock for instances and hubs of the project.
+   */
+  "project-nodes": [projectId: string]
+
+  /**
+   * Locks for the evaluation of the project.
+   */
+  "project-evaluation": [projectId: string]
+
+  /**
+   * Lock for all instance states of the project.
+   */
+  "project-instance-states": [projectId: string]
 
   /**
    * Lock for a specific instance within a project.

package/src/lock/test.ts

@@ -0,0 +1,108 @@
+import type { Fixtures } from "@vitest/runner"
+import type { LockKey } from "./manager"
+import { linkTraceEntry, renderTraceEntry, type TestTracer, type TraceEntry } from "../common"
+import { LockManager } from "./manager"
+import { MemoryLockBackend } from "./memory"
+
+class LockAcquireEntry implements TraceEntry {
+  releaseEntry?: TraceEntry
+
+  constructor(
+    readonly id: number,
+    private readonly keys: string[],
+  ) {}
+
+  render(): string {
+    return renderTraceEntry({
+      icon: "🔒",
+      title: this.releaseEntry
+        ? `locked, released at ${linkTraceEntry(this.releaseEntry)}`
+        : "locked",
+      fields: {
+        keys: {
+          value: this.keys,
+        },
+      },
+    })
+  }
+}
+
+class LockReleaseEntry implements TraceEntry {
+  constructor(
+    readonly id: number,
+    private readonly keys: string[],
+    private readonly acquireEntry: TraceEntry,
+  ) {}
+
+  render(): string {
+    return renderTraceEntry({
+      icon: "🔓",
+      title: `released, locked at ${linkTraceEntry(this.acquireEntry)}`,
+      fields: {
+        keys: {
+          value: this.keys,
+        },
+      },
+    })
+  }
+}
+
+class TestLockManager {
+  constructor(
+    private readonly lockManager: LockManager,
+    private readonly tracer: TestTracer,
+  ) {}
+
+  async acquire<T>(keys: LockKey | LockKey[], fn: () => Promise<T> | T): Promise<T> {
+    const keyStrings = this.formatKeys(keys)
+
+    // add acquire trace entry
+    const acquireEntry = new LockAcquireEntry(this.tracer.nextEntryId(), keyStrings)
+    this.tracer.addEntry(acquireEntry)
+
+    try {
+      const result = await this.lockManager.acquire(keys, fn)
+
+      // add release trace entry
+      const releaseEntry = new LockReleaseEntry(this.tracer.nextEntryId(), keyStrings, acquireEntry)
+      this.tracer.addEntry(releaseEntry)
+
+      // link acquire entry to release entry
+      acquireEntry.releaseEntry = releaseEntry
+
+      return result
+    } catch (error) {
+      // add release trace entry even on error
+      const releaseEntry = new LockReleaseEntry(this.tracer.nextEntryId(), keyStrings, acquireEntry)
+      this.tracer.addEntry(releaseEntry)
+
+      // link acquire entry to release entry
+      acquireEntry.releaseEntry = releaseEntry
+
+      throw error
+    }
+  }
+
+  private formatKeys(keys: LockKey | LockKey[]): string[] {
+    if (typeof keys[0] === "string") {
+      return [keys.join(":")]
+    }
+
+    return keys.map(key => key.join(":"))
+  }
+}
+
+export const lockFixtures: Fixtures<
+  {
+    lockManager: LockManager
+  },
+  { tracer: TestTracer }
+> = {
+  lockManager: async ({ tracer }, use) => {
+    const backend = MemoryLockBackend.create()
+    const baseLockManager = new LockManager(backend)
+    const testLockManager = new TestLockManager(baseLockManager, tracer) as unknown as LockManager
+
+    await use(testLockManager)
+  },
+}

package/src/orchestrator/manager.ts

@@ -10,10 +10,16 @@ import type {
   SecretService,
   ProjectUnlockService,
   InstanceStateService,
+  WorkerService,
 } from "../business"
 import type { PubSubManager } from "../pubsub"
 import { v7 as uuidv7 } from "uuid"
-import { isFinalOperationStatus, type Operation, type OperationRequest } from "../shared"
+import {
+  isFinalOperationStatus,
+  type Operation,
+  type OperationRequest,
+  type Project,
+} from "../shared"
 import { RuntimeOperation } from "./operation"
 
 export class OperationManager {
@@ -29,6 +35,7 @@ export class OperationManager {
     private readonly secretService: SecretService,
     private readonly instanceStateService: InstanceStateService,
     private readonly pubsubManager: PubSubManager,
+    private readonly workerService: WorkerService,
     private readonly logger: Logger,
   ) {
     this.stateUnlockService.registerUnlockTask(
@@ -74,7 +81,12 @@ export class OperationManager {
 
     await this.operationService.updateOperation(request.projectId, operation)
 
-    this.startOperation(request.projectId, operation)
+    const project = await this.stateManager.getProjectRepository().get(request.projectId)
+    if (!project) {
+      throw new Error(`Project with ID "${request.projectId}" not found`)
+    }
+
+    this.startOperation(project, operation)
 
     return operation
   }
@@ -97,9 +109,9 @@ export class OperationManager {
     }
   }
 
-  private startOperation(projectId: string, operation: Operation): void {
+  private startOperation(project: Project, operation: Operation): void {
     const runtimeOperation = new RuntimeOperation(
-      projectId,
+      project,
       operation,
       this.runnerBackend,
       this.libraryBackend,
@@ -111,6 +123,7 @@ export class OperationManager {
       this.secretService,
       this.instanceStateService,
       this.pubsubManager,
+      this.workerService,
       this.logger.child({ operationId: operation.id }),
     )
 
@@ -182,36 +195,4 @@ export class OperationManager {
       projectId,
     )
   }
-
-  static create(
-    runnerBackend: RunnerBackend,
-    libraryBackend: LibraryBackend,
-    projectBackend: ProjectBackend,
-    artifactManager: ArtifactService,
-    stateManager: StateManager,
-    instanceLockService: InstanceLockService,
-    stateUnlockService: ProjectUnlockService,
-    operationService: OperationService,
-    secretService: SecretService,
-    instanceService: InstanceStateService,
-    pubsubManager: PubSubManager,
-    logger: Logger,
-  ): OperationManager {
-    const operator = new OperationManager(
-      runnerBackend,
-      libraryBackend,
-      projectBackend,
-      artifactManager,
-      stateManager,
-      instanceLockService,
-      stateUnlockService,
-      operationService,
-      secretService,
-      instanceService,
-      pubsubManager,
-      logger.child({ service: "OperationManager" }),
-    )
-
-    return operator
-  }
 }

package/src/orchestrator/operation-workset.ts

@@ -23,6 +23,7 @@ import {
   type InstanceStatePatch,
   type LibraryModel,
   type Operation,
+  type Project,
   type ResolvedInstanceInput,
 } from "../shared"
 
@@ -62,10 +63,9 @@ export class OperationWorkset {
   private readonly lockEventEmitter = new EventEmitter()
 
   private constructor(
-    public readonly projectId: string,
+    public readonly project: Project,
     public readonly operation: Operation,
     public readonly library: LibraryModel,
-    public readonly libraryId: string,
     private readonly instanceLockService: InstanceLockService,
     private readonly instanceStateService: InstanceStateService,
     private readonly logger: Logger,
@@ -116,7 +116,7 @@ export class OperationWorkset {
 
   public async patchState(patch: InstanceStatePatch): Promise<InstanceState> {
     const state = await this.instanceStateService.patchOperationInstanceState(
-      this.projectId,
+      this.project.id,
       this.operation,
       patch,
     )
@@ -168,7 +168,12 @@ export class OperationWorkset {
       instanceId,
     )
 
-    await this.instanceStateService.updateInstanceStates(this.projectId, [initialState], true)
+    await this.instanceStateService.updateInstanceStates(
+      this.project.id,
+      [initialState],
+      true,
+      false,
+    )
 
     if (initialState.parentId) {
       await this.recalculateCompositeInstanceState(initialState.parentId)
@@ -185,7 +190,12 @@ export class OperationWorkset {
       }
     }
 
-    await this.instanceStateService.updateInstanceStates(this.projectId, instanceStates)
+    await this.instanceStateService.updateInstanceStates(
+      this.project.id,
+      instanceStates,
+      false,
+      false,
+    )
   }
 
   public getAffectedCompositeChildren(instanceId: string): InstanceModel[] {
@@ -532,7 +542,7 @@ export class OperationWorkset {
    */
   public async tryLock(instancesToLock?: string[]): Promise<void> {
     const [, lockedInstanceIds] = await this.instanceLockService.tryLockInstances(
-      this.projectId,
+      this.project.id,
       instancesToLock ?? Array.from(this.instanceIdsToLockIds),
       {
         title: "Operation Lock",
@@ -552,7 +562,7 @@ export class OperationWorkset {
   }
 
   public static async load(
-    projectId: string,
+    project: Project,
     operation: Operation,
     projectBackend: ProjectBackend,
     libraryBackend: LibraryBackend,
@@ -562,59 +572,46 @@ export class OperationWorkset {
     logger: Logger,
     signal: AbortSignal,
   ): Promise<OperationWorkset> {
-    // First get project info to obtain libraryId
-    const projectInfo = await projectBackend.getProjectInfo(projectId, signal)
-
-    // Use StateManager for instance states, StateBackend directly for composite instances
-    // TODO: use hotstate for composite instances as well
-    const compositeInstanceRepo = stateManager.getCompositeInstanceRepository(projectId)
-
-    const [library, project, compositeInstances, states] = await Promise.all([
-      libraryBackend.loadLibrary(projectInfo.libraryId, signal),
-      projectBackend.getProject(projectId, signal),
-      compositeInstanceRepo.getAllItems(),
-      instanceStateService.getInstanceStates(projectId),
+    // TODO: use hotstate for virtual instances
+    const [library, { instances, hubs }, virtualInstances, states] = await Promise.all([
+      libraryBackend.loadLibrary(project.libraryId, signal),
+      projectBackend.getProjectModel(project, signal),
+      stateManager.getVirtualInstanceRepository(project.id).getAllItems(),
+      instanceStateService.getInstanceStates(project.id),
     ])
 
     const workset = new OperationWorkset(
-      projectId,
+      project,
       operation,
       library,
-      projectInfo.libraryId,
       instanceLockService,
       instanceStateService,
       logger.child({ operationId: operation.id, service: "OperationWorkset" }),
     )
 
     // prepare instances
-    for (const instance of project.instances) {
+    for (const instance of instances) {
       workset.addInstance(instance)
     }
 
-    for (const instance of compositeInstances) {
-      const worksetInstance = workset.instanceMap.get(instance.instance.id)
+    for (const instance of virtualInstances) {
+      const worksetInstance = workset.instanceMap.get(instance.id)
 
       if (worksetInstance) {
-        worksetInstance.outputs = instance.instance.outputs
-        worksetInstance.resolvedOutputs = instance.instance.resolvedOutputs
-      } else if (instance.instance.parentId) {
-        workset.addInstance(instance.instance)
+        worksetInstance.outputs = instance.outputs
+        worksetInstance.resolvedOutputs = instance.resolvedOutputs
+      } else if (instance.parentId) {
+        workset.addInstance(instance)
       } else {
         workset.logger.warn(
-          `ignoring instance "${instance.instance.id}" from composite instance state because it is not in the project or is not a part of known composite instance`,
+          `ignoring virtual instance "${instance.id}" because it is not in the project or is not a part of known composite instance`,
         )
         continue
       }
-
-      for (const child of instance.children) {
-        if (!workset.instanceMap.has(child.id)) {
-          workset.addInstance(child)
-        }
-      }
     }
 
     const unitSources = await libraryBackend.getResolvedUnitSources(
-      projectInfo.libraryId,
+      project.libraryId,
       unique(Array.from(workset.instanceMap.values()).map(i => i.type)),
     )
 
@@ -642,7 +639,7 @@ export class OperationWorkset {
       })
     }
 
-    for (const hub of project.hubs) {
+    for (const hub of hubs) {
       workset.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
     }