@highstate/backend 0.19.1 → 0.20.0

package/src/business/evaluation.test.ts

@@ -1,6 +1,7 @@
 import type { LibraryBackend, ProjectEvaluationResult } from "../library"
 import type { PubSubManager } from "../pubsub"
 import type { LibraryModel, ProjectModel } from "../shared"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import type { ProjectModelService } from "./project-model"
 import type { ProjectUnlockService } from "./project-unlock"
 import { defineComponent, defineUnit, getInstanceId, type InstanceModel } from "@highstate/contract"
@@ -141,6 +142,9 @@ describe("ProjectEvaluationSubsystem", () => {
         projectModelService,
         pubsubManager,
         projectUnlockService,
+        vi.mockObject({
+          track: vi.fn().mockResolvedValue(undefined),
+        } as unknown as ObjectRefIndexService),
         logger,
       )
 
@@ -317,4 +321,56 @@ describe("ProjectEvaluationSubsystem", () => {
       expect(payload?.deletedGhostInstanceIds).toEqual([virtualInstance.id])
     },
   )
+
+  evaluationTest(
+    "deduplicates evaluation states and prioritizes errors",
+    async ({
+      subsystem,
+      project,
+      projectDatabase,
+      projectModelService,
+      libraryBackend,
+      expect,
+    }) => {
+      // arrange
+      const virtualInstance = createVirtualInstance("overlap")
+
+      const evaluationResult: ProjectEvaluationResult = {
+        success: true,
+        virtualInstances: [virtualInstance],
+        topLevelErrors: {
+          [virtualInstance.id]: "failed to evaluate subtree",
+        },
+      }
+
+      projectModelService.resolveProject.mockResolvedValue({
+        project,
+        library: clone(libraryModel),
+        instances: [
+          {
+            ...virtualInstance,
+            kind: "composite",
+            type: "composite.v1",
+          },
+        ],
+        stateMap: new Map(),
+        resolvedInputs: {},
+      })
+
+      libraryBackend.evaluateCompositeInstances.mockResolvedValue(evaluationResult)
+
+      // act
+      await subsystem.evaluateProject(project.id)
+
+      // assert
+      const state = await projectDatabase.instanceState.findUniqueOrThrow({
+        where: { instanceId: virtualInstance.id },
+        include: { evaluationState: true },
+      })
+
+      expect(state.evaluationState).toBeDefined()
+      expect(state.evaluationState?.status).toBe("error")
+      expect(state.evaluationState?.message).toBe("failed to evaluate subtree")
+    },
+  )
 })

package/src/business/evaluation.ts

@@ -2,6 +2,7 @@ import type { InstanceId, InstanceModel } from "@highstate/contract"
 import type { Logger } from "pino"
 import type { LibraryBackend } from "../library"
 import type { PubSubManager } from "../pubsub"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import type { ProjectModelService } from "./project-model"
 import type { ProjectUnlockService } from "./project-unlock"
 import { isNonNullish } from "remeda"
@@ -13,6 +14,7 @@ import {
   type InstanceEvaluationStateUpdateInput,
   type InstanceStatus,
 } from "../database"
+import { stableJsonStringify } from "../shared"
 
 type EvaluatedInstance = {
   instanceId: InstanceId
@@ -23,6 +25,8 @@ type EvaluatedInstance = {
 
 export class ProjectEvaluationSubsystem {
   private readonly projectWatchers = new Map<string, AbortController>()
+  private readonly evaluationPromises = new Map<string, Promise<void>>()
+  private readonly evaluationRequested = new Set<string>()
 
   constructor(
     private readonly database: DatabaseManager,
@@ -30,6 +34,7 @@ export class ProjectEvaluationSubsystem {
     private readonly projectModelService: ProjectModelService,
     private readonly pubsubManager: PubSubManager,
     private readonly projectUnlockService: ProjectUnlockService,
+    private readonly objectRefIndexService: ObjectRefIndexService,
     private readonly logger: Logger,
   ) {
     this.projectUnlockService.registerUnlockTask(
@@ -46,6 +51,30 @@ export class ProjectEvaluationSubsystem {
   }
 
   async evaluateProject(projectId: string): Promise<void> {
+    const existingPromise = this.evaluationPromises.get(projectId)
+    if (existingPromise) {
+      this.evaluationRequested.add(projectId)
+      await existingPromise
+      return
+    }
+
+    const promise = this.evaluateProjectLoop(projectId).finally(() => {
+      this.evaluationPromises.delete(projectId)
+      this.evaluationRequested.delete(projectId)
+    })
+
+    this.evaluationPromises.set(projectId, promise)
+    await promise
+  }
+
+  private async evaluateProjectLoop(projectId: string): Promise<void> {
+    do {
+      this.evaluationRequested.delete(projectId)
+      await this.evaluateProjectNow(projectId)
+    } while (this.evaluationRequested.has(projectId))
+  }
+
+  private async evaluateProjectNow(projectId: string): Promise<void> {
     const { project, instances, resolvedInputs } =
       await this.projectModelService.resolveProject(projectId)
 
@@ -88,10 +117,13 @@ export class ProjectEvaluationSubsystem {
           model: null,
         }))
 
-        await this.setInstanceEvaluationStates(project.id, [
-          ...evaluatedInstances,
-          ...errorEvaluatedInstances,
-        ])
+        await this.setInstanceEvaluationStates(
+          project.id,
+          ProjectEvaluationSubsystem.mergeEvaluatedInstances(
+            evaluatedInstances,
+            errorEvaluatedInstances,
+          ),
+        )
       } else {
         // set all composite instances to error state in evaluation
         const errorEvaluationStates: EvaluatedInstance[] = compositeInstanceIds.map(instanceId => ({
@@ -135,7 +167,7 @@ export class ProjectEvaluationSubsystem {
   ): Promise<void> {
     const database = await this.database.forProject(projectId)
 
-    await database.$transaction(async tx => {
+    const { stateIds } = await database.$transaction(async tx => {
       const previousVirtualStates = await tx.instanceState.findMany({
         where: { source: "virtual" },
         select: {
@@ -186,6 +218,9 @@ export class ProjectEvaluationSubsystem {
       const existingStates = await tx.instanceEvaluationState.findMany({
         select: {
           stateId: true,
+          status: true,
+          message: true,
+          model: true,
           state: {
             select: {
               instanceId: true,
@@ -197,9 +232,21 @@ export class ProjectEvaluationSubsystem {
 
       const existingStateIds = new Set(existingStates.map(state => state.stateId))
       const actualStateIds = new Set(states.map(state => state.stateId))
+      const existingStateById = new Map(existingStates.map(state => [state.stateId, state]))
 
       const newStates = states.filter(state => !existingStateIds.has(state.stateId))
-      const statesToUpdate = states.filter(state => existingStateIds.has(state.stateId))
+      const statesToUpdate = states.filter(state => {
+        const existingState = existingStateById.get(state.stateId)
+        if (!existingState) {
+          return false
+        }
+
+        return (
+          existingState.status !== state.status ||
+          (existingState.message ?? null) !== (state.message ?? null) ||
+          stableJsonStringify(existingState.model) !== stableJsonStringify(state.model)
+        )
+      })
       const statesToDelete = existingStates.filter(state => !actualStateIds.has(state.stateId))
 
       // create new states
@@ -225,13 +272,15 @@ export class ProjectEvaluationSubsystem {
       })
 
       // 5. publish evaluation state updates
-      for (const state of updatedStates) {
+      if (updatedStates.length > 0) {
         void this.pubsubManager.publish(["instance-state", projectId], {
-          type: "patched",
-          stateId: state.stateId,
-          patch: {
-            evaluationState: state,
-          },
+          type: "patched-batch",
+          patches: updatedStates.map(state => ({
+            stateId: state.stateId,
+            patch: {
+              evaluationState: state,
+            },
+          })),
         })
       }
 
@@ -276,19 +325,33 @@ export class ProjectEvaluationSubsystem {
         }
       }
 
-      // 6. publish project model update
-      void this.pubsubManager.publish(["project-model", projectId], {
-        updatedVirtualInstances: [...newStates, ...updatedStates]
-          .map(state => state.model as InstanceModel)
-          .filter(isNonNullish),
-
-        deletedVirtualInstanceIds: statesToDelete.map(state => state.state.instanceId),
+      const updatedVirtualInstances = [...newStates, ...updatedStates]
+        .map(state => state.model as InstanceModel)
+        .filter(isNonNullish)
+      const deletedVirtualInstanceIds = statesToDelete.map(state => state.state.instanceId)
+      const updatedGhostInstances = newGhostInstances.filter(isNonNullish)
 
-        updatedGhostInstances: newGhostInstances.filter(isNonNullish),
+      // 6. publish project model update
+      if (
+        updatedVirtualInstances.length > 0 ||
+        deletedVirtualInstanceIds.length > 0 ||
+        updatedGhostInstances.length > 0 ||
+        resolvedGhostInstanceIds.length > 0
+      ) {
+        void this.pubsubManager.publish(["project-model", projectId], {
+          updatedVirtualInstances,
+          deletedVirtualInstanceIds,
+          updatedGhostInstances,
+          deletedGhostInstanceIds: resolvedGhostInstanceIds,
+        })
+      }
 
-        deletedGhostInstanceIds: resolvedGhostInstanceIds,
-      })
+      return { stateIds: Array.from(instanceIdToStateMap.values()).map(state => state.id) }
     })
+
+    if (stateIds.length > 0) {
+      await this.objectRefIndexService.track(projectId, stateIds)
+    }
   }
 
   private async trackUnlockedProject(projectId: string): Promise<void> {
@@ -384,4 +447,21 @@ export class ProjectEvaluationSubsystem {
 
     return `Composite instance evaluation completed successfully.\n\nInstance Tree:\n${tree}`
   }
+
+  private static mergeEvaluatedInstances(
+    evaluatedInstances: EvaluatedInstance[],
+    errorEvaluatedInstances: EvaluatedInstance[],
+  ): EvaluatedInstance[] {
+    const merged = new Map<InstanceId, EvaluatedInstance>()
+
+    for (const state of evaluatedInstances) {
+      merged.set(state.instanceId, state)
+    }
+
+    for (const state of errorEvaluatedInstances) {
+      merged.set(state.instanceId, state)
+    }
+
+    return Array.from(merged.values())
+  }
 }

package/src/business/global-search.test.ts

@@ -0,0 +1,344 @@
+import type { ProjectUnlockBackend } from "../unlock"
+import { createId } from "@paralleldrive/cuid2"
+import { describe, type MockedObject, vi } from "vitest"
+import { test } from "../test-utils"
+import { GlobalSearchService } from "./global-search"
+
+const globalSearchTest = test.extend<{
+  projectUnlockBackend: MockedObject<ProjectUnlockBackend>
+  globalSearchService: GlobalSearchService
+}>({
+  projectUnlockBackend: async ({}, use) => {
+    const projectUnlockBackend = vi.mockObject({
+      checkProjectUnlocked: vi.fn().mockResolvedValue(false),
+      getProjectMasterKey: vi.fn().mockResolvedValue(null),
+      unlockProject: vi.fn().mockResolvedValue(undefined),
+      lockProject: vi.fn().mockResolvedValue(undefined),
+    } as unknown as ProjectUnlockBackend)
+
+    await use(projectUnlockBackend)
+  },
+
+  globalSearchService: async ({ database, projectUnlockBackend, logger }, use) => {
+    const service = new GlobalSearchService(
+      database,
+      projectUnlockBackend,
+      logger.child({ service: "GlobalSearchService" }),
+    )
+
+    await use(service)
+  },
+})
+
+describe("searchByText", () => {
+  globalSearchTest(
+    "returns hits only from unlocked projects",
+    async ({
+      database,
+      createProject,
+      project,
+      projectUnlockBackend,
+      globalSearchService,
+      expect,
+    }) => {
+      // arrange
+      const lockedProject = await createProject("locked")
+
+      projectUnlockBackend.checkProjectUnlocked.mockImplementation(async projectId => {
+        return projectId === project.id
+      })
+
+      const unlockedDb = await database.forProject(project.id)
+      const lockedDb = await database.forProject(lockedProject.id)
+
+      const unlockedOperationId = createId()
+      await unlockedDb.operation.create({
+        data: {
+          id: unlockedOperationId,
+          meta: { title: "Deploy application" },
+          type: "update",
+          options: {},
+          requestedInstanceIds: [],
+        },
+      })
+
+      const lockedOperationId = createId()
+      await lockedDb.operation.create({
+        data: {
+          id: lockedOperationId,
+          meta: { title: "Deploy locked project" },
+          type: "update",
+          options: {},
+          requestedInstanceIds: [],
+        },
+      })
+
+      // act
+      const result = await globalSearchService.searchByText("deploy")
+
+      // assert
+      expect(result.projects).toHaveLength(1)
+      expect(result.projects[0].projectId).toBe(project.id)
+
+      const operationHits = result.projects[0].hits.filter(h => h.kind === "operation")
+      expect(operationHits.map(h => h.id)).toEqual([unlockedOperationId])
+    },
+  )
+
+  globalSearchTest(
+    "matches instance states by instanceId",
+    async ({ database, project, projectUnlockBackend, globalSearchService, expect }) => {
+      // arrange
+      projectUnlockBackend.checkProjectUnlocked.mockResolvedValue(true)
+
+      const db = await database.forProject(project.id)
+
+      const stateId = createId()
+      const instanceId = "component.v1:my-instance-123"
+
+      await db.instanceState.create({
+        data: {
+          id: stateId,
+          instanceId,
+          status: "undeployed",
+          source: "resident",
+          kind: "unit",
+        },
+      })
+
+      // act
+      const result = await globalSearchService.searchByText("my-instance")
+
+      // assert
+      expect(result.projects).toHaveLength(1)
+      expect(result.projects[0].projectId).toBe(project.id)
+
+      const hits = result.projects[0].hits.filter(h => h.kind === "instanceState")
+      expect(hits).toHaveLength(1)
+      expect(hits[0].id).toBe(stateId)
+      expect(hits[0].meta.title).toBe(instanceId)
+      expect(hits[0].meta.description).toBe("undeployed")
+    },
+  )
+
+  globalSearchTest(
+    "returns empty result for empty query",
+    async ({ projectUnlockBackend, globalSearchService, expect }) => {
+      // arrange
+      projectUnlockBackend.checkProjectUnlocked.mockResolvedValue(true)
+
+      // act
+      const result = await globalSearchService.searchByText("   ")
+
+      // assert
+      expect(result.projects).toEqual([])
+    },
+  )
+})
+
+describe("searchByIds", () => {
+  globalSearchTest(
+    "returns empty result for empty ids",
+    async ({ globalSearchService, expect }) => {
+      // act
+      const result = await globalSearchService.searchByIds([])
+
+      // assert
+      expect(result).toEqual([])
+    },
+  )
+
+  globalSearchTest(
+    "marks projects as locked when the project is locked",
+    async ({ database, project, projectUnlockBackend, globalSearchService, expect }) => {
+      // arrange
+      projectUnlockBackend.checkProjectUnlocked.mockResolvedValue(false)
+
+      const objectId = createId()
+      await database.backend.object.create({
+        data: {
+          id: objectId,
+          projectId: project.id,
+        },
+      })
+
+      // act
+      const result = await globalSearchService.searchByIds([objectId])
+
+      // assert
+      expect(result).toHaveLength(1)
+      expect(result[0].id).toBe(objectId)
+      expect(result[0].projects).toEqual([{ projectId: project.id, unlockState: "locked" }])
+    },
+  )
+
+  globalSearchTest(
+    "returns hits for unlocked projects and does not leak details for locked ones",
+    async ({
+      database,
+      createProject,
+      project,
+      projectUnlockBackend,
+      globalSearchService,
+      expect,
+    }) => {
+      // arrange
+      const lockedProject = await createProject(`locked-ids-${createId()}`)
+
+      projectUnlockBackend.checkProjectUnlocked.mockImplementation(async projectId => {
+        return projectId === project.id
+      })
+
+      const objectId = createId()
+      await database.backend.object.createMany({
+        data: [
+          { id: objectId, projectId: project.id },
+          { id: objectId, projectId: lockedProject.id },
+        ],
+      })
+
+      const unlockedDb = await database.forProject(project.id)
+      await unlockedDb.operation.create({
+        data: {
+          id: objectId,
+          meta: { title: "Deploy application" },
+          type: "update",
+          options: {},
+          requestedInstanceIds: [],
+        },
+      })
+
+      // act
+      const result = await globalSearchService.searchByIds([objectId])
+
+      // assert
+      expect(result).toHaveLength(1)
+
+      const projectResults = result[0].projects
+      expect(projectResults).toHaveLength(2)
+
+      const unlocked = projectResults.find(p => p.projectId === project.id)
+      expect(unlocked).toEqual({
+        projectId: project.id,
+        unlockState: "unlocked",
+        hits: [
+          {
+            kind: "operation",
+            id: objectId,
+            meta: { title: "Deploy application" },
+          },
+        ],
+      })
+
+      const locked = projectResults.find(p => p.projectId === lockedProject.id)
+      expect(locked).toEqual({ projectId: lockedProject.id, unlockState: "locked" })
+    },
+  )
+
+  globalSearchTest(
+    "returns unlocked projects with empty hits when the indexed object is not found in curated collections",
+    async ({ database, project, projectUnlockBackend, globalSearchService, expect }) => {
+      // arrange
+      projectUnlockBackend.checkProjectUnlocked.mockResolvedValue(true)
+
+      const objectId = createId()
+      await database.backend.object.create({
+        data: {
+          id: objectId,
+          projectId: project.id,
+        },
+      })
+
+      // act
+      const result = await globalSearchService.searchByIds([objectId])
+
+      // assert
+      expect(result).toHaveLength(1)
+      expect(result[0].projects).toEqual([
+        {
+          projectId: project.id,
+          unlockState: "unlocked",
+          hits: [],
+        },
+      ])
+    },
+  )
+
+  globalSearchTest(
+    "returns unlocked projects with empty hits when project database access fails",
+    async ({ database, createProject, projectUnlockBackend, logger, expect }) => {
+      // arrange
+      const projectA = await createProject(`aa-${createId()}`)
+      const projectB = await createProject(`bb-${createId()}`)
+
+      projectUnlockBackend.checkProjectUnlocked.mockResolvedValue(true)
+
+      const objectId = createId()
+      await database.backend.object.createMany({
+        data: [
+          { id: objectId, projectId: projectA.id },
+          { id: objectId, projectId: projectB.id },
+        ],
+      })
+
+      const unlockedDb = await database.forProject(projectA.id)
+      await unlockedDb.operation.create({
+        data: {
+          id: objectId,
+          meta: { title: "Deploy application" },
+          type: "update",
+          options: {},
+          requestedInstanceIds: [],
+        },
+      })
+
+      const failingDatabase = {
+        backend: database.backend,
+        isEncryptionEnabled: database.isEncryptionEnabled,
+        updateBackendUnlockRecipients: async () => {},
+        getProjectMasterKey: async () => undefined,
+        setupDatabase: async () => {
+          throw new Error("not implemented")
+        },
+        forProject: async (projectId: string) => {
+          if (projectId === projectB.id) {
+            throw new Error("boom")
+          }
+
+          return await database.forProject(projectId)
+        },
+      }
+
+      const service = new GlobalSearchService(
+        failingDatabase as never,
+        projectUnlockBackend,
+        logger.child({ service: "GlobalSearchService" }),
+      )
+
+      // act
+      const result = await service.searchByIds([objectId])
+
+      // assert
+      expect(result).toHaveLength(1)
+
+      const byProjectId = new Map(result[0].projects.map(p => [p.projectId, p]))
+      expect(byProjectId.get(projectA.id)).toEqual({
+        projectId: projectA.id,
+        unlockState: "unlocked",
+        hits: [
+          {
+            kind: "operation",
+            id: objectId,
+            meta: { title: "Deploy application" },
+          },
+        ],
+      })
+
+      expect(byProjectId.get(projectB.id)).toEqual({
+        projectId: projectB.id,
+        unlockState: "unlocked",
+        hits: [],
+      })
+    },
+  )
+})