@highstate/backend 0.19.1 → 0.21.1

package/src/business/object-ref-index.ts

@@ -0,0 +1,193 @@
+import type { Logger } from "pino"
+import type { DatabaseManager } from "../database"
+import type { ProjectDatabase } from "../database/prisma"
+
+const objectRefBatchSize = 500
+
+type IdRow = { id: string }
+
+type CuratedIdResolver = (database: ProjectDatabase) => Promise<IdRow[]>
+
+const curatedIdResolvers: CuratedIdResolver[] = [
+  async database => await database.operation.findMany({ select: { id: true } }),
+  async database => await database.instanceState.findMany({ select: { id: true } }),
+  async database => await database.artifact.findMany({ select: { id: true } }),
+  async database => await database.page.findMany({ select: { id: true } }),
+  async database => await database.terminal.findMany({ select: { id: true } }),
+  async database => await database.terminalSession.findMany({ select: { id: true } }),
+  async database => await database.secret.findMany({ select: { id: true } }),
+  async database => await database.serviceAccount.findMany({ select: { id: true } }),
+  async database => await database.apiKey.findMany({ select: { id: true } }),
+  async database => await database.trigger.findMany({ select: { id: true } }),
+  async database => await database.unlockMethod.findMany({ select: { id: true } }),
+  async database => await database.worker.findMany({ select: { id: true } }),
+  async database => await database.workerVersion.findMany({ select: { id: true } }),
+  async database => await database.entitySnapshot.findMany({ select: { id: true } }),
+  async database => await database.entity.findMany({ select: { id: true } }),
+]
+
+function chunk<T>(items: T[], size: number): T[][] {
+  if (size <= 0) {
+    throw new Error("chunk size must be positive")
+  }
+
+  const result: T[][] = []
+
+  for (let i = 0; i < items.length; i += size) {
+    result.push(items.slice(i, i + size))
+  }
+
+  return result
+}
+
+function normalizeIds(ids: string[]): string[] {
+  return Array.from(new Set(ids.map(id => id.trim()).filter(Boolean)))
+}
+
+/**
+ * Maintains the global backend object reference index.
+ *
+ * The index is stored in the backend database as `(id, projectId)` pairs.
+ * It is used by `GlobalSearchService.searchByIds()` to quickly find which projects
+ * may contain a given object ID.
+ *
+ * This service only indexes the curated list of project collections used by global object search.
+ */
+export class ObjectRefIndexService {
+  constructor(
+    private readonly database: DatabaseManager,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Tracks the provided object IDs for the given project.
+   *
+   * This is a best-effort helper for incremental updates.
+   * It never deletes references.
+   *
+   * @param projectId The ID of the project that knows the objects.
+   * @param ids The list of object IDs.
+   */
+  async track(projectId: string, ids: string[]): Promise<void> {
+    const uniqueIds = normalizeIds(ids)
+
+    if (uniqueIds.length === 0) {
+      return
+    }
+
+    try {
+      for (const batch of chunk(uniqueIds, objectRefBatchSize)) {
+        const existing = await this.database.backend.object.findMany({
+          where: {
+            projectId,
+            id: {
+              in: batch,
+            },
+          },
+          select: {
+            id: true,
+          },
+        })
+
+        const existingIds = new Set(existing.map(r => r.id))
+        const toCreate = batch.filter(id => !existingIds.has(id))
+        if (toCreate.length === 0) {
+          continue
+        }
+
+        await this.database.backend.object.createMany({
+          data: toCreate.map(id => ({ id, projectId })),
+        })
+      }
+    } catch (error) {
+      this.logger.warn(
+        { error, projectId },
+        'failed to track object refs for project "%s"',
+        projectId,
+      )
+    }
+  }
+
+  /**
+   * Reconciles backend object references for the given project.
+   *
+   * It queries the curated set of project collections and makes backend.object match.
+   * This method requires the project database to be accessible (project must be unlocked).
+   *
+   * @param projectId The ID of the project to sync.
+   */
+  async syncProject(projectId: string): Promise<void> {
+    let projectDatabase: ProjectDatabase
+
+    try {
+      projectDatabase = await this.database.forProject(projectId)
+    } catch (error) {
+      this.logger.debug({ error, projectId }, "failed to open project database for object ref sync")
+      return
+    }
+
+    const [expectedIds, existingRows] = await Promise.all([
+      this.collectCuratedProjectObjectIds(projectDatabase, projectId),
+      this.database.backend.object.findMany({
+        where: { projectId },
+        select: { id: true },
+      }),
+    ])
+
+    const existingIds = new Set(existingRows.map(r => r.id))
+
+    const toCreate: string[] = []
+    for (const id of expectedIds) {
+      if (!existingIds.has(id)) {
+        toCreate.push(id)
+      }
+    }
+
+    const expectedSet = new Set(expectedIds)
+    const toDelete: string[] = []
+    for (const id of existingIds) {
+      if (!expectedSet.has(id)) {
+        toDelete.push(id)
+      }
+    }
+
+    for (const batch of chunk(toCreate, objectRefBatchSize)) {
+      await this.database.backend.object.createMany({
+        data: batch.map(id => ({ id, projectId })),
+      })
+    }
+
+    for (const batch of chunk(toDelete, objectRefBatchSize)) {
+      await this.database.backend.object.deleteMany({
+        where: {
+          projectId,
+          id: {
+            in: batch,
+          },
+        },
+      })
+    }
+  }
+
+  private async collectCuratedProjectObjectIds(
+    database: ProjectDatabase,
+    projectId: string,
+  ): Promise<string[]> {
+    const settled = await Promise.allSettled(curatedIdResolvers.map(async r => await r(database)))
+
+    const ids = new Set<string>()
+
+    for (const result of settled) {
+      if (result.status === "rejected") {
+        this.logger.debug({ error: result.reason, projectId }, "curated object id resolver failed")
+        continue
+      }
+
+      for (const row of result.value) {
+        ids.add(row.id)
+      }
+    }
+
+    return Array.from(ids)
+  }
+}

package/src/business/operation.test.ts

@@ -1,6 +1,7 @@
 import type { InstanceId } from "@highstate/contract"
 import type { PubSubManager } from "../pubsub"
 import type { OperationOptions } from "../shared"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import { createId } from "@paralleldrive/cuid2"
 import { describe, type MockedObject, vi } from "vitest"
 import { test } from "../test-utils"
@@ -8,6 +9,7 @@ import { OperationService } from "./operation"
 
 const operationTest = test.extend<{
   pubsubManager: MockedObject<PubSubManager>
+  objectRefIndexService: MockedObject<ObjectRefIndexService>
   operationService: OperationService
 }>({
   pubsubManager: async ({}, use) => {
@@ -20,10 +22,19 @@ const operationTest = test.extend<{
     await use(pubsubManager)
   },
 
-  operationService: async ({ database, pubsubManager, logger }, use) => {
+  objectRefIndexService: async ({}, use) => {
+    const objectRefIndexService = vi.mockObject({
+      track: vi.fn().mockResolvedValue(undefined),
+    } as unknown as ObjectRefIndexService)
+
+    await use(objectRefIndexService)
+  },
+
+  operationService: async ({ database, pubsubManager, objectRefIndexService, logger }, use) => {
     const service = new OperationService(
       database,
       pubsubManager,
+      objectRefIndexService,
       logger.child({ service: "OperationService" }),
     )
 
@@ -39,6 +50,7 @@ describe("createOperation", () => {
       projectDatabase,
       project,
       pubsubManager,
+      objectRefIndexService,
       createInstanceState,
       expect,
     }) => {
@@ -79,6 +91,8 @@ describe("createOperation", () => {
         type: "updated",
         operation,
       })
+
+      expect(objectRefIndexService.track).toHaveBeenCalledWith(project.id, [operation.id])
     },
   )
 })

package/src/business/operation.ts

@@ -2,6 +2,7 @@ import type { InstanceId } from "@highstate/contract"
 import type { Logger } from "pino"
 import type { DatabaseManager, Operation, OperationStatus, OperationUpdateInput } from "../database"
 import type { PubSubManager } from "../pubsub"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import { ulid } from "ulid"
 import {
   type OperationMeta,
@@ -14,6 +15,7 @@ export class OperationService {
   constructor(
     private readonly database: DatabaseManager,
     private readonly pubsubManager: PubSubManager,
+    private readonly objectRefIndexService: ObjectRefIndexService,
     private readonly logger: Logger,
   ) {}
 
@@ -46,6 +48,8 @@ export class OperationService {
       },
     })
 
+    await this.objectRefIndexService.track(projectId, [operation.id])
+
     await this.pubsubManager.publish(["operation", projectId], {
       type: "updated",
       operation,

package/src/business/project-model.ts

@@ -1,4 +1,4 @@
-import type { InstanceModel } from "@highstate/contract"
+import type { InstanceInput, InstanceModel } from "@highstate/contract"
 import type { Logger } from "pino"
 import type { DatabaseManager } from "../database"
 import type { LibraryBackend } from "../library"
@@ -42,6 +42,12 @@ export class ProjectModelService {
     private readonly projectUnlockService: ProjectUnlockService,
     private readonly logger: Logger,
   ) {
+    this.projectUnlockService.registerUnlockTask(
+      //
+      "repair-instance-inputs",
+      projectId => this.repairInstanceInputs(projectId),
+    )
+
     this.projectUnlockService.registerUnlockTask(
       //
       "sync-instance-states",
@@ -61,18 +67,28 @@ export class ProjectModelService {
   ): Promise<[projectModel: FullProjectModel, project: ProjectOutput]> {
     const { project, backend, spec } = await this.getProjectWithBackend(projectId)
 
-    // get base model from storage backend
-    const { instances, hubs } = await backend.getProjectModel(project, spec)
+    const [{ instances, hubs }, library] = await Promise.all([
+      backend.getProjectModel(project, spec),
+      this.libraryBackend.loadLibrary(project.libraryId),
+    ])
+
+    const filteredInstances = this.filterInstancesWithKnownComponents(
+      project.id,
+      project.libraryId,
+      instances,
+      library.components,
+      "project-model",
+    )
 
     return [
       {
-        instances,
+        instances: filteredInstances,
         hubs,
         virtualInstances: includeVirtualInstances ? await this.getVirtualInstances(projectId) : [],
         ghostInstances: includeGhostInstances
           ? await this.getGhostInstances(
               projectId,
-              instances.map(instance => instance.id),
+              filteredInstances.map(instance => instance.id),
             )
           : [],
       },
@@ -94,21 +110,21 @@ export class ProjectModelService {
 
     const library = await this.libraryBackend.loadLibrary(project.libraryId)
 
-    const filteredInstances = instances.filter(instance => instance.type in library.components)
     const stateMap = new Map(states.map(state => [state.id, state]))
 
     const inputResolverNodes = new Map<string, InputResolverNode>()
 
-    for (const instance of filteredInstances) {
+    for (const instance of instances) {
       inputResolverNodes.set(`instance:${instance.id}`, {
         kind: "instance",
         instance,
-        component: library.components[instance.type],
+        component: library.components[instance.type]!,
+        entities: library.entities,
       })
     }
 
     for (const hub of hubs) {
-      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
+      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub, entities: library.entities })
     }
 
     const inputResolver = new InputResolver(inputResolverNodes, this.logger)
@@ -118,7 +134,7 @@ export class ProjectModelService {
 
     await inputResolver.process()
 
-    for (const instance of filteredInstances) {
+    for (const instance of instances) {
       const output = inputResolver.requireOutput(`instance:${instance.id}`)
       if (output.kind !== "instance") {
         throw new Error("Expected instance node")
@@ -130,12 +146,42 @@ export class ProjectModelService {
     return {
       project,
       library,
-      instances: filteredInstances,
+      instances,
       stateMap,
       resolvedInputs,
     }
   }
 
+  private filterInstancesWithKnownComponents(
+    projectId: string,
+    libraryId: string,
+    instances: InstanceModel[],
+    components: Record<string, unknown>,
+    source: string,
+  ): InstanceModel[] {
+    const filteredInstances: InstanceModel[] = []
+
+    for (const instance of instances) {
+      if (instance.type in components) {
+        filteredInstances.push(instance)
+        continue
+      }
+
+      this.logger.warn(
+        {
+          projectId,
+          libraryId,
+          instanceId: instance.id,
+          instanceType: instance.type,
+          source,
+        },
+        "ignoring instance because its component is not defined in the library",
+      )
+    }
+
+    return filteredInstances
+  }
+
   /**
   /**
    * Get the appropriate project model backend for the given project.
@@ -219,8 +265,12 @@ export class ProjectModelService {
           // the resident instance is ghost if it is not in the model
           { source: "resident", instanceId: { notIn: residentInstanceIds } },
 
-          // the virtual instance is ghost if it is has no evaluation state
-          { source: "virtual", evaluationState: null },
+          // the virtual instance is ghost if it has no evaluation state and is not represented in the model
+          {
+            source: "virtual",
+            evaluationState: null,
+            instanceId: { notIn: residentInstanceIds },
+          },
         ],
       },
       select: { model: true },
@@ -255,4 +305,95 @@ export class ProjectModelService {
       this.logger.info({ projectId }, "created missing %s instance states", missingInstances.length)
     })
   }
+
+  private async repairInstanceInputs(projectId: string): Promise<void> {
+    const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+
+    const [{ instances, hubs }, library] = await Promise.all([
+      backend.getProjectModel(project, spec),
+      this.libraryBackend.loadLibrary(project.libraryId),
+    ])
+
+    const instanceById = new Map(instances.map(instance => [instance.id, instance]))
+
+    const isValidInstanceInput = (input: InstanceInput): boolean => {
+      const target = instanceById.get(input.instanceId)
+      if (!target) {
+        return false
+      }
+
+      const targetComponent = library.components[target.type]
+      if (!targetComponent) {
+        return false
+      }
+
+      return Boolean(targetComponent.outputs?.[input.output])
+    }
+
+    let repairedInstances = 0
+    let removedInputs = 0
+    let repairedHubs = 0
+    let removedHubInputs = 0
+
+    for (const instance of instances) {
+      const component = library.components[instance.type]
+      if (!component || !instance.inputs) {
+        continue
+      }
+
+      const componentInputNames = new Set(Object.keys(component.inputs ?? {}))
+      const nextInputs: Record<string, InstanceInput[]> = Object.fromEntries(
+        Object.entries(instance.inputs)
+          .filter(([inputName]) => componentInputNames.has(inputName))
+          .map(([inputName, inputValues]) => [inputName, inputValues.filter(isValidInstanceInput)])
+          .filter(([, inputValues]) => inputValues.length > 0),
+      )
+
+      const removed =
+        Object.values(instance.inputs).reduce((sum, values) => sum + values.length, 0) -
+        Object.values(nextInputs).reduce((sum, values) => sum + values.length, 0)
+
+      if (removed === 0) {
+        continue
+      }
+
+      await backend.updateInstance(project, spec, instance.id, {
+        inputs: nextInputs,
+      })
+
+      repairedInstances += 1
+      removedInputs += removed
+    }
+
+    for (const hub of hubs) {
+      const inputs = hub.inputs ?? []
+      if (inputs.length === 0) {
+        continue
+      }
+
+      const nextInputs = inputs.filter(isValidInstanceInput)
+      const removed = inputs.length - nextInputs.length
+      if (removed === 0) {
+        continue
+      }
+
+      await backend.updateHub(project, spec, hub.id, {
+        inputs: nextInputs,
+      })
+
+      repairedHubs += 1
+      removedHubInputs += removed
+    }
+
+    if (repairedInstances > 0 || repairedHubs > 0) {
+      this.logger.warn(
+        { projectId },
+        "repaired %s instances and %s hubs, removed %s instance inputs and %s hub inputs during unlock",
+        repairedInstances,
+        repairedHubs,
+        removedInputs,
+        removedHubInputs,
+      )
+    }
+  }
 }

package/src/business/project-unlock.ts

@@ -2,6 +2,7 @@ import type { Logger } from "pino"
 import type { DatabaseManager } from "../database"
 import type { PubSubManager } from "../pubsub"
 import type { ProjectUnlockBackend } from "../unlock"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import { randomBytes } from "node:crypto"
 import { armor, Decrypter, Encrypter } from "age-encryption"
 import { z } from "zod"
@@ -32,6 +33,7 @@ export class ProjectUnlockService {
     private readonly database: DatabaseManager,
     private readonly pubsubManager: PubSubManager,
     private readonly projectUnlockBackend: ProjectUnlockBackend,
+    private readonly objectRefIndexService: ObjectRefIndexService,
     private readonly config: z.infer<typeof projectUnlockServiceConfig>,
     private readonly logger: Logger,
   ) {}
@@ -92,7 +94,12 @@ export class ProjectUnlockService {
     const database = await this.database.setupDatabase(projectId)
 
     // persist unlock method (now we can do it since the database is set up and unlocked)
-    await database.unlockMethod.create({ data: unlockMethodInput })
+    const unlockMethod = await database.unlockMethod.create({
+      data: unlockMethodInput,
+      select: { id: true },
+    })
+
+    await this.objectRefIndexService.track(projectId, [unlockMethod.id])
 
     const unlockSuite: ProjectUnlockSuite = {
       encryptedIdentities: [unlockMethodInput.encryptedIdentity],
@@ -175,6 +182,8 @@ export class ProjectUnlockService {
   ): Promise<void> {
     const database = await this.database.forProject(projectId)
 
+    let createdUnlockMethodId: string | null = null
+
     await database.$transaction(async tx => {
       // 1. fetch all unlock method recipients for the project
       const unlockMethods = await tx.unlockMethod.findMany({
@@ -187,7 +196,12 @@ export class ProjectUnlockService {
       const encryptedMasterKey = await this.encryptProjectMasterKey(projectId, allUnlockMethods)
 
       // 3. persist the new unlock method
-      await tx.unlockMethod.create({ data: inputUnlockMethod })
+      const created = await tx.unlockMethod.create({
+        data: inputUnlockMethod,
+        select: { id: true },
+      })
+
+      createdUnlockMethodId = created.id
 
       // 4. update the project with the new master key and unlock suite
       await this.database.backend.project.update({
@@ -198,6 +212,10 @@ export class ProjectUnlockService {
         },
       })
     })
+
+    if (createdUnlockMethodId) {
+      await this.objectRefIndexService.track(projectId, [createdUnlockMethodId])
+    }
   }
 
   /**
@@ -246,6 +264,11 @@ export class ProjectUnlockService {
    * @param handler The handler function for the unlock task. It receives the project ID as an argument.
    */
   registerUnlockTask(name: string, handler: (projectId: string) => Promise<void> | void): void {
+    const existingIndex = this.unlockTasks.findIndex(task => task.name === name)
+    if (existingIndex !== -1) {
+      this.unlockTasks.splice(existingIndex, 1)
+    }
+
     this.unlockTasks.push({ name, handler })
   }
 

package/src/business/project.ts

@@ -2,6 +2,7 @@ import type { InputJsonValue } from "@prisma/client/runtime/client"
 import type { Logger } from "pino"
 import type { LibraryBackend } from "../library"
 import type { PubSubManager } from "../pubsub"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import type { ProjectModelService } from "./project-model"
 import type { ProjectUnlockService } from "./project-unlock"
 import {
@@ -45,6 +46,7 @@ export class ProjectService {
     private readonly projectModelBackends: Record<string, ProjectModelBackend>,
     private readonly libraryBackend: LibraryBackend,
     private readonly pubsubManager: PubSubManager,
+    private readonly objectRefIndexService: ObjectRefIndexService,
     private readonly logger: Logger,
   ) {}
 
@@ -452,6 +454,13 @@ export class ProjectService {
         )
       })
 
+      if (states.length > 0) {
+        await this.objectRefIndexService.track(
+          projectId,
+          states.map(state => state.id),
+        )
+      }
+
       void this.pubsubManager.publish(["project-model", projectId], {
         updatedHubs: hubs,
         updatedInstances: instances,

package/src/business/secret.test.ts

@@ -1,8 +1,9 @@
 import type { LibraryBackend } from "../library"
 import type { PubSubManager } from "../pubsub"
+import type { ObjectRefIndexService } from "./object-ref-index"
 import { defineEntity, defineUnit, z } from "@highstate/contract"
 import { createId } from "@paralleldrive/cuid2"
-import { describe, vi } from "vitest"
+import { describe, type MockedObject, vi } from "vitest"
 import { InstanceStateNotFoundError, InvalidInstanceKindError, SystemSecretNames } from "../shared"
 import { test } from "../test-utils"
 import { SecretService } from "./secret"
@@ -10,6 +11,7 @@ import { SecretService } from "./secret"
 const secretTest = test.extend<{
   pubsubManager: PubSubManager
   libraryBackend: LibraryBackend
+  objectRefIndexService: MockedObject<ObjectRefIndexService>
   secretService: SecretService
 }>({
   pubsubManager: async ({}, use) => {
@@ -94,11 +96,23 @@ const secretTest = test.extend<{
     await use(libraryBackend)
   },
 
-  secretService: async ({ database, pubsubManager, libraryBackend, logger }, use) => {
+  objectRefIndexService: async ({}, use) => {
+    const objectRefIndexService = vi.mockObject({
+      track: vi.fn().mockResolvedValue(undefined),
+    } as unknown as ObjectRefIndexService)
+
+    await use(objectRefIndexService)
+  },
+
+  secretService: async (
+    { database, pubsubManager, libraryBackend, objectRefIndexService, logger },
+    use,
+  ) => {
     const service = new SecretService(
       database,
       pubsubManager,
       libraryBackend,
+      objectRefIndexService,
       logger.child({ service: "SecretService" }),
     )
 
@@ -452,6 +466,25 @@ describe("getInstanceSecretValues", () => {
 })
 
 describe("getPulumiPassword", () => {
+  secretTest(
+    "tracks created pulumi password secret",
+    async ({ secretService, database, createProject, objectRefIndexService, expect }) => {
+      const otherProject = await createProject(`aa-${createId()}`)
+
+      const password = await secretService.getPulumiPassword(otherProject.id)
+      expect(password).toBeTypeOf("string")
+
+      const otherProjectDatabase = await database.forProject(otherProject.id)
+      const secret = await otherProjectDatabase.secret.findUnique({
+        where: { systemName: SystemSecretNames.PulumiPassword },
+        select: { id: true },
+      })
+
+      expect(secret).toBeDefined()
+      expect(objectRefIndexService.track).toHaveBeenCalledWith(otherProject.id, [secret!.id])
+    },
+  )
+
   secretTest(
     "returns existing Pulumi password",
     async ({ secretService, projectDatabase, project, expect }) => {