@highstate/backend 0.9.33 → 0.9.34

package/src/orchestrator/operation.ts

@@ -566,9 +566,6 @@ export class RuntimeOperation {
 
       await this.watchStateStream(state, type, name, logger)
 
-      // clean up artifacts after successful destruction
-      await this.artifactService.clearInstanceArtifactReferences(this.project.id, instance.id)
-
       logger.info("unit destroyed")
     })
   }
@@ -736,6 +733,10 @@ export class RuntimeOperation {
       statusFields: update.statusFields ?? null,
     }
 
+    const artifactIds = update.exportedArtifactIds
+      ? Object.values(update.exportedArtifactIds).flat()
+      : []
+
     if (update.operationType !== "destroy") {
       // давайте еще больше усложним и без того сложную штуку
       // set output hash before calculating input hash to capture up-to-date output hash for dependencies
@@ -749,6 +750,8 @@ export class RuntimeOperation {
       data.dependencyOutputHash = dependencyOutputHash
       data.outputHash = update.outputHash
 
+      data.exportedArtifactIds = update.exportedArtifactIds
+
       // also update the parent ID
       if (instance.parentId) {
         const parentState = this.context.getState(instance.parentId)
@@ -763,6 +766,7 @@ export class RuntimeOperation {
       data.parentId = null
       data.model = null
       data.resolvedInputs = null
+      data.exportedArtifactIds = null
     }
 
     // update the operation state
@@ -786,6 +790,7 @@ export class RuntimeOperation {
             triggers: update.triggers ?? [],
             workers: update.workers ?? [],
             secrets: update.secrets ?? {},
+            artifactIds,
           }
         : undefined,
     })

package/src/runner/abstractions.ts

@@ -90,6 +90,13 @@ export type UnitStateUpdate = {
        * The workers produced by the unit.
        */
       workers?: UnitWorker[] | null
+
+      /**
+       * The IDs of the artifacts exported by the unit.
+       *
+       * The keys are the output names, and the values are the IDs of the artifacts exported for that output.
+       */
+      exportedArtifactIds?: Record<string, string[]> | null
     }
 )
 

package/src/runner/artifact-env.ts

@@ -1,4 +1,3 @@
-import type { UnitArtifact } from "@highstate/contract"
 import type { Logger } from "pino"
 import type { ArtifactBackend, ArtifactService } from "../artifact"
 import type { Artifact } from "../database"
@@ -7,6 +6,7 @@ import { createReadStream, createWriteStream } from "node:fs"
 import { mkdtemp, readdir } from "node:fs/promises"
 import { join } from "node:path"
 import { pipeline } from "node:stream/promises"
+import { type UnitArtifact, unitArtifactId } from "@highstate/contract"
 
 export interface ArtifactEnvironment {
   readPath: string | null
@@ -45,7 +45,7 @@ export async function setupArtifactEnvironment(
     await Promise.all(
       artifacts.map(async artifact => {
         const artifactPath = join(readDir!, `${artifact.hash}.tgz`)
-        const stream = await artifactBackend.retrieve(projectId, artifact.hash, artifact.chunkSize)
+        const stream = await artifactBackend.retrieve(projectId, artifact.id, artifact.chunkSize)
 
         if (!stream) {
           throw new Error(`artifact not found: ${artifact.hash}`)
@@ -54,7 +54,7 @@ export async function setupArtifactEnvironment(
         const writeStream = createWriteStream(artifactPath)
         await pipeline(stream, writeStream)
 
-        logger.debug({ msg: "downloaded artifact", hash: artifact.hash, path: artifactPath })
+        logger.debug({ msg: "downloaded artifact", id: artifact.id, path: artifactPath })
       }),
     )
   } else {
@@ -123,7 +123,7 @@ export async function collectAndStoreArtifacts(
 
           await using readStream = createReadStream(filePath)
 
-          await artifactManager.store(
+          const storedArtifact = await artifactManager.store(
             projectId,
             expectedHash,
             fileSize,
@@ -141,6 +141,8 @@ export async function collectAndStoreArtifacts(
             },
           )
 
+          artifact[unitArtifactId] = storedArtifact.id
+
           logger.debug({ msg: "stored artifact", hash: expectedHash, file: filePath })
         } catch (error) {
           logger.error({ msg: "failed to store artifact", file: filePath, error })

package/src/runner/local.ts

@@ -24,6 +24,7 @@ import {
   HighstateConfigKey,
   type InstanceId,
   instanceStatusFieldSchema,
+  unitArtifactId,
   unitArtifactSchema,
   unitPageSchema,
   unitTerminalSchema,
@@ -33,7 +34,7 @@ import {
 import { encode } from "@msgpack/msgpack"
 import { sha256 } from "@noble/hashes/sha2"
 import { ensureDependencyInstalled } from "nypm"
-import { omitBy } from "remeda"
+import { mapValues, omitBy } from "remeda"
 import { z } from "zod"
 import { runWithRetryOnError } from "../common"
 import {
@@ -109,7 +110,7 @@ export class LocalRunnerBackend implements RunnerBackend {
 
     try {
       // create unit-specific temp directory for better cleanup reliability
-      unitTempPath = await mkdtemp(join(tmpdir(), `highstate-unit-${unitId}-`))
+      unitTempPath = await mkdtemp(join(tmpdir(), `highstate-unit-${options.stateId}-`))
       childLogger.debug({ msg: "created unit temp directory", unitTempPath })
       options.signal?.throwIfAborted()
 
@@ -221,6 +222,18 @@ export class LocalRunnerBackend implements RunnerBackend {
                   Object.values(artifacts).flat(),
                   childLogger,
                 )
+
+                completionUpdate.exportedArtifactIds = mapValues(artifacts, artifacts => {
+                  return artifacts.map(artifact => {
+                    if (artifact[unitArtifactId]) {
+                      return artifact[unitArtifactId]
+                    }
+
+                    throw new Error(
+                      `Failed to determine artifact ID for artifact with hash ${artifact.hash}`,
+                    )
+                  })
+                })
               } else if (preview && outputs["$artifacts"]) {
                 childLogger.debug({ msg: "skipping artifact persistence for preview" })
               }