@highstate/pulumi 0.9.15 → 0.9.18

package/src/unit.ts

@@ -5,20 +5,24 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import type { DeepInput, InputArray, InputMap } from "./utils"
+import type { ComponentSecret } from "../../contract/src/unit"
 import {
-  type ArgumentValue,
   type ComponentInputSpec,
-  type Entity,
+  type EntityModel,
   type Unit,
   type ComponentInput,
   type InstanceInput,
   parseInstanceId,
-  type ArgumentValueSchema,
   getInstanceId,
   type ComponentArgumentSpec,
   type ComponentArgumentSpecToStatic,
+  HighstateSignature,
+  camelCaseToHumanReadable,
+  z,
+  type UnitSecretModel,
+  unitSecretSchema,
+  unitArtifactSchema,
 } from "@highstate/contract"
-import { Type, type Static } from "@sinclair/typebox"
 import { mapValues, pickBy, pipe } from "remeda"
 import {
   Config,
@@ -31,59 +35,71 @@ import {
   type Unwrap,
 } from "@pulumi/pulumi"
 import { Ajv } from "ajv"
-import { createdSecrets } from "./secret"
+import { updatedSecretValues, type UnitSecret } from "./secret"
 
 const ajv = new Ajv({ strict: false })
 
-export type InstanceTerminalFile = {
-  content: Input<string | undefined>
-  mode?: Input<number | undefined>
-  isBinary?: Input<boolean>
+export type ObjectMeta = {
+  title?: Input<string | undefined>
+  description?: Input<string | undefined>
+  icon?: Input<string | undefined>
+  iconColor?: Input<string | undefined>
 }
 
-export type InstanceTerminal = {
+export type InstanceFileMeta = {
   name: Input<string>
-  title: Input<string>
-  description?: Input<string>
-  icon?: Input<string>
+  contentType?: Input<string>
+  size?: Input<number>
+  isBinary?: Input<boolean>
+  mode?: Input<number>
+}
+
+export type UnitArtifact = {
+  hash: Input<string>
+  meta?: Input<ObjectMeta>
+}
+
+export type InstanceFile = {
+  meta: Input<InstanceFileMeta>
+  content:
+    | { type: "embedded"; value: Input<string> }
+    | {
+        type: "artifact"
+        [HighstateSignature.Artifact]: UnitArtifact
+      }
+}
+
+export type InstanceTerminalSpec = {
   image: Input<string>
   command: InputArray<string>
   cwd?: Input<string | undefined>
   env?: InputMap<string | undefined>
-  files?: InputMap<InstanceTerminalFile | string | undefined>
+  files?: InputMap<InstanceFile | string | undefined>
+}
+
+export type InstanceTerminal = {
+  name: Input<string>
+  meta: Input<ObjectMeta>
+  spec: Input<InstanceTerminalSpec>
 }
 
 export type StatusFieldValue = string | number | boolean | string[]
 
 export type StatusField<TArgName extends string = string> = {
   name: Input<string>
-  value?: Input<StatusFieldValue | undefined>
-  displayName?: Input<string | undefined>
-  sensitive?: Input<boolean | undefined>
-  url?: Input<string | undefined>
+  meta?: Input<ObjectMeta>
   complementaryTo?: Input<TArgName | undefined>
-}
-
-export type InstanceFileMeta = {
-  name: Input<string>
-  contentType: Input<string>
-  isBinary?: Input<boolean>
-  size: Input<number>
-}
-
-export type InstanceFile = {
-  meta: Input<InstanceFileMeta>
-  content: Input<string>
+  value?: Input<StatusFieldValue | undefined>
 }
 
 export type InstancePageBlock =
   | { type: "markdown"; content: Input<string> }
   | { type: "qr"; content: Input<string>; showContent?: boolean; language?: string }
-  | { type: "file"; fileMeta: Input<InstanceFileMeta> }
+  | ({ type: "file" } & InstanceFile)
 
 export type InstancePage = {
   name: Input<string>
-  title: Input<string>
+  meta: Input<ObjectMeta>
   content: InputArray<InstancePageBlock>
 }
 
@@ -103,8 +119,14 @@ export type InstanceTrigger = {
   spec: Input<InstanceTriggerSpec>
 }
 
+export type InstanceWorker = {
+  name: Input<string>
+  image: Input<string>
+  params?: InputMap<unknown>
+}
+
 export type ExtraOutputs<TArgName extends string = string> = {
-  $status?:
+  $statusFields?:
     | InputMap<Omit<StatusField<TArgName>, "name"> | StatusFieldValue | undefined>
     | InputArray<StatusField<TArgName> | undefined>
 
@@ -113,11 +135,14 @@ export type ExtraOutputs<TArgName extends string = string> = {
     | InputArray<InstanceTerminal | undefined>
 
   $pages?: InputMap<Omit<InstancePage, "name"> | undefined> | InputArray<InstancePage | undefined>
-  $files?: InputArray<InstanceFile | undefined>
 
   $triggers?:
     | InputMap<Omit<InstanceTrigger, "name"> | undefined>
     | InputArray<InstanceTrigger | undefined>
+
+  $workers?:
+    | InputMap<Omit<InstanceWorker, "name"> | undefined>
+    | InputArray<InstanceWorker | undefined>
 }
 
 export type InstanceTriggerInvocation = {
@@ -129,17 +154,21 @@ type OutputMapToDeepInputMap<T extends Record<string, unknown>, TArgName extends
     ? ExtraOutputs
     : { [K in keyof T]: DeepInput<T[K]> } & ExtraOutputs<TArgName>
 
+export type SecretValueMapToSecretMap<T extends Record<string, unknown>> = {
+  [K in keyof T]-?: UnitSecret<T[K]>
+}
+
 export interface UnitContext<
-  TArgs extends Record<string, ArgumentValue>,
+  TArgs extends Record<string, unknown>,
   TInputs extends Record<string, unknown>,
   TOutputs extends Record<string, unknown>,
-  TSecrets extends Record<string, ArgumentValue>,
+  TSecrets extends Record<string, unknown>,
 > {
   args: TArgs
   instanceId: string
   type: string
   name: string
-  secrets: Output<TSecrets>
+  secrets: SecretValueMapToSecretMap<TSecrets>
 
   inputs: TInputs extends Record<string, never>
     ? never
@@ -158,10 +187,10 @@ export interface UnitContext<
 }
 
 type InputSpecToValue<T extends ComponentInputSpec> = T[2] extends true
-  ? Static<T[0]["schema"]>[]
+  ? z.infer<T[0]["schema"]>[]
   : T[1] extends true
-    ? Static<T[0]["schema"]>
-    : Static<T[0]["schema"]> | undefined
+    ? z.infer<T[0]["schema"]>
+    : z.infer<T[0]["schema"]> | undefined
 
 type InputSpecMapToValueMap<T extends Record<string, ComponentInputSpec>> =
   T extends Record<string, never>
@@ -210,18 +239,20 @@ function getOutput(unit: Unit, input: ComponentInput, refs: InstanceInput[]) {
     const value = getStackRef(ref).requireOutput(ref.output)
 
     return value.apply(value => {
-      let schema = entity.schema
-
-      if (input.multiple) {
-        schema = Type.Union([schema, Type.Array(schema)])
-      }
-
-      if (!ajv.validate(schema, value)) {
-        throw new Error(`Invalid output for '${input.type}': ${ajv.errorsText()}`)
+      if (Array.isArray(value)) {
+        for (const [index, item] of value.entries()) {
+          if (!ajv.validate(entity.schema, item)) {
+            throw new Error(`Invalid output for '${input.type}[${index}]': ${ajv.errorsText()}`)
+          }
+        }
+      } else {
+        if (!ajv.validate(entity.schema, value)) {
+          throw new Error(`Invalid output for '${input.type}': ${ajv.errorsText()}`)
+        }
       }
 
       if (Array.isArray(value)) {
-        return value as unknown
+        return value
       }
 
       return input.multiple ? [value] : value
@@ -237,17 +268,15 @@ function getOutput(unit: Unit, input: ComponentInput, refs: InstanceInput[]) {
   return values
 }
 
-function isAnyOfSchema(schema: ArgumentValueSchema, itemType: string): boolean {
+function isAnyOfSchema(schema: z.core.JSONSchema.BaseSchema, itemType: string): boolean {
   if (schema.anyOf) {
-    return Object.values(schema.anyOf).every(schema =>
-      isAnyOfSchema(schema as ArgumentValueSchema, itemType),
-    )
+    return Object.values(schema.anyOf).every(schema => isAnyOfSchema(schema, itemType))
   }
 
   return schema.type === itemType
 }
 
-function isStringSchema(schema: ArgumentValueSchema): boolean {
+function isStringSchema(schema: z.core.JSONSchema.BaseSchema): boolean {
   if (schema.type === "string") {
     return true
   }
@@ -259,7 +288,7 @@ function isStringSchema(schema: ArgumentValueSchema): boolean {
   return false
 }
 
-function isNumberSchema(schema: ArgumentValueSchema): boolean {
+function isNumberSchema(schema: z.core.JSONSchema.BaseSchema): boolean {
   if (schema.type === "number") {
     return true
   }
@@ -271,7 +300,7 @@ function isNumberSchema(schema: ArgumentValueSchema): boolean {
   return false
 }
 
-function isBooleanSchema(schema: ArgumentValueSchema): boolean {
+function isBooleanSchema(schema: z.core.JSONSchema.BaseSchema): boolean {
   if (schema.type === "boolean") {
     return true
   }
@@ -302,19 +331,48 @@ export function forUnit<
   const args = mapValues(unit.model.args, (arg, argName) => {
     switch (true) {
       case isStringSchema(arg.schema): {
-        return arg.required ? config.require(argName) : (config.get(argName) ?? arg.schema.default)
+        if (arg.required) {
+          return config.require(argName)
+        }
+
+        // handle empty strings as undefined
+        return config.get(argName) || arg.schema.default
       }
       case isNumberSchema(arg.schema): {
-        return arg.required
-          ? config.requireNumber(argName)
-          : (config.getNumber(argName) ?? arg.schema.default)
+        if (arg.required) {
+          return config.requireNumber(argName)
+        }
+
+        // handle empty strings as undefined
+        const value = config.get(argName)
+        if (!value) {
+          return arg.schema.default
+        }
+
+        return config.getNumber(argName) ?? arg.schema.default
       }
       case isBooleanSchema(arg.schema): {
-        return arg.required
-          ? config.requireBoolean(argName)
-          : (config.getBoolean(argName) ?? arg.schema.default)
+        if (arg.required) {
+          return config.requireBoolean(argName)
+        }
+
+        // handle empty strings as undefined
+        const value = config.get(argName)
+        if (!value) {
+          return arg.schema.default
+        }
+
+        return config.getBoolean(argName) ?? arg.schema.default
       }
       default: {
+        if (!arg.required) {
+          const value = config.get(argName)
+          // handle empty strings as undefined
+          if (!value) {
+            return arg.schema.default
+          }
+        }
+
         const value = arg.required ? config.requireObject(argName) : config.getObject(argName)
         if (value === undefined) return arg.schema.default
 
@@ -327,36 +385,54 @@ export function forUnit<
     }
   }) as ComponentArgumentSpecToStatic<TArgs>
 
-  const secrets = output(
-    mapValues(unit.model.secrets, (secret, secretName) => {
-      switch (true) {
-        case isStringSchema(secret.schema): {
-          return secret.required ? config.requireSecret(secretName) : config.getSecret(secretName)
-        }
-        case isNumberSchema(secret.schema): {
-          return secret.required
-            ? config.requireSecretNumber(secretName)
-            : config.getSecretNumber(secretName)
-        }
-        case isBooleanSchema(secret.schema): {
-          return secret.required
-            ? config.requireSecretBoolean(secretName)
-            : config.getSecretBoolean(secretName)
-        }
-        default: {
-          const value = secret.required
-            ? config.requireSecretObject(secretName)
-            : config.getSecretObject(secretName)
+  const secretIds = config.requireObject<Record<string, string>>("$secretIds")
 
-          if (!ajv.validate(secret.schema, value)) {
-            throw new Error(`Invalid secret for '${secretName}': ${ajv.errorsText()}`)
-          }
+  const getSecretValue = (
+    secretName: string,
+    secret: ComponentSecret,
+  ): Output<unknown> | undefined => {
+    switch (true) {
+      case isStringSchema(secret.schema): {
+        return secret.required ? config.requireSecret(secretName) : config.getSecret(secretName)
+      }
+      case isNumberSchema(secret.schema): {
+        return secret.required
+          ? config.requireSecretNumber(secretName)
+          : config.getSecretNumber(secretName)
+      }
+      case isBooleanSchema(secret.schema): {
+        return secret.required
+          ? config.requireSecretBoolean(secretName)
+          : config.getSecretBoolean(secretName)
+      }
+      default: {
+        const value = secret.required
+          ? config.requireSecretObject(secretName)
+          : config.getSecretObject(secretName)
 
-          return value
+        if (!ajv.validate(secret.schema, value)) {
+          throw new Error(`Invalid secret for '${secretName}': ${ajv.errorsText()}`)
         }
+
+        return value
       }
-    }),
-  ) as unknown as Output<ComponentArgumentSpecToStatic<TSecrets>>
+    }
+  }
+
+  const secrets = mapValues(unit.model.secrets, (secret, secretName): UnitSecret<unknown> => {
+    const secretId = secretIds[secretName]
+    if (!secretId) {
+      throw new Error(`Secret '${secretName}' not found in the config.`)
+    }
+
+    return {
+      [HighstateSignature.Secret]: true,
+      id: secretId,
+      value: secret.required
+        ? config.requireSecret(secretName)
+        : output(getSecretValue(secretName, secret)),
+    }
+  })
 
   const inputs = mapValues(unit.model.inputs, (input, inputName) => {
     const value = input.required
@@ -382,24 +458,20 @@ export function forUnit<
     instanceId,
     type,
     name: instanceName,
-    secrets,
+    secrets: secrets as SecretValueMapToSecretMap<ComponentArgumentSpecToStatic<TSecrets>>,
     inputs: inputs as any,
     invokedTriggers: config.getObject<InstanceTriggerInvocation[]>("$invokedTriggers") ?? [],
 
     outputs: async (outputs: any = {}) => {
       const result: any = mapValues(outputs, (outputValue, outputName) => {
-        if (outputName === "$status") {
-          return output(outputValue).apply(mapStatus)
+        if (outputName === "$statusFields") {
+          return output(outputValue).apply(mapStatusFields)
         }
 
         if (outputName === "$pages") {
           return output(outputValue).apply(mapPages)
         }
 
-        if (outputName === "$files") {
-          return output(outputValue).apply(mapFiles)
-        }
-
         if (outputName === "$terminals") {
           return output(outputValue).apply(mapTerminals)
         }
@@ -433,7 +505,7 @@ export function forUnit<
             return undefined
           }
 
-          const schema = outputModel.multiple ? Type.Array(entity.schema) : entity.schema
+          const schema = outputModel.multiple ? entity.schema.array() : entity.schema
 
           if (!ajv.validate(schema, value)) {
             throw new Error(`Invalid output for '${outputName}': ${ajv.errorsText()}`)
@@ -445,8 +517,38 @@ export function forUnit<
 
       await Promise.all(Object.values(result).map(o => outputToPromise(o)))
 
-      if (Object.keys(createdSecrets).length > 0) {
-        result.$secrets = createdSecrets
+      result.$secrets = updatedSecretValues
+
+      // collect secrets from all outputs
+      const secretsMap: Record<string, UnitSecretModel[]> = {}
+      for (const [outputName, outputValue] of Object.entries(outputs)) {
+        if (!outputName.startsWith("$")) {
+          const resolvedValue = await outputToPromise(outputValue)
+          const secrets = extractObjectsFromValue(unitSecretSchema, resolvedValue)
+          if (secrets.length > 0) {
+            secretsMap[outputName] = secrets
+          }
+        }
+      }
+
+      // collect artifacts from all outputs
+      const artifactsMap: Record<string, UnitArtifact[]> = {}
+      for (const [outputName, outputValue] of Object.entries(outputs)) {
+        if (!outputName.startsWith("$")) {
+          const resolvedValue = await outputToPromise(outputValue)
+          const artifacts = extractObjectsFromValue(unitArtifactSchema, resolvedValue)
+          if (artifacts.length > 0) {
+            artifactsMap[outputName] = artifacts
+          }
+        }
+      }
+
+      if (Object.keys(artifactsMap).length > 0) {
+        result.$exportedArtifacts = artifactsMap
+      }
+
+      if (Object.keys(secretsMap).length > 0) {
+        result.$exportedSecretIds = mapValues(secretsMap, v => v.map(secret => secret.id))
       }
 
       return result
@@ -454,20 +556,30 @@ export function forUnit<
   }
 }
 
-export type EntityValue<T extends Entity> = Static<T["schema"]>
-export type EntityInput<T extends Entity> = Output<EntityValue<T>>
+export type EntityValue<T extends EntityModel> = z.infer<T["schema"]>
+export type EntityInput<T extends EntityModel> = Output<EntityValue<T>>
 
 function outputToPromise(o: unknown): Promise<unknown> {
   return new Promise(resolve => (output(o) as Output<unknown>).apply(resolve))
 }
 
-function mapStatus(status: Unwrap<ExtraOutputs["$status"]>): StatusField[] {
+function mapStatusFields(status: Unwrap<ExtraOutputs["$statusFields"]>): StatusField[] {
   if (!status) {
     return []
   }
 
   if (Array.isArray(status)) {
-    return status.filter(field => !!field?.value) as StatusField[]
+    return status
+      .filter(field => !!field?.value)
+      .map(field => {
+        return {
+          name: field!.name,
+          meta: {
+            title: field!.meta?.title ?? camelCaseToHumanReadable(field!.name),
+          },
+          value: field!.value,
+        } as StatusField
+      })
   }
 
   return Object.entries(status)
@@ -482,10 +594,23 @@ function mapStatus(status: Unwrap<ExtraOutputs["$status"]>): StatusField[] {
         typeof field === "boolean" ||
         Array.isArray(field)
       ) {
-        return { name, value: field }
+        return {
+          name,
+          meta: {
+            title: camelCaseToHumanReadable(name),
+          },
+          value: field,
+        }
       }
 
-      return { ...(field as StatusField), name }
+      return {
+        ...(field as StatusField),
+        meta: {
+          ...field.meta,
+          title: field.meta?.title ?? camelCaseToHumanReadable(name),
+        },
+        name,
+      }
     })
     .filter(field => !!field?.value) as StatusField[]
 }
@@ -516,7 +641,10 @@ export function fileFromString(
       contentType,
       size: Buffer.byteLength(content, "utf8"),
     },
-    content: isSecret ? secret(content) : content,
+    content: {
+      type: "embedded",
+      value: isSecret ? secret(content) : content,
+    },
   }
 }
 
@@ -533,14 +661,13 @@ export function fileFromBuffer(
       size: content.byteLength,
       isBinary: true,
     },
-    content: isSecret ? secret(content.toString("base64")) : content.toString("base64"),
+    content: {
+      type: "embedded",
+      value: isSecret ? secret(content.toString("base64")) : content.toString("base64"),
+    },
   }
 }
 
-function mapFiles(files: Unwrap<ExtraOutputs["$files"]>): InstanceFile[] {
-  return files?.filter(file => !!file) ?? []
-}
-
 function mapTerminals(terminals: Unwrap<ExtraOutputs["$terminals"]>): InstanceTerminal[] {
   if (!terminals) {
     return []
@@ -559,24 +686,38 @@ function mapTerminals(terminals: Unwrap<ExtraOutputs["$terminals"]>): InstanceTe
   return terminals
     .filter(terminal => !!terminal)
     .map(terminal => {
-      if (!terminal.files) {
+      if (!terminal.spec.files) {
         return terminal
       }
 
       return {
         ...terminal,
 
-        files: pipe(
-          terminal.files,
-          mapValues(file => {
-            if (typeof file === "string") {
-              return { content: file }
-            }
-
-            return file
-          }),
-          pickBy(value => !!value?.content),
-        ),
+        spec: {
+          ...terminal.spec,
+
+          files: pipe(
+            terminal.spec.files,
+            mapValues(file => {
+              if (typeof file === "string") {
+                return {
+                  meta: {
+                    name: "content",
+                    contentType: "text/plain",
+                    size: Buffer.byteLength(file, "utf8"),
+                  },
+                  content: {
+                    type: "embedded" as const,
+                    value: file,
+                  },
+                }
+              }
+
+              return file
+            }),
+            pickBy(value => !!value),
+          ),
+        },
       }
     })
 }
@@ -594,3 +735,40 @@ function mapTriggers(triggers: Unwrap<ExtraOutputs["$triggers"]>): InstanceTrigg
     .filter(([, trigger]) => !!trigger)
     .map(([name, trigger]) => ({ ...(trigger as InstanceTrigger), name }))
 }
+
+/**
+ * Extracts all objects with the specified schema from a value.
+ */
+function extractObjectsFromValue<TSchema extends z.ZodType>(
+  schema: TSchema,
+  data: unknown,
+): z.infer<TSchema>[] {
+  const result: z.infer<TSchema>[] = []
+
+  function traverse(obj: unknown): void {
+    if (obj === null || obj === undefined || typeof obj !== "object") {
+      return
+    }
+
+    if (Array.isArray(obj)) {
+      for (const item of obj) {
+        traverse(item)
+      }
+      return
+    }
+
+    const parseResult = schema.safeParse(obj)
+    if (parseResult.success) {
+      result.push(parseResult.data)
+      return
+    }
+
+    // recursively traverse all properties
+    for (const value of Object.values(obj)) {
+      traverse(value)
+    }
+  }
+
+  traverse(data)
+  return result
+}