@highstate/common 0.9.16 → 0.9.19

package/src/shared/dns.ts

@@ -1,10 +1,11 @@
 import type { ArrayPatchMode, dns, network } from "@highstate/library"
+import { getOrCreate, z } from "@highstate/contract"
 import {
   ComponentResource,
-  normalize,
+  interpolate,
+  normalizeInputsAndMap,
   output,
-  Output,
-  Resource,
+  type Output,
   toPromise,
   type Input,
   type InputArray,
@@ -12,7 +13,7 @@ import {
   type ResourceOptions,
   type Unwrap,
 } from "@highstate/pulumi"
-import { capitalize, groupBy, uniqueBy } from "remeda"
+import { flat, groupBy, uniqueBy } from "remeda"
 import { Command, type CommandHost } from "./command"
 import {
   filterEndpoints,
@@ -21,6 +22,13 @@ import {
   parseL3Endpoint,
   type InputL3Endpoint,
 } from "./network"
+import { ImplementationMediator } from "./impl-ref"
+
+export const dnsRecordMediator = new ImplementationMediator(
+  "dns-record",
+  z.object({ name: z.string(), args: z.custom<ResolvedDnsRecordArgs>() }),
+  z.instanceof(ComponentResource),
+)
 
 export type DnsRecordArgs = {
   /**
@@ -74,23 +82,23 @@ export type DnsRecordArgs = {
   waitAt?: InputOrArray<CommandHost>
 }
 
-export type ResolvedDnsRecordArgs = Unwrap<Omit<DnsRecordArgs, "value" | "type">> & {
+export type ResolvedDnsRecordArgs = Pick<DnsRecordArgs, "name" | "priority" | "ttl" | "proxied"> & {
   /**
    * The value of the DNS record.
    */
-  value: string
+  value: Input<string>
 
   /**
    * The type of the DNS record.
    */
-  type: string
+  type: Input<string>
 }
 
 export type DnsRecordSetArgs = Omit<DnsRecordArgs, "provider" | "value"> & {
   /**
    * The DNS providers to use to create the DNS records.
    *
-   * If multiple providers matched the specified domain, multiple DNS records will be created.
+   * If multiple providers matched the specified domain, records will be created for each of them.
    */
   providers: Input<dns.Provider[]>
 
@@ -116,125 +124,177 @@ function getTypeByEndpoint(endpoint: network.L3Endpoint): string {
   }
 }
 
-export abstract class DnsRecord extends ComponentResource {
+/**
+ * Creates a DNS record for the specified value and waits for it to be resolved.
+ *
+ * Uses the specified DNS provider to create the record.
+ */
+export class DnsRecord extends ComponentResource {
   /**
    * The underlying dns record resource.
    */
-  public readonly dnsRecord: Output<Resource>
+  readonly dnsRecord: Output<ComponentResource>
 
   /**
-   * The wait commands to be executed after the DNS record is created/updated.
+   * The commands to be executed after the DNS record is created/updated.
    *
-   * Use this field as a dependency for other resources.
+   * These commands will wait for the DNS record to be resolved to the specified value.
    */
-  public readonly waitCommands: Output<Command[]>
+  readonly waitCommands: Output<Command[]>
 
-  protected constructor(name: string, args: DnsRecordArgs, opts?: ResourceOptions) {
+  constructor(name: string, args: DnsRecordArgs, opts?: ResourceOptions) {
     super("highstate:common:DnsRecord", name, args, opts)
 
-    this.dnsRecord = output(args).apply(args => {
-      const l3Endpoint = parseL3Endpoint(args.value)
-      const type = args.type ?? getTypeByEndpoint(l3Endpoint)
-
-      return output(
-        this.create(
-          name,
-          {
-            ...args,
-            type,
-            value: l3EndpointToString(l3Endpoint),
-          },
-          { ...opts, parent: this },
-        ),
-      )
+    const l3Endpoint = output(args.value).apply(value => parseL3Endpoint(value))
+    const resolvedValue = l3Endpoint.apply(l3EndpointToString)
+    const resolvedType = args.type ? output(args.type) : l3Endpoint.apply(getTypeByEndpoint)
+
+    this.dnsRecord = output(args.provider).apply(provider => {
+      return dnsRecordMediator.call(provider.implRef, {
+        name,
+        args: {
+          name: args.name,
+          priority: args.priority,
+          ttl: args.ttl,
+          value: resolvedValue,
+          type: resolvedType,
+        },
+      })
     })
 
-    this.waitCommands = output(args).apply(args => {
-      const waitAt = args.waitAt ? (Array.isArray(args.waitAt) ? args.waitAt : [args.waitAt]) : []
-
-      return waitAt.map(host => {
+    this.waitCommands = output({
+      waitAt: args.waitAt,
+      resolvedType,
+      proxied: args.proxied,
+    }).apply(({ waitAt, resolvedType, proxied }) => {
+      if (resolvedType === "CNAME") {
+        // TODO: handle CNAME records
+        return []
+      }
+
+      const resolvedHosts = waitAt ? [waitAt].flat() : []
+
+      if (proxied) {
+        // for proxied records do not verify the value since we do not know the actual IP addressa
+
+        return (resolvedHosts as Unwrap<CommandHost>[]).map(host => {
+          const hostname = host === "local" ? "local" : host.hostname
+
+          return new Command(
+            `${name}.wait-for-dns.${hostname}`,
+            {
+              host,
+              create: [
+                interpolate`while ! getent hosts "${args.name}";`,
+                interpolate`do echo "Waiting for DNS record \"${args.name}\" to be available...";`,
+                `sleep 5;`,
+                `done`,
+              ],
+            },
+            { parent: this },
+          )
+        })
+      }
+
+      return (resolvedHosts as Unwrap<CommandHost>[]).map(host => {
         const hostname = host === "local" ? "local" : host.hostname
 
         return new Command(
-          `${name}-wait-${hostname}`,
+          `${name}.wait-for-dns.${hostname}`,
           {
             host,
-            create: `while ! getent hosts ${args.name} >/dev/null; do echo "Waiting for DNS record ${args.name} to be created"; sleep 5; done`,
-            triggers: [args.type, args.ttl, args.priority, args.proxied],
+            create: [
+              interpolate`while ! getent hosts "${args.name}" | grep "${resolvedValue}";`,
+              interpolate`do echo "Waiting for DNS record \"${args.name}" to resolve to "${resolvedValue}"...";`,
+              `sleep 5;`,
+              `done`,
+            ],
           },
           { parent: this },
         )
       })
     })
   }
-
-  protected abstract create(
-    name: string,
-    args: ResolvedDnsRecordArgs,
-    opts?: ResourceOptions,
-  ): Input<Resource>
-
-  static create(name: string, args: DnsRecordArgs, opts?: ResourceOptions): Output<DnsRecord> {
-    return output(args).apply(async args => {
-      const providerType = args.provider.type
-      const implName = `${capitalize(providerType)}DnsRecord`
-      const implModule = (await import(`@highstate/${providerType}`)) as Record<string, unknown>
-
-      const implClass = implModule[implName] as new (
-        name: string,
-        args: Unwrap<DnsRecordArgs>,
-        opts?: ResourceOptions,
-      ) => DnsRecord
-
-      return new implClass(name, args, opts)
-    })
-  }
 }
 
+/**
+ * Creates a set of DNS records for the specified values and waits for them to be resolved.
+ */
 export class DnsRecordSet extends ComponentResource {
   /**
    * The underlying dns record resources.
    */
-  public readonly dnsRecords: Output<DnsRecord[]>
+  readonly dnsRecords: Output<DnsRecord[]>
 
   /**
-   * The wait commands to be executed after the DNS records are created/updated.
+   * The flat list of all wait commands for the DNS records.
    */
-  public readonly waitCommands: Output<Command[]>
+  readonly waitCommands: Output<Command[]>
 
-  private constructor(name: string, records: Output<DnsRecord[]>, opts?: ResourceOptions) {
-    super("highstate:common:DnsRecordSet", name, records, opts)
+  constructor(name: string, args: DnsRecordSetArgs, opts?: ResourceOptions) {
+    super("highstate:common:DnsRecordSet", name, args, opts)
 
-    this.dnsRecords = records
+    const matchedProviders = output({
+      providers: args.providers,
+      name: args.name ?? name,
+    }).apply(({ providers }) => {
+      const matchedProviders = providers.filter(provider => name.endsWith(provider.domain))
 
-    this.waitCommands = records.apply(records =>
-      records.flatMap(record => record.waitCommands),
-    ) as unknown as Output<Command[]>
-  }
+      if (matchedProviders.length === 0) {
+        throw new Error(`No DNS provider matched the domain "${name}"`)
+      }
 
-  static create(name: string, args: DnsRecordSetArgs, opts?: ResourceOptions): DnsRecordSet {
-    const records = output(args).apply(args => {
-      const recordName = args.name ?? name
-      const values = normalize(args.value, args.values)
-
-      return output(
-        args.providers
-          .filter(provider => recordName.endsWith(provider.domain))
-          .flatMap(provider => {
-            return values.map(value => {
-              const l3Endpoint = parseL3Endpoint(value)
-
-              return DnsRecord.create(
-                `${provider.type}-from-${recordName}-to-${l3EndpointToString(l3Endpoint)}`,
-                { name: recordName, ...args, value: l3Endpoint, provider },
-                opts,
-              )
-            })
-          }),
-      )
+      return matchedProviders
     })
 
-    return new DnsRecordSet(name, records, opts)
+    this.dnsRecords = normalizeInputsAndMap(args.value, args.values, value => {
+      return output({
+        name: args.name ?? name,
+        providers: matchedProviders,
+      }).apply(({ name, providers }) => {
+        return providers.flatMap(provider => {
+          const l3Endpoint = parseL3Endpoint(value)
+
+          return new DnsRecord(
+            `${name}.${provider.id}.${l3EndpointToString(l3Endpoint)}`,
+            {
+              name,
+              provider,
+              value: l3Endpoint,
+              type: args.type ?? getTypeByEndpoint(l3Endpoint),
+              proxied: args.proxied,
+              ttl: args.ttl,
+              priority: args.priority,
+              waitAt: args.waitAt,
+            },
+            { parent: this },
+          )
+        })
+      })
+    }).apply(flat)
+
+    this.waitCommands = this.dnsRecords
+      .apply(records => records.flatMap(record => record.waitCommands))
+      .apply(flat)
+  }
+
+  private static readonly dnsRecordSetCache = new Map<string, DnsRecordSet>()
+
+  /**
+   * Creates a DNS record set for the specified endpoints and waits for it to be resolved.
+   *
+   * If a DNS record set with the same name already exists, it will be reused.
+   *
+   * @param name The name of the DNS record set.
+   * @param args The arguments for the DNS record set.
+   * @param opts The options for the resource.
+   */
+  static createOnce(name: string, args: DnsRecordSetArgs, opts?: ResourceOptions): DnsRecordSet {
+    return getOrCreate(
+      DnsRecordSet.dnsRecordSetCache,
+      name,
+      () => new DnsRecordSet(name, args, opts),
+    )
   }
 }
 
@@ -268,7 +328,7 @@ export async function updateEndpointsWithFqdn<TEndpoint extends network.L34Endpo
 
   const filteredEndpoints = filterEndpoints(resolvedEndpoints, fqdnEndpointFilter)
 
-  const dnsRecordSet = DnsRecordSet.create(fqdn, {
+  const dnsRecordSet = new DnsRecordSet(fqdn, {
     providers: dnsProviders,
     values: filteredEndpoints,
     waitAt: "local",

package/src/shared/files.ts

@@ -1,18 +1,17 @@
 import type { common, network } from "@highstate/library"
-import { tmpdir } from "node:os"
-import { cp, mkdir, mkdtemp, rm, writeFile, rename, stat } from "node:fs/promises"
-import { join, extname, basename, dirname } from "node:path"
+import { createHash } from "node:crypto"
 import { createReadStream } from "node:fs"
-import { pipeline } from "node:stream/promises"
+import { cp, mkdir, mkdtemp, rename, rm, stat, writeFile } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { basename, dirname, extname, join } from "node:path"
 import { Readable } from "node:stream"
-import { createHash } from "node:crypto"
+import { pipeline } from "node:stream/promises"
+import { type CommonObjectMeta, type File, HighstateSignature } from "@highstate/contract"
+import { asset, toPromise } from "@highstate/pulumi"
 import { minimatch } from "minimatch"
-
-import { HighstateSignature, type File } from "@highstate/contract"
-import { asset, toPromise, type ObjectMeta } from "@highstate/pulumi"
 import * as tar from "tar"
 import unzipper from "unzipper"
-import { l7EndpointToString, parseL7Endpoint, type InputL7Endpoint } from "./network"
+import { type InputL7Endpoint, l7EndpointToString, parseL7Endpoint } from "./network"
 
 export type FolderPackOptions = {
   /**
@@ -50,7 +49,7 @@ export function assetFromFile(file: common.File): asset.Asset {
     )
   }
 
-  if (file.meta.isBinary) {
+  if (file.content.isBinary) {
     throw new Error(
       "Cannot create asset from inline binary file content. Please open an issue if you need this feature.",
     )
@@ -171,12 +170,16 @@ export class MaterializedFile implements AsyncDisposable {
   private _path!: string
   private _disposed = false
 
-  readonly artifactMeta: ObjectMeta = {}
+  readonly artifactMeta: CommonObjectMeta
 
   constructor(
     readonly entity: common.File,
     readonly parent?: MaterializedFolder,
-  ) {}
+  ) {
+    this.artifactMeta = {
+      title: `Materialized file "${entity.meta.name}"`,
+    }
+  }
 
   get path(): string {
     return this._path
@@ -196,7 +199,7 @@ export class MaterializedFile implements AsyncDisposable {
 
     switch (this.entity.content.type) {
       case "embedded": {
-        const content = this.entity.meta.isBinary
+        const content = this.entity.content.isBinary
           ? Buffer.from(this.entity.content.value, "base64")
           : this.entity.content.value
 
@@ -208,7 +211,7 @@ export class MaterializedFile implements AsyncDisposable {
         break
       }
       case "remote": {
-        const response = await load(l7EndpointToString(this.entity.content.endpoint))
+        const response = await fetch(l7EndpointToString(this.entity.content.endpoint))
         if (!response.ok) throw new Error(`Failed to fetch: ${response.statusText}`)
 
         const arrayBuffer = await response.arrayBuffer()
@@ -217,16 +220,14 @@ export class MaterializedFile implements AsyncDisposable {
         break
       }
       case "artifact": {
-        const artifactData = this.entity.content[HighstateSignature.Artifact]
         const artifactPath = process.env.HIGHSTATE_ARTIFACT_READ_PATH
-
         if (!artifactPath) {
           throw new Error(
             "HIGHSTATE_ARTIFACT_READ_PATH environment variable is not set but required for artifact content",
           )
         }
 
-        const tgzPath = join(artifactPath, `${artifactData.hash}.tgz`)
+        const tgzPath = join(artifactPath, `${this.entity.content.hash}.tgz`)
 
         // extract the tgz file directly to the target path
         const readStream = createReadStream(tgzPath)
@@ -303,7 +304,6 @@ export class MaterializedFile implements AsyncDisposable {
         name: this.entity.meta.name,
         mode: fileStats.mode & 0o777, // extract only permission bits
         size: fileStats.size,
-        isBinary: this.entity.meta.isBinary, // keep original binary flag as we can't reliably detect this from filesystem
       }
 
       // return file entity with artifact content using actual filesystem stats
@@ -311,10 +311,9 @@ export class MaterializedFile implements AsyncDisposable {
         meta: newMeta,
         content: {
           type: "artifact",
-          [HighstateSignature.Artifact]: {
-            hash: hashValue,
-            meta: await toPromise(this.artifactMeta),
-          },
+          [HighstateSignature.Artifact]: true,
+          hash: hashValue,
+          meta: await toPromise(this.artifactMeta),
         },
       }
     } finally {
@@ -341,7 +340,6 @@ export class MaterializedFile implements AsyncDisposable {
         name,
         mode,
         size: 0,
-        isBinary: false,
       },
       content: {
         type: "embedded",
@@ -392,12 +390,16 @@ export class MaterializedFolder implements AsyncDisposable {
 
   private readonly _disposables: AsyncDisposable[] = []
 
-  readonly artifactMeta: ObjectMeta = {}
+  readonly artifactMeta: CommonObjectMeta
 
   constructor(
     readonly entity: common.Folder,
     readonly parent?: MaterializedFolder,
-  ) {}
+  ) {
+    this.artifactMeta = {
+      title: `Materialized folder "${entity.meta.name}"`,
+    }
+  }
 
   get path(): string {
     return this._path
@@ -451,7 +453,7 @@ export class MaterializedFolder implements AsyncDisposable {
         break
       }
       case "remote": {
-        const response = await load(l7EndpointToString(this.entity.content.endpoint))
+        const response = await fetch(l7EndpointToString(this.entity.content.endpoint))
         if (!response.ok) throw new Error(`Failed to fetch: ${response.statusText}`)
         if (!response.body) throw new Error("Response body is empty")
 
@@ -491,7 +493,6 @@ export class MaterializedFolder implements AsyncDisposable {
         break
       }
       case "artifact": {
-        const artifactData = this.entity.content[HighstateSignature.Artifact]
         const artifactPath = process.env.HIGHSTATE_ARTIFACT_READ_PATH
 
         if (!artifactPath) {
@@ -500,7 +501,7 @@ export class MaterializedFolder implements AsyncDisposable {
           )
         }
 
-        const tgzPath = join(artifactPath, `${artifactData.hash}.tgz`)
+        const tgzPath = join(artifactPath, `${this.entity.content.hash}.tgz`)
 
         // extract the tgz file directly to the target path
         const readStream = createReadStream(tgzPath)
@@ -615,11 +616,10 @@ export class MaterializedFolder implements AsyncDisposable {
       return {
         meta: newMeta,
         content: {
+          [HighstateSignature.Artifact]: true,
           type: "artifact",
-          [HighstateSignature.Artifact]: {
-            hash: hashValue,
-            meta: await toPromise(this.artifactMeta),
-          },
+          hash: hashValue,
+          meta: await toPromise(this.artifactMeta),
         },
       }
     } finally {
@@ -701,7 +701,7 @@ export async function fetchFileSize(endpoint: network.L7Endpoint): Promise<numbe
   }
 
   const url = l7EndpointToString(endpoint)
-  const response = await load(url, { method: "HEAD" })
+  const response = await fetch(url, { method: "HEAD" })
 
   if (!response.ok) {
     throw new Error(`Failed to fetch file size: ${response.statusText}`)
@@ -713,7 +713,7 @@ export async function fetchFileSize(endpoint: network.L7Endpoint): Promise<numbe
   }
 
   const size = parseInt(contentLength, 10)
-  if (isNaN(size)) {
+  if (Number.isNaN(size)) {
     throw new Error(`Invalid Content-Length value: ${contentLength}`)
   }
 

package/src/shared/gateway.ts

@@ -0,0 +1,117 @@
+import type { TlsCertificate } from "./tls"
+import { z } from "@highstate/contract"
+import { type common, network } from "@highstate/library"
+import {
+  ComponentResource,
+  type ComponentResourceOptions,
+  type Input,
+  type Output,
+  output,
+  Resource,
+} from "@highstate/pulumi"
+import { ImplementationMediator } from "./impl-ref"
+
+export const gatewayRouteMediator = new ImplementationMediator(
+  "gateway-route",
+  z.object({
+    name: z.string(),
+    spec: z.custom<GatewayRouteSpec>(),
+    opts: z.custom<ComponentResourceOptions>().optional(),
+  }),
+  z.object({
+    resource: z.instanceof(Resource),
+    endpoints: network.l3EndpointEntity.schema.array(),
+  }),
+)
+
+export type GatewayRouteSpec = {
+  /**
+   * The FQDN to expose the workload on.
+   */
+  fqdn?: Input<string>
+
+  /**
+   * The endpoints of the backend workload to route traffic to.
+   */
+  endpoints: Input<network.L4Endpoint[]>
+
+  /**
+   * The native data to pass to the implementation.
+   *
+   * This is used for data which implementation may natively understand,
+   * such as Kubernetes `Service` resources.
+   *
+   * Implementations may use this data to create more efficient routes
+   * using native resources.
+   */
+  nativeData?: Input<unknown>
+
+  /**
+   * The TLS certificate to use for the route.
+   */
+  tlsCertificate?: Input<TlsCertificate | undefined>
+} & (
+  | {
+      type: "http"
+
+      /**
+       * Whether to expose the workload over plain HTTP.
+       *
+       * By default, the workload will be exposed over HTTPS.
+       */
+      insecure?: Input<boolean>
+
+      /**
+       * The relative path to expose the workload on.
+       *
+       * By default, the workload will be exposed on the root path (`/`).
+       */
+      path?: Input<string>
+    }
+  | {
+      type: "l3"
+
+      /**
+       * The ports to request for the workload.
+       *
+       * The order must match the order of the ports in the service.
+       *
+       * If not provided, random ports will be assigned.
+       */
+      ports?: number
+    }
+)
+
+export type GatewayRouteArgs = GatewayRouteSpec & {
+  /**
+   * The gateway to attach the route to.
+   */
+  gateway: Input<common.Gateway>
+}
+
+export class GatewayRoute extends ComponentResource {
+  /**
+   * The underlying resource created by the implementation.
+   */
+  readonly resource: Output<Resource>
+
+  /**
+   * The endpoints of the gateway which serve this route.
+   *
+   * In most cases, this will be a single endpoint of the gateway shared for all routes.
+   */
+  readonly endpoints: Output<network.L3Endpoint[]>
+
+  constructor(name: string, args: GatewayRouteArgs, opts?: ComponentResourceOptions) {
+    super("highstate:common:GatewayRoute", name, args, opts)
+
+    const { resource, endpoints } = gatewayRouteMediator.callOutput(output(args.gateway).implRef, {
+      name,
+      spec: args,
+      opts: { ...opts, parent: this },
+    })
+
+    this.resource = resource
+    this.endpoints = endpoints
+  }
+}