@highstate/common 0.9.18 → 0.9.20

package/src/shared/dns.ts

@@ -1,10 +1,11 @@
 import type { ArrayPatchMode, dns, network } from "@highstate/library"
+import { getOrCreate, z } from "@highstate/contract"
 import {
   ComponentResource,
-  normalize,
+  interpolate,
+  normalizeInputsAndMap,
   output,
-  Output,
-  Resource,
+  type Output,
   toPromise,
   type Input,
   type InputArray,
@@ -12,7 +13,7 @@ import {
   type ResourceOptions,
   type Unwrap,
 } from "@highstate/pulumi"
-import { capitalize, groupBy, uniqueBy } from "remeda"
+import { flat, groupBy, uniqueBy } from "remeda"
 import { Command, type CommandHost } from "./command"
 import {
   filterEndpoints,
@@ -21,6 +22,13 @@ import {
   parseL3Endpoint,
   type InputL3Endpoint,
 } from "./network"
+import { ImplementationMediator } from "./impl-ref"
+
+export const dnsRecordMediator = new ImplementationMediator(
+  "dns-record",
+  z.object({ name: z.string(), args: z.custom<ResolvedDnsRecordArgs>() }),
+  z.instanceof(ComponentResource),
+)
 
 export type DnsRecordArgs = {
   /**
@@ -74,23 +82,23 @@ export type DnsRecordArgs = {
   waitAt?: InputOrArray<CommandHost>
 }
 
-export type ResolvedDnsRecordArgs = Unwrap<Omit<DnsRecordArgs, "value" | "type">> & {
+export type ResolvedDnsRecordArgs = Pick<DnsRecordArgs, "name" | "priority" | "ttl" | "proxied"> & {
   /**
    * The value of the DNS record.
    */
-  value: string
+  value: Input<string>
 
   /**
    * The type of the DNS record.
    */
-  type: string
+  type: Input<string>
 }
 
 export type DnsRecordSetArgs = Omit<DnsRecordArgs, "provider" | "value"> & {
   /**
    * The DNS providers to use to create the DNS records.
    *
-   * If multiple providers matched the specified domain, multiple DNS records will be created.
+   * If multiple providers matched the specified domain, records will be created for each of them.
    */
   providers: Input<dns.Provider[]>
 
@@ -116,125 +124,177 @@ function getTypeByEndpoint(endpoint: network.L3Endpoint): string {
   }
 }
 
-export abstract class DnsRecord extends ComponentResource {
+/**
+ * Creates a DNS record for the specified value and waits for it to be resolved.
+ *
+ * Uses the specified DNS provider to create the record.
+ */
+export class DnsRecord extends ComponentResource {
   /**
    * The underlying dns record resource.
    */
-  public readonly dnsRecord: Output<Resource>
+  readonly dnsRecord: Output<ComponentResource>
 
   /**
-   * The wait commands to be executed after the DNS record is created/updated.
+   * The commands to be executed after the DNS record is created/updated.
    *
-   * Use this field as a dependency for other resources.
+   * These commands will wait for the DNS record to be resolved to the specified value.
    */
-  public readonly waitCommands: Output<Command[]>
+  readonly waitCommands: Output<Command[]>
 
-  protected constructor(name: string, args: DnsRecordArgs, opts?: ResourceOptions) {
+  constructor(name: string, args: DnsRecordArgs, opts?: ResourceOptions) {
     super("highstate:common:DnsRecord", name, args, opts)
 
-    this.dnsRecord = output(args).apply(args => {
-      const l3Endpoint = parseL3Endpoint(args.value)
-      const type = args.type ?? getTypeByEndpoint(l3Endpoint)
-
-      return output(
-        this.create(
-          name,
-          {
-            ...args,
-            type,
-            value: l3EndpointToString(l3Endpoint),
-          },
-          { ...opts, parent: this },
-        ),
-      )
+    const l3Endpoint = output(args.value).apply(value => parseL3Endpoint(value))
+    const resolvedValue = l3Endpoint.apply(l3EndpointToString)
+    const resolvedType = args.type ? output(args.type) : l3Endpoint.apply(getTypeByEndpoint)
+
+    this.dnsRecord = output(args.provider).apply(provider => {
+      return dnsRecordMediator.call(provider.implRef, {
+        name,
+        args: {
+          name: args.name,
+          priority: args.priority,
+          ttl: args.ttl,
+          value: resolvedValue,
+          type: resolvedType,
+        },
+      })
     })
 
-    this.waitCommands = output(args).apply(args => {
-      const waitAt = args.waitAt ? (Array.isArray(args.waitAt) ? args.waitAt : [args.waitAt]) : []
-
-      return waitAt.map(host => {
+    this.waitCommands = output({
+      waitAt: args.waitAt,
+      resolvedType,
+      proxied: args.proxied,
+    }).apply(({ waitAt, resolvedType, proxied }) => {
+      if (resolvedType === "CNAME") {
+        // TODO: handle CNAME records
+        return []
+      }
+
+      const resolvedHosts = waitAt ? [waitAt].flat() : []
+
+      if (proxied) {
+        // for proxied records do not verify the value since we do not know the actual IP addressa
+
+        return (resolvedHosts as Unwrap<CommandHost>[]).map(host => {
+          const hostname = host === "local" ? "local" : host.hostname
+
+          return new Command(
+            `${name}.wait-for-dns.${hostname}`,
+            {
+              host,
+              create: [
+                interpolate`while ! getent hosts "${args.name}";`,
+                interpolate`do echo "Waiting for DNS record \"${args.name}\" to be available...";`,
+                `sleep 5;`,
+                `done`,
+              ],
+            },
+            { parent: this },
+          )
+        })
+      }
+
+      return (resolvedHosts as Unwrap<CommandHost>[]).map(host => {
         const hostname = host === "local" ? "local" : host.hostname
 
         return new Command(
-          `${name}-wait-${hostname}`,
+          `${name}.wait-for-dns.${hostname}`,
           {
             host,
-            create: `while ! getent hosts ${args.name} >/dev/null; do echo "Waiting for DNS record ${args.name} to be created"; sleep 5; done`,
-            triggers: [args.type, args.ttl, args.priority, args.proxied],
+            create: [
+              interpolate`while ! getent hosts "${args.name}" | grep "${resolvedValue}";`,
+              interpolate`do echo "Waiting for DNS record \"${args.name}" to resolve to "${resolvedValue}"...";`,
+              `sleep 5;`,
+              `done`,
+            ],
           },
           { parent: this },
         )
       })
     })
   }
-
-  protected abstract create(
-    name: string,
-    args: ResolvedDnsRecordArgs,
-    opts?: ResourceOptions,
-  ): Input<Resource>
-
-  static create(name: string, args: DnsRecordArgs, opts?: ResourceOptions): Output<DnsRecord> {
-    return output(args).apply(async args => {
-      const providerType = args.provider.type
-      const implName = `${capitalize(providerType)}DnsRecord`
-      const implModule = (await import(`@highstate/${providerType}`)) as Record<string, unknown>
-
-      const implClass = implModule[implName] as new (
-        name: string,
-        args: Unwrap<DnsRecordArgs>,
-        opts?: ResourceOptions,
-      ) => DnsRecord
-
-      return new implClass(name, args, opts)
-    })
-  }
 }
 
+/**
+ * Creates a set of DNS records for the specified values and waits for them to be resolved.
+ */
 export class DnsRecordSet extends ComponentResource {
   /**
    * The underlying dns record resources.
    */
-  public readonly dnsRecords: Output<DnsRecord[]>
+  readonly dnsRecords: Output<DnsRecord[]>
 
   /**
-   * The wait commands to be executed after the DNS records are created/updated.
+   * The flat list of all wait commands for the DNS records.
    */
-  public readonly waitCommands: Output<Command[]>
+  readonly waitCommands: Output<Command[]>
 
-  private constructor(name: string, records: Output<DnsRecord[]>, opts?: ResourceOptions) {
-    super("highstate:common:DnsRecordSet", name, records, opts)
+  constructor(name: string, args: DnsRecordSetArgs, opts?: ResourceOptions) {
+    super("highstate:common:DnsRecordSet", name, args, opts)
 
-    this.dnsRecords = records
+    const matchedProviders = output({
+      providers: args.providers,
+      name: args.name ?? name,
+    }).apply(({ providers }) => {
+      const matchedProviders = providers.filter(provider => name.endsWith(provider.domain))
 
-    this.waitCommands = records.apply(records =>
-      records.flatMap(record => record.waitCommands),
-    ) as unknown as Output<Command[]>
-  }
+      if (matchedProviders.length === 0) {
+        throw new Error(`No DNS provider matched the domain "${name}"`)
+      }
 
-  static create(name: string, args: DnsRecordSetArgs, opts?: ResourceOptions): DnsRecordSet {
-    const records = output(args).apply(args => {
-      const recordName = args.name ?? name
-      const values = normalize(args.value, args.values)
-
-      return output(
-        args.providers
-          .filter(provider => recordName.endsWith(provider.domain))
-          .flatMap(provider => {
-            return values.map(value => {
-              const l3Endpoint = parseL3Endpoint(value)
-
-              return DnsRecord.create(
-                `${provider.type}-from-${recordName}-to-${l3EndpointToString(l3Endpoint)}`,
-                { name: recordName, ...args, value: l3Endpoint, provider },
-                opts,
-              )
-            })
-          }),
-      )
+      return matchedProviders
     })
 
-    return new DnsRecordSet(name, records, opts)
+    this.dnsRecords = normalizeInputsAndMap(args.value, args.values, value => {
+      return output({
+        name: args.name ?? name,
+        providers: matchedProviders,
+      }).apply(({ name, providers }) => {
+        return providers.flatMap(provider => {
+          const l3Endpoint = parseL3Endpoint(value)
+
+          return new DnsRecord(
+            `${name}.${provider.id}.${l3EndpointToString(l3Endpoint)}`,
+            {
+              name,
+              provider,
+              value: l3Endpoint,
+              type: args.type ?? getTypeByEndpoint(l3Endpoint),
+              proxied: args.proxied,
+              ttl: args.ttl,
+              priority: args.priority,
+              waitAt: args.waitAt,
+            },
+            { parent: this },
+          )
+        })
+      })
+    }).apply(flat)
+
+    this.waitCommands = this.dnsRecords
+      .apply(records => records.flatMap(record => record.waitCommands))
+      .apply(flat)
+  }
+
+  private static readonly dnsRecordSetCache = new Map<string, DnsRecordSet>()
+
+  /**
+   * Creates a DNS record set for the specified endpoints and waits for it to be resolved.
+   *
+   * If a DNS record set with the same name already exists, it will be reused.
+   *
+   * @param name The name of the DNS record set.
+   * @param args The arguments for the DNS record set.
+   * @param opts The options for the resource.
+   */
+  static createOnce(name: string, args: DnsRecordSetArgs, opts?: ResourceOptions): DnsRecordSet {
+    return getOrCreate(
+      DnsRecordSet.dnsRecordSetCache,
+      name,
+      () => new DnsRecordSet(name, args, opts),
+    )
   }
 }
 
@@ -268,7 +328,7 @@ export async function updateEndpointsWithFqdn<TEndpoint extends network.L34Endpo
 
   const filteredEndpoints = filterEndpoints(resolvedEndpoints, fqdnEndpointFilter)
 
-  const dnsRecordSet = DnsRecordSet.create(fqdn, {
+  const dnsRecordSet = new DnsRecordSet(fqdn, {
     providers: dnsProviders,
     values: filteredEndpoints,
     waitAt: "local",

package/src/shared/files.ts

@@ -1,18 +1,17 @@
 import type { common, network } from "@highstate/library"
-import { tmpdir } from "node:os"
-import { cp, mkdir, mkdtemp, rm, writeFile, rename, stat } from "node:fs/promises"
-import { join, extname, basename, dirname } from "node:path"
+import { createHash } from "node:crypto"
 import { createReadStream } from "node:fs"
-import { pipeline } from "node:stream/promises"
+import { cp, mkdir, mkdtemp, rename, rm, stat, writeFile } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { basename, dirname, extname, join } from "node:path"
 import { Readable } from "node:stream"
-import { createHash } from "node:crypto"
+import { pipeline } from "node:stream/promises"
+import { type CommonObjectMeta, type File, HighstateSignature } from "@highstate/contract"
+import { asset, toPromise } from "@highstate/pulumi"
 import { minimatch } from "minimatch"
-
-import { HighstateSignature, type File } from "@highstate/contract"
-import { asset, toPromise, type ObjectMeta } from "@highstate/pulumi"
 import * as tar from "tar"
 import unzipper from "unzipper"
-import { l7EndpointToString, parseL7Endpoint, type InputL7Endpoint } from "./network"
+import { type InputL7Endpoint, l7EndpointToString, parseL7Endpoint } from "./network"
 
 export type FolderPackOptions = {
   /**
@@ -50,7 +49,7 @@ export function assetFromFile(file: common.File): asset.Asset {
     )
   }
 
-  if (file.meta.isBinary) {
+  if (file.content.isBinary) {
     throw new Error(
       "Cannot create asset from inline binary file content. Please open an issue if you need this feature.",
     )
@@ -171,12 +170,16 @@ export class MaterializedFile implements AsyncDisposable {
   private _path!: string
   private _disposed = false
 
-  readonly artifactMeta: ObjectMeta = {}
+  readonly artifactMeta: CommonObjectMeta
 
   constructor(
     readonly entity: common.File,
     readonly parent?: MaterializedFolder,
-  ) {}
+  ) {
+    this.artifactMeta = {
+      title: `Materialized file "${entity.meta.name}"`,
+    }
+  }
 
   get path(): string {
     return this._path
@@ -196,7 +199,7 @@ export class MaterializedFile implements AsyncDisposable {
 
     switch (this.entity.content.type) {
       case "embedded": {
-        const content = this.entity.meta.isBinary
+        const content = this.entity.content.isBinary
           ? Buffer.from(this.entity.content.value, "base64")
           : this.entity.content.value
 
@@ -301,7 +304,6 @@ export class MaterializedFile implements AsyncDisposable {
         name: this.entity.meta.name,
         mode: fileStats.mode & 0o777, // extract only permission bits
         size: fileStats.size,
-        isBinary: this.entity.meta.isBinary, // keep original binary flag as we can't reliably detect this from filesystem
       }
 
       // return file entity with artifact content using actual filesystem stats
@@ -338,7 +340,6 @@ export class MaterializedFile implements AsyncDisposable {
         name,
         mode,
         size: 0,
-        isBinary: false,
       },
       content: {
         type: "embedded",
@@ -389,12 +390,16 @@ export class MaterializedFolder implements AsyncDisposable {
 
   private readonly _disposables: AsyncDisposable[] = []
 
-  readonly artifactMeta: ObjectMeta = {}
+  readonly artifactMeta: CommonObjectMeta
 
   constructor(
     readonly entity: common.Folder,
     readonly parent?: MaterializedFolder,
-  ) {}
+  ) {
+    this.artifactMeta = {
+      title: `Materialized folder "${entity.meta.name}"`,
+    }
+  }
 
   get path(): string {
     return this._path
@@ -708,7 +713,7 @@ export async function fetchFileSize(endpoint: network.L7Endpoint): Promise<numbe
   }
 
   const size = parseInt(contentLength, 10)
-  if (isNaN(size)) {
+  if (Number.isNaN(size)) {
     throw new Error(`Invalid Content-Length value: ${contentLength}`)
   }
 

package/src/shared/gateway.ts

@@ -0,0 +1,117 @@
+import type { TlsCertificate } from "./tls"
+import { z } from "@highstate/contract"
+import { type common, network } from "@highstate/library"
+import {
+  ComponentResource,
+  type ComponentResourceOptions,
+  type Input,
+  type Output,
+  output,
+  Resource,
+} from "@highstate/pulumi"
+import { ImplementationMediator } from "./impl-ref"
+
+export const gatewayRouteMediator = new ImplementationMediator(
+  "gateway-route",
+  z.object({
+    name: z.string(),
+    spec: z.custom<GatewayRouteSpec>(),
+    opts: z.custom<ComponentResourceOptions>().optional(),
+  }),
+  z.object({
+    resource: z.instanceof(Resource),
+    endpoints: network.l3EndpointEntity.schema.array(),
+  }),
+)
+
+export type GatewayRouteSpec = {
+  /**
+   * The FQDN to expose the workload on.
+   */
+  fqdn?: Input<string>
+
+  /**
+   * The endpoints of the backend workload to route traffic to.
+   */
+  endpoints: Input<network.L4Endpoint[]>
+
+  /**
+   * The native data to pass to the implementation.
+   *
+   * This is used for data which implementation may natively understand,
+   * such as Kubernetes `Service` resources.
+   *
+   * Implementations may use this data to create more efficient routes
+   * using native resources.
+   */
+  nativeData?: Input<unknown>
+
+  /**
+   * The TLS certificate to use for the route.
+   */
+  tlsCertificate?: Input<TlsCertificate | undefined>
+} & (
+  | {
+      type: "http"
+
+      /**
+       * Whether to expose the workload over plain HTTP.
+       *
+       * By default, the workload will be exposed over HTTPS.
+       */
+      insecure?: Input<boolean>
+
+      /**
+       * The relative path to expose the workload on.
+       *
+       * By default, the workload will be exposed on the root path (`/`).
+       */
+      path?: Input<string>
+    }
+  | {
+      type: "l3"
+
+      /**
+       * The ports to request for the workload.
+       *
+       * The order must match the order of the ports in the service.
+       *
+       * If not provided, random ports will be assigned.
+       */
+      ports?: number
+    }
+)
+
+export type GatewayRouteArgs = GatewayRouteSpec & {
+  /**
+   * The gateway to attach the route to.
+   */
+  gateway: Input<common.Gateway>
+}
+
+export class GatewayRoute extends ComponentResource {
+  /**
+   * The underlying resource created by the implementation.
+   */
+  readonly resource: Output<Resource>
+
+  /**
+   * The endpoints of the gateway which serve this route.
+   *
+   * In most cases, this will be a single endpoint of the gateway shared for all routes.
+   */
+  readonly endpoints: Output<network.L3Endpoint[]>
+
+  constructor(name: string, args: GatewayRouteArgs, opts?: ComponentResourceOptions) {
+    super("highstate:common:GatewayRoute", name, args, opts)
+
+    const { resource, endpoints } = gatewayRouteMediator.callOutput(output(args.gateway).implRef, {
+      name,
+      spec: args,
+      opts: { ...opts, parent: this },
+    })
+
+    this.resource = resource
+    this.endpoints = endpoints
+  }
+}

package/src/shared/impl-ref.ts

@@ -0,0 +1,123 @@
+import type { ImplementationReference } from "@highstate/library"
+import type { z } from "@highstate/contract"
+import { type Output, output, toPromise, type Input, type Unwrap } from "@highstate/pulumi"
+
+/**
+ * The ImplementationMediator is used as a contract between the calling code and the implementation.
+ *
+ * From the calling code perspective, it provides a way to define the input and output schemas for the implementation
+ * and call the implementation with the provided input.
+ *
+ * From the implementation perspective, it provides a way to get zod function with automatic type inference and validation.
+ */
+export class ImplementationMediator<
+  TInputSchema extends z.ZodType,
+  TOutputSchema extends z.ZodType,
+> {
+  constructor(
+    readonly path: string,
+    private readonly inputSchema: TInputSchema,
+    private readonly outputSchema: TOutputSchema,
+  ) {}
+
+  implement<TDataSchema extends z.ZodType>(
+    dataSchema: TDataSchema,
+    func: (
+      input: z.infer<TInputSchema>,
+      data: z.infer<TDataSchema>,
+    ) => z.infer<TOutputSchema> | Promise<z.infer<TOutputSchema>>,
+  ) {
+    return async (
+      input: z.infer<TInputSchema>,
+      data: z.infer<TDataSchema>,
+    ): Promise<z.infer<TOutputSchema>> => {
+      const parsedInput = this.inputSchema.safeParse(input)
+      if (!parsedInput.success) {
+        throw new Error(
+          `Invalid input for implementation "${this.path}": ${parsedInput.error.message}`,
+        )
+      }
+
+      const parsedData = dataSchema.safeParse(data)
+      if (!parsedData.success) {
+        throw new Error(
+          `Invalid data for implementation "${this.path}": ${parsedData.error.message}`,
+        )
+      }
+
+      const result = await func(parsedInput.data, parsedData.data)
+      const parsedResult = this.outputSchema.safeParse(result)
+
+      if (!parsedResult.success) {
+        throw new Error(
+          `Invalid output from implementation "${this.path}": ${parsedResult.error.message}`,
+        )
+      }
+
+      return parsedResult.data
+    }
+  }
+
+  async call(
+    implRef: Input<ImplementationReference>,
+    input: Input<z.infer<TInputSchema>>,
+  ): Promise<z.infer<TOutputSchema>> {
+    const resolvedImplRef = await toPromise(implRef)
+    const resolvedInput = await toPromise(input)
+
+    const importPath = `${resolvedImplRef.package}/impl/${this.path}`
+
+    let impl: Record<string, unknown>
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+      impl = await import(importPath)
+    } catch (error) {
+      throw new Error(`Failed to import module "${importPath}" required by implementation.`, {
+        cause: error,
+      })
+    }
+
+    const funcs = Object.entries(impl).filter(value => typeof value[1] === "function") as [
+      string,
+      Function,
+    ][]
+
+    if (funcs.length === 0) {
+      throw new Error(`No implementation functions found in module "${importPath}".`)
+    }
+
+    if (funcs.length > 1) {
+      throw new Error(
+        `Multiple implementation functions found in module "${importPath}": ${funcs.map(func => func[0]).join(", ")}. ` +
+          "Ensure only one function is exported.",
+      )
+    }
+
+    const [funcName, implFunc] = funcs[0]
+
+    let result: unknown
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+      result = await implFunc(resolvedInput, resolvedImplRef.data)
+    } catch (error) {
+      console.error(`Error in implementation function "${funcName}":`, error)
+      throw new Error(`Implementation function "${funcName}" failed`)
+    }
+
+    const parsedResult = this.outputSchema.safeParse(result)
+    if (!parsedResult.success) {
+      throw new Error(
+        `Implementation function "${funcName}" returned invalid result: ${parsedResult.error.message}`,
+      )
+    }
+
+    return parsedResult.data
+  }
+
+  callOutput(
+    implRef: Input<ImplementationReference>,
+    input: Input<z.infer<TInputSchema>>,
+  ): Output<Unwrap<z.infer<TOutputSchema>>> {
+    return output(this.call(implRef, input))
+  }
+}

package/src/shared/index.ts

@@ -4,3 +4,7 @@ export * from "./passwords"
 export * from "./ssh"
 export * from "./network"
 export * from "./files"
+export * from "./impl-ref"
+export * from "./gateway"
+export * from "./tls"
+export * from "./access-point"