@start9labs/start-sdk 0.3.6-alpha1 → 0.3.6-alpha2

package/lib/mainFn/CommandController.ts

@@ -0,0 +1,148 @@
+import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
+import { MountOptions, Overlay } from "../util/Overlay"
+import { splitCommand } from "../util/splitCommand"
+import { cpExecFile, cpExec } from "./Daemons"
+
+export class CommandController {
+  private constructor(
+    readonly runningAnswer: Promise<unknown>,
+    readonly overlay: Overlay,
+    readonly pid: number | undefined,
+  ) {}
+  static of<Manifest extends SDKManifest>() {
+    return async <A extends string>(
+      effects: Effects,
+      imageId: {
+        id: keyof Manifest["images"] & ImageId
+        sharedRun?: boolean
+      },
+      command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+      options: {
+        mounts?: { path: string; options: MountOptions }[]
+        overlay?: Overlay
+        env?:
+          | {
+              [variable: string]: string
+            }
+          | undefined
+        cwd?: string | undefined
+        user?: string | undefined
+        onStdout?: (x: Buffer) => void
+        onStderr?: (x: Buffer) => void
+      },
+    ) => {
+      const commands = splitCommand(command)
+      const overlay = options.overlay || (await Overlay.of(effects, imageId))
+      for (let mount of options.mounts || []) {
+        await overlay.mount(mount.options, mount.path)
+      }
+      const childProcess = await overlay.spawn(commands, {
+        env: options.env,
+      })
+      const answer = new Promise<null>((resolve, reject) => {
+        childProcess.stdout.on(
+          "data",
+          options.onStdout ??
+            ((data: any) => {
+              console.log(data.toString())
+            }),
+        )
+        childProcess.stderr.on(
+          "data",
+          options.onStderr ??
+            ((data: any) => {
+              console.error(data.toString())
+            }),
+        )
+
+        childProcess.on("exit", (code: any) => {
+          if (code === 0) {
+            return resolve(null)
+          }
+          return reject(new Error(`${commands[0]} exited with code ${code}`))
+        })
+      })
+
+      const pid = childProcess.pid
+
+      return new CommandController(answer, overlay, pid)
+    }
+  }
+  async wait() {
+    try {
+      return await this.runningAnswer
+    } finally {
+      if (this.pid !== undefined) {
+        await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch(
+          (_) => {},
+        )
+      }
+      await this.overlay.destroy().catch((_) => {})
+    }
+  }
+  async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) {
+    if (this.pid === undefined) return
+    try {
+      await cpExecFile("pkill", [
+        `-${signal.replace("SIG", "")}`,
+        "-s",
+        String(this.pid),
+      ])
+
+      const didTimeout = await waitSession(this.pid, timeout)
+      if (didTimeout) {
+        await cpExecFile("pkill", [`-9`, "-s", String(this.pid)]).catch(
+          (_) => {},
+        )
+      }
+    } finally {
+      await this.overlay.destroy()
+    }
+  }
+}
+
+function waitSession(
+  sid: number,
+  timeout = NO_TIMEOUT,
+  interval = 100,
+): Promise<boolean> {
+  let nextInterval = interval * 2
+  if (timeout >= 0 && timeout < nextInterval) {
+    nextInterval = timeout
+  }
+  let nextTimeout = timeout
+  if (timeout > 0) {
+    if (timeout >= interval) {
+      nextTimeout -= interval
+    } else {
+      nextTimeout = 0
+    }
+  }
+  return new Promise((resolve, reject) => {
+    let next: NodeJS.Timeout | null = null
+    if (timeout !== 0) {
+      next = setTimeout(() => {
+        waitSession(sid, nextTimeout, nextInterval).then(resolve, reject)
+      }, interval)
+    }
+    cpExecFile("ps", [`--sid=${sid}`, "-o", "--pid="]).then(
+      (_) => {
+        if (timeout === 0) {
+          resolve(true)
+        }
+      },
+      (e) => {
+        if (next) {
+          clearTimeout(next)
+        }
+        if (typeof e === "object" && e && "code" in e && e.code) {
+          resolve(false)
+        } else {
+          reject(e)
+        }
+      },
+    )
+  })
+}

package/lib/mainFn/Daemon.ts

@@ -0,0 +1,79 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
+import { MountOptions, Overlay } from "../util/Overlay"
+import { CommandController } from "./CommandController"
+
+const TIMEOUT_INCREMENT_MS = 1000
+const MAX_TIMEOUT_MS = 30000
+/**
+ * This is a wrapper around CommandController that has a state of off, where the command shouldn't be running
+ * and the others state of running, where it will keep a living running command
+ */
+
+export class Daemon {
+  private commandController: CommandController | null = null
+  private shouldBeRunning = false
+  private constructor(private startCommand: () => Promise<CommandController>) {}
+  static of<Manifest extends SDKManifest>() {
+    return async <A extends string>(
+      effects: Effects,
+      imageId: {
+        id: keyof Manifest["images"] & ImageId
+        sharedRun?: boolean
+      },
+      command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+      options: {
+        mounts?: { path: string; options: MountOptions }[]
+        overlay?: Overlay
+        env?:
+          | {
+              [variable: string]: string
+            }
+          | undefined
+        cwd?: string | undefined
+        user?: string | undefined
+        onStdout?: (x: Buffer) => void
+        onStderr?: (x: Buffer) => void
+      },
+    ) => {
+      const startCommand = () =>
+        CommandController.of<Manifest>()(effects, imageId, command, options)
+      return new Daemon(startCommand)
+    }
+  }
+
+  async start() {
+    if (this.commandController) {
+      return
+    }
+    this.shouldBeRunning = true
+    let timeoutCounter = 0
+    new Promise(async () => {
+      while (this.shouldBeRunning) {
+        this.commandController = await this.startCommand()
+        await this.commandController.wait().catch((err) => console.error(err))
+        await new Promise((resolve) => setTimeout(resolve, timeoutCounter))
+        timeoutCounter += TIMEOUT_INCREMENT_MS
+        timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
+      }
+    }).catch((err) => {
+      console.error(err)
+    })
+  }
+  async term(termOptions?: {
+    signal?: NodeJS.Signals | undefined
+    timeout?: number | undefined
+  }) {
+    return this.stop(termOptions)
+  }
+  async stop(termOptions?: {
+    signal?: NodeJS.Signals | undefined
+    timeout?: number | undefined
+  }) {
+    this.shouldBeRunning = false
+    await this.commandController
+      ?.term(termOptions)
+      .catch((e) => console.error(e))
+    this.commandController = null
+  }
+}

package/lib/mainFn/Daemons.ts

@@ -0,0 +1,175 @@
+import { NO_TIMEOUT, SIGKILL, SIGTERM, Signals } from "../StartSdk"
+import { HealthReceipt } from "../health/HealthReceipt"
+import { CheckResult } from "../health/checkFns"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Trigger } from "../trigger"
+import { TriggerInput } from "../trigger/TriggerInput"
+import { defaultTrigger } from "../trigger/defaultTrigger"
+import {
+  DaemonReturned,
+  Effects,
+  ImageId,
+  ValidIfNoStupidEscape,
+} from "../types"
+import { Mounts } from "./Mounts"
+import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
+import { splitCommand } from "../util/splitCommand"
+
+import { promisify } from "node:util"
+import * as CP from "node:child_process"
+
+export { Daemon } from "./Daemon"
+export { CommandController } from "./CommandController"
+import { HealthDaemon } from "./HealthDaemon"
+import { Daemon } from "./Daemon"
+import { CommandController } from "./CommandController"
+
+export const cpExec = promisify(CP.exec)
+export const cpExecFile = promisify(CP.execFile)
+export type Ready = {
+  display: string | null
+  fn: () => Promise<CheckResult> | CheckResult
+  trigger?: Trigger
+}
+
+type DaemonsParams<
+  Manifest extends SDKManifest,
+  Ids extends string,
+  Command extends string,
+  Id extends string,
+> = {
+  command: ValidIfNoStupidEscape<Command> | [string, ...string[]]
+  image: { id: keyof Manifest["images"] & ImageId; sharedRun?: boolean }
+  mounts: Mounts<Manifest>
+  env?: Record<string, string>
+  ready: Ready
+  requires: Exclude<Ids, Id>[]
+}
+
+type ErrorDuplicateId<Id extends string> = `The id '${Id}' is already used`
+
+export const runCommand = <Manifest extends SDKManifest>() =>
+  CommandController.of<Manifest>()
+
+/**
+ * A class for defining and controlling the service daemons
+```ts
+Daemons.of({
+  effects,
+  started,
+  interfaceReceipt, // Provide the interfaceReceipt to prove it was completed
+  healthReceipts, // Provide the healthReceipts or [] to prove they were at least considered
+}).addDaemon('webui', {
+  command: 'hello-world', // The command to start the daemon
+  ready: {
+    display: 'Web Interface',
+    // The function to run to determine the health status of the daemon
+    fn: () =>
+      checkPortListening(effects, 80, {
+        successMessage: 'The web interface is ready',
+        errorMessage: 'The web interface is not ready',
+      }),
+  },
+  requires: [],
+})
+```
+ */
+export class Daemons<Manifest extends SDKManifest, Ids extends string> {
+  private constructor(
+    readonly effects: Effects,
+    readonly started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>,
+    readonly daemons: Promise<Daemon>[],
+    readonly ids: Ids[],
+    readonly healthDaemons: HealthDaemon[],
+  ) {}
+  /**
+   * Returns an empty new Daemons class with the provided config.
+   *
+   * Call .addDaemon() on the returned class to add a daemon.
+   *
+   * Daemons run in the order they are defined, with latter daemons being capable of
+   * depending on prior daemons
+   * @param config
+   * @returns
+   */
+  static of<Manifest extends SDKManifest>(config: {
+    effects: Effects
+    started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>
+    healthReceipts: HealthReceipt[]
+  }) {
+    return new Daemons<Manifest, never>(
+      config.effects,
+      config.started,
+      [],
+      [],
+      [],
+    )
+  }
+  /**
+   * Returns the complete list of daemons, including the one defined here
+   * @param id
+   * @param newDaemon
+   * @returns
+   */
+  addDaemon<Id extends string, Command extends string>(
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: DaemonsParams<Manifest, Ids, Command, Id>,
+  ) {
+    const daemonIndex = this.daemons.length
+    const daemon = Daemon.of()(this.effects, options.image, options.command, {
+      ...options,
+      mounts: options.mounts.build(),
+    })
+    const healthDaemon = new HealthDaemon(
+      daemon,
+      daemonIndex,
+      options.requires
+        .map((x) => this.ids.indexOf(id as any))
+        .filter((x) => x >= 0)
+        .map((id) => this.healthDaemons[id]),
+      id,
+      this.ids,
+      options.ready,
+      this.effects,
+    )
+    const daemons = this.daemons.concat(daemon)
+    const ids = [...this.ids, id] as (Ids | Id)[]
+    const healthDaemons = [...this.healthDaemons, healthDaemon]
+    return new Daemons<Manifest, Ids | Id>(
+      this.effects,
+      this.started,
+      daemons,
+      ids,
+      healthDaemons,
+    )
+  }
+
+  async build() {
+    this.updateMainHealth()
+    this.healthDaemons.forEach((x) =>
+      x.addWatcher(() => this.updateMainHealth()),
+    )
+    return {
+      term: async (options?: { signal?: Signals; timeout?: number }) => {
+        try {
+          await Promise.all(this.healthDaemons.map((x) => x.term(options)))
+        } finally {
+          this.effects.setMainStatus({ status: "stopped" })
+        }
+      },
+    }
+  }
+
+  private updateMainHealth() {
+    if (this.healthDaemons.every((x) => x.health.status === "success")) {
+      this.effects.setMainStatus({ status: "running" })
+    } else {
+      this.effects.setMainStatus({ status: "starting" })
+    }
+  }
+}

package/lib/mainFn/HealthDaemon.ts

@@ -0,0 +1,152 @@
+import { CheckResult } from "../health/checkFns"
+import { defaultTrigger } from "../trigger/defaultTrigger"
+import { Ready } from "./Daemons"
+import { Daemon } from "./Daemon"
+import { Effects } from "../types"
+
+const oncePromise = <T>() => {
+  let resolve: (value: T) => void
+  const promise = new Promise<T>((res) => {
+    resolve = res
+  })
+  return { resolve: resolve!, promise }
+}
+
+/**
+ * Wanted a structure that deals with controlling daemons by their health status
+ * States:
+ * -- Waiting for dependencies to be success
+ * -- Running: Daemon is running and the status is in the health
+ *
+ */
+export class HealthDaemon {
+  #health: CheckResult = { status: "starting", message: null }
+  #healthWatchers: Array<() => unknown> = []
+  #running = false
+  #hadSuccess = false
+  constructor(
+    readonly daemon: Promise<Daemon>,
+    readonly daemonIndex: number,
+    readonly dependencies: HealthDaemon[],
+    readonly id: string,
+    readonly ids: string[],
+    readonly ready: Ready,
+    readonly effects: Effects,
+  ) {
+    this.updateStatus()
+    this.dependencies.forEach((d) => d.addWatcher(() => this.updateStatus()))
+  }
+
+  /** Run after we want to do cleanup */
+  async term(termOptions?: {
+    signal?: NodeJS.Signals | undefined
+    timeout?: number | undefined
+  }) {
+    this.#healthWatchers = []
+    this.#running = false
+    this.#healthCheckCleanup?.()
+
+    await this.daemon.then((d) => d.stop(termOptions))
+  }
+
+  /** Want to add another notifier that the health might have changed */
+  addWatcher(watcher: () => unknown) {
+    this.#healthWatchers.push(watcher)
+  }
+
+  get health() {
+    return Object.freeze(this.#health)
+  }
+
+  private async changeRunning(newStatus: boolean) {
+    if (this.#running === newStatus) return
+
+    this.#running = newStatus
+
+    if (newStatus) {
+      ;(await this.daemon).start()
+      this.setupHealthCheck()
+    } else {
+      ;(await this.daemon).stop()
+      this.turnOffHealthCheck()
+
+      this.setHealth({ status: "starting", message: null })
+    }
+  }
+
+  #healthCheckCleanup: (() => void) | null = null
+  private turnOffHealthCheck() {
+    this.#healthCheckCleanup?.()
+  }
+  private async setupHealthCheck() {
+    if (this.#healthCheckCleanup) return
+    const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({
+      hadSuccess: this.#hadSuccess,
+      lastResult: this.#health.status,
+    }))
+
+    const { promise: status, resolve: setStatus } = oncePromise<{
+      done: true
+    }>()
+    new Promise(async () => {
+      for (
+        let res = await Promise.race([status, trigger.next()]);
+        !res.done;
+        res = await Promise.race([status, trigger.next()])
+      ) {
+        const response: CheckResult = await Promise.resolve(
+          this.ready.fn(),
+        ).catch((err) => {
+          console.error(err)
+          return {
+            status: "failure",
+            message: "message" in err ? err.message : String(err),
+          }
+        })
+        this.setHealth(response)
+        if (response.status === "success") {
+          this.#hadSuccess = true
+        }
+      }
+    }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
+
+    this.#healthCheckCleanup = () => {
+      setStatus({ done: true })
+      this.#healthCheckCleanup = null
+    }
+  }
+
+  private setHealth(health: CheckResult) {
+    this.#health = health
+    this.#healthWatchers.forEach((watcher) => watcher())
+    const display = this.ready.display
+    const status = health.status
+    if (!display) {
+      return
+    }
+    if (
+      status === "success" ||
+      status === "disabled" ||
+      status === "starting"
+    ) {
+      this.effects.setHealth({
+        result: status,
+        message: health.message,
+        id: display,
+        name: display,
+      })
+    } else {
+      this.effects.setHealth({
+        result: health.status,
+        message: health.message || "",
+        id: display,
+        name: display,
+      })
+    }
+  }
+
+  private async updateStatus() {
+    const healths = this.dependencies.map((d) => d.#health)
+    this.changeRunning(healths.every((x) => x.status === "success"))
+  }
+}

package/lib/mainFn/Mounts.ts

@@ -0,0 +1,126 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects } from "../types"
+import { MountOptions } from "../util/Overlay"
+
+type MountArray = { path: string; options: MountOptions }[]
+
+export class Mounts<Manifest extends SDKManifest> {
+  private constructor(
+    readonly volumes: {
+      id: Manifest["volumes"][number]
+      subpath: string | null
+      mountpoint: string
+      readonly: boolean
+    }[],
+    readonly assets: {
+      id: Manifest["assets"][number]
+      subpath: string | null
+      mountpoint: string
+    }[],
+    readonly dependencies: {
+      dependencyId: string
+      volumeId: string
+      subpath: string | null
+      mountpoint: string
+      readonly: boolean
+    }[],
+  ) {}
+
+  static of<Manifest extends SDKManifest>() {
+    return new Mounts<Manifest>([], [], [])
+  }
+
+  addVolume(
+    id: Manifest["volumes"][number],
+    subpath: string | null,
+    mountpoint: string,
+    readonly: boolean,
+  ) {
+    this.volumes.push({
+      id,
+      subpath,
+      mountpoint,
+      readonly,
+    })
+    return this
+  }
+
+  addAssets(
+    id: Manifest["assets"][number],
+    subpath: string | null,
+    mountpoint: string,
+  ) {
+    this.assets.push({
+      id,
+      subpath,
+      mountpoint,
+    })
+    return this
+  }
+
+  addDependency<DependencyManifest extends SDKManifest>(
+    dependencyId: keyof Manifest["dependencies"] & string,
+    volumeId: DependencyManifest["volumes"][number],
+    subpath: string | null,
+    mountpoint: string,
+    readonly: boolean,
+  ) {
+    this.dependencies.push({
+      dependencyId,
+      volumeId,
+      subpath,
+      mountpoint,
+      readonly,
+    })
+    return this
+  }
+
+  build(): MountArray {
+    const mountpoints = new Set()
+    for (let mountpoint of this.volumes
+      .map((v) => v.mountpoint)
+      .concat(this.assets.map((a) => a.mountpoint))
+      .concat(this.dependencies.map((d) => d.mountpoint))) {
+      if (mountpoints.has(mountpoint)) {
+        throw new Error(
+          `cannot mount more than once to mountpoint ${mountpoint}`,
+        )
+      }
+      mountpoints.add(mountpoint)
+    }
+    return ([] as MountArray)
+      .concat(
+        this.volumes.map((v) => ({
+          path: v.mountpoint,
+          options: {
+            type: "volume",
+            id: v.id,
+            subpath: v.subpath,
+            readonly: v.readonly,
+          },
+        })),
+      )
+      .concat(
+        this.assets.map((a) => ({
+          path: a.mountpoint,
+          options: {
+            type: "assets",
+            id: a.id,
+            subpath: a.subpath,
+          },
+        })),
+      )
+      .concat(
+        this.dependencies.map((d) => ({
+          path: d.mountpoint,
+          options: {
+            type: "pointer",
+            packageId: d.dependencyId,
+            volumeId: d.volumeId,
+            subpath: d.subpath,
+            readonly: d.readonly,
+          },
+        })),
+      )
+  }
+}

package/lib/mainFn/index.ts

@@ -0,0 +1,30 @@
+import { ExpectedExports } from "../types"
+import { Daemons } from "./Daemons"
+import "../interfaces/ServiceInterfaceBuilder"
+import "../interfaces/Origin"
+
+import "./Daemons"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { MainEffects } from "../StartSdk"
+
+/**
+ * Used to ensure that the main function is running with the valid proofs.
+ * We first do the folowing order of things
+ * 1. We get the interfaces
+ * 2. We setup all the commands to setup the system
+ * 3. We create the health checks
+ * 4. We setup the daemons init system
+ * @param fn
+ * @returns
+ */
+export const setupMain = <Manifest extends SDKManifest, Store>(
+  fn: (o: {
+    effects: MainEffects
+    started(onTerm: () => PromiseLike<void>): PromiseLike<void>
+  }) => Promise<Daemons<Manifest, any>>,
+): ExpectedExports.main => {
+  return async (options) => {
+    const result = await fn(options)
+    return result
+  }
+}