@highstate/backend 0.7.2 → 0.7.3

package/src/terminal/docker.ts

@@ -0,0 +1,90 @@
+import type { TerminalRunOptions, TerminalBackend } from "./shared"
+import type { Logger } from "pino"
+import { Readable } from "node:stream"
+import { tmpdir } from "node:os"
+import { resolve } from "node:path"
+import { mkdir, writeFile } from "node:fs/promises"
+import { z } from "zod"
+import spawn from "nano-spawn"
+import { runScript } from "./run.sh"
+
+export const dockerTerminalBackendConfig = z.object({
+  HIGHSTATE_BACKEND_TERMINAL_DOCKER_BINARY: z.string().default("docker"),
+  HIGHSTATE_BACKEND_TERMINAL_DOCKER_USE_SUDO: z.coerce.boolean().default(false),
+  HIGHSTATE_BACKEND_TERMINAL_DOCKER_HOST: z.string().optional(),
+})
+
+export class DockerTerminalBackend implements TerminalBackend {
+  constructor(
+    private readonly binary: string,
+    private readonly useSudo: boolean,
+    private readonly host: string | undefined,
+    private readonly logger: Logger,
+  ) {}
+
+  async run({ factory, stdin, stdout, screenSize, signal }: TerminalRunOptions): Promise<void> {
+    const hsTempDir = resolve(tmpdir(), "highstate")
+    await mkdir(hsTempDir, { recursive: true })
+
+    const runScriptPath = resolve(hsTempDir, "run.sh")
+    await writeFile(runScriptPath, runScript, { mode: 0o755 })
+
+    const args = [
+      "run",
+      "-i",
+      "-v",
+      `${runScriptPath}:/run.sh:ro`,
+      factory.image,
+      "/bin/bash",
+      "/run.sh",
+    ]
+
+    const initData = {
+      command: factory.command,
+      cwd: factory.cwd,
+      env: {
+        ...factory.env,
+        TERM: "xterm-256color",
+      },
+      files: factory.files ?? {},
+      screenSize,
+    }
+
+    const initDataStream = Readable.from(JSON.stringify(initData) + "\n")
+
+    if (this.useSudo) {
+      args.unshift(this.binary)
+    }
+
+    const process = spawn(this.useSudo ? "sudo" : this.binary, args, {
+      env: {
+        DOCKER_HOST: this.host,
+      },
+      signal,
+    })
+
+    const childProcess = await process.nodeChildProcess
+
+    initDataStream.pipe(childProcess.stdin!, { end: false })
+    initDataStream.on("end", () => stdin.pipe(childProcess.stdin!))
+
+    childProcess.stdout!.pipe(stdout)
+    childProcess.stderr!.pipe(stdout)
+
+    this.logger.info({ pid: childProcess.pid }, "process started")
+
+    await process
+  }
+
+  static create(
+    config: z.infer<typeof dockerTerminalBackendConfig>,
+    logger: Logger,
+  ): DockerTerminalBackend {
+    return new DockerTerminalBackend(
+      config.HIGHSTATE_BACKEND_TERMINAL_DOCKER_BINARY,
+      config.HIGHSTATE_BACKEND_TERMINAL_DOCKER_USE_SUDO,
+      config.HIGHSTATE_BACKEND_TERMINAL_DOCKER_HOST,
+      logger.child({ backend: "TerminalBackend", service: "DockerTerminalBackend" }),
+    )
+  }
+}

package/src/terminal/factory.ts

@@ -0,0 +1,20 @@
+import type { Logger } from "pino"
+import type { TerminalBackend } from "./shared"
+import { z } from "zod"
+import { DockerTerminalBackend, dockerTerminalBackendConfig } from "./docker"
+
+export const terminalBackendConfig = z.object({
+  HIGHSTATE_BACKEND_TERMINAL_TYPE: z.enum(["docker"]).default("docker"),
+  ...dockerTerminalBackendConfig.shape,
+})
+
+export function createTerminalBackend(
+  config: z.infer<typeof terminalBackendConfig>,
+  logger: Logger,
+): TerminalBackend {
+  switch (config.HIGHSTATE_BACKEND_TERMINAL_TYPE) {
+    case "docker": {
+      return DockerTerminalBackend.create(config, logger)
+    }
+  }
+}

package/src/terminal/index.ts

@@ -0,0 +1,3 @@
+export * from "./shared"
+export * from "./factory"
+export * from "./manager"

package/src/terminal/manager.ts

@@ -0,0 +1,330 @@
+import type { ScreenSize, TerminalBackend } from "./shared"
+import type { InstanceTerminal, TerminalSession } from "../shared"
+import type { Logger } from "pino"
+import type { StateBackend, TerminalHistoryEntry } from "../state"
+import type { RunnerBackend } from "../runner"
+import { PassThrough, type Stream, type Writable } from "node:stream"
+import { EventEmitter, on } from "node:events"
+import { parseInstanceId } from "@highstate/contract"
+import { nanoid } from "nanoid"
+import { createAsyncBatcher, isAbortErrorLike } from "../common"
+
+type ManagedTerminal = {
+  readonly session: TerminalSession
+  readonly abortController: AbortController
+  readonly stdin: PassThrough
+  readonly stdout: PassThrough
+
+  refCount: number
+  started: boolean
+  start(screenSize: ScreenSize): void
+}
+
+type TerminalEvents = {
+  [sessionId: string]: [terminal: TerminalSession]
+}
+
+const notAttachedTerminalLifetime = 5 * 60 * 1000 // 5 minutes
+
+export class TerminalManager {
+  private readonly managedTerminals = new Map<string, ManagedTerminal>()
+  private readonly existingSessions = new Map<string, TerminalSession>()
+  private readonly terminalEE = new EventEmitter<TerminalEvents>()
+
+  private constructor(
+    private readonly terminalBackend: TerminalBackend,
+    private readonly stateBackend: StateBackend,
+    private readonly runnerBackend: RunnerBackend,
+    private readonly logger: Logger,
+  ) {
+    void this.markFinishedSessions()
+  }
+
+  public isSessionActive(sessionId: string): boolean {
+    return this.managedTerminals.has(sessionId)
+  }
+
+  public async *watchSession(
+    projectId: string,
+    instanceId: string,
+    sessionId: string,
+    signal?: AbortSignal,
+  ): AsyncIterable<TerminalSession> {
+    const managedTerminal = this.managedTerminals.get(sessionId)
+
+    // if terminal session is not active, just fetch it, return and stop watching
+    if (!managedTerminal) {
+      const session = await this.stateBackend.getTerminalSession(projectId, instanceId, sessionId)
+
+      if (!session) {
+        throw new Error(`Terminal session "${sessionId}" not found`)
+      }
+
+      yield session
+      return
+    }
+
+    yield managedTerminal.session
+
+    for await (const [terminalSession] of on(this.terminalEE, sessionId, {
+      signal,
+    }) as AsyncIterable<[TerminalSession]>) {
+      yield terminalSession
+
+      if (terminalSession.finishedAt) {
+        // terminal session is finished, stop watching
+        return
+      }
+    }
+  }
+
+  public async createSession(
+    projectId: string,
+    instanceId: string,
+    terminalName: string,
+  ): Promise<TerminalSession> {
+    const [instanceType, instanceName] = parseInstanceId(instanceId)
+
+    const factory = await this.runnerBackend.getTerminalFactory(
+      { projectId, instanceType, instanceName },
+      terminalName,
+    )
+
+    if (!factory) {
+      throw new Error(
+        `Terminal factory for instance "${instanceId}" with name "${terminalName}" not found`,
+      )
+    }
+
+    const terminalSession: TerminalSession = {
+      id: nanoid(),
+      projectId,
+      instanceId,
+      terminalName,
+      terminalTitle: factory.title,
+      createdAt: new Date(),
+    }
+
+    await this.stateBackend.putTerminalSession(projectId, instanceId, terminalSession)
+    this.logger.info({ msg: "terminal session created", id: terminalSession.id })
+
+    this.createManagedTerminal(factory, terminalSession)
+
+    return terminalSession
+  }
+
+  public async getOrCreateSession(
+    projectId: string,
+    instanceId: string,
+    terminalName: string,
+  ): Promise<TerminalSession> {
+    const existingSession = this.existingSessions.get(`${projectId}:${instanceId}.${terminalName}`)
+
+    if (existingSession) {
+      return existingSession
+    }
+
+    return await this.createSession(projectId, instanceId, terminalName)
+  }
+
+  public close(sessionId: string): void {
+    this.logger.info({ msg: "closing terminal session", id: sessionId })
+
+    const managedTerminal = this.managedTerminals.get(sessionId)
+
+    if (managedTerminal) {
+      managedTerminal.abortController.abort()
+    }
+  }
+
+  public attach(
+    sessionId: string,
+    stdin: Stream,
+    stdout: Writable,
+    screenSize: ScreenSize,
+    signal: AbortSignal,
+  ): void {
+    const terminal = this.managedTerminals.get(sessionId)
+    if (!terminal) {
+      throw new Error(`Terminal session "${sessionId}" not found, check if it's still active`)
+    }
+
+    const handleStdin = (data: unknown) => {
+      terminal.stdin.write(data)
+    }
+
+    const handleStdout = (data: unknown) => {
+      stdout.write(data)
+    }
+
+    terminal.stdout.on("data", handleStdout)
+    stdin.on("data", handleStdin)
+
+    terminal.refCount += 1
+
+    this.logger.info(
+      "terminal attached (sessionId: %s, refCount: %d)",
+      sessionId,
+      terminal.refCount,
+    )
+
+    signal.addEventListener("abort", () => {
+      terminal.refCount -= 1
+      terminal.stdout.off("data", handleStdout)
+      stdin.off("data", handleStdin)
+
+      this.logger.info(
+        "terminal detached (sessionId: %s, refCount: %d)",
+        sessionId,
+        terminal.refCount,
+      )
+    })
+
+    if (!terminal.started) {
+      terminal.start(screenSize)
+      terminal.started = true
+    }
+  }
+
+  private async updateActiveTerminalSessions(): Promise<void> {
+    await this.stateBackend.putActiveTerminalSessions(
+      Array.from(this.managedTerminals.values()).map(t => t.session),
+    )
+  }
+
+  private createManagedTerminal(
+    factory: InstanceTerminal,
+    terminalSession: TerminalSession,
+  ): ManagedTerminal {
+    const managedTerminal: ManagedTerminal = {
+      session: terminalSession,
+      abortController: new AbortController(),
+      stdin: new PassThrough(),
+      stdout: new PassThrough(),
+
+      refCount: 0,
+      started: false,
+
+      start: (screenSize: ScreenSize) => {
+        void this.terminalBackend
+          .run({
+            factory,
+            stdin: managedTerminal.stdin,
+            stdout: managedTerminal.stdout,
+            screenSize,
+            signal: managedTerminal.abortController.signal,
+          })
+          .catch(error => {
+            if (isAbortErrorLike(error)) {
+              return
+            }
+
+            this.logger.error({
+              msg: "managed terminal failed",
+              id: managedTerminal.session.id,
+              error: error as unknown,
+            })
+          })
+          .finally(() => {
+            this.logger.info({ msg: "managed terminal closed", id: managedTerminal.session.id })
+            this.managedTerminals.delete(managedTerminal.session.id)
+            this.existingSessions.delete(
+              `${managedTerminal.session.projectId}:${managedTerminal.session.instanceId}.${managedTerminal.session.terminalName}`,
+            )
+
+            managedTerminal.session.finishedAt = new Date()
+            this.terminalEE.emit(managedTerminal.session.id, managedTerminal.session)
+
+            void this.stateBackend.putTerminalSession(
+              managedTerminal.session.projectId,
+              managedTerminal.session.instanceId,
+              managedTerminal.session,
+            )
+
+            void this.updateActiveTerminalSessions()
+          })
+
+        setTimeout(
+          () => this.closeTerminalIfNotAttached(managedTerminal),
+          notAttachedTerminalLifetime,
+        )
+
+        this.logger.info({ msg: "managed terminal created", id: managedTerminal.session.id })
+      },
+    }
+
+    managedTerminal.stdout.on("data", data => {
+      const entry = String(data)
+
+      void this.persistHistory.call([terminalSession.id, entry])
+    })
+
+    this.managedTerminals.set(managedTerminal.session.id, managedTerminal)
+    this.existingSessions.set(
+      `${managedTerminal.session.projectId}:${managedTerminal.session.instanceId}.${managedTerminal.session.terminalName}`,
+      managedTerminal.session,
+    )
+
+    void this.updateActiveTerminalSessions()
+
+    return managedTerminal
+  }
+
+  private closeTerminalIfNotAttached(terminal: ManagedTerminal) {
+    if (!this.managedTerminals.has(terminal.session.id)) {
+      // Already closed
+      return
+    }
+
+    if (terminal.refCount <= 0) {
+      this.logger.info({
+        msg: "terminal not attached for too long, closing",
+        id: terminal.session.id,
+      })
+
+      terminal.abortController.abort()
+      this.managedTerminals.delete(terminal.session.id)
+      this.existingSessions.delete(
+        `${terminal.session.projectId}:${terminal.session.instanceId}.${terminal.session.terminalName}`,
+      )
+      return
+    }
+
+    setTimeout(() => this.closeTerminalIfNotAttached(terminal), notAttachedTerminalLifetime)
+  }
+
+  static create(
+    terminalBackend: TerminalBackend,
+    stateBackend: StateBackend,
+    runnerBackend: RunnerBackend,
+    logger: Logger,
+  ): TerminalManager {
+    return new TerminalManager(
+      terminalBackend,
+      stateBackend,
+      runnerBackend,
+      logger.child({ service: "TerminalManager" }),
+    )
+  }
+
+  private persistHistory = createAsyncBatcher(async (entries: TerminalHistoryEntry[]) => {
+    this.logger.trace({ msg: "persisting history entries", count: entries.length })
+
+    // TODO: buffer entries and save them as lines
+
+    await this.stateBackend.appendTerminalSessionHistory(entries)
+  })
+
+  private async markFinishedSessions(): Promise<void> {
+    const sessions = await this.stateBackend.getActiveTerminalSessions()
+
+    for (const session of sessions) {
+      await this.stateBackend.putTerminalSession(session.projectId, session.instanceId, {
+        ...session,
+        finishedAt: new Date(),
+      })
+    }
+
+    await this.updateActiveTerminalSessions()
+  }
+}

package/src/terminal/run.sh.ts

@@ -0,0 +1,37 @@
+export const runScript = `set -e -o pipefail
+read -r data
+
+# Extract env and files as key-value pairs, and command as an array
+envKeys=($(jq -r '.env | keys[]' <<<"$data"))
+filesKeys=($(jq -r '.files | keys[]' <<<"$data"))
+commandArr=($(jq -r '.command[]' <<<"$data"))
+cols=$(jq -r '.screenSize.cols' <<<"$data")
+rows=$(jq -r '.screenSize.rows' <<<"$data")
+
+# Set environment variables
+for key in "\${envKeys[@]}"; do
+  value=$(jq -r ".env[\\"$key\\"]" <<<"$data")
+  export "$key=$value"
+done
+
+# Create files
+for key in "\${filesKeys[@]}"; do
+  isBinary=$(jq -r ".files[\\"$key\\"].isBinary // false" <<<"$data")
+  content=$(jq -r ".files[\\"$key\\"].content" <<<"$data")
+  mode=$(jq -r ".files[\\"$key\\"].mode // 0" <<<"$data")
+
+  if [ "$isBinary" = "true" ]; then
+    echo "$content" | base64 -d > "$key"
+  else
+    echo "$content" > "$key"
+  fi
+
+  if [ "$mode" -ne 0 ]; then
+    chmod $(printf "%o" "$mode") "$key"
+  fi
+done
+
+# Execute the command, keeping stdin/stdout open and spawnin a new TTY
+cmd=$(printf "%q " "\${commandArr[@]}")
+exec script -q -c "stty cols $cols rows $rows; $cmd" /dev/null
+`

package/src/terminal/shared.ts

@@ -0,0 +1,50 @@
+import type { Stream } from "node:stream"
+import type { InstanceTerminal } from "../shared"
+
+export type ScreenSize = {
+  /**
+   * The number of columns.
+   */
+  cols: number
+
+  /**
+   * The number of rows.
+   */
+  rows: number
+}
+
+export type TerminalRunOptions = {
+  /**
+   * The factory to use.
+   */
+  factory: InstanceTerminal
+
+  /**
+   * The input stream.
+   */
+  stdin: Stream
+
+  /**
+   * The output stream.
+   */
+  stdout: NodeJS.WritableStream
+
+  /**
+   * The size of the screen to set before running the terminal.
+   */
+  screenSize: ScreenSize
+
+  /**
+   * The signal to abort the terminal.
+   */
+  signal?: AbortSignal
+}
+
+export interface TerminalBackend {
+  /**
+   * Creates a new terminal and runs it.
+   *
+   * @param options The options.
+   */
+  run(options: TerminalRunOptions): Promise<void>
+}

package/src/workspace/abstractions.ts

@@ -0,0 +1,41 @@
+export interface WorkspaceBackend {
+  /**
+   * Gets the layout of the workspace.
+   */
+  getWorkspaceLayout(): Promise<unknown>
+
+  /**
+   * Sets the layout of the workspace.
+   */
+  setWorkspaceLayout(layout: unknown): Promise<void>
+
+  /**
+   * Gets the viewport of the project.
+   *
+   * @param projectId The ID of the project.
+   */
+  getProjectViewport(projectId: string): Promise<unknown>
+
+  /**
+   * Sets the viewport of the project.
+   *
+   * @param projectId The ID of the project.
+   */
+  setProjectViewport(projectId: string, viewport: unknown): Promise<void>
+
+  /**
+   * Gets the viewport of the composite instance.
+   *
+   * @param projectId The ID of the project.
+   * @param instanceId The ID of the instance.
+   */
+  getInstanceViewport(projectId: string, instanceId: string): Promise<unknown>
+
+  /**
+   * Sets the viewport of the composite instance.
+   *
+   * @param projectId The ID of the project.
+   * @param instanceId The ID of the instance.
+   */
+  setInstanceViewport(projectId: string, instanceId: string, viewport: unknown): Promise<void>
+}

package/src/workspace/factory.ts

@@ -0,0 +1,14 @@
+import type { WorkspaceBackend } from "./abstractions"
+import { z } from "zod"
+import { LocalWorkspaceBackend, localWorkspaceBackendConfig } from "./local"
+
+export const workspaceBackendConfig = z.object({
+  HIGHSTATE_BACKEND_WORKSPACE_TYPE: z.enum(["local"]).default("local"),
+  ...localWorkspaceBackendConfig.shape,
+})
+
+export function createWorkspaceBackend(
+  config: z.infer<typeof workspaceBackendConfig>,
+): Promise<WorkspaceBackend> {
+  return LocalWorkspaceBackend.create(config)
+}

package/src/workspace/index.ts

@@ -0,0 +1,2 @@
+export * from "./abstractions"
+export * from "./factory"

package/src/workspace/local.ts

@@ -0,0 +1,54 @@
+import type { ClassicLevel } from "classic-level"
+import type { WorkspaceBackend } from "./abstractions"
+import { homedir } from "node:os"
+import { resolve } from "node:path"
+import { z } from "zod"
+
+export const localWorkspaceBackendConfig = z.object({
+  HIGHSTATE_BACKEND_WORKSPACE_LOCAL_DIR: z.string().optional(),
+})
+
+export class LocalWorkspaceBackend implements WorkspaceBackend {
+  constructor(private readonly db: ClassicLevel<string, unknown>) {}
+
+  async getWorkspaceLayout(): Promise<unknown> {
+    return await this.db.get("layout")
+  }
+
+  async setWorkspaceLayout(layout: unknown): Promise<void> {
+    await this.db.put("layout", layout)
+  }
+
+  async getProjectViewport(projectId: string): Promise<unknown> {
+    return await this.db.get(`viewports/projects/${projectId}`)
+  }
+
+  async setProjectViewport(projectId: string, viewport: unknown): Promise<void> {
+    await this.db.put(`viewports/projects/${projectId}`, viewport)
+  }
+
+  async getInstanceViewport(projectId: string, instanceId: string): Promise<unknown> {
+    return await this.db.get(`viewports/instances/${projectId}/${instanceId}`)
+  }
+
+  async setInstanceViewport(
+    projectId: string,
+    instanceId: string,
+    viewport: unknown,
+  ): Promise<void> {
+    await this.db.put(`viewports/instances/${projectId}/${instanceId}`, viewport)
+  }
+
+  static async create(config: z.infer<typeof localWorkspaceBackendConfig>) {
+    let location = config.HIGHSTATE_BACKEND_WORKSPACE_LOCAL_DIR
+    if (!location) {
+      const home = homedir()
+      location = resolve(home, ".highstate/workspace")
+    }
+
+    const { ClassicLevel } = await import("classic-level")
+    const db = new ClassicLevel<string, unknown>(location, { valueEncoding: "json" })
+
+    return new LocalWorkspaceBackend(db)
+  }
+}