@start9labs/start-sdk 0.3.6-alpha1 → 0.3.6-alpha2

package/lib/util/Overlay.ts

@@ -0,0 +1,239 @@
+import * as fs from "fs/promises"
+import * as T from "../types"
+import * as cp from "child_process"
+import { promisify } from "util"
+import { Buffer } from "node:buffer"
+export const execFile = promisify(cp.execFile)
+const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
+export class Overlay {
+  private constructor(
+    readonly effects: T.Effects,
+    readonly imageId: T.ImageId,
+    readonly rootfs: string,
+    readonly guid: T.Guid,
+  ) {}
+  static async of(
+    effects: T.Effects,
+    image: { id: T.ImageId; sharedRun?: boolean },
+  ) {
+    const { id, sharedRun } = image
+    const [rootfs, guid] = await effects.createOverlayedImage({
+      imageId: id as string,
+    })
+
+    const shared = ["dev", "sys", "proc"]
+    if (!!sharedRun) {
+      shared.push("run")
+    }
+
+    for (const dirPart of shared) {
+      await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true })
+      await execFile("mount", [
+        "--rbind",
+        `/${dirPart}`,
+        `${rootfs}/${dirPart}`,
+      ])
+    }
+
+    return new Overlay(effects, id, rootfs, guid)
+  }
+
+  async mount(options: MountOptions, path: string): Promise<Overlay> {
+    path = path.startsWith("/")
+      ? `${this.rootfs}${path}`
+      : `${this.rootfs}/${path}`
+    if (options.type === "volume") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/volumes/${options.id}${subpath}`,
+        path,
+      ])
+    } else if (options.type === "assets") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/assets/${options.id}${subpath}`,
+        path,
+      ])
+    } else if (options.type === "pointer") {
+      await this.effects.mount({ location: path, target: options })
+    } else if (options.type === "backup") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/backup${subpath}`,
+        path,
+      ])
+    } else {
+      throw new Error(`unknown type ${(options as any).type}`)
+    }
+    return this
+  }
+
+  async destroy() {
+    const imageId = this.imageId
+    const guid = this.guid
+    await this.effects.destroyOverlayedImage({ guid })
+  }
+
+  async exec(
+    command: string[],
+    options?: CommandOptions,
+    timeoutMs: number | null = 30000,
+  ): Promise<{
+    exitCode: number | null
+    exitSignal: NodeJS.Signals | null
+    stdout: string | Buffer
+    stderr: string | Buffer
+  }> {
+    const imageMeta: T.ImageMetadata = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    const child = cp.spawn(
+      "start-cli",
+      [
+        "chroot",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      options || {},
+    )
+    const pid = child.pid
+    const stdout = { data: "" as string | Buffer }
+    const stderr = { data: "" as string | Buffer }
+    const appendData =
+      (appendTo: { data: string | Buffer }) =>
+      (chunk: string | Buffer | any) => {
+        if (typeof appendTo.data === "string" && typeof chunk === "string") {
+          appendTo.data += chunk
+        } else if (typeof chunk === "string" || chunk instanceof Buffer) {
+          appendTo.data = Buffer.concat([
+            Buffer.from(appendTo.data),
+            Buffer.from(chunk),
+          ])
+        } else {
+          console.error("received unexpected chunk", chunk)
+        }
+      }
+    return new Promise((resolve, reject) => {
+      child.on("error", reject)
+      if (timeoutMs !== null && pid) {
+        setTimeout(
+          () => execFile("pkill", ["-9", "-s", String(pid)]).catch((_) => {}),
+          timeoutMs,
+        )
+      }
+      child.stdout.on("data", appendData(stdout))
+      child.stderr.on("data", appendData(stderr))
+      child.on("exit", (code, signal) =>
+        resolve({
+          exitCode: code,
+          exitSignal: signal,
+          stdout: stdout.data,
+          stderr: stderr.data,
+        }),
+      )
+    })
+  }
+
+  async spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    const imageMeta: any = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    return cp.spawn(
+      "start-cli",
+      [
+        "chroot",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      options,
+    )
+  }
+}
+
+export type CommandOptions = {
+  env?: { [variable: string]: string }
+  cwd?: string
+  user?: string
+}
+
+export type MountOptions =
+  | MountOptionsVolume
+  | MountOptionsAssets
+  | MountOptionsPointer
+  | MountOptionsBackup
+
+export type MountOptionsVolume = {
+  type: "volume"
+  id: string
+  subpath: string | null
+  readonly: boolean
+}
+
+export type MountOptionsAssets = {
+  type: "assets"
+  id: string
+  subpath: string | null
+}
+
+export type MountOptionsPointer = {
+  type: "pointer"
+  packageId: string
+  volumeId: string
+  subpath: string | null
+  readonly: boolean
+}
+
+export type MountOptionsBackup = {
+  type: "backup"
+  subpath: string | null
+}

package/lib/util/deepEqual.ts

@@ -0,0 +1,19 @@
+import { object } from "ts-matches"
+
+export function deepEqual(...args: unknown[]) {
+  if (!object.test(args[args.length - 1])) return args[args.length - 1]
+  const objects = args.filter(object.test)
+  if (objects.length === 0) {
+    for (const x of args) if (x !== args[0]) return false
+    return true
+  }
+  if (objects.length !== args.length) return false
+  const allKeys = new Set(objects.flatMap((x) => Object.keys(x)))
+  for (const key of allKeys) {
+    for (const x of objects) {
+      if (!(key in x)) return false
+      if (!deepEqual((objects[0] as any)[key], (x as any)[key])) return false
+    }
+  }
+  return true
+}

package/lib/util/deepMerge.ts

@@ -0,0 +1,17 @@
+import { object } from "ts-matches"
+
+export function deepMerge(...args: unknown[]): unknown {
+  const lastItem = (args as any)[args.length - 1]
+  if (!object.test(lastItem)) return lastItem
+  const objects = args.filter(object.test).filter((x) => !Array.isArray(x))
+  if (objects.length === 0) return lastItem as any
+  if (objects.length === 1) objects.unshift({})
+  const allKeys = new Set(objects.flatMap((x) => Object.keys(x)))
+  for (const key of allKeys) {
+    const filteredValues = objects.flatMap((x) =>
+      key in x ? [(x as any)[key]] : [],
+    )
+    ;(objects as any)[0][key] = deepMerge(...filteredValues)
+  }
+  return objects[0] as any
+}

package/lib/util/fileHelper.ts

@@ -0,0 +1,150 @@
+import * as matches from "ts-matches"
+import * as YAML from "yaml"
+import * as TOML from "@iarna/toml"
+import _ from "lodash"
+import * as T from "../types"
+import * as fs from "node:fs/promises"
+
+const previousPath = /(.+?)\/([^/]*)$/
+
+/**
+ * Used in the get config and the set config exported functions.
+ * The idea is that we are going to be reading/ writing to a file, or multiple files. And then we use this tool
+ * to keep the same path on the read and write, and have methods for helping with structured data.
+ * And if we are not using a structured data, we can use the raw method which forces the construction of a BiMap
+ * ```ts
+        import {InputSpec} from './InputSpec.ts'
+        import {matches, T} from '../deps.ts';
+        const { object, string, number, boolean, arrayOf, array, anyOf, allOf } = matches
+        const someValidator = object({
+        data: string
+        })
+        const jsonFile = FileHelper.json({
+        path: 'data.json',
+        validator: someValidator,
+        volume: 'main'
+        })
+        const  tomlFile = FileHelper.toml({
+        path: 'data.toml',
+        validator: someValidator,
+        volume: 'main'
+        })
+        const rawFile = FileHelper.raw({
+        path: 'data.amazingSettings',
+        volume: 'main'
+        fromData(dataIn: Data): string {
+            return `myDatais ///- ${dataIn.data}`
+        },
+        toData(rawData: string): Data {
+        const [,data] = /myDatais \/\/\/- (.*)/.match(rawData)
+        return {data}
+        }
+        })
+
+        export const setConfig : T.ExpectedExports.setConfig= async (effects, config) => {
+        await  jsonFile.write({ data: 'here lies data'}, effects)
+        }
+
+        export const getConfig: T.ExpectedExports.getConfig = async (effects, config) => ({
+        spec: InputSpec,
+        config: nullIfEmpty({
+            ...jsonFile.get(effects)
+        })
+    ```
+ */
+export class FileHelper<A> {
+  protected constructor(
+    readonly path: string,
+    readonly writeData: (dataIn: A) => string,
+    readonly readData: (stringValue: string) => A,
+  ) {}
+  async write(data: A, effects: T.Effects) {
+    const parent = previousPath.exec(this.path)
+    if (parent) {
+      await fs.mkdir(parent[1], { recursive: true })
+    }
+
+    await fs.writeFile(this.path, this.writeData(data))
+  }
+  async read(effects: T.Effects) {
+    if (
+      !(await fs.access(this.path).then(
+        () => true,
+        () => false,
+      ))
+    ) {
+      return null
+    }
+    return this.readData(
+      await fs.readFile(this.path).then((data) => data.toString("utf-8")),
+    )
+  }
+
+  async merge(data: A, effects: T.Effects) {
+    const fileData = (await this.read(effects).catch(() => ({}))) || {}
+    const mergeData = _.merge({}, fileData, data)
+    return await this.write(mergeData, effects)
+  }
+  /**
+   * Create a File Helper for an arbitrary file type.
+   *
+   * Provide custom functions for translating data to the file format and visa versa.
+   */
+  static raw<A>(
+    path: string,
+    toFile: (dataIn: A) => string,
+    fromFile: (rawData: string) => A,
+  ) {
+    return new FileHelper<A>(path, toFile, fromFile)
+  }
+  /**
+   * Create a File Helper for a .json file
+   */
+  static json<A>(path: string, shape: matches.Validator<unknown, A>) {
+    return new FileHelper<A>(
+      path,
+      (inData) => {
+        return JSON.stringify(inData, null, 2)
+      },
+      (inString) => {
+        return shape.unsafeCast(JSON.parse(inString))
+      },
+    )
+  }
+  /**
+   * Create a File Helper for a .toml file
+   */
+  static toml<A extends Record<string, unknown>>(
+    path: string,
+    shape: matches.Validator<unknown, A>,
+  ) {
+    return new FileHelper<A>(
+      path,
+      (inData) => {
+        return TOML.stringify(inData as any)
+      },
+      (inString) => {
+        return shape.unsafeCast(TOML.parse(inString))
+      },
+    )
+  }
+  /**
+   * Create a File Helper for a .yaml file
+   */
+  static yaml<A extends Record<string, unknown>>(
+    path: string,
+    shape: matches.Validator<unknown, A>,
+  ) {
+    return new FileHelper<A>(
+      path,
+      (inData) => {
+        return YAML.stringify(inData, null, 2)
+      },
+      (inString) => {
+        return shape.unsafeCast(YAML.parse(inString))
+      },
+    )
+  }
+}
+
+export default FileHelper

package/lib/util/getDefaultString.ts

@@ -0,0 +1,10 @@
+import { DefaultString } from "../config/configTypes"
+import { getRandomString } from "./getRandomString"
+
+export function getDefaultString(defaultSpec: DefaultString): string {
+  if (typeof defaultSpec === "string") {
+    return defaultSpec
+  } else {
+    return getRandomString(defaultSpec)
+  }
+}

package/lib/util/getRandomCharInSet.ts

@@ -0,0 +1,98 @@
+// a,g,h,A-Z,,,,-
+
+import * as crypto from "crypto"
+export function getRandomCharInSet(charset: string): string {
+  const set = stringToCharSet(charset)
+  let charIdx = crypto.randomInt(0, set.len)
+  for (let range of set.ranges) {
+    if (range.len > charIdx) {
+      return String.fromCharCode(range.start.charCodeAt(0) + charIdx)
+    }
+    charIdx -= range.len
+  }
+  throw new Error("unreachable")
+}
+function stringToCharSet(charset: string): CharSet {
+  let set: CharSet = { ranges: [], len: 0 }
+  let start: string | null = null
+  let end: string | null = null
+  let in_range = false
+  for (let char of charset) {
+    switch (char) {
+      case ",":
+        if (start !== null && end !== null) {
+          if (start!.charCodeAt(0) > end!.charCodeAt(0)) {
+            throw new Error("start > end of charset")
+          }
+          const len = end.charCodeAt(0) - start.charCodeAt(0) + 1
+          set.ranges.push({
+            start,
+            end,
+            len,
+          })
+          set.len += len
+          start = null
+          end = null
+          in_range = false
+        } else if (start !== null && !in_range) {
+          set.len += 1
+          set.ranges.push({ start, end: start, len: 1 })
+          start = null
+        } else if (start !== null && in_range) {
+          end = ","
+        } else if (start === null && end === null && !in_range) {
+          start = ","
+        } else {
+          throw new Error('unexpected ","')
+        }
+        break
+      case "-":
+        if (start === null) {
+          start = "-"
+        } else if (!in_range) {
+          in_range = true
+        } else if (in_range && end === null) {
+          end = "-"
+        } else {
+          throw new Error('unexpected "-"')
+        }
+        break
+      default:
+        if (start === null) {
+          start = char
+        } else if (in_range && end === null) {
+          end = char
+        } else {
+          throw new Error(`unexpected "${char}"`)
+        }
+    }
+  }
+  if (start !== null && end !== null) {
+    if (start!.charCodeAt(0) > end!.charCodeAt(0)) {
+      throw new Error("start > end of charset")
+    }
+    const len = end.charCodeAt(0) - start.charCodeAt(0) + 1
+    set.ranges.push({
+      start,
+      end,
+      len,
+    })
+    set.len += len
+  } else if (start !== null) {
+    set.len += 1
+    set.ranges.push({
+      start,
+      end: start,
+      len: 1,
+    })
+  }
+  return set
+}
+type CharSet = {
+  ranges: {
+    start: string
+    end: string
+    len: number
+  }[]
+  len: number
+}

package/lib/util/getRandomString.ts

@@ -0,0 +1,11 @@
+import { RandomString } from "../config/configTypes"
+import { getRandomCharInSet } from "./getRandomCharInSet"
+
+export function getRandomString(generator: RandomString): string {
+  let s = ""
+  for (let i = 0; i < generator.len; i++) {
+    s = s + getRandomCharInSet(generator.charset)
+  }
+
+  return s
+}