@highstate/cli 0.7.1 → 0.7.3

package/package.json

@@ -1,10 +1,12 @@
 {
   "name": "@highstate/cli",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "description": "The CLI for the Highstate project.",
   "type": "module",
   "files": [
-    "dist"
+    "assets",
+    "dist",
+    "src"
   ],
   "bin": {
     "highstate": "dist/main.js"
@@ -13,19 +15,27 @@
     "access": "public"
   },
   "scripts": {
-    "build": "pkgroll --tsconfig=tsconfig.build.json"
+    "build": "pkgroll --clean --tsconfig=tsconfig.build.json"
   },
   "dependencies": {
-    "@highstate/backend": "^0.7.1",
-    "consola": "^3.4.0"
-  },
-  "devDependencies": {
     "clipanion": "^4.0.0-rc.4",
+    "consola": "^3.4.0",
+    "crypto-hash": "^3.1.0",
     "get-port-please": "^3.1.2",
+    "import-meta-resolve": "^4.1.0",
     "nypm": "^0.4.1",
+    "oxc-parser": "^0.63.0",
+    "oxc-walker": "^0.2.4",
     "pino": "^9.6.0",
     "pkg-types": "^1.2.1",
+    "remeda": "^2.21.2",
+    "tsup": "^8.4.0"
+  },
+  "devDependencies": {
     "pkgroll": "^2.5.1"
   },
-  "gitHead": "76c38ce5dbf7a710cf0e6339f52e86358537a99a"
+  "peerDependencies": {
+    "@highstate/backend": "workspace:^0.7.2"
+  },
+  "gitHead": "5cf7cec27262c8fa1d96f6478833b94841459d64"
 }

package/src/commands/build.ts

@@ -0,0 +1,94 @@
+import { Command, Option } from "clipanion"
+import { readPackageJSON } from "pkg-types"
+import { mapKeys, mapValues, pipe } from "remeda"
+import { build } from "tsup"
+import { logger, schemaTransformerPlugin, SourceHashCalculator } from "../shared"
+
+export class BuildCommand extends Command {
+  static paths = [["build"]]
+
+  static usage = Command.Usage({
+    category: "Builder",
+    description: "Builds the Highstate library or unit package.",
+  })
+
+  watch = Option.Boolean("--watch", false)
+  library = Option.Boolean("--library", false)
+
+  async execute(): Promise<void> {
+    const packageJson = await readPackageJSON()
+    const exports = packageJson.exports
+
+    if (!exports) {
+      logger.warn("no exports found in package.json")
+      return
+    }
+
+    if (typeof exports !== "object" || Array.isArray(exports)) {
+      throw new Error("Exports field in package.json must be an object")
+    }
+
+    const entry = pipe(
+      exports,
+      mapValues((value, key) => {
+        let distPath
+
+        if (typeof value === "string") {
+          distPath = value
+        } else if (typeof value === "object" && !Array.isArray(value)) {
+          if (!value.default) {
+            throw new Error(`Export "${key}" must have a default field in package.json`)
+          }
+
+          if (typeof value.default !== "string") {
+            throw new Error(`Export "${key}" default field must be a string in package.json`)
+          }
+
+          distPath = value.default
+        } else {
+          throw new Error(`Export "${key}" must be a string or an object in package.json`)
+        }
+
+        if (!distPath.startsWith("./dist/")) {
+          throw new Error(
+            `The default value of export "${key}" must start with "./dist/" in package.json, got "${distPath}"`,
+          )
+        }
+
+        if (!distPath.endsWith(".js")) {
+          throw new Error(
+            `The default value of export "${key}" must end with ".js" in package.json, got "${distPath}"`,
+          )
+        }
+
+        const targetName = distPath.slice(7).slice(0, -3)
+
+        return {
+          entryPoint: `./src/${targetName}.ts`,
+          targetName,
+          distPath,
+        }
+      }),
+      mapKeys((_, value) => value.targetName),
+    )
+
+    await build({
+      entry: mapValues(entry, value => value.entryPoint),
+      outDir: "dist",
+      watch: this.watch,
+      sourcemap: true,
+      clean: true,
+      format: "esm",
+      target: "esnext",
+      external: ["@pulumi/pulumi"],
+      esbuildPlugins: this.library ? [schemaTransformerPlugin] : [],
+    })
+
+    const upToDatePackageJson = await readPackageJSON()
+
+    const sourceHashCalculator = new SourceHashCalculator(upToDatePackageJson, logger)
+    const distPaths = Object.values(entry).map(value => value.distPath)
+
+    await sourceHashCalculator.writeHighstateManifest("./dist", distPaths)
+  }
+}

package/src/commands/designer.ts

@@ -0,0 +1,73 @@
+import { Command, UsageError } from "clipanion"
+import { readPackageJSON } from "pkg-types"
+import { addDevDependency } from "nypm"
+import { consola } from "consola"
+import { colorize } from "consola/utils"
+import { getPort } from "get-port-please"
+import { getBackendServices, logger } from "../shared"
+
+export class DesignerCommand extends Command {
+  static paths = [["designer"]]
+
+  static usage = Command.Usage({
+    category: "Designer",
+    description: "Starts the Highstate designer in the current project.",
+  })
+
+  async execute(): Promise<void> {
+    const packageJson = await readPackageJSON()
+    if (!packageJson.devDependencies?.["@highstate/cli"]) {
+      throw new UsageError(
+        "This project is not a Highstate project.\n@highstate/cli must be installed as a devDependency.",
+      )
+    }
+
+    if (!packageJson.devDependencies?.["@highstate/designer"]) {
+      logger.info("Installing @highstate/designer...")
+
+      await addDevDependency(["@highstate/designer", "classic-level"])
+    }
+
+    logger.info("starting highstate designer...")
+
+    await getBackendServices()
+
+    const oldConsoleLog = console.log
+
+    const port = await getPort()
+
+    process.env.NITRO_PORT = port.toString()
+    process.env.NITRO_HOST = "0.0.0.0"
+
+    await new Promise<void>(resolve => {
+      console.log = (message: string) => {
+        if (message.startsWith("Listening on")) {
+          resolve()
+        }
+      }
+
+      const path = "@highstate/designer/.output/server/index.mjs"
+      void import(path)
+    })
+
+    console.log = oldConsoleLog
+
+    consola.log(
+      [
+        "\n  ",
+        colorize("bold", colorize("cyanBright", "Highstate Designer")),
+        "\n  ",
+        colorize("greenBright", "➜ Local:  "),
+        colorize("underline", colorize("cyanBright", `http://localhost:${port}`)),
+        "\n",
+      ].join(""),
+    )
+
+    process.on("SIGINT", () => {
+      process.stdout.write("\r")
+      consola.info("shutting down highstate designer...")
+
+      setTimeout(() => process.exit(0), 1000)
+    })
+  }
+}

package/src/main.ts

@@ -0,0 +1,17 @@
+import { Builtins, Cli } from "clipanion"
+import { version } from "../package.json"
+import { DesignerCommand } from "./commands/designer"
+import { BuildCommand } from "./commands/build"
+
+const cli = new Cli({
+  binaryName: "highstate",
+  binaryLabel: "Highstate",
+  binaryVersion: version,
+})
+
+cli.register(BuildCommand)
+cli.register(DesignerCommand)
+cli.register(Builtins.HelpCommand)
+cli.register(Builtins.VersionCommand)
+
+await cli.runExit(process.argv.slice(2))

package/src/shared/index.ts

@@ -0,0 +1,4 @@
+export * from "./services"
+export * from "./logger"
+export * from "./schema-transformer"
+export * from "./source-hash-calculator"

package/src/shared/logger.ts

@@ -0,0 +1,54 @@
+import { PassThrough } from "node:stream"
+import pino, { levels } from "pino"
+import { consola, LogLevels } from "consola"
+
+export const logger = pino(
+  {
+    name: "highstate-cli",
+    level: process.env.LOG_LEVEL ?? "info",
+  },
+  createConsolaStream(),
+)
+
+consola.level = LogLevels[(process.env.LOG_LEVEL as keyof typeof LogLevels) ?? "info"]
+
+function createConsolaStream() {
+  const stream = new PassThrough()
+
+  stream.on("data", data => {
+    const { level, msg, error } = JSON.parse(String(data)) as {
+      msg: string
+      level: number
+      error?: unknown
+    }
+
+    const levelLabel = levels.labels[level]
+
+    switch (levelLabel) {
+      case "info":
+        consola.info(msg)
+        break
+      case "warn":
+        consola.warn(msg)
+        break
+      case "error":
+        if (error) {
+          consola.error(msg, error)
+        } else {
+          consola.error(msg)
+        }
+        break
+      case "debug":
+        consola.debug(msg)
+        break
+      case "fatal":
+        consola.fatal(msg)
+        break
+      case "trace":
+        consola.trace(msg)
+        break
+    }
+  })
+
+  return stream
+}

package/src/shared/schema-transformer.ts

@@ -0,0 +1,75 @@
+import type { Plugin } from "esbuild"
+import { readFile } from "node:fs/promises"
+import { parseAsync, type Comment } from "oxc-parser"
+import { walk, type Node } from "oxc-walker"
+import MagicString from "magic-string"
+
+export const schemaTransformerPlugin: Plugin = {
+  name: "schema-transformer",
+  setup(build) {
+    build.onLoad({ filter: /src\/.*\.ts$/ }, async args => {
+      const content = await readFile(args.path, "utf-8")
+
+      return {
+        contents: await applySchemaTransformations(content),
+        loader: "ts",
+      }
+    })
+  },
+}
+
+export async function applySchemaTransformations(content: string): Promise<string> {
+  const magicString = new MagicString(content)
+  const { program, comments } = await parseAsync("file.ts", content)
+
+  walk(program, {
+    enter(node) {
+      if (node.type !== "Property" || node.key.type !== "Identifier") {
+        return
+      }
+
+      const jsdoc = comments.find(comment => isLeadingComment(content, node, comment))
+      if (!jsdoc || !jsdoc.value.includes("@schema")) {
+        return
+      }
+
+      magicString.update(
+        node.value.start,
+        node.value.end,
+        `{
+  ...${content.substring(node.value.start, node.value.end)},
+  description: \`${cleanJsdoc(jsdoc.value)}\`,
+}`,
+      )
+    },
+  })
+
+  return magicString.toString()
+}
+
+function isLeadingComment(content: string, node: Node, comment: Comment) {
+  if (comment.end > node.start) {
+    return false
+  }
+
+  const contentRange = content.substring(comment.end, node.start)
+
+  return contentRange.trim().length === 0
+}
+
+function cleanJsdoc(str: string) {
+  return (
+    str
+      // remove leading asterisks
+      .replace(/^\s*\*/gm, "")
+
+      // remove @schema tag
+      .replace("@schema", "")
+
+      // escape backticks and dollar signs
+      .replace(/\\/g, "\\\\")
+      .replace(/`/g, "\\`")
+      .replace(/\${/g, "\\${")
+      .trim()
+  )
+}

package/src/shared/services.ts

@@ -0,0 +1,20 @@
+import type { Services } from "@highstate/backend"
+import { logger } from "./logger"
+
+let services: Promise<Services> | undefined
+
+export function getBackendServices() {
+  if (services) {
+    return services
+  }
+
+  services = import("@highstate/backend").then(({ getSharedServices }) => {
+    return getSharedServices({
+      services: {
+        logger: logger.child({}, { msgPrefix: "[backend] " }),
+      },
+    })
+  })
+
+  return services
+}

package/src/shared/source-hash-calculator.ts

@@ -0,0 +1,219 @@
+import type { Logger } from "pino"
+import { dirname, relative, resolve } from "node:path"
+import { readFile, writeFile } from "node:fs/promises"
+import { fileURLToPath } from "node:url"
+import { readPackageJSON, resolvePackageJSON, type PackageJson } from "pkg-types"
+import { sha256 } from "crypto-hash"
+import { resolve as importMetaResolve } from "import-meta-resolve"
+
+export type HighstateManifestJson = {
+  sourceHashes?: Record<string, string>
+}
+
+type FileDependency =
+  | {
+      type: "relative"
+      id: string
+      fullPath: string
+    }
+  | {
+      type: "npm"
+      id: string
+      package: string
+    }
+
+export class SourceHashCalculator {
+  private readonly dependencyHashes = new Map<string, Promise<string>>()
+  private readonly fileHashes = new Map<string, Promise<string>>()
+
+  constructor(
+    private readonly packageJson: PackageJson,
+    private readonly logger: Logger,
+  ) {}
+
+  async writeHighstateManifest(distBasePath: string, distPaths: string[]): Promise<void> {
+    const promises: Promise<{ distPath: string; hash: string }>[] = []
+
+    for (const distPath of distPaths) {
+      const fullPath = resolve(distPath)
+
+      promises.push(
+        this.getFileHash(fullPath).then(hash => ({
+          distPath,
+          hash,
+        })),
+      )
+    }
+
+    const manifest: HighstateManifestJson = {
+      sourceHashes: {},
+    }
+
+    const hashes = await Promise.all(promises)
+    for (const { distPath, hash } of hashes) {
+      manifest.sourceHashes![distPath] = hash
+    }
+
+    const manifestPath = resolve(distBasePath, "highstate.manifest.json")
+    await writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8")
+  }
+
+  private async getFileHash(fullPath: string): Promise<string> {
+    const existingHash = this.fileHashes.get(fullPath)
+    if (existingHash) {
+      return existingHash
+    }
+
+    const hash = this.calculateFileHash(fullPath)
+    this.fileHashes.set(fullPath, hash)
+
+    return hash
+  }
+
+  private async calculateFileHash(fullPath: string): Promise<string> {
+    const content = await readFile(fullPath, "utf8")
+    const fileDeps = this.parseDependencies(fullPath, content)
+
+    const hashes = await Promise.all([
+      sha256(content),
+      ...fileDeps.map(dep => this.getDependencyHash(dep)),
+    ])
+
+    return await sha256(hashes.join(""))
+  }
+
+  getDependencyHash(dependency: FileDependency): Promise<string> {
+    const existingHash = this.dependencyHashes.get(dependency.id)
+    if (existingHash) {
+      return existingHash
+    }
+
+    const hash = this.calculateDependencyHash(dependency)
+    this.dependencyHashes.set(dependency.id, hash)
+
+    return hash
+  }
+
+  private async calculateDependencyHash(dependency: FileDependency): Promise<string> {
+    switch (dependency.type) {
+      case "relative": {
+        return await this.getFileHash(dependency.fullPath)
+      }
+      case "npm": {
+        let resolvedUrl
+        try {
+          resolvedUrl = importMetaResolve(dependency.package, import.meta.url)
+        } catch (error) {
+          this.logger.error(`failed to resolve package "%s"`, dependency.package)
+          throw error
+        }
+
+        if (resolvedUrl.startsWith("node:")) {
+          throw new Error(`"${dependency.package}" imported without "node:" prefix`)
+        }
+
+        const resolvedPath = fileURLToPath(resolvedUrl)
+
+        const [depPackageJsonPath, depPackageJson] = await this.getPackageJson(resolvedPath)
+        const packageName = depPackageJson.name!
+
+        this.logger.debug(
+          `resolved package.json for "%s": "%s"`,
+          dependency.package,
+          depPackageJsonPath,
+        )
+
+        if (
+          !this.packageJson.dependencies?.[packageName] &&
+          !this.packageJson.peerDependencies?.[packageName]
+        ) {
+          this.logger.warn(`package "%s" is not listed in package.json dependencies`, packageName)
+        }
+
+        let relativePath = relative(dirname(depPackageJsonPath), resolvedPath)
+        relativePath = relativePath.startsWith(".") ? relativePath : `./${relativePath}`
+
+        const highstateManifestPath = resolve(
+          dirname(depPackageJsonPath),
+          "dist",
+          "highstate.manifest.json",
+        )
+
+        let manifest: HighstateManifestJson | undefined
+        try {
+          const manifestContent = await readFile(highstateManifestPath, "utf8")
+          manifest = JSON.parse(manifestContent) as HighstateManifestJson
+        } catch (error) {
+          this.logger.debug(
+            { error },
+            `failed to read highstate manifest for package "%s"`,
+            packageName,
+          )
+        }
+
+        const sourceHash = manifest?.sourceHashes?.[relativePath]
+
+        if (sourceHash) {
+          this.logger.debug(`resolved source hash for package "%s"`, packageName)
+          return sourceHash
+        }
+
+        // use the package version as a fallback hash
+        // this case will be applied for most npm packages
+        this.logger.debug(`using package version as a fallback hash for "%s"`, packageName)
+        return depPackageJson.version ?? "0.0.0"
+      }
+    }
+  }
+
+  private async getPackageJson(basePath: string): Promise<[string, PackageJson]> {
+    while (true) {
+      const packageJson = await readPackageJSON(basePath)
+      if (packageJson.name) {
+        const packageJsonPath = await resolvePackageJSON(basePath)
+
+        return [packageJsonPath, packageJson]
+      }
+
+      basePath = resolve(dirname(basePath), "..")
+    }
+  }
+
+  private parseDependencies(filePath: string, content: string): FileDependency[] {
+    type DependencyMatch = {
+      relativePath?: string
+      nodeBuiltin?: string
+      npmPackage?: string
+    }
+
+    const dependencyRegex =
+      /^[ \t]*import[\s\S]*?\bfrom\s*["']((?<relativePath>\.\.?\/[^"']+)|(?<nodeBuiltin>node:[^"']+)|(?<npmPackage>[^"']+))["']/gm
+
+    const matches = content.matchAll(dependencyRegex)
+    const dependencies: FileDependency[] = []
+
+    for (const match of matches) {
+      const { nodeBuiltin, npmPackage, relativePath } = match.groups as DependencyMatch
+
+      if (relativePath) {
+        const fullPath = resolve(dirname(filePath), relativePath)
+
+        dependencies.push({
+          type: "relative",
+          id: `relative:${fullPath}`,
+          fullPath,
+        })
+      } else if (npmPackage) {
+        dependencies.push({
+          type: "npm",
+          id: `npm:${npmPackage}`,
+          package: npmPackage,
+        })
+      } else if (nodeBuiltin) {
+        // ignore node built-in modules
+      }
+    }
+
+    return dependencies
+  }
+}