@highstate/cli 0.9.15 → 0.9.18

package/src/commands/build.ts

@@ -1,13 +1,17 @@
 import type { Plugin } from "esbuild"
+import { resolve } from "node:path"
+import { writeFile } from "node:fs/promises"
 import { Command, Option } from "clipanion"
 import { readPackageJSON, resolvePackageJSON } from "pkg-types"
 import { mapKeys, mapValues, pipe } from "remeda"
 import { build } from "tsup"
+import { encode } from "@msgpack/msgpack"
 import {
   createBinTransformerPlugin,
   logger,
   schemaTransformerPlugin,
   SourceHashCalculator,
+  highstateConfigSchema,
 } from "../shared"
 
 export class BuildCommand extends Command {
@@ -21,9 +25,20 @@ export class BuildCommand extends Command {
   watch = Option.Boolean("--watch", false)
   library = Option.Boolean("--library", false)
   silent = Option.Boolean("--silent", true)
+  noSourceHash = Option.Boolean("--no-source-hash", false)
 
   async execute(): Promise<void> {
     const packageJson = await readPackageJSON()
+
+    const highstateConfig = highstateConfigSchema.parse(packageJson.highstate ?? {})
+    if (highstateConfig.type === "library") {
+      this.library = true
+    }
+
+    if (highstateConfig.type === "worker") {
+      this.noSourceHash = true
+    }
+
     const exports = packageJson.exports
     let bin = packageJson.bin
 
@@ -87,6 +102,7 @@ export class BuildCommand extends Command {
           targetName,
           distPath,
           isBin,
+          key,
         }
       }),
       mapKeys((_, value) => value.targetName),
@@ -113,25 +129,46 @@ export class BuildCommand extends Command {
       sourcemap: true,
       clean: true,
       format: "esm",
-      target: "esnext",
+      target: "es2024",
+      platform: "node",
       external: ["@pulumi/pulumi"],
       esbuildPlugins,
+      treeshake: true,
+      removeNodeProtocol: false,
       silent: this.silent || ["warn", "error", "fatal"].includes(logger.level),
     })
 
     const packageJsonPath = await resolvePackageJSON()
     const upToDatePackageJson = await readPackageJSON()
 
-    // do not write highstate manifest for library packages since their changes are tracked more granularly
-    if (!this.library) {
+    if (!this.noSourceHash) {
       const sourceHashCalculator = new SourceHashCalculator(
         packageJsonPath,
         upToDatePackageJson,
         logger,
       )
-      const distPaths = Object.values(entry).map(value => value.distPath)
 
-      await sourceHashCalculator.writeHighstateManifest("./dist", distPaths)
+      const distPathToExportKey = new Map<string, string>()
+      for (const value of Object.values(entry)) {
+        distPathToExportKey.set(value.distPath, value.key)
+      }
+
+      await sourceHashCalculator.writeHighstateManifest("./dist", distPathToExportKey)
     }
+
+    // write the "highstate.library.json" file if the library flag is set
+    if (this.library) {
+      const { loadLibrary } = await import("../shared/library-loader.js")
+      const fullModulePaths = Object.values(entry).map(value => resolve(value.distPath))
+
+      logger.info("evaluating library components from modules: %s", fullModulePaths.join(", "))
+
+      const library = await loadLibrary(logger, fullModulePaths)
+      const libraryPath = resolve("./dist", "highstate.library.msgpack")
+
+      await writeFile(libraryPath, encode(library), "utf8")
+    }
+
+    logger.info("build completed successfully")
   }
 }

package/src/main.ts

@@ -2,6 +2,7 @@ import { Builtins, Cli } from "clipanion"
 import { version } from "../package.json"
 import { DesignerCommand } from "./commands/designer"
 import { BuildCommand } from "./commands/build"
+import { BackendIdentityCommand } from "./commands/backend/identity"
 
 const cli = new Cli({
   binaryName: "highstate",
@@ -11,6 +12,7 @@ const cli = new Cli({
 
 cli.register(BuildCommand)
 cli.register(DesignerCommand)
+cli.register(BackendIdentityCommand)
 cli.register(Builtins.HelpCommand)
 cli.register(Builtins.VersionCommand)
 

package/src/shared/index.ts

@@ -3,3 +3,4 @@ export * from "./logger"
 export * from "./schema-transformer"
 export * from "./source-hash-calculator"
 export * from "./bin-transformer"
+export * from "./schemas"

package/src/shared/library-loader.ts

@@ -0,0 +1,129 @@
+import type { Logger } from "pino"
+import console from "console"
+import {
+  type Component,
+  type ComponentModel,
+  type Entity,
+  type EntityModel,
+  isComponent,
+  isEntity,
+  isUnitModel,
+} from "@highstate/contract"
+import { Crc32, crc32 } from "@aws-crypto/crc32"
+import { encode } from "@msgpack/msgpack"
+import { int32ToBytes } from "./utils"
+
+export type Library = Readonly<{
+  components: Readonly<Record<string, ComponentModel>>
+  entities: Readonly<Record<string, EntityModel>>
+}>
+
+export async function loadLibrary(logger: Logger, modulePaths: string[]): Promise<Library> {
+  const modules: Record<string, unknown> = {}
+  for (const modulePath of modulePaths) {
+    try {
+      logger.debug({ modulePath }, "loading module")
+      modules[modulePath] = await import(modulePath)
+
+      logger.debug({ modulePath }, "module loaded")
+    } catch (error) {
+      console.error(error)
+
+      throw new Error(`Failed to load module "${modulePath}"`, { cause: error })
+    }
+  }
+
+  const components: Record<string, ComponentModel> = {}
+  const entities: Record<string, EntityModel> = {}
+
+  await _loadLibrary(modules, components, entities)
+
+  logger.info(
+    {
+      componentCount: Object.keys(components).length,
+      entityCount: Object.keys(entities).length,
+    },
+    "library loaded",
+  )
+
+  logger.trace({ components, entities }, "library content")
+
+  return { components, entities }
+}
+
+async function _loadLibrary(
+  value: unknown,
+  components: Record<string, ComponentModel>,
+  entities: Record<string, EntityModel>,
+): Promise<void> {
+  if (isComponent(value)) {
+    const entityHashes: number[] = []
+    for (const entity of value.entities.values()) {
+      entity.model.definitionHash ??= calculateEntityDefinitionHash(entity)
+      entityHashes.push(entity.model.definitionHash)
+    }
+
+    components[value.model.type] = value.model
+    value.model.definitionHash = await calculateComponentDefinitionHash(value, entityHashes)
+
+    return
+  }
+
+  if (isEntity(value)) {
+    entities[value.type] = value.model
+    entities[value.type].definitionHash ??= calculateEntityDefinitionHash(value)
+
+    // @ts-expect-error remove the schema since it's not needed in the designer
+    delete value.model.schema
+    return
+  }
+
+  if (typeof value !== "object" || value === null) {
+    return
+  }
+
+  if ("_zod" in value) {
+    // this is a zod schema, we can skip it
+    return
+  }
+
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      await _loadLibrary(item, components, entities)
+    }
+
+    return
+  }
+
+  for (const key in value) {
+    await _loadLibrary((value as Record<string, unknown>)[key], components, entities)
+  }
+}
+
+async function calculateComponentDefinitionHash(
+  component: Component,
+  entityHashes: number[],
+): Promise<number> {
+  const result = new Crc32()
+
+  // 1. include the full component model
+  result.update(encode(component.model))
+
+  if (!isUnitModel(component.model)) {
+    // 2. for composite components, include the content of the serialized create function
+    // const serializedCreate = await serializeFunction(component[originalCreate])
+    const serializedCreate = { text: "TODO: investigate why serializeFunction hangs" }
+    result.update(Buffer.from(serializedCreate.text))
+  }
+
+  // 3. include the hashes of all entities
+  for (const entityHash of entityHashes) {
+    result.update(int32ToBytes(entityHash))
+  }
+
+  return result.digest()
+}
+
+function calculateEntityDefinitionHash(entity: Entity): number {
+  return crc32(encode(entity.model))
+}