@dbx-tools/cli 0.9.60 → 0.9.61

package/src/genie-code/config.ts

@@ -0,0 +1,161 @@
+/**
+ * Generate one persistent Genie Code home per exact profile-model pairing.
+ *
+ * @module
+ */
+
+import { createHash, randomUUID } from "node:crypto";
+import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { withFileLock } from "@dbx-tools/core/file-lock";
+import { object } from "@dbx-tools/shared-core";
+import {
+  GenieCodeBaseConfigSchema,
+  GenieCodeConfigSchema,
+  GenieCodeProviderOverlaySchema,
+  type GenieCodeBaseConfig,
+  type GenieCodeConfig,
+  type GenieCodeProviderOverlay,
+} from "@dbx-tools/shared-genie-code/config";
+import { genieCodePairingName } from "@dbx-tools/shared-genie-code/options";
+import { parse, stringify } from "smol-toml";
+
+/** Provider identifier written to the base and invocation overlay configurations. */
+export const GENIE_CODE_GATEWAY_PROVIDER = "dbx_tools_model_gateway";
+
+/** Inputs used to update a persistent home and create one invocation overlay. */
+export interface WriteGenieCodeConfigOptions {
+  bearerToken: string;
+  gatewayBaseUrl: string;
+  homeDirectory?: string;
+  model: string;
+  profile: string;
+  projectDirectory: string;
+}
+
+/** Stable persistent home for one exact profile-model pairing. */
+export interface GenieCodePairingHome {
+  configPath: string;
+  home: string;
+  name: string;
+}
+
+/** Persistent pairing home and invocation-specific provider overlay. */
+export interface GenieCodeHome extends GenieCodePairingHome {
+  overlayName: string;
+  overlayPath: string;
+}
+
+/** Derive the stable home for one exact profile-model pair. */
+export function genieCodeHome(
+  profile: string,
+  model: string,
+  homeDirectory: string = homedir(),
+): GenieCodePairingHome {
+  const digest = createHash("sha256")
+    .update(JSON.stringify([profile, model]))
+    .digest("hex")
+    .slice(0, 12);
+  const name = genieCodePairingName({ profile, model, digest });
+  const home = join(homeDirectory, ".dbx-tools", "genie", "profiles", name);
+  return { name, home, configPath: join(home, "config.toml") };
+}
+
+/** Render the persistent profile-model configuration. */
+export function genieCodeBaseConfig(
+  options: WriteGenieCodeConfigOptions,
+  existingProjects: Readonly<Record<string, unknown>> = {},
+): GenieCodeBaseConfig {
+  const projects = Object.fromEntries(
+    Object.entries(existingProjects).filter(
+      ([, value]) => object.isRecord(value) && value.trust_level === "trusted",
+    ),
+  );
+  return GenieCodeBaseConfigSchema.parse({
+    model_provider: GENIE_CODE_GATEWAY_PROVIDER,
+    model: options.model,
+    databricks_profile: options.profile,
+    projects: {
+      ...projects,
+      [resolve(options.projectDirectory)]: {
+        trust_level: "trusted",
+      },
+    },
+    tui: {
+      model_availability_nux: {
+        [options.model]: 1,
+      },
+    },
+  });
+}
+
+/** Render the invocation-specific guarded provider configuration. */
+export function genieCodeProviderOverlay(
+  options: WriteGenieCodeConfigOptions,
+): GenieCodeProviderOverlay {
+  return GenieCodeProviderOverlaySchema.parse({
+    model_providers: {
+      [GENIE_CODE_GATEWAY_PROVIDER]: {
+        name: "dbx-tools model gateway",
+        base_url: options.gatewayBaseUrl,
+        wire_api: "responses",
+        requires_openai_auth: false,
+        supports_websockets: false,
+        http_headers: {
+          Authorization: `Bearer ${options.bearerToken}`,
+          Originator: "codex",
+        },
+      },
+    },
+  });
+}
+
+/** Render the merged configuration used by validation and tests. */
+export function genieCodeConfig(options: WriteGenieCodeConfigOptions): GenieCodeConfig {
+  return GenieCodeConfigSchema.parse({
+    ...genieCodeBaseConfig(options),
+    ...genieCodeProviderOverlay(options),
+  });
+}
+
+async function existingProjects(path: string): Promise<Record<string, unknown>> {
+  try {
+    const value = parse(await readFile(path, "utf8"));
+    return object.isRecord(value.projects) ? value.projects : {};
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return {};
+    throw error;
+  }
+}
+
+async function writePrivateToml(path: string, value: unknown): Promise<void> {
+  const staged = join(dirname(path), `.${randomUUID()}.toml`);
+  try {
+    await writeFile(staged, stringify(value), { mode: 0o600 });
+    await chmod(staged, 0o600);
+    await rename(staged, path);
+  } finally {
+    await rm(staged, { force: true });
+  }
+}
+
+/** Update the persistent base config and write one private invocation overlay. */
+export async function writeGenieCodeConfig(
+  options: WriteGenieCodeConfigOptions,
+): Promise<GenieCodeHome> {
+  const base = genieCodeHome(options.profile, options.model, options.homeDirectory);
+  const overlayName = `dbx-${randomUUID().replaceAll("-", "")}`;
+  const destination = {
+    ...base,
+    overlayName,
+    overlayPath: join(base.home, `${overlayName}.config.toml`),
+  };
+  await mkdir(destination.home, { recursive: true });
+  await withFileLock(["genie-code-config", destination.configPath], async () => {
+    const projects = await existingProjects(destination.configPath);
+    await writePrivateToml(destination.configPath, genieCodeBaseConfig(options, projects));
+  });
+  await writePrivateToml(destination.overlayPath, genieCodeProviderOverlay(options));
+  return destination;
+}

package/src/genie-code/install.ts

@@ -0,0 +1,103 @@
+/**
+ * Install the pinned multi-file Genie Code runtime under dbx-tools ownership.
+ *
+ * @module
+ */
+
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { bin } from "@dbx-tools/core";
+import { GENIE_CODE_VERSION } from "@dbx-tools/shared-genie-code/options";
+
+interface GenieCodeAsset {
+  target: string;
+  sha256: string;
+}
+
+const GENIE_CODE_ASSETS = {
+  "darwin:arm64": {
+    target: "aarch64-apple-darwin",
+    sha256: "78cc7e358c8bfc62d6da26364c762be37fc2e2a10b7e83f21a3b166f901f528e",
+  },
+  "darwin:x64": {
+    target: "x86_64-apple-darwin",
+    sha256: "2021b8b27ec3116fc015c1676b5f65c3ce11d57f1dff9c302aae135f6e4f1a5f",
+  },
+  "linux:arm64": {
+    target: "aarch64-unknown-linux-musl",
+    sha256: "dfd278c267dde43e1fe7e7d7a0da8d7a1a148cdf3d2004bd30b351645c4fe0a7",
+  },
+  "linux:x64": {
+    target: "x86_64-unknown-linux-musl",
+    sha256: "e3d9a55f6d51bdc745eda4744fb827ba1df6dacf1069518f7347a59f48c89430",
+  },
+  "win32:arm64": {
+    target: "aarch64-pc-windows-msvc",
+    sha256: "aeac95bd0148f942194270ba932245cec32e98da67292ba7b008bcab675ba7f8",
+  },
+  "win32:x64": {
+    target: "x86_64-pc-windows-msvc",
+    sha256: "c9f92c8d60273250262527bca525451501c4e8351e59927a1686c305be29a922",
+  },
+} as const satisfies Record<string, GenieCodeAsset>;
+
+/** Host overrides used by installation tests and embedded callers. */
+export interface InstallGenieCodeOptions {
+  homeDirectory?: string;
+  platform?: NodeJS.Platform;
+  architecture?: NodeJS.Architecture;
+}
+
+/** Resolve the pinned release asset for one supported host. */
+export function genieCodeAsset(
+  platform: NodeJS.Platform = process.platform,
+  architecture: NodeJS.Architecture = process.arch,
+): GenieCodeAsset {
+  const asset = GENIE_CODE_ASSETS[`${platform}:${architecture}` as keyof typeof GENIE_CODE_ASSETS];
+  if (!asset) {
+    throw new Error(
+      `Genie Code ${GENIE_CODE_VERSION} does not support ${platform}/${architecture}`,
+    );
+  }
+  return asset;
+}
+
+/** Install and return the shared Genie Code package entrypoint. */
+export async function installGenieCode(
+  options: InstallGenieCodeOptions = {},
+): Promise<bin.BinContext> {
+  const platform = options.platform ?? process.platform;
+  const asset = genieCodeAsset(platform, options.architecture ?? process.arch);
+  const executable = platform === "win32" ? "genie.exe" : "genie";
+  const helper = platform === "win32" ? "genie-code-mode-host.exe" : "genie-code-mode-host";
+  const entrypoint = join("bin", executable);
+  const root = join(
+    options.homeDirectory ?? homedir(),
+    ".dbx-tools",
+    "genie",
+    "releases",
+    `${GENIE_CODE_VERSION}-${asset.target}`,
+  );
+  const destination: bin.BinContext = {
+    root,
+    binDir: join(root, dirname(entrypoint)),
+    path: join(root, entrypoint),
+  };
+  const url =
+    `https://github.com/databricks/genie-code-cli/releases/download/v${GENIE_CODE_VERSION}/` +
+    `genie-package-${asset.target}.tar.gz`;
+  return bin.ensure(
+    "genie",
+    { url, sha256: asset.sha256 },
+    {
+      destination,
+      minVersion: "0.1.0",
+      versionParser: ({ stdout }) =>
+        stdout.trim() === `genie ${GENIE_CODE_VERSION}` ? "0.1.0" : undefined,
+      package: {
+        entrypoint,
+        requiredPaths: ["genie-package.json", join("bin", helper), "genie-path", "genie-resources"],
+      },
+    },
+  );
+}

package/src/genie-code/options.ts

@@ -0,0 +1,14 @@
+/**
+ * CLI projection of the shared managed Genie Code options.
+ *
+ * @module
+ */
+
+export {
+  GenieCodeOptionsSchema as GenieCodeCliOptionsSchema,
+  resolveGenieCodeOptions as resolveGenieCodeCliOptions,
+} from "@dbx-tools/shared-genie-code/options";
+export type {
+  GenieCodeOptions as GenieCodeCliOptions,
+  ResolvedGenieCodeOptions as ResolvedGenieCodeCliOptions,
+} from "@dbx-tools/shared-genie-code/options";

package/src/genie-code/runner.ts

@@ -0,0 +1,106 @@
+/**
+ * Internal readiness gate and interactive Genie Code child process.
+ *
+ * @module
+ */
+
+import { AppKitChildProcess } from "@dbx-tools/appkit/child-process";
+import { asyncUtils, log } from "@dbx-tools/shared-core";
+import { GenieCodeRunnerOptionsSchema } from "@dbx-tools/shared-genie-code/options";
+
+/** Private environment field carrying serialized runner options. */
+const RUNNER_OPTIONS_ENV = "DBX_TOOLS_GENIE_RUNNER_OPTIONS";
+const logger = log.logger("genie-code:runner");
+
+/** Parse the serialized runner boundary from its private environment field. */
+export function genieCodeRunnerOptions(environment: NodeJS.ProcessEnv = process.env) {
+  const serialized = environment[RUNNER_OPTIONS_ENV];
+  if (!serialized) throw new Error(`${RUNNER_OPTIONS_ENV} is required`);
+  return GenieCodeRunnerOptionsSchema.parse(JSON.parse(serialized));
+}
+
+/** Wait until the guarded model-gateway health route accepts the runner token. */
+export async function waitForGenieCodeGateway(
+  healthUrl: string,
+  bearerToken: string,
+  timeoutMs: number,
+): Promise<void> {
+  for await (const ready of asyncUtils.poll(
+    async ({ signal }) => {
+      try {
+        const response = await fetch(healthUrl, {
+          headers: { authorization: `Bearer ${bearerToken}` },
+          signal,
+        });
+        return response.ok;
+      } catch {
+        return false;
+      }
+    },
+    {
+      intervalMs: 200,
+      timeoutMs,
+      predicate: (ready) => !ready,
+    },
+  )) {
+    if (ready) return;
+  }
+}
+
+/** Run Genie Code with inherited terminal streams after its gateway is ready. */
+export async function runGenieCodeChild(
+  environment: NodeJS.ProcessEnv = process.env,
+): Promise<void> {
+  const options = genieCodeRunnerOptions(environment);
+  logger.info("waiting for model gateway", {
+    gateway: options.gatewayHealthUrl,
+    timeoutMs: options.startupTimeoutMs,
+  });
+  await waitForGenieCodeGateway(
+    options.gatewayHealthUrl,
+    options.bearerToken,
+    options.startupTimeoutMs,
+  );
+  logger.info("model gateway ready; starting Genie Code", {
+    executable: options.executable,
+    home: options.home,
+  });
+  const child = new AppKitChildProcess(
+    [
+      options.executable,
+      [...options.arguments],
+      {
+        cwd: process.cwd(),
+        env: {
+          ...environment,
+          GENIE_HOME: options.home,
+        },
+        stdin: "inherit",
+        stdout: "inherit",
+        stderr: "inherit",
+      },
+    ],
+    {
+      gracefulTimeoutMs: 5_000,
+      forceTimeoutMs: 1_000,
+    },
+  );
+  const running = child.start();
+  const shutdown = () => {
+    void child.shutdown();
+  };
+  for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
+    process.once(signal, shutdown);
+  }
+  try {
+    const result = await running;
+    if (result.exitCode !== 0) process.exitCode = result.exitCode;
+  } finally {
+    for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
+      process.off(signal, shutdown);
+    }
+    await child.shutdown();
+  }
+}
+
+export { RUNNER_OPTIONS_ENV };

package/src/genie-code/runtime.ts

@@ -0,0 +1,212 @@
+/**
+ * Supervise Genie Code and its authenticated local model gateway.
+ *
+ * @module
+ */
+
+import { randomBytes } from "node:crypto";
+import { rm } from "node:fs/promises";
+import { delimiter, dirname } from "node:path";
+import { DatabricksModelRegistry, models } from "@dbx-tools/appkit-model-gateway";
+import * as auth from "@dbx-tools/auth";
+import { resolveServicePackageBin } from "@dbx-tools/cli-service/definition";
+import type { BinContext } from "@dbx-tools/core/bin";
+import { workspaceClient } from "@dbx-tools/databricks";
+import { log } from "@dbx-tools/shared-core";
+import {
+  resolveGenieCodeOptions,
+  type GenieCodeOptions,
+  type GenieCodeRunnerOptions,
+} from "@dbx-tools/shared-genie-code/options";
+import concurrently from "concurrently";
+import getPort from "get-port";
+
+import { writeGenieCodeConfig, type GenieCodeHome } from "./config.ts";
+import { installGenieCode } from "./install.ts";
+import { RUNNER_OPTIONS_ENV } from "./runner.ts";
+
+const logger = log.logger("genie-code");
+
+/** Injectable owners used to prepare and supervise managed Genie Code. */
+export interface GenieCodeRuntimeDependencies {
+  createAuthClient: typeof auth.client.createAuthClient;
+  install: typeof installGenieCode;
+  port(options: { host: string; port?: number }): Promise<number>;
+  resolveBin(packageReference: string, binName: string): string;
+  resolveCodexModel(model: string, profile: string): Promise<string>;
+  supervise: typeof concurrently;
+  token(): string;
+  writeConfig: typeof writeGenieCodeConfig;
+}
+
+const DEFAULT_DEPENDENCIES: GenieCodeRuntimeDependencies = {
+  createAuthClient: auth.client.createAuthClient,
+  install: installGenieCode,
+  port: ({ host, port }) => getPort({ host, ...(port ? { port } : {}) }),
+  resolveBin: resolveServicePackageBin,
+  resolveCodexModel,
+  supervise: concurrently,
+  token: () => randomBytes(32).toString("base64url"),
+  writeConfig: writeGenieCodeConfig,
+};
+
+/** Resolve a Genie `--model` value to the Codex catalogue slug the gateway publishes. */
+async function resolveCodexModel(model: string, profile: string): Promise<string> {
+  const client = await workspaceClient.createWorkspaceClient({ profile });
+  const target = await new DatabricksModelRegistry({ client }).resolve(model, {
+    requiresTools: true,
+  });
+  if (!target) throw new Error(`Genie Code model not found: ${model}`);
+  return models.codexModelSlug(target);
+}
+
+/** Inputs accepted by managed Genie Code preparation and execution. */
+export interface RunGenieCodeOptions {
+  cwd?: string;
+  dependencies?: Partial<GenieCodeRuntimeDependencies>;
+  genieArgs?: readonly string[];
+  options?: GenieCodeOptions;
+}
+
+/** Resolved installation, auth, listener, and home for one invocation. */
+export interface PreparedGenieCodeRuntime {
+  bearerToken: string;
+  gatewayBaseUrl: string;
+  gatewayHealthUrl: string;
+  gatewayListen: string;
+  gatewayPort: number;
+  home: GenieCodeHome;
+  installation: BinContext;
+  model: string;
+  profile: string;
+}
+
+/** Resolve auth, installation, listener, and configuration before spawning children. */
+export async function prepareGenieCodeRuntime(
+  input: RunGenieCodeOptions = {},
+): Promise<PreparedGenieCodeRuntime> {
+  const dependencies = { ...DEFAULT_DEPENDENCIES, ...input.dependencies };
+  const options = resolveGenieCodeOptions(input.options);
+  logger.info("resolving Databricks authentication", {
+    requestedProfile: options.profile,
+  });
+  const authClient = await dependencies.createAuthClient(
+    options.profile ? { profile: options.profile } : {},
+  );
+  const profile = authClient.profile;
+  if (!profile) {
+    throw new Error("Genie Code requires a configured Databricks profile");
+  }
+  logger.info("preparing Genie Code runtime", {
+    profile,
+    model: options.model,
+  });
+  const installation = await dependencies.install();
+  const host = options.gatewayListen.host;
+  const requestedPort = options.gatewayListen.port;
+  const gatewayPort = await dependencies.port({
+    host,
+    ...(requestedPort > 0 ? { port: requestedPort } : {}),
+  });
+  if (requestedPort > 0 && gatewayPort !== requestedPort) {
+    throw new Error(`Genie Code gateway port ${requestedPort} is already in use`);
+  }
+  const model = await dependencies.resolveCodexModel(options.model, profile);
+  logger.info("resolved Genie Code model", { requested: options.model, model });
+  const urlHost = host.includes(":") ? `[${host}]` : host;
+  const gatewayBaseUrl = `http://${urlHost}:${gatewayPort}/v1`;
+  const gatewayHealthUrl = `http://${urlHost}:${gatewayPort}/api/healthz`;
+  const gatewayListen = `tcp://${urlHost}:${gatewayPort}`;
+  const bearerToken = dependencies.token();
+  const home = await dependencies.writeConfig({
+    bearerToken,
+    gatewayBaseUrl,
+    model,
+    profile,
+    projectDirectory: input.cwd ?? process.cwd(),
+  });
+  return {
+    bearerToken,
+    gatewayBaseUrl,
+    gatewayHealthUrl,
+    gatewayListen,
+    gatewayPort,
+    home,
+    installation,
+    model,
+    profile,
+  };
+}
+
+function shellExecutable(path: string): string {
+  if (process.platform === "win32") return `"${path.replaceAll('"', '""')}"`;
+  return `'${path.replaceAll("'", "'\\''")}'`;
+}
+
+function childPath(...paths: string[]): string {
+  return [...new Set([...paths.map(dirname), process.env.PATH].filter(Boolean))].join(delimiter);
+}
+
+/** Run Genie Code and the model gateway until either process exits. */
+export async function runGenieCode(input: RunGenieCodeOptions = {}): Promise<void> {
+  const dependencies = { ...DEFAULT_DEPENDENCIES, ...input.dependencies };
+  const prepared = await prepareGenieCodeRuntime({
+    ...input,
+    dependencies,
+  });
+  const gatewayBin = dependencies.resolveBin(import.meta.url, "dbx-model-gateway");
+  const runnerBin = dependencies.resolveBin(import.meta.url, "dbx-genie");
+  const path = childPath(gatewayBin, runnerBin);
+  const runnerOptions: GenieCodeRunnerOptions = {
+    executable: prepared.installation.path,
+    arguments: ["-p", prepared.home.overlayName, ...(input.genieArgs ?? [])],
+    home: prepared.home.home,
+    gatewayHealthUrl: prepared.gatewayHealthUrl,
+    bearerToken: prepared.bearerToken,
+    startupTimeoutMs: 60_000,
+  };
+  const { result } = dependencies.supervise(
+    [
+      {
+        command: shellExecutable(gatewayBin),
+        name: "gateway",
+        env: {
+          ...process.env,
+          PATH: path,
+          LISTEN: prepared.gatewayListen,
+          DATABRICKS_CONFIG_PROFILE: prepared.profile,
+          DBX_TOOLS_MODEL_GATEWAY_BEARER_TOKEN: prepared.bearerToken,
+        },
+      },
+      {
+        command: shellExecutable(runnerBin),
+        name: "genie",
+        env: {
+          ...process.env,
+          PATH: path,
+          DATABRICKS_CONFIG_PROFILE: prepared.profile,
+          [RUNNER_OPTIONS_ENV]: JSON.stringify(runnerOptions),
+        },
+      },
+    ],
+    {
+      cwd: input.cwd ?? process.cwd(),
+      raw: true,
+      killOthersOn: ["failure", "success"],
+      killSignal: "SIGTERM",
+      killTimeout: 17_000,
+      successCondition: "command-genie",
+    },
+  );
+  logger.info("started Genie Code and model gateway", {
+    profile: prepared.profile,
+    model: prepared.model,
+    gateway: prepared.gatewayBaseUrl,
+    home: prepared.home.home,
+  });
+  try {
+    await result;
+  } finally {
+    await rm(prepared.home.overlayPath, { force: true });
+  }
+}

package/src/model-gateway/cli.ts

@@ -15,8 +15,8 @@ import {
 } from "@dbx-tools/shared-model-gateway/options";
 import { Command } from "commander";
 
-import { PACKAGE_VERSION } from "../../index.ts";
 import { startModelGateway } from "./server.ts";
+import { PACKAGE_VERSION } from "../../index.ts";
 
 /** Injectable foreground gateway boundary for CLI tests. */
 export interface ModelGatewayCliDependencies {
@@ -33,7 +33,11 @@ const DEFAULT_DEPENDENCIES: ModelGatewayCliDependencies = {
 export function modelGatewayServiceDefinition(
   options: ModelGatewayOptions = {},
 ): CliServiceDefinition {
-  const { runtimeInfo: _, ...resolved } = resolveModelGatewayCliOptions(options);
+  const {
+    runtimeInfo: _,
+    bearerToken: _bearerToken,
+    ...resolved
+  } = resolveModelGatewayCliOptions(options);
   const host = urlHost(resolved.listen.host);
   return defineService(import.meta.url, {
     id: "dbx-tools.cli-model-gateway",