@dbx-tools/appkit 0.1.9

package/src/config.ts

@@ -0,0 +1,445 @@
+/**
+ * Layered configuration resolution for local development against a Databricks
+ * App / Asset Bundle.
+ *
+ * Default sources: `env`, then Databricks App `config.env` (from {@link bundle})
+ * and hard-coded `app.yaml` env entries (from {@link appYaml}). Opt in to `cli`
+ * when a dev command wants flag overrides.
+ *
+ * Server-only (`node:child_process`, bundle root discovery). Databricks-app
+ * specific, so it lives in node-appkit rather than shared-core.
+ */
+
+import { readFile } from "node:fs/promises";
+import { spawnSync } from "node:child_process";
+import { dirname, resolve } from "node:path";
+import { functionModule, iterable, log, string } from "@dbx-tools/shared-core";
+import { project } from "@dbx-tools/core";
+import { parse as parseYamlText } from "yaml";
+import { z } from "zod";
+
+import { isAppEnv } from "./databricks";
+
+const logger = log.logger("config");
+
+/** Parsed payload from `databricks bundle validate --output json`. */
+export type BundleValidateJson = Record<string, unknown>;
+
+/** A config file discovered on disk with its parsed contents. */
+export interface ConfigFile {
+  path: string;
+  data: Record<string, unknown>;
+}
+
+/** Supported configuration sources, consulted in array order. */
+export type ConfigSource = "explicit" | "cli" | "env" | "bundle";
+
+const defaultConfigSources: ConfigSource[] = ["env", "bundle"];
+
+const APP_YAML_NAMES = ["app.yaml", "app.yml"] as const;
+
+const bundleValidateCache = new Map<string, BundleValidateJson | undefined>();
+
+const normalizedString = z
+  .string()
+  .transform((value) => value.trim())
+  .pipe(z.string().min(1));
+
+const bundleAppEnvEntrySchema = z.object({
+  name: normalizedString.optional(),
+  value: z.string().optional(),
+  value_from: normalizedString.optional(),
+});
+
+const bundleAppResourceSchema = z.object({
+  name: normalizedString.optional(),
+  sql_warehouse: z.object({ id: normalizedString.optional() }).optional(),
+  genie_space: z.object({ space_id: normalizedString.optional() }).optional(),
+  postgres: z
+    .object({
+      database: normalizedString.optional(),
+      branch: normalizedString.optional(),
+      endpoint: normalizedString.optional(),
+    })
+    .optional(),
+});
+
+const bundleAppSchema = z.object({
+  name: normalizedString.optional(),
+  source_code_path: z.string().optional(),
+  config: z.object({ env: z.array(bundleAppEnvEntrySchema).optional() }).optional(),
+  resources: z.array(bundleAppResourceSchema).optional(),
+});
+
+const bundleValidateAppsSchema = z.object({
+  resources: z.object({ apps: z.record(z.string(), bundleAppSchema).optional() }).optional(),
+});
+
+const appYamlEnvEntrySchema = z.object({
+  name: normalizedString,
+  value: z.string().optional(),
+  valueFrom: normalizedString.optional(),
+});
+
+const appYamlResourceSchema = z
+  .object({
+    name: normalizedString,
+    sql_warehouse: z.object({ id: normalizedString.optional() }).optional(),
+    genie_space: z.object({ space_id: normalizedString.optional() }).optional(),
+    postgres: z
+      .object({
+        database: normalizedString.optional(),
+        branch: normalizedString.optional(),
+        endpoint: normalizedString.optional(),
+      })
+      .optional(),
+  })
+  .passthrough();
+
+const appYamlSchema = z.object({
+  env: z.array(appYamlEnvEntrySchema).optional(),
+  resources: z.array(appYamlResourceSchema).optional(),
+});
+
+type BundleApp = z.infer<typeof bundleAppSchema>;
+
+/** Single config map entry (string or repeated values, like headers). */
+export type ConfigMapValue = string | string[] | undefined;
+
+export interface ResolveConfigValueOptions {
+  /** Bundle validate JSON. Defaults to {@link bundle} (skipped inside a Databricks App). */
+  bundleData?: ConfigFile;
+  /** Parsed `app.yaml` contents. Defaults to {@link appYaml} (skipped inside a Databricks App). */
+  appData?: ConfigFile;
+  /** Sources to consult, first truthy string wins. Defaults to `env`, then `bundle`. */
+  sources?: ConfigSource[];
+  /** Programmatic overrides. When set, `explicit` is appended to `sources` unless already listed. */
+  explicit?: Record<string, ConfigMapValue>;
+  /** CLI flag values (when `cli` is listed in `sources`). */
+  cli?: Record<string, ConfigMapValue>;
+}
+
+const bundleDefault = functionModule.memoize(() => loadBundle(process.cwd()));
+const appYamlDefault = functionModule.memoize(() => loadAppYaml(process.cwd()));
+
+function envKeysForName(name: string): iterable.Sequence<string> {
+  const trimmed = name.trim();
+  if (!trimmed) {
+    return iterable.sequence();
+  }
+  const keys = (function* () {
+    const modifiers: (((value: string) => string) | null)[] = [
+      null,
+      () => trimmed.toUpperCase(),
+      () => Array.from(string.tokenize(trimmed)).join("_").toUpperCase(),
+    ];
+    for (const modifier of modifiers) {
+      yield modifier ? modifier(trimmed) : trimmed;
+    }
+  })();
+  return iterable.sequence(keys).cache().filter(Boolean).distinct();
+}
+
+function readEnv(keys: Iterable<string>): string | undefined {
+  for (const key of keys) {
+    const value = process.env[key]?.trim();
+    if (value) return value;
+  }
+  return undefined;
+}
+
+function readConfigMapValue(value: ConfigMapValue): string | undefined {
+  if (value == null) return undefined;
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      const trimmed = item?.trim();
+      if (trimmed) return trimmed;
+    }
+    return undefined;
+  }
+  const trimmed = value.trim();
+  return trimmed || undefined;
+}
+
+function readMap(
+  keys: Iterable<string>,
+  map: Record<string, ConfigMapValue> | undefined,
+): string | undefined {
+  if (!map) return undefined;
+  for (const key of keys) {
+    const value = readConfigMapValue(map[key]);
+    if (value) return value;
+  }
+  return undefined;
+}
+
+function readAppEnv(keys: Iterable<string>, envMap: Record<string, string>): string | undefined {
+  for (const key of keys) {
+    const value = envMap[key]?.trim();
+    if (value) return value;
+  }
+  return undefined;
+}
+
+function parseYaml(text: string): unknown {
+  return parseYamlText(text) as unknown;
+}
+
+function pickAppResourceId(apps: Record<string, BundleApp>): string | undefined {
+  const keys = Object.keys(apps);
+  return keys.length === 1 ? keys[0] : undefined;
+}
+
+/**
+ * Flatten `env` entries from parsed `app.yaml` content. Literal `value` entries
+ * are returned as-is; `valueFrom` entries resolve against the sibling
+ * `resources` array when possible.
+ */
+export function flattenAppYamlEnv(data: unknown): Record<string, string> {
+  const parsed = appYamlSchema.safeParse(data);
+  if (!parsed.success || !parsed.data.env?.length) {
+    return {};
+  }
+
+  const resourceByName = new Map(
+    (parsed.data.resources ?? []).map((resource) => [resource.name, resource]),
+  );
+
+  const out: Record<string, string> = {};
+  for (const entry of parsed.data.env) {
+    if (entry.value?.trim()) {
+      out[entry.name] = entry.value.trim();
+      continue;
+    }
+    if (!entry.valueFrom) continue;
+    const resource = resourceByName.get(entry.valueFrom);
+    const resolved =
+      resource?.sql_warehouse?.id ??
+      resource?.genie_space?.space_id ??
+      resource?.postgres?.endpoint ??
+      resource?.postgres?.database ??
+      resource?.postgres?.branch;
+    if (resolved) out[entry.name] = resolved;
+  }
+  return out;
+}
+
+/**
+ * Flatten `resources.apps.<key>.config.env` into a `name -> value` map.
+ * Auto-picks the app only when the bundle defines exactly one.
+ */
+export function flattenAppEnv(data: unknown): Record<string, string> {
+  const parsed = bundleValidateAppsSchema.safeParse(data);
+  if (!parsed.success) return {};
+
+  const apps = parsed.data.resources?.apps;
+  if (!apps || Object.keys(apps).length === 0) return {};
+
+  const key = pickAppResourceId(apps);
+  if (!key) return {};
+
+  const app = apps[key];
+  if (!app?.config?.env?.length) return {};
+
+  const resourceByName = new Map(
+    (app.resources ?? [])
+      .filter((resource) => resource.name)
+      .map((resource) => [resource.name!, resource]),
+  );
+
+  const out: Record<string, string> = {};
+  for (const entry of app.config.env) {
+    if (!entry.name) continue;
+    if (entry.value) {
+      out[entry.name] = entry.value;
+      continue;
+    }
+    if (!entry.value_from) continue;
+    const resource = resourceByName.get(entry.value_from);
+    const resolved =
+      resource?.sql_warehouse?.id ??
+      resource?.genie_space?.space_id ??
+      resource?.postgres?.endpoint ??
+      resource?.postgres?.database ??
+      resource?.postgres?.branch;
+    if (resolved) out[entry.name] = resolved;
+  }
+  return out;
+}
+
+function validateBundle(root: string): BundleValidateJson | undefined {
+  const key = root;
+  if (bundleValidateCache.has(key)) {
+    return bundleValidateCache.get(key);
+  }
+
+  const args = ["bundle", "validate", "--output", "json"];
+  try {
+    const proc = spawnSync("databricks", args, {
+      cwd: root,
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const text = proc.stdout?.trim();
+    if (!text) {
+      bundleValidateCache.set(key, undefined);
+      return undefined;
+    }
+    const data = JSON.parse(text) as BundleValidateJson;
+    bundleValidateCache.set(key, data);
+    return data;
+  } catch {
+    bundleValidateCache.set(key, undefined);
+    return undefined;
+  }
+}
+
+async function loadBundle(cwd: string): Promise<ConfigFile | undefined> {
+  const configFile = resolveConfigFile(cwd, "databricks.yml");
+  if (configFile) {
+    const data = validateBundle(dirname(configFile));
+    if (data) return { path: configFile, data };
+  }
+  return undefined;
+}
+
+async function loadAppYaml(cwd: string): Promise<ConfigFile | undefined> {
+  for (const fileName of APP_YAML_NAMES) {
+    const configFile = resolveConfigFile(cwd, fileName);
+    if (!configFile) continue;
+    try {
+      const text = await readFile(configFile, "utf8");
+      const data = parseYaml(text);
+      if (typeof data !== "object" || data === null || Array.isArray(data)) {
+        return undefined;
+      }
+      return { path: configFile, data: data as Record<string, unknown> };
+    } catch {
+      logger.warn("failed to parse app yaml", { path: configFile });
+    }
+  }
+  return undefined;
+}
+
+function resolveConfigFile(cwd: string, configFile: string): string | undefined {
+  if (isAppEnv()) return undefined;
+  for (const rootDir of project.resolveProjectRoots(cwd)) {
+    const bundlePath = resolve(rootDir, configFile);
+    if (project.stat(bundlePath)?.isFile()) return bundlePath;
+  }
+  return undefined;
+}
+
+/**
+ * Locate the bundle root and run `databricks bundle validate --output json`.
+ * When `cwd` is omitted or equals `process.cwd()`, the result is memoized for
+ * the process lifetime. Returns `undefined` inside a Databricks App.
+ */
+export function bundle(cwd?: string): Promise<ConfigFile | undefined> {
+  if (isAppEnv()) return Promise.resolve(undefined);
+  return cwd && resolve(cwd) !== process.cwd() ? loadBundle(cwd) : bundleDefault();
+}
+
+/**
+ * Locate and parse `app.yaml` / `app.yml` from the bundle or project root. When
+ * `cwd` is omitted or equals `process.cwd()`, the result is memoized for the
+ * process lifetime. Returns `undefined` inside a Databricks App.
+ */
+export function appYaml(cwd?: string): Promise<ConfigFile | undefined> {
+  if (isAppEnv()) return Promise.resolve(undefined);
+  return cwd && resolve(cwd) !== process.cwd() ? loadAppYaml(cwd) : appYamlDefault();
+}
+
+/**
+ * Walk a dot-separated path through bundle validate JSON. When the terminal
+ * node is a bundle variable object (`{ value: "..." }`), the `value` field is
+ * returned.
+ */
+export function getBundlePath(data: BundleValidateJson, path: string): string | undefined {
+  const parts = path.split(".").filter(Boolean);
+  if (parts.length === 0) return undefined;
+
+  let current: unknown = data;
+  for (let i = 0; i < parts.length; i++) {
+    if (typeof current !== "object" || current === null) return undefined;
+    const record = current as Record<string, unknown>;
+    const part = parts[i]!;
+    const next = record[part];
+    if (i === parts.length - 1) {
+      if (typeof next === "string" && next) return next;
+      if (typeof next === "object" && next !== null && "value" in next) {
+        const value = (next as { value?: unknown }).value;
+        return typeof value === "string" && value ? value : undefined;
+      }
+      return undefined;
+    }
+    current = next;
+  }
+  return undefined;
+}
+
+async function resolveAppEnvMap(
+  options: ResolveConfigValueOptions,
+): Promise<Record<string, string>> {
+  const appData = options.appData ?? (await appYaml());
+  const bundleData = options.bundleData ?? (await bundle());
+  const fromYaml = appData ? flattenAppYamlEnv(appData.data) : {};
+  const fromBundle = bundleData ? flattenAppEnv(bundleData.data) : {};
+  return { ...fromYaml, ...fromBundle };
+}
+
+function resolveSources(options: ResolveConfigValueOptions): ConfigSource[] {
+  const sources = [...(options.sources ?? defaultConfigSources)];
+  if (options.explicit !== undefined && !sources.includes("explicit")) {
+    sources.push("explicit");
+  }
+  return sources;
+}
+
+/**
+ * Resolve a configuration string from the configured sources. Returns the first
+ * non-empty value, or `undefined` when nothing matches.
+ */
+export async function resolveConfigValue(
+  name: string,
+  options: ResolveConfigValueOptions = {},
+): Promise<string | undefined> {
+  const keys = envKeysForName(name).toArray();
+  if (keys.length === 0) return undefined;
+  const sources = resolveSources(options);
+  let appEnvMap: Record<string, string> | undefined;
+  const values = (async function* () {
+    for (const source of sources) {
+      switch (source) {
+        case "explicit":
+          yield readMap(keys, options.explicit);
+          break;
+        case "cli":
+          yield readMap(keys, options.cli);
+          break;
+        case "env":
+          yield readEnv(keys);
+          break;
+        case "bundle":
+          if (appEnvMap === undefined) appEnvMap = await resolveAppEnvMap(options);
+          yield readAppEnv(keys, appEnvMap);
+          break;
+        default:
+          throw new Error(`Unknown config source: ${source}`);
+      }
+    }
+  })();
+  for await (const value of values) {
+    if (value) return value;
+  }
+  return undefined;
+}
+
+/**
+ * Sources with `cli` included, in CLI-first order. Use for dev commands that
+ * accept flag overrides.
+ */
+export function withCliSources(sources: ConfigSource[] = defaultConfigSources): ConfigSource[] {
+  const rest = sources.filter((source) => source !== "cli" && source !== "explicit");
+  return ["cli", "explicit", ...rest];
+}

package/src/create-app.ts

@@ -0,0 +1,95 @@
+/**
+ * `createApp` wrapper: runs dbx-tools auto-configuration, then delegates to
+ * AppKit's own `createApp` with the exact same arguments.
+ *
+ * Drop it in as a one-for-one replacement for `@databricks/appkit`'s
+ * `createApp` - same parameters, same return type, full plugin-export inference
+ * preserved:
+ *
+ * ```ts
+ * import { createApp } from "@dbx-tools/appkit";
+ * import { lakebase, server } from "@databricks/appkit";
+ *
+ * await createApp({ plugins: [server(), lakebase()] });
+ * ```
+ *
+ * Auto-configuration runs BEFORE delegating so plugins see a fully populated
+ * `process.env` during their synchronous `setup()`. Lakebase Postgres runs when
+ * a `lakebase` plugin is present, or when `autoConfigure: true` is set on the
+ * config object.
+ */
+
+import { log } from "@dbx-tools/shared-core";
+import { createApp as appkitCreateApp, getUsernameWithApiLookup } from "@databricks/appkit";
+
+import {
+  applyLakebaseToEnv,
+  resolveLakebaseConnection,
+  type LakebaseConnection,
+} from "./lakebase-resolver";
+import { provisionCacheSchema } from "./provision";
+
+type CreateAppConfig = Parameters<typeof appkitCreateApp>[0] & {
+  autoConfigure?: boolean | "provision";
+};
+
+const logger = log.logger("create-app");
+
+const LAKEBASE_PLUGIN = "lakebase";
+
+function usesPlugin(config: CreateAppConfig | undefined, name: string): boolean {
+  return Boolean(config?.plugins?.some((entry) => entry.name === name));
+}
+
+/**
+ * Run enabled auto-configuration steps without calling AppKit's `createApp`.
+ * Lakebase Postgres resolves when `autoConfigure` is `true` or a `lakebase`
+ * plugin is listed in `config.plugins`. Defaults to `"provision"` (Lakebase env
+ * + optional cache schema). Pass `autoConfigure: false` to skip entirely.
+ */
+export async function autoConfigure(config?: CreateAppConfig): Promise<void> {
+  const { autoConfigure = "provision" } = config ?? {};
+  if (autoConfigure !== false) {
+    if (autoConfigure === true || usesPlugin(config, LAKEBASE_PLUGIN)) {
+      await autoConfigureLakebase(autoConfigure === "provision");
+    }
+  }
+}
+
+/**
+ * Resolve Lakebase Postgres connection info, write the resolved values to
+ * `process.env`, and return the record. Used by {@link autoConfigure}; call
+ * {@link resolveLakebaseConnection} and {@link applyLakebaseToEnv} directly when
+ * finer control is needed.
+ */
+async function autoConfigureLakebase(provision: boolean): Promise<LakebaseConnection> {
+  const resolved = await resolveLakebaseConnection();
+  applyLakebaseToEnv(resolved);
+  const user = await getUsernameWithApiLookup({});
+  if (user) process.env.PGUSER ??= user;
+  logger.info("env updated", { ...redactLakebaseConnection(resolved), user });
+  if (provision) {
+    await provisionCacheSchema(logger, user);
+  }
+  return resolved;
+}
+
+const create = async (config?: CreateAppConfig) => {
+  await autoConfigure(config);
+  return appkitCreateApp(config);
+};
+
+function redactLakebaseConnection(resolved: LakebaseConnection): Record<string, unknown> {
+  return {
+    project: resolved.project,
+    branch: resolved.branch,
+    endpoint: resolved.endpoint,
+    database: resolved.database,
+    host: resolved.host,
+    port: resolved.port,
+    sslMode: resolved.sslMode,
+  };
+}
+
+/** Auto-configuring drop-in for AppKit's `createApp`. */
+export const createApp = create as unknown as typeof appkitCreateApp;

package/src/databricks.ts

@@ -0,0 +1,121 @@
+/**
+ * Generic Databricks SDK glue (no AppKit): adapt WHATWG cancellation
+ * (`AbortSignal` / `AbortController`) into the SDK's `Context` /
+ * `CancellationToken` shapes so a single `AbortController` can drive every
+ * in-flight SDK call, plus Databricks runtime-environment detection.
+ *
+ * Server-only: leans on the Databricks SDK `Context`. Lives in node-appkit so
+ * the browser-safe shared-core stays SDK-free.
+ */
+
+import { async } from "@dbx-tools/shared-core";
+import type { CancellationToken } from "@databricks/sdk-experimental";
+import { Context } from "@databricks/sdk-experimental";
+
+/**
+ * Detect the Databricks App runtime from environment shape: requires a
+ * non-empty `DATABRICKS_APP_NAME`, a `DATABRICKS_HOST` that parses as an
+ * `http`/`https` URL, and a `DATABRICKS_APP_PORT` that is a valid TCP port.
+ * Reads `process.env` when no `env` is passed.
+ */
+export function isAppEnv(env: Record<string, string | undefined> = process.env): boolean {
+  const appName = env.DATABRICKS_APP_NAME?.trim();
+  const host = env.DATABRICKS_HOST?.trim();
+  const port = env.DATABRICKS_APP_PORT?.trim();
+
+  if (!appName || !host || !port) {
+    return false;
+  }
+
+  try {
+    const url = new URL(host);
+    if (!["http:", "https:"].includes(url.protocol)) {
+      return false;
+    }
+  } catch {
+    return false;
+  }
+
+  const portNumber = Number(port);
+  if (!Number.isInteger(portNumber) || portNumber < 1 || portNumber > 65535) {
+    return false;
+  }
+
+  return true;
+}
+
+/** Either an SDK `Context` or a WHATWG `AbortSignal`. */
+export type ContextLike = Context | AbortSignal;
+
+/** Wrap a `Context` (returned as-is) or `AbortSignal` (adapted) as an SDK `Context`. */
+export function toContext(input: ContextLike): Context;
+/**
+ * Derive an SDK `Context` from `controller.signal`, optionally tying `input`
+ * into the controller so the controller becomes the single cancellation
+ * source for downstream SDK calls:
+ *
+ *   - `AbortSignal`: aborting it propagates into `controller` (and from there
+ *     into every SDK call you pass the returned context to).
+ *   - `Context`: its `cancellationToken` is tied into `controller`, and its
+ *     other fields (`logger`, `opName`, `rootClassName`, `rootFnName`, `opId`)
+ *     are preserved in the returned `Context`. The returned context's
+ *     `cancellationToken` is replaced with one backed by `controller.signal`.
+ *
+ * The tie is one-way (parent -> child): aborting `controller` directly does
+ * NOT cancel `input`. So a request-level cancel (your loop's `try/finally {
+ * controller.abort() }`) won't tear down a caller-supplied AbortSignal it
+ * didn't own.
+ */
+export function toContext(controller: AbortController, input?: ContextLike): Context;
+export function toContext(source: AbortController | ContextLike, input?: ContextLike): Context {
+  if (!(source instanceof AbortController)) {
+    if (source instanceof Context) return source;
+    return new Context({ cancellationToken: signalToCancellationToken(source) });
+  }
+  if (input instanceof AbortSignal) {
+    async.tieAbortSignal(source, input);
+  } else if (input instanceof Context) {
+    const token = input.cancellationToken;
+    if (token) tieCancellationToken(source, token);
+    const merged = input.copy();
+    merged.setItems({ cancellationToken: signalToCancellationToken(source.signal) });
+    return merged;
+  }
+  return new Context({ cancellationToken: signalToCancellationToken(source.signal) });
+}
+
+/**
+ * Adapt a WHATWG `AbortSignal` to the Databricks SDK's `CancellationToken`
+ * interface. The SDK's `api-client.ts` internally creates an `AbortController`
+ * and wires `cancellationToken.onCancellationRequested` to it, so this adapter
+ * is the one-line bridge from "platform-standard cancellation" to "the SDK
+ * aborts the fetch on your behalf".
+ */
+function signalToCancellationToken(signal: AbortSignal): CancellationToken {
+  return {
+    get isCancellationRequested() {
+      return signal.aborted;
+    },
+    onCancellationRequested(cb) {
+      if (signal.aborted) {
+        cb(signal.reason);
+        return;
+      }
+      signal.addEventListener("abort", () => cb(signal.reason), { once: true });
+    },
+  };
+}
+
+/**
+ * Tie the SDK's `CancellationToken` interface back into an `AbortController`.
+ * Mirrors `async.tieAbortSignal` but for the SDK's cancellation shape, used
+ * when a caller hands us a pre-built `Context` whose token we want to fold into
+ * our own controller.
+ */
+function tieCancellationToken(controller: AbortController, token: CancellationToken): void {
+  if (token.isCancellationRequested) {
+    controller.abort();
+    return;
+  }
+  token.onCancellationRequested((reason) => controller.abort(reason));
+}