@dbx-tools/core 0.6.89 → 0.6.91

package/src/config.ts

@@ -1,29 +1,25 @@
 /**
- * Layered configuration lookup: environment, `.env`, Databricks bundle.
+ * Layered configuration lookup: environment, `.env`, bundle, and `app.yaml`.
  *
  * Every package resolves settings the same way: take the caller's value, else
  * an environment variable, else a default. Local development adds two fallback
- * locations: `.env` files and one App's `resources.apps.<app>.config.env` in
- * `databricks.yml`.
+ * locations: `.env` files, one App's `resources.apps.<app>.config.env` in
+ * `databricks.yml`, and literal env values in `app.yaml` / `app.yml`.
  *
- * Two things make it cheap to call from a hot path:
+ * {@link values} is LAZY (an `object.Sequence`), so `databricks bundle validate`
+ * is only spawned when the environment and `.env` both missed; app YAML is only
+ * read after the bundle source also missed.
  *
- *   - {@link values} is LAZY (an `object.Sequence`), so `databricks bundle
- *     validate` is only spawned when the environment and `.env` both missed.
- *   - the parsed `.env` and bundle are cached through {@link context.cached}, so
- *     the spawn happens once per working directory and a `cwd` change misses
- *     rather than returning another project's config.
- *
- * Inside a deployed Databricks App both file sources are skipped by default
+ * Inside a deployed Databricks App all three file sources are skipped by default
  * ({@link isDatabricksAppEnv}): the platform has already turned them into real
  * environment variables, there is no bundle to validate, and the `databricks`
  * CLI is not on the image. Boolean environment overrides can force either file
  * source on or off when a tool needs different behavior.
  *
- * Only the single app's `config.env` is consulted. Root bundle `variables` are
- * not: they are authoring inputs for the bundle itself (interpolated into
- * targets, resources, and paths), so treating one as a process setting resolves
- * names the deployed app never sees.
+ * Only the single bundle app's `config.env` and literal app YAML `env[].value`
+ * entries are consulted. Root bundle `variables` are not: they are authoring
+ * inputs for the bundle itself (interpolated into targets, resources, and paths),
+ * so treating one as a process setting resolves names the deployed app never sees.
  *
  * Node-only (`child_process`, `fs`, `process`).
  *
@@ -34,32 +30,37 @@ import { spawnSync } from "node:child_process";
 import { readFileSync } from "node:fs";
 import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
 import { parseEnv } from "node:util";
-import {
-  context,
-  json,
-  log,
-  object,
-  string as sharedString,
-} from "@dbx-tools/shared-core";
+import { json, log, object, string as sharedString } from "@dbx-tools/shared-core";
+import { parse as parseYaml } from "yaml";
 import { z } from "zod";
-import { statSync } from "./file.ts";
-import { root as resolveProjectRoot } from "./project.ts";
+import { cachedRecord, statSync } from "./file.ts";
+import { resolveWorkingDirectory, root as resolveProjectRoot } from "./project.ts";
 
 const logger = log.logger("config");
+const configFileCache = new Map<string, string | undefined>();
 
 type ConfigKey = string | readonly string[];
 
+export type ConfigMapValue = string | readonly string[] | null | undefined;
+export type ConfigData = Readonly<Record<string, ConfigMapValue>>;
+
 /** Where a value may come from, consulted in the order given. */
-type ConfigSource = "env" | "dotenv" | "bundle";
+export type ConfigSource = "config" | "env" | "dotenv" | "bundle" | "app";
 
-interface ConfigOptions {
+export interface ConfigOptions {
   /**
    * Outermost namespaces tried before each key. Defaults to `DBX_TOOLS`.
    */
   scope?: string | readonly string[];
   /** Capability namespaces inserted after the scope and before each key. */
   prefix?: string | readonly string[];
-  /** Sources in precedence order. Default: `env`, `dotenv`, `bundle`. */
+  /** Constant config maps read by the `config` source. */
+  data?: ConfigData | readonly ConfigData[];
+  /** Parsed bundle data used instead of loading `databricks.yml`. */
+  bundleData?: ConfigFile | Record<string, unknown>;
+  /** Parsed app YAML data used instead of loading `app.yaml`. */
+  appData?: ConfigFile | Record<string, unknown>;
+  /** Sources in precedence order. Default: `config`, `env`, `dotenv`, `bundle`, `app`. */
   sources?: ConfigSource | readonly ConfigSource[];
   /** Directory to resolve `.env` and the bundle from. Default: `process.cwd()`. */
   cwd?: string;
@@ -73,14 +74,15 @@ interface ConfigValue {
 }
 
 /** A config file found on disk, with its parsed contents. */
-interface ConfigFile {
+export interface ConfigFile {
   path: string;
   data: Record<string, unknown>;
 }
 
 const DEFAULT_SCOPE = "DBX_TOOLS";
-const DEFAULT_SOURCES: readonly ConfigSource[] = ["env", "dotenv", "bundle"];
+const DEFAULT_SOURCES: readonly ConfigSource[] = ["config", "env", "dotenv", "bundle", "app"];
 const BUNDLE_FILE_NAMES = ["databricks.yml", "databricks.yaml"] as const;
+const APP_FILE_NAMES = ["app.yaml", "app.yml"] as const;
 const DOTENV_FILE_NAME = ".env";
 const NODE_ENV_ALTERNATIVES = {
   production: ["prod"],
@@ -99,6 +101,9 @@ const CONFIG_DOTENV_KEY = "DBX_TOOLS_CONFIG_DOTENV";
 /** Boolean environment override for Databricks bundle reads. */
 const CONFIG_BUNDLE_KEY = "DBX_TOOLS_CONFIG_BUNDLE";
 
+/** Boolean environment override for Databricks App YAML reads. */
+const CONFIG_APP_KEY = "DBX_TOOLS_CONFIG_APP";
+
 /** Exact process-environment lookup for callers that do not read local config files. */
 export const ENV_ONLY = { scope: [] as const, sources: "env" as const };
 
@@ -135,13 +140,7 @@ const databricksAppEnvSchema = z.object({
   DATABRICKS_APP_PORT: portValueSchema,
 });
 
-/**
- * The GENERIC shape of a bundle resource: a name, and whatever else the resource
- * type carries. Deliberately unopinionated and `passthrough()` - the concrete
- * resource kinds (`sql_warehouse`, `genie_space`, `postgres`, ...) are
- * Databricks-App concepts that belong to the package that resolves them, so
- * node-appkit `.extend()`s this rather than this module knowing about them.
- */
+/** A named bundle or App resource; concrete resource fields pass through. */
 export const bundleResourceSchema = z.object({ name: valueSchema.optional() }).passthrough();
 
 export const bundleEnvEntrySchema = z.object({
@@ -156,21 +155,108 @@ export const bundleAppSchema = z.object({
   resources: z.array(bundleResourceSchema).optional(),
 });
 
-const bundleAppConfigEnvSchema = z
-  .object({
-    resources: z.object({ apps: z.record(z.string(), bundleAppSchema).optional() }).optional(),
-  })
-  .transform((bundle): Record<string, string> => {
-    const apps = Object.values(bundle.resources?.apps ?? {});
-    if (apps.length !== 1) return {};
-    return Object.fromEntries(
-      (apps[0]?.config?.env ?? []).flatMap((entry) => {
-        if (!entry.name) return [];
-        const parsed = valueSchema.safeParse(entry.value);
-        return parsed.success ? ([[entry.name, parsed.data]] as const) : [];
-      }),
-    );
-  });
+export const appEnvEntrySchema = z.object({
+  name: valueSchema,
+  value: z.string().optional(),
+  valueFrom: valueSchema.optional(),
+});
+
+export const appSchema = z.object({
+  env: z.array(appEnvEntrySchema).optional(),
+  resources: z.array(bundleResourceSchema).optional(),
+});
+
+const bundleConfigSchema = z.object({
+  resources: z.object({ apps: z.record(z.string(), bundleAppSchema).optional() }).optional(),
+});
+
+/** Flatten the single bundle App's `config.env`, resolving resource references. */
+export function flattenBundleEnv(data: unknown): Record<string, string> {
+  const parsed = bundleConfigSchema.safeParse(data);
+  if (!parsed.success) return {};
+  const apps = Object.values(parsed.data.resources?.apps ?? {});
+  if (apps.length !== 1) return {};
+  const app = apps[0];
+  const resources = new Map(
+    (app?.resources ?? [])
+      .filter((resource) => resource.name)
+      .map((resource) => [resource.name!, resource] as const),
+  );
+  const result: Record<string, string> = {};
+  for (const entry of app?.config?.env ?? []) {
+    if (!entry.name) continue;
+    const literal = valueSchema.safeParse(entry.value);
+    if (literal.success) {
+      result[entry.name] = literal.data;
+      continue;
+    }
+    const referenced = entry.value_from
+      ? resourceValue(resources.get(entry.value_from))
+      : undefined;
+    if (referenced) result[entry.name] = referenced;
+  }
+  return result;
+}
+
+/** Flatten `app.yaml` env entries, resolving `valueFrom` resource references. */
+export function flattenAppEnv(data: unknown): Record<string, string> {
+  const parsed = appSchema.safeParse(data);
+  if (!parsed.success) return {};
+  const resources = new Map(
+    (parsed.data.resources ?? [])
+      .filter((resource) => resource.name)
+      .map((resource) => [resource.name!, resource] as const),
+  );
+  const result: Record<string, string> = {};
+  for (const entry of parsed.data.env ?? []) {
+    const literal = valueSchema.safeParse(entry.value);
+    if (literal.success) {
+      result[entry.name] = literal.data;
+      continue;
+    }
+    const referenced = entry.valueFrom ? resourceValue(resources.get(entry.valueFrom)) : undefined;
+    if (referenced) result[entry.name] = referenced;
+  }
+  return result;
+}
+
+function resourceValue(resource: unknown): string | undefined {
+  for (const path of [
+    ["sql_warehouse", "id"],
+    ["genie_space", "space_id"],
+    ["postgres", "endpoint"],
+    ["postgres", "database"],
+    ["postgres", "branch"],
+  ] as const) {
+    let value = resource;
+    for (const part of path) value = object.isRecord(value) ? value[part] : undefined;
+    const parsed = valueSchema.safeParse(value);
+    if (parsed.success) return parsed.data;
+  }
+  return undefined;
+}
+
+/** Walk a dot-separated path through parsed bundle data. */
+export function getBundlePath(data: Record<string, unknown>, path: string): string | undefined {
+  const parts = path.split(".").filter(Boolean);
+  if (parts.length === 0) return undefined;
+  let current: unknown = data;
+  for (let index = 0; index < parts.length; index++) {
+    if (!object.isRecord(current)) return undefined;
+    const next = current[parts[index]!];
+    if (index === parts.length - 1) {
+      const direct = valueSchema.safeParse(next);
+      if (direct.success) return direct.data;
+      if (object.isRecord(next)) {
+        const nested = valueSchema.safeParse(next.value);
+        return nested.success ? nested.data : undefined;
+      }
+      return undefined;
+    }
+    current = next;
+  }
+  return undefined;
+}
 
 /**
  * Detect a Databricks App runtime from its required name, host, and port.
@@ -189,6 +275,21 @@ export function isDatabricksAppEnv(
   return parsed.success;
 }
 
+/** Exact, uppercase, and tokenized-uppercase names for a human-friendly key. */
+export function environmentKeys(name: string): readonly string[] {
+  const trimmed = name.trim();
+  if (!trimmed) return [];
+  return object
+    .sequence(
+      trimmed,
+      trimmed.toUpperCase(),
+      Array.from(sharedString.tokenize(trimmed)).join("_").toUpperCase(),
+    )
+    .filter(Boolean)
+    .distinct()
+    .toArray();
+}
+
 /** Candidate names in scope, prefix, and input precedence order. */
 function keys(
   input: ConfigKey,
@@ -235,13 +336,13 @@ function keys(
  */
 function values(input: ConfigKey, options: ConfigOptions = {}): object.Sequence<ConfigValue> {
   const candidates = keys(input, options);
-  const sources = object.sequence<ConfigSource>(options.sources ?? DEFAULT_SOURCES);
+  const sources = configSources(options);
   return object.sequence({
     *[Symbol.iterator](): Generator<ConfigValue> {
       for (const source of sources) {
-        for (const map of read(source, options.cwd)) {
+        for (const map of read(source, options)) {
           for (const key of candidates) {
-            const value = sharedString.trimToNull(map[key]);
+            const value = configMapValue(map[key]);
             if (value !== null) yield { key, source, value };
           }
         }
@@ -250,6 +351,27 @@ function values(input: ConfigKey, options: ConfigOptions = {}): object.Sequence<
   });
 }
 
+function configSources(options: ConfigOptions): readonly ConfigSource[] {
+  const sources: ConfigSource[] = [
+    ...object.sequence<ConfigSource>(options.sources ?? DEFAULT_SOURCES).distinct(),
+  ];
+  if (options.data !== undefined && options.sources !== undefined && !sources.includes("config")) {
+    sources.push("config");
+  }
+  return sources;
+}
+
+function configMapValue(value: ConfigMapValue | unknown): string | null {
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const resolved = sharedString.trimToNull(entry);
+      if (resolved !== null) return resolved;
+    }
+    return null;
+  }
+  return sharedString.trimToNull(value);
+}
+
 /**
  * The first value that resolves for `input`, or `undefined`.
  *
@@ -260,6 +382,11 @@ export function text(input: ConfigKey, options: ConfigOptions = {}): string | un
   return values(input, options).at(0)?.value;
 }
 
+/** Resolve a human-friendly name through {@link environmentKeys} and {@link text}. */
+export function resolveValue(name: string, options: ConfigOptions = {}): string | undefined {
+  return text(environmentKeys(name), options);
+}
+
 /**
  * The PRIMARY (fully-scoped) name for `input` - what to print in a log line or an
  * error, so the message names the variable a reader should set. Do not index
@@ -394,10 +521,8 @@ export function list(
  * or a deployed App without an explicit override, there is no bundle, or the
  * CLI produces no JSON.
  *
- * Cached once per resolved working-directory context and
- * `DATABRICKS_CONFIG_PROFILE` through {@link context.cached}, so repeated
- * lookups do not rerun validation and changing either cannot return another
- * context's bundle.
+ * Parsed validation output is cached by bundle path and Databricks profile.
+ *
  */
 export function bundleFile(cwd?: string | null): ConfigFile | undefined {
   const override = object.toBoolean(process.env[CONFIG_BUNDLE_KEY]);
@@ -408,60 +533,81 @@ export function bundleFile(cwd?: string | null): ConfigFile | undefined {
   ) {
     return undefined;
   }
-  const profile = sharedString.trimToNull(process.env.DATABRICKS_CONFIG_PROFILE);
-  return cachedConfig(["bundle", profile ?? ""], cwd, (resolved) =>
-    loadBundleFile(resolved, profile),
-  );
+  const resolved = resolveWorkingDirectory(cwd);
+  return loadBundleFile(resolved, sharedString.trimToNull(process.env.DATABRICKS_CONFIG_PROFILE));
+}
+
+/** The parsed `app.yaml` / `app.yml` for `cwd`, when local App config reads are enabled. */
+export function appFile(cwd?: string | null): ConfigFile | undefined {
+  const enabled = object.toBoolean(process.env[CONFIG_APP_KEY]) ?? !isDatabricksAppEnv();
+  if (!enabled) return undefined;
+  return loadAppFile(resolveWorkingDirectory(cwd));
+}
+
+/** The single bundle App's resolved environment, when available. */
+function bundleEnvironment(options: ConfigOptions): Record<string, string> | undefined {
+  const data = sourceData(options.bundleData) ?? bundleFile(options.cwd)?.data;
+  return data ? flattenBundleEnv(data) : undefined;
+}
+
+function appEnvironment(options: ConfigOptions): Record<string, string> | undefined {
+  const data = sourceData(options.appData) ?? appFile(options.cwd)?.data;
+  return data ? flattenAppEnv(data) : undefined;
 }
 
-/** The single bundle App's resolved literal environment, when available. */
-function bundleEnvironment(cwd?: string): Record<string, string> | undefined {
-  const file = bundleFile(cwd);
-  if (!file) return undefined;
-  const parsed = bundleAppConfigEnvSchema.safeParse(file.data);
-  return parsed.success ? parsed.data : undefined;
+function sourceData(
+  source: ConfigFile | Record<string, unknown> | undefined,
+): Record<string, unknown> | undefined {
+  if (!source) return undefined;
+  if ("path" in source && typeof source.path === "string" && object.isRecord(source.data)) {
+    return source.data;
+  }
+  return source as Record<string, unknown>;
 }
 
-/** Parsed `.env` for `cwd`, or `{}`. Read once per resolved context. */
+/** Parsed `.env` for `cwd`, or `{}`; parsed file data is cached by path. */
 function dotenv(cwd?: string | null): Record<string, string | undefined> {
   const enabled = object.toBoolean(process.env[CONFIG_DOTENV_KEY]) ?? !isDatabricksAppEnv();
   if (!enabled) return {};
-  const environments = nodeEnvNames(process.env.NODE_ENV);
-  return cachedConfig(["dotenv", ...environments], cwd, (resolved) =>
-    loadDotenv(resolved, environments),
-  );
-}
-
-/** Cache every target directory separately within the active process context. */
-function cachedConfig<T>(
-  name: readonly string[],
-  cwd: string | null | undefined,
-  loader: (resolved: string) => T,
-): T {
-  const active = context.getContext() ?? "";
-  const resolved = resolve(cwd ?? ".");
-  return context.cached(["config", active, resolved, ...name], () => loader(resolved));
+  const resolved = resolveWorkingDirectory(cwd);
+  return loadDotenv(resolved, nodeEnvNames(process.env.NODE_ENV));
 }
 
 /** Maps for one source in precedence order. */
 function read(
   source: ConfigSource,
-  cwd?: string,
-): object.Sequence<Record<string, string | undefined>> {
+  options: ConfigOptions,
+): object.Sequence<Readonly<Record<string, unknown>>> {
   return object.sequence({
-    *[Symbol.iterator](): Generator<Record<string, string | undefined>> {
+    *[Symbol.iterator](): Generator<Readonly<Record<string, unknown>>> {
       switch (source) {
+        case "config": {
+          const data = options.data;
+          if (Array.isArray(data)) {
+            yield* data;
+          } else if (data !== undefined) {
+            yield data as ConfigData;
+          }
+          break;
+        }
         case "env":
           yield process.env;
           break;
         case "dotenv":
-          yield dotenv(cwd);
+          yield dotenv(options.cwd);
           break;
         case "bundle": {
-          const environment = bundleEnvironment(cwd);
+          const environment = bundleEnvironment(options);
+          if (environment) yield environment;
+          break;
+        }
+        case "app": {
+          const environment = appEnvironment(options);
           if (environment) yield environment;
           break;
         }
+        default:
+          throw new TypeError(`Unknown config source: ${source}`);
       }
     },
   });
@@ -473,6 +619,8 @@ function read(
  */
 function findConfigFile(cwd: string, names: readonly string[]): string | undefined {
   const start = resolve(cwd);
+  const key = JSON.stringify([start, ...names]);
+  if (configFileCache.has(key)) return configFileCache.get(key);
   const root = resolveProjectRoot(start);
   const pathFromRoot = root ? relative(root, start) : undefined;
   const boundary =
@@ -486,10 +634,14 @@ function findConfigFile(cwd: string, names: readonly string[]): string | undefin
   for (let dir = start; ; dir = dirname(dir)) {
     for (const name of names) {
       const path = resolve(dir, name);
-      if (statSync(path)?.isFile()) return path;
+      if (statSync(path)?.isFile()) {
+        configFileCache.set(key, path);
+        return path;
+      }
     }
     if (boundary === undefined || dir === boundary) break;
   }
+  configFileCache.set(key, undefined);
   return undefined;
 }
 
@@ -500,42 +652,64 @@ function loadDotenv(
   const names = [...environments.map((name) => `${DOTENV_FILE_NAME}.${name}`), DOTENV_FILE_NAME];
   const path = findConfigFile(cwd, names);
   if (!path) return {};
-  try {
-    return parseEnv(readFileSync(path, "utf8"));
-  } catch {
-    logger.warn("failed to read dotenv file", { path });
-    return {};
-  }
+  return (
+    cachedRecord(JSON.stringify(["dotenv", path]), () => {
+      try {
+        return parseEnv(readFileSync(path, "utf8"));
+      } catch {
+        logger.warn("failed to read dotenv file", { path });
+        return {};
+      }
+    }) ?? {}
+  );
 }
 
 function loadBundleFile(cwd: string, profile: string | null): ConfigFile | undefined {
   const path = findConfigFile(cwd, BUNDLE_FILE_NAMES);
   if (!path) return undefined;
-  const args = [
-    "bundle",
-    "validate",
-    "--output",
-    "json",
-    ...(profile === null ? [] : ["--profile", profile]),
-  ];
-  const result = spawnSync("databricks", args, {
-    cwd: resolve(path, ".."),
-    encoding: "utf8",
-    stdio: ["ignore", "pipe", "pipe"],
+  const data = cachedRecord(JSON.stringify(["bundle", path, profile]), () => {
+    const args = [
+      "bundle",
+      "validate",
+      "--output",
+      "json",
+      ...(profile === null ? [] : ["--profile", profile]),
+    ];
+    const result = spawnSync("databricks", args, {
+      cwd: resolve(path, ".."),
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const output = sharedString.trimToNull(result.stdout);
+    const error = sharedString.trimToNull(result.stderr);
+    if (output === null) {
+      logger.debug("bundle validate produced no JSON", { path, status: result.status, error });
+      return undefined;
+    }
+    const parsed = json.parseRecord(output);
+    if (!parsed) {
+      logger.warn("failed to parse bundle validate output", { path, status: result.status });
+      return undefined;
+    }
+    if (result.status !== 0) {
+      logger.debug("using partial bundle output", { path, status: result.status, error });
+    }
+    return parsed;
   });
-  const output = sharedString.trimToNull(result.stdout);
-  const error = sharedString.trimToNull(result.stderr);
-  if (output === null) {
-    logger.debug("bundle validate produced no JSON", { path, status: result.status, error });
-    return undefined;
-  }
-  const data = json.parseRecord(output);
-  if (!data) {
-    logger.warn("failed to parse bundle validate output", { path, status: result.status });
-    return undefined;
-  }
-  if (result.status !== 0) {
-    logger.debug("using partial bundle output", { path, status: result.status, error });
-  }
-  return { path, data };
+  return data === undefined ? undefined : { path, data };
+}
+
+function loadAppFile(cwd: string): ConfigFile | undefined {
+  const path = findConfigFile(cwd, APP_FILE_NAMES);
+  if (!path) return undefined;
+  const data = cachedRecord(JSON.stringify(["app", path]), () => {
+    try {
+      const parsed = parseYaml(readFileSync(path, "utf8"));
+      return object.isRecord(parsed) ? parsed : undefined;
+    } catch {
+      logger.warn("failed to parse app yaml", { path });
+      return undefined;
+    }
+  });
+  return data === undefined ? undefined : { path, data };
 }

package/src/file.ts

@@ -5,6 +5,8 @@
  */
 import { Stats, statSync as nodeStatSync } from "node:fs";
 
+const recordCache = new Map<string, Record<string, unknown> | undefined>();
+
 /**
  * Best-effort `fs.stat` (sync). Returns `undefined` for a blank path or when the
  * path can't be stat'd (missing, permission denied, ...), so callers can treat
@@ -18,3 +20,20 @@ export function statSync(path: string): Stats | undefined {
   }
   return undefined;
 }
+
+/**
+ * Load and cache a parsed record by caller-defined source identity.
+ *
+ * Every result is cached, including empty records and `undefined`. Include every
+ * input that affects parsing in `key`, such as a file path plus a Databricks
+ * profile.
+ */
+export function cachedRecord<T extends Record<string, unknown>>(
+  key: string,
+  loader: () => T | undefined,
+): T | undefined {
+  if (recordCache.has(key)) return recordCache.get(key) as T | undefined;
+  const value = loader();
+  recordCache.set(key, value);
+  return value;
+}