@malloydata/malloy 0.0.374 → 0.0.376

package/dist/api/foundation/config.d.ts

@@ -161,10 +161,12 @@ export declare class MalloyConfig {
      *
      * Returns the value the named overlay produces for the given path,
      * or `undefined` if the overlay doesn't exist or the path has no value.
+     * Async because overlays may be async (secret stores, session readers);
+     * `await` tolerates both sync and Promise return types.
      *
-     *   config.readOverlay('config', 'rootDirectory')
-     *   config.readOverlay('config', 'configURL')
-     *   config.readOverlay('env', 'PG_PASSWORD')
+     *   await config.readOverlay('config', 'rootDirectory')
+     *   await config.readOverlay('config', 'configURL')
+     *   await config.readOverlay('env', 'PG_PASSWORD')
      */
-    readOverlay(overlayName: string, ...path: string[]): unknown;
+    readOverlay(overlayName: string, ...path: string[]): Promise<unknown>;
 }

package/dist/api/foundation/config.js

@@ -5,9 +5,9 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MalloyConfig = exports.Manifest = void 0;
-const registry_1 = require("../../connection/registry");
 const config_compile_1 = require("./config_compile");
 const config_resolve_1 = require("./config_resolve");
+const config_lookup_1 = require("./config_lookup");
 const config_overlays_1 = require("./config_overlays");
 /**
  * In-memory manifest store. Reads, updates, and serializes manifest data.
@@ -175,24 +175,25 @@ class MalloyConfig {
         // Compile → typed tree + validation warnings.
         const compiled = (0, config_compile_1.compileConfig)(pojo);
         log.push(...compiled.log);
-        // Resolve against the merged config overlays. Entries from the passed
-        // overlays replace same-named entries in the defaults; anything new
-        // is added.
+        // Merge the host-supplied overlays onto the defaults. Same-named
+        // entries replace defaults; new keys are added.
         const mergedOverlays = {
             ...(0, config_overlays_1.defaultConfigOverlays)(),
             ...overlays,
         };
-        const resolved = (0, config_resolve_1.resolveConfig)(compiled.compiled, mergedOverlays, log);
-        // Hand fully-resolved connection entries to the registry — factories
-        // never see raw references.
-        this._managedLookup = (0, registry_1.createConnectionsFromConfig)({
-            connections: resolved.connections,
-        });
+        // Synchronous prep: extract literal sections (manifestPath, virtualMap),
+        // pull out compiled connection subtrees, fabricate bare entries for
+        // missing registered types when `includeDefaultConnections` is set.
+        // Reference resolution for connection properties is *not* done here —
+        // it happens async at `lookupConnection` time, so overlays that touch
+        // IO (secret stores, session reads) have a natural async seam.
+        const prepared = (0, config_resolve_1.prepareConfig)(compiled.compiled, log);
+        this._managedLookup = (0, config_lookup_1.buildManagedLookup)(prepared.compiledConnections, mergedOverlays, log);
         this._connections = this._managedLookup;
         this._overlays = mergedOverlays;
-        this.virtualMap = toVirtualMap(resolved.virtualMap);
-        this.manifestPath = resolved.manifestPath;
-        this.manifestURL = computeManifestURL(resolved.manifestPath, mergedOverlays);
+        this.virtualMap = toVirtualMap(prepared.virtualMap);
+        this.manifestPath = prepared.manifestPath;
+        this.manifestURL = computeManifestURL(prepared.manifestPath, mergedOverlays, log);
         this.log = log;
     }
     /**
@@ -245,14 +246,16 @@ class MalloyConfig {
      *
      * Returns the value the named overlay produces for the given path,
      * or `undefined` if the overlay doesn't exist or the path has no value.
+     * Async because overlays may be async (secret stores, session readers);
+     * `await` tolerates both sync and Promise return types.
      *
-     *   config.readOverlay('config', 'rootDirectory')
-     *   config.readOverlay('config', 'configURL')
-     *   config.readOverlay('env', 'PG_PASSWORD')
+     *   await config.readOverlay('config', 'rootDirectory')
+     *   await config.readOverlay('config', 'configURL')
+     *   await config.readOverlay('env', 'PG_PASSWORD')
      */
-    readOverlay(overlayName, ...path) {
+    async readOverlay(overlayName, ...path) {
         var _a, _b;
-        return (_b = (_a = this._overlays)[overlayName]) === null || _b === void 0 ? void 0 : _b.call(_a, path);
+        return await ((_b = (_a = this._overlays)[overlayName]) === null || _b === void 0 ? void 0 : _b.call(_a, path));
     }
 }
 exports.MalloyConfig = MalloyConfig;
@@ -276,6 +279,14 @@ const MANIFEST_FILENAME = 'malloy-manifest.json';
  *
  * Rules:
  *   - If there's no `configURL` in the `config` overlay → undefined.
+ *   - **The `config` overlay MUST resolve `configURL` synchronously.** This
+ *     is the one construction-time overlay call; it runs before the first
+ *     `lookupConnection` and must return a plain string (or undefined). If
+ *     it returns a Promise, `manifestURL` would be `undefined` — persistence
+ *     would silently stop working. To avoid the silent failure, we warn on
+ *     Promise returns so hosts discover the mistake. Other fields inside the
+ *     `config` overlay (`rootDirectory`, etc.) can still be async; only
+ *     `configURL` is sync-only.
  *   - `manifestPath` defaults to `'MANIFESTS'`.
  *   - `new URL(manifestPath, configURL)` handles three shapes uniformly:
  *       "MANIFESTS"               → relative to configURL's directory
@@ -285,9 +296,17 @@ const MANIFEST_FILENAME = 'malloy-manifest.json';
  *   - A trailing slash is forced onto the directory portion so that the
  *     final `new URL(MANIFEST_FILENAME, dir)` joins correctly.
  */
-function computeManifestURL(manifestPath, overlays) {
+function computeManifestURL(manifestPath, overlays, log) {
     const configOverlay = overlays['config'];
     const configURLValue = configOverlay === null || configOverlay === void 0 ? void 0 : configOverlay(['configURL']);
+    if (isThenable(configURLValue)) {
+        log.push({
+            message: 'the `config` overlay returned a Promise for `configURL`; `configURL` must be resolved synchronously. manifestURL will be undefined and persistence will not work.',
+            severity: 'warn',
+            code: 'config-overlay',
+        });
+        return undefined;
+    }
     if (typeof configURLValue !== 'string')
         return undefined;
     let configURL;
@@ -336,6 +355,11 @@ function toVirtualMap(raw) {
 function isRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
+function isThenable(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.then === 'function');
+}
 function isBuildManifestEntry(value) {
     return isRecord(value) && typeof value['tableName'] === 'string';
 }

package/dist/api/foundation/config_lookup.d.ts

@@ -0,0 +1,31 @@
+import type { LogMessage } from '../../lang/parse-log';
+import type { ManagedConnectionLookup } from '../../connection/registry';
+import type { ConfigDict } from './config_compile';
+import type { ConfigOverlays } from './config_overlays';
+/**
+ * Build the `ManagedConnectionLookup` a `MalloyConfig` hands to Runtime.
+ *
+ * Reference resolution and property-default application run *per lookup*,
+ * asynchronously. This is the point where overlays are actually called —
+ * which means secret stores, session readers, and any other IO-backed
+ * overlay has a natural async seam to live in. Construction of the
+ * `MalloyConfig` stays synchronous and zero-IO.
+ *
+ * Lookup flow for a connection name:
+ *   1. Find its compiled entry. Throw if the name is unknown.
+ *   2. Walk the entry tree, `await`-ing each overlay reference.
+ *   3. Fill in any property whose definition carries a `default` and that
+ *      the user didn't specify (including post-step-2 drops from unresolved
+ *      inline references).
+ *   4. Hand the resolved POJO to the registered factory.
+ *   5. Cache the resulting `Connection` by name so subsequent lookups skip
+ *      the whole pipeline.
+ *
+ * Warnings collected during steps 2–3 (unknown overlay source) are pushed
+ * into the shared `log` array — same array exposed as `MalloyConfig.log`.
+ * The log grows the first time a connection is looked up; callers that
+ * read `log` before any lookup won't see resolution warnings. That's an
+ * intentional consequence of deferred resolution: we don't pay for
+ * warnings on connections nobody asks about.
+ */
+export declare function buildManagedLookup(compiledConnections: Record<string, ConfigDict>, overlays: ConfigOverlays, log: LogMessage[]): ManagedConnectionLookup;

package/dist/api/foundation/config_lookup.js

@@ -0,0 +1,180 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildManagedLookup = buildManagedLookup;
+const registry_1 = require("../../connection/registry");
+/**
+ * Build the `ManagedConnectionLookup` a `MalloyConfig` hands to Runtime.
+ *
+ * Reference resolution and property-default application run *per lookup*,
+ * asynchronously. This is the point where overlays are actually called —
+ * which means secret stores, session readers, and any other IO-backed
+ * overlay has a natural async seam to live in. Construction of the
+ * `MalloyConfig` stays synchronous and zero-IO.
+ *
+ * Lookup flow for a connection name:
+ *   1. Find its compiled entry. Throw if the name is unknown.
+ *   2. Walk the entry tree, `await`-ing each overlay reference.
+ *   3. Fill in any property whose definition carries a `default` and that
+ *      the user didn't specify (including post-step-2 drops from unresolved
+ *      inline references).
+ *   4. Hand the resolved POJO to the registered factory.
+ *   5. Cache the resulting `Connection` by name so subsequent lookups skip
+ *      the whole pipeline.
+ *
+ * Warnings collected during steps 2–3 (unknown overlay source) are pushed
+ * into the shared `log` array — same array exposed as `MalloyConfig.log`.
+ * The log grows the first time a connection is looked up; callers that
+ * read `log` before any lookup won't see resolution warnings. That's an
+ * intentional consequence of deferred resolution: we don't pay for
+ * warnings on connections nobody asks about.
+ */
+function buildManagedLookup(compiledConnections, overlays, log) {
+    const entries = Object.entries(compiledConnections);
+    const firstConnectionName = entries.length > 0 ? entries[0][0] : undefined;
+    const cache = new Map();
+    return {
+        async lookupConnection(connectionName) {
+            if (connectionName === undefined) {
+                connectionName = firstConnectionName;
+            }
+            if (connectionName === undefined) {
+                throw new Error('No connections defined in config');
+            }
+            const cached = cache.get(connectionName);
+            if (cached)
+                return cached;
+            const compiledEntry = compiledConnections[connectionName];
+            if (!compiledEntry) {
+                throw new Error(`No connection named "${connectionName}" found in config`);
+            }
+            const resolved = await resolveCompiledEntry(compiledEntry, overlays, log);
+            // compileConnections guarantees `is` is present and a string-valued
+            // literal node — resolveCompiledEntry preserves it. Defensive check
+            // in case a compiler bug sneaks through.
+            if (typeof resolved['is'] !== 'string') {
+                throw new Error(`Connection "${connectionName}" is missing a valid "is" field`);
+            }
+            const typeDef = (0, registry_1.getConnectionTypeDef)(resolved.is);
+            if (!typeDef) {
+                throw new Error(`No registered connection type "${resolved.is}" for connection "${connectionName}". ` +
+                    'Did you forget to import the connection package?');
+            }
+            const connConfig = { name: connectionName };
+            for (const [key, value] of Object.entries(resolved)) {
+                if (key === 'is')
+                    continue;
+                if (value !== undefined && value !== null) {
+                    connConfig[key] = value;
+                }
+            }
+            const connection = await typeDef.factory(connConfig);
+            cache.set(connectionName, connection);
+            return connection;
+        },
+        async close() {
+            const connections = [...cache.values()];
+            cache.clear();
+            for (const conn of connections) {
+                await conn.close();
+            }
+        },
+    };
+}
+/**
+ * Async walk of a single connection's compiled entry against the overlays,
+ * followed by property-default application. Returns the plain POJO that
+ * gets handed to the factory.
+ */
+async function resolveCompiledEntry(entry, overlays, log) {
+    const resolved = (await resolveNode(entry, overlays, log));
+    await applyPropertyDefaults(resolved, overlays);
+    return resolved;
+}
+/**
+ * Walk a node, awaiting any overlay calls. References that fail to resolve
+ * return `undefined`; the parent dict walker drops the corresponding
+ * property. Unknown overlay sources push a warning to the log (case 1);
+ * overlays returning undefined are silently dropped (case 2).
+ */
+async function resolveNode(node, overlays, log) {
+    switch (node.kind) {
+        case 'value':
+            return node.value;
+        case 'reference':
+            return resolveReference(node, overlays, log);
+        case 'dict': {
+            const out = {};
+            for (const [k, child] of Object.entries(node.entries)) {
+                const r = await resolveNode(child, overlays, log);
+                if (r !== undefined)
+                    out[k] = r;
+            }
+            return out;
+        }
+    }
+}
+async function resolveReference(ref, overlays, log) {
+    const overlay = overlays[ref.source];
+    if (!overlay) {
+        // Case 1: unknown overlay source — warn and drop.
+        log.push({
+            message: `unknown overlay source "${ref.source}" for reference path ${JSON.stringify(ref.path)}`,
+            severity: 'warn',
+            code: 'config-overlay',
+        });
+        return undefined;
+    }
+    // Case 2: overlay returns undefined (sync or via a Promise) — silent drop.
+    // `await` tolerates both sync and Promise return types.
+    return await overlay(ref.path);
+}
+/**
+ * Fill in missing properties on a resolved entry from the registry's
+ * declared defaults. Reference-shaped defaults (like DuckDB's
+ * `workingDirectory: {config: 'rootDirectory'}`) resolve through the same
+ * overlays as inline refs. Unresolved defaults are silently dropped — a
+ * default is a hint, not a requirement (case 3).
+ *
+ * Interaction with inline references: if the user wrote a property as a
+ * reference that resolved to `undefined`, `resolveNode` already dropped it
+ * before we see the entry. From here the property looks absent, and the
+ * default applies — effectively "try the inline ref, else fall back to the
+ * default." Almost always what users want.
+ */
+async function applyPropertyDefaults(entry, overlays) {
+    var _a;
+    const typeName = entry['is'];
+    if (typeof typeName !== 'string')
+        return;
+    const props = (_a = (0, registry_1.getConnectionProperties)(typeName)) !== null && _a !== void 0 ? _a : [];
+    for (const prop of props) {
+        if (prop.default === undefined)
+            continue;
+        if (entry[prop.name] !== undefined)
+            continue;
+        const v = await resolveDefault(prop.default, overlays);
+        if (v !== undefined)
+            entry[prop.name] = v;
+    }
+}
+async function resolveDefault(def, overlays) {
+    if (typeof def !== 'object')
+        return def;
+    const keys = Object.keys(def);
+    if (keys.length !== 1)
+        return undefined;
+    const source = keys[0];
+    const raw = def[source];
+    const path = typeof raw === 'string' ? [raw] : raw;
+    if (!Array.isArray(path))
+        return undefined;
+    const overlay = overlays[source];
+    if (!overlay)
+        return undefined;
+    return await overlay(path);
+}
+//# sourceMappingURL=config_lookup.js.map

package/dist/api/foundation/config_overlays.d.ts

@@ -2,13 +2,20 @@
  * An overlay is a lambda that takes a compiled reference path and returns
  * a value (or undefined if the path is not present in the overlay source).
  *
+ * Overlays may return synchronously or asynchronously — the return type is
+ * `unknown | Promise<unknown>`. The reference resolver awaits every overlay
+ * result, so a sync overlay and an async overlay are interchangeable from
+ * the caller's perspective. Use sync for purely in-memory values (env vars,
+ * host context dicts) and async for anything that touches IO (secret stores,
+ * session fetches, enterprise-managed injected values).
+ *
  * The path is an array because references can address nested data:
  *   {env: "HOME"}             -> path = ["HOME"]
  *   {user: ["address", "zip"]} -> path = ["address", "zip"]
  *
  * Overlay values should resolve to ordinary JSON-compatible values.
  */
-export type Overlay = (path: string[]) => unknown;
+export type Overlay = (path: string[]) => unknown | Promise<unknown>;
 /**
  * A dict keyed by overlay name. A config reference like
  * `{env: "PG_PASSWORD"}` is resolved by looking up `overlays["env"]` and

package/dist/api/foundation/config_resolve.d.ts

@@ -1,49 +1,39 @@
 import type { LogMessage } from '../../lang/parse-log';
-import type { ConnectionConfigEntry } from '../../connection/registry';
 import type { ConfigDict } from './config_compile';
-import type { ConfigOverlays } from './config_overlays';
 /**
- * The shape the class body consumes: a plain POJO with the same top-level
- * sections as the input, but with all references resolved and defaults
- * applied. This is fed into the connection registry to build connections.
+ * The synchronous slice of config preparation. What the `MalloyConfig`
+ * constructor needs *before* any overlay IO happens:
  *
- * `connections` uses the registry's `ConnectionConfigEntry` shape directly —
- * a resolved entry is precisely what the registry consumes, so there's no
- * reason to invent a local alias and cast across the boundary.
+ *   - `compiledConnections` — per-connection compiled subtrees. References
+ *     inside these are resolved lazily at `lookupConnection` time, not here.
+ *     Fabricated bare `{is: typeName}` entries are included when the POJO
+ *     opts in via `includeDefaultConnections`.
+ *   - `manifestPath` — raw string (never a reference; section compiler only
+ *     produces value nodes).
+ *   - `virtualMap` — raw literal POJO (same — virtualMap is a literal slot).
  *
- * Note: `includeDefaultConnections` is an input directive, not a resolved
- * value — by the time `resolveConfig` returns, fabrication has already
- * happened. It intentionally does not appear on this interface.
+ * `includeDefaultConnections` is an input directive, not an output value —
+ * fabrication has already happened by the time this returns.
  */
-export interface ResolvedConfig {
-    connections: Record<string, ConnectionConfigEntry>;
+export interface PreparedConfig {
+    compiledConnections: Record<string, ConfigDict>;
     manifestPath?: string;
     virtualMap?: unknown;
 }
 /**
- * Walk a compiled tree against the overlay dict and produce a plain
- * resolved POJO.
+ * Synchronous top-level walk of a compiled config tree. Extracts the
+ * non-connection sections (which only contain literals — see the section
+ * compilers) and hands back the compiled connection subtrees untouched.
  *
- * Two distinct "defaults" mechanisms, deliberately separated:
+ * Reference resolution for connection properties is *not* done here. It is
+ * deferred until `lookupConnection` is called, at which point the async
+ * walker in `config_lookup.ts` can `await` overlays that do IO (secret
+ * stores, session fetches, etc.). This keeps `MalloyConfig` construction
+ * synchronous and zero-IO.
  *
- *   1. **Property defaults** (`applyPropertyDefaults`) — fill in missing
- *      properties on *every* connection entry, user-listed or fabricated.
- *      This is a uniform per-property rule; there is no asymmetry between
- *      explicit and auto-generated entries.
- *
- *   2. **`includeDefaultConnections`** (`fabricateMissingConnections`) —
- *      fabricate a bare `{is: typeName}` entry for each registered backend
- *      not already represented. Property defaults then fill in their
- *      properties via (1).
- *
- *   Order matters: fabrication runs before property defaults so that
- *   fabricated entries pick up defaults in the same pass as user-listed
- *   ones.
- *
- * Three unresolved-reference cases, each with different handling:
- *   1. Unknown overlay source    → warning, drop property
- *   2. Known overlay → undefined → silent drop
- *   3. Property default → unresolved (either of the above inside a default)
- *                                → silent drop (a default is a hint, not a requirement)
+ * Fabrication of bare `{is: typeName}` compiled entries for registered
+ * backends not otherwise represented happens here when the config opts in
+ * via `includeDefaultConnections`. Property defaults are filled in at
+ * lookup time alongside reference resolution.
  */
-export declare function resolveConfig(compiled: ConfigDict, overlays: ConfigOverlays, log: LogMessage[]): ResolvedConfig;
+export declare function prepareConfig(compiled: ConfigDict, _log: LogMessage[]): PreparedConfig;