@malloydata/malloy 0.0.431 → 0.0.433

package/dist/api/foundation/config.js

@@ -6,6 +6,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MalloyConfig = exports.Manifest = void 0;
 exports.isBuildManifestEntry = isBuildManifestEntry;
+const registry_1 = require("../../connection/registry");
 const config_compile_1 = require("./config_compile");
 const config_resolve_1 = require("./config_resolve");
 const config_lookup_1 = require("./config_lookup");
@@ -165,7 +166,7 @@ class MalloyConfig {
         // it happens async at `lookupConnection` time, so overlays that touch
         // IO (secret stores, session reads) have a natural async seam.
         const prepared = (0, config_resolve_1.prepareConfig)(compiled.compiled, mergedOverlays, log);
-        this._managedLookup = (0, config_lookup_1.buildManagedLookup)(prepared.compiledConnections, mergedOverlays, log);
+        this._managedLookup = (0, config_lookup_1.buildManagedLookup)(prepared.compiledConnections, rawConnectionEntries(pojo), mergedOverlays, log);
         this._connections = this._managedLookup;
         this._overlays = mergedOverlays;
         this.virtualMap = toVirtualMap(prepared.virtualMap, log);
@@ -476,6 +477,27 @@ function toVirtualMap(raw, log) {
 function isRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
+/**
+ * The connection entries exactly as authored, kept alongside the compiled tree
+ * and handed to each factory as `rawConfigData`. Compilation replaces an
+ * overlay reference with a `ConfigReference` node and resolution replaces that
+ * with the value, so this is the only place a connector can still see *which*
+ * overlay a value came from — which is what BigQuery's `getDigest` needs to
+ * tell two impersonated identities apart.
+ *
+ * No validation here beyond the shape: entries the compiler rejected are
+ * absent from `compiledConnections` and so are never looked up.
+ */
+function rawConnectionEntries(pojo) {
+    const entries = {};
+    if (!isRecord(pojo) || !isRecord(pojo['connections']))
+        return entries;
+    for (const [name, entry] of Object.entries(pojo['connections'])) {
+        if ((0, registry_1.isConnectionConfigEntry)(entry))
+            entries[name] = entry;
+    }
+    return entries;
+}
 function isBuildManifestEntry(value) {
     return isRecord(value) && typeof value['tableName'] === 'string';
 }

package/dist/api/foundation/config_compile.d.ts

@@ -3,7 +3,7 @@ import type { LogMessage } from '../../lang/parse-log';
  * A compiled config node. The compiler produces a tree of these; the resolver
  * walks the tree against a ConfigOverlays dict and produces a plain POJO.
  */
-export type ConfigNode = ConfigDict | ConfigLiteral | ConfigReference;
+export type ConfigNode = ConfigDict | ConfigLiteral | ConfigReference | ConfigInvalid;
 export interface ConfigDict {
     kind: 'dict';
     entries: Record<string, ConfigNode>;
@@ -23,6 +23,17 @@ export interface ConfigReference {
     /** Path into the overlay. */
     path: string[];
 }
+/**
+ * A property the user wrote that can never produce a value — it failed its
+ * type check. Compilation records it rather than dropping it so that a
+ * `mustHaveValue` property can tell "the user wrote something unusable" apart
+ * from "the user wrote nothing." It resolves to `undefined` like any other
+ * failure; the difference is only that the key is still visible in the
+ * compiled tree.
+ */
+export interface ConfigInvalid {
+    kind: 'invalid';
+}
 export type SectionCompiler = (value: unknown, log: LogMessage[]) => ConfigNode | undefined;
 /**
  * Compile a POJO into a typed dictionary tree, collecting validation warnings.

package/dist/api/foundation/config_compile.js

@@ -110,34 +110,49 @@ function compileConnectionEntry(prefix, typeName, rawEntry, log) {
     return entry;
 }
 /**
- * Compile a single connection property value. At non-`json` property slots,
- * a single-key object whose value is a string or string[] is recognized as
- * an overlay reference. `json`-typed slots always pass through as literal
- * data — this is the security invariant that prevents reference injection
- * into structured config.
+ * Compile a single connection property value.
+ *
+ * At non-`json` property slots, a single-key object whose value is a string or
+ * string[] is recognized as an overlay reference. `json`-typed slots always
+ * pass through as literal data — this is the security invariant that prevents
+ * reference injection into structured config.
+ *
+ * A property's `source` overrides that reading in both directions. `'literal'`
+ * refuses a reference; `'overlay'` refuses a literal, which is also what lets
+ * a reference be recognized in a slot that would otherwise hold a dictionary.
  */
 function compileConnectionProperty(path, propDef, value, log) {
-    if (value === undefined || value === null)
-        return undefined;
-    if (propDef.type === 'json') {
-        return { kind: 'value', value };
+    if (value === undefined || value === null) {
+        return propDef.mustHaveValue ? { kind: 'invalid' } : undefined;
     }
     const ref = asReferenceShape(value);
-    if (ref !== undefined) {
-        if (propDef.requireLiteralString) {
-            log.push(makeWarning(path, 'must be a literal string and cannot use an overlay reference'));
-            return { kind: 'value', value };
-        }
-        return ref;
+    switch (propDef.source) {
+        case 'overlay':
+            if (ref === undefined) {
+                log.push(makeWarning(path, 'must name an overlay, as in {env: "NAME"}; a value written here ' +
+                    'directly is not allowed'));
+                return { kind: 'invalid' };
+            }
+            return ref;
+        case 'literal':
+            if (ref !== undefined) {
+                log.push(makeWarning(path, 'must be written here directly and cannot name an overlay'));
+                return { kind: 'invalid' };
+            }
+            break;
+        default:
+            if (propDef.type === 'json') {
+                return { kind: 'value', value };
+            }
+            if (ref !== undefined) {
+                return ref;
+            }
+            break;
     }
     const typeError = checkValueType(value, propDef.type);
     if (typeError) {
-        if (propDef.requireLiteralString) {
-            log.push(makeWarning(path, `must be a literal string, got ${describeConfigValue(value)}`));
-            return { kind: 'value', value };
-        }
         log.push(makeWarning(path, `${typeError} (expected ${propDef.type})`));
-        return undefined;
+        return propDef.mustHaveValue ? { kind: 'invalid' } : undefined;
     }
     return { kind: 'value', value };
 }
@@ -221,11 +236,6 @@ function makeWarning(path, message) {
         code: 'config-validation',
     };
 }
-function describeConfigValue(value) {
-    if (Array.isArray(value))
-        return 'array';
-    return typeof value;
-}
 function checkValueType(value, expectedType) {
     switch (expectedType) {
         case 'number':
@@ -245,6 +255,7 @@ function checkValueType(value, expectedType) {
                 return `should be a string, got ${typeof value}`;
             break;
         case 'json':
+        case 'opaque':
             break;
     }
     return undefined;

package/dist/api/foundation/config_lookup.d.ts

@@ -1,5 +1,5 @@
 import type { LogMessage } from '../../lang/parse-log';
-import type { ManagedConnectionLookup } from '../../connection/registry';
+import type { ConnectionConfigEntry, ManagedConnectionLookup } from '../../connection/registry';
 import type { ConfigDict } from './config_compile';
 import type { ConfigOverlays } from './config_overlays';
 /**
@@ -28,4 +28,4 @@ import type { ConfigOverlays } from './config_overlays';
  * intentional consequence of deferred resolution: we don't pay for
  * warnings on connections nobody asks about.
  */
-export declare function buildManagedLookup(compiledConnections: Record<string, ConfigDict>, overlays: ConfigOverlays, log: LogMessage[]): ManagedConnectionLookup;
+export declare function buildManagedLookup(compiledConnections: Record<string, ConfigDict>, rawConnections: Record<string, ConnectionConfigEntry>, overlays: ConfigOverlays, log: LogMessage[]): ManagedConnectionLookup;

package/dist/api/foundation/config_lookup.js

@@ -32,7 +32,7 @@ const registry_1 = require("../../connection/registry");
  * intentional consequence of deferred resolution: we don't pay for
  * warnings on connections nobody asks about.
  */
-function buildManagedLookup(compiledConnections, overlays, log) {
+function buildManagedLookup(compiledConnections, rawConnections, overlays, log) {
     const entries = Object.entries(compiledConnections);
     const firstConnectionName = entries.length > 0 ? entries[0][0] : undefined;
     const cache = new Map();
@@ -51,7 +51,7 @@ function buildManagedLookup(compiledConnections, overlays, log) {
             if (!compiledEntry) {
                 throw new Error(`No connection named "${connectionName}" found in config`);
             }
-            const resolved = await resolveCompiledEntry(compiledEntry, overlays, log);
+            const resolved = await resolveCompiledEntry(connectionName, compiledEntry, overlays, log);
             // compileConnections guarantees `is` is present and a string-valued
             // literal node — resolveCompiledEntry preserves it. Defensive check
             // in case a compiler bug sneaks through.
@@ -71,8 +71,7 @@ function buildManagedLookup(compiledConnections, overlays, log) {
                     connConfig[key] = value;
                 }
             }
-            (0, registry_1.validateConnectionConfigProperties)(connectionName, resolved.is, connConfig);
-            const connection = await typeDef.factory(connConfig);
+            const connection = await typeDef.factory(connConfig, rawConnections[connectionName]);
             cache.set(connectionName, connection);
             return connection;
         },
@@ -97,7 +96,7 @@ function buildManagedLookup(compiledConnections, overlays, log) {
  * followed by property-default application. Returns the plain POJO that
  * gets handed to the factory.
  */
-async function resolveCompiledEntry(entry, overlays, log) {
+async function resolveCompiledEntry(connectionName, entry, overlays, log) {
     const resolved = await resolveNode(entry, overlays, log);
     // resolveNode returns `unknown`. The compileConnections pipeline
     // guarantees every connection entry is an object dict with `is` set to
@@ -106,9 +105,35 @@ async function resolveCompiledEntry(entry, overlays, log) {
     if (!(0, registry_1.isConnectionConfigEntry)(resolved)) {
         throw new Error('Connection entry did not resolve to a valid {is: string, ...} dict');
     }
+    // Before defaults, so that a declared default can't quietly satisfy a
+    // promise the user's own value failed to keep.
+    requireAuthoredValues(connectionName, entry, resolved);
     await applyPropertyDefaults(resolved, overlays);
     return resolved;
 }
+/**
+ * Enforce `mustHaveValue`: a property the user wrote must survive resolution.
+ *
+ * The compiled entry is the record of what was authored — a reference that
+ * resolved to nothing, or a value that failed its type check, is gone from
+ * `resolved` but still present here. Absence from both is fine; that's the
+ * user declining to set the property, which is what `optional` already means.
+ */
+function requireAuthoredValues(connectionName, entry, resolved) {
+    var _a;
+    for (const prop of (_a = (0, registry_1.getConnectionProperties)(resolved.is)) !== null && _a !== void 0 ? _a : []) {
+        if (!prop.mustHaveValue)
+            continue;
+        if (entry.entries[prop.name] === undefined)
+            continue;
+        if (resolved[prop.name] !== undefined)
+            continue;
+        throw new Error(`Connection "${connectionName}" sets "${prop.name}", but no value ` +
+            'arrived — an overlay reference that did not resolve, or a value that ' +
+            'was rejected. Omitting the property is allowed; this one cannot fall ' +
+            'back silently.');
+    }
+}
 /**
  * Walk a node, awaiting any overlay calls. References that fail to resolve
  * return `undefined`; the parent dict walker drops the corresponding
@@ -119,6 +144,8 @@ async function resolveNode(node, overlays, log) {
     switch (node.kind) {
         case 'value':
             return node.value;
+        case 'invalid':
+            return undefined;
         case 'reference':
             return resolveReference(node, overlays, log);
         case 'dict': {

package/dist/connection/registry.d.ts

@@ -1,12 +1,29 @@
 import type { Connection, ConnectionConfig, LookupConnection } from './types';
 /**
  * A factory function that creates a Connection from a config object.
+ *
+ * `rawConfigData` is the connection's entry exactly as the user wrote it,
+ * before overlay references were resolved — so an overlay-supplied property
+ * still reads as `{tenantAuth: "acme"}` rather than as the value it produced.
+ * A connector needs it when a resolved value has no stable identity of its own
+ * but the connection's digest depends on which one it is; BigQuery's
+ * `authClient` is the case that exists today. It is undefined for hosts that
+ * build connections from an already-resolved config.
+ *
+ * Being unresolved, it is the *non-secret* twin of `config`: it holds
+ * `{env: "PGPASSWORD"}` where `config` holds the password.
  */
-export type ConnectionTypeFactory = (config: ConnectionConfig) => Promise<Connection>;
+export type ConnectionTypeFactory = (config: ConnectionConfig, rawConfigData?: ConnectionConfigEntry) => Promise<Connection>;
 /**
  * The type of a connection property value.
  */
-export type ConnectionPropertyType = 'string' | 'number' | 'boolean' | 'password' | 'secret' | 'file' | 'json' | 'text';
+export type ConnectionPropertyType = 'string' | 'number' | 'boolean' | 'password' | 'secret' | 'file' | 'json' | 'text'
+/**
+ * A value with no declared shape that Malloy never inspects. Only reachable
+ * through an overlay — pair it with `source: 'overlay'`, since a config file
+ * has no way to write a live object down.
+ */
+ | 'opaque';
 /**
  * Describes a single configuration property for a connection type.
  *
@@ -35,12 +52,31 @@ export interface ConnectionPropertyDefinition {
     /** For type 'file': extension filters for picker dialogs. */
     fileFilters?: Record<string, string[]>;
     /**
-     * For security-sensitive string slots, preserve malformed/reference-shaped
-     * raw values so registry lookup can fail closed instead of silently dropping
-     * the property during generic compilation. Factories must not rely on this
-     * metadata as their only validation layer.
+     * Where this property's value is allowed to come from. Unset means either —
+     * a literal in the config file, or an overlay reference.
+     *
+     * `'literal'` refuses overlay references. For a property whose value is only
+     * meaningful if it was written down deliberately.
+     *
+     * `'overlay'` refuses literals: the config file can only *name* an overlay
+     * the host registered, never supply the value itself. This is also what
+     * makes a reference unambiguous in a slot that would otherwise hold a
+     * dictionary — if a literal isn't legal there, a single-key object can only
+     * be a reference.
      */
-    requireLiteralString?: true;
+    source?: 'literal' | 'overlay';
+    /**
+     * Writing this property is a promise that it will produce a value. Omitting
+     * it is fine; setting it and having nothing survive resolution is an error
+     * rather than a silent drop.
+     *
+     * For properties whose absence triggers a fallback that isn't safe — DuckDB's
+     * `securityPolicy`, where absent means the `none` policy, or a credential
+     * whose absence means the ambient one. Without this, a misspelled overlay
+     * source or an unset environment variable removes the property and the
+     * connection is built as if the user had never asked for it.
+     */
+    mustHaveValue?: true;
 }
 /**
  * A connection type definition: factory plus property metadata.
@@ -106,11 +142,6 @@ export declare function getRegisteredConnectionTypes(): string[];
  * lookup to hand fully-resolved configs to the right factory.
  */
 export declare function getConnectionTypeDef(typeName: string): ConnectionTypeDef | undefined;
-/**
- * Enforce registry-level literal-string requirements after overlay resolution
- * and before a connection factory sees the config.
- */
-export declare function validateConnectionConfigProperties(connectionName: string, typeName: string, config: ConnectionConfig): void;
 /**
  * Parse a JSON config string into a ConnectionsConfig.
  * Entries without a valid `is` field are silently dropped.

package/dist/connection/registry.js

@@ -10,7 +10,6 @@ exports.getConnectionProperties = getConnectionProperties;
 exports.getConnectionTypeDisplayName = getConnectionTypeDisplayName;
 exports.getRegisteredConnectionTypes = getRegisteredConnectionTypes;
 exports.getConnectionTypeDef = getConnectionTypeDef;
-exports.validateConnectionConfigProperties = validateConnectionConfigProperties;
 exports.readConnectionsConfig = readConnectionsConfig;
 exports.writeConnectionsConfig = writeConnectionsConfig;
 exports.createConnectionsFromConfig = createConnectionsFromConfig;
@@ -67,22 +66,6 @@ function getRegisteredConnectionTypes() {
 function getConnectionTypeDef(typeName) {
     return registry.get(typeName);
 }
-/**
- * Enforce registry-level literal-string requirements after overlay resolution
- * and before a connection factory sees the config.
- */
-function validateConnectionConfigProperties(connectionName, typeName, config) {
-    var _a, _b;
-    const props = (_b = (_a = registry.get(typeName)) === null || _a === void 0 ? void 0 : _a.properties) !== null && _b !== void 0 ? _b : [];
-    for (const prop of props) {
-        if (!prop.requireLiteralString)
-            continue;
-        const value = config[prop.name];
-        if (value !== undefined && typeof value !== 'string') {
-            throw new Error(`Connection "${connectionName}" property "${prop.name}" must be a literal string`);
-        }
-    }
-}
 /**
  * Parse a JSON config string into a ConnectionsConfig.
  * Entries without a valid `is` field are silently dropped.
@@ -144,7 +127,6 @@ function createConnectionsFromConfig(config, onConnectionCreated) {
                     connConfig[key] = value;
                 }
             }
-            validateConnectionConfigProperties(connectionName, entry.is, connConfig);
             const connection = await typeDef.factory(connConfig);
             if (onConnectionCreated) {
                 onConnectionCreated(connectionName, connection);

package/dist/connection/types.d.ts

@@ -49,9 +49,15 @@ export interface InfoConnection {
 export type ConnectionParameterValue = string | number | boolean | null | Array<ConnectionParameterValue> | {
     [key: string]: ConnectionParameterValue;
 };
+/**
+ * A value with no declared shape, supplied by a host overlay rather than
+ * written in a config file — an auth client, a session handle, anything live.
+ * Malloy never inspects one; it carries it from the overlay to the factory.
+ */
+export type OpaqueConnectionValue = object;
 export interface ConnectionConfig {
     name: string;
-    [key: string]: ConnectionParameterValue | undefined;
+    [key: string]: ConnectionParameterValue | OpaqueConnectionValue | undefined;
 }
 export interface ConnectionMetadata {
     url?: string;

package/dist/dialect/mysql/mysql.js

@@ -22,38 +22,65 @@ const inSeconds = {
     day: 24 * 3600,
     week: 7 * 24 * 3600,
 };
+/**
+ * MySQL reports a column type as
+ *
+ *     base [ '(' params ')' ] [ 'unsigned' ] [ 'zerofill' ]
+ *
+ * and normalizes every alias away before reporting, so a schema read only
+ * ever produces canonical base names. A raw cast reaches this map without
+ * that normalization, and any alias it spells -- REAL, INTEGER, NUMERIC --
+ * falls through to `sql native`.
+ *
+ * No modifier changes the Malloy type: `unsigned` only widens the range, and
+ * the sole type whose range crosses a Malloy boundary is bigint, which is
+ * already `bigint`; `zerofill` is presentation. So the map is keyed on the
+ * base name alone. DECIMAL is not in the map because its Malloy type depends
+ * on its parameters.
+ */
 const mysqlToMalloyTypes = {
-    // TODO: This assumes tinyint is always going to be a boolean.
-    'tinyint': { type: 'boolean' },
+    'tinyint': { type: 'number', numberType: 'integer' },
     'smallint': { type: 'number', numberType: 'integer' },
     'mediumint': { type: 'number', numberType: 'integer' },
     'int': { type: 'number', numberType: 'integer' },
     'bigint': { type: 'number', numberType: 'bigint' },
-    'tinyint unsigned': { type: 'number', numberType: 'integer' },
-    'smallint unsigned': { type: 'number', numberType: 'integer' },
-    'mediumint unsigned': { type: 'number', numberType: 'integer' },
-    'int unsigned': { type: 'number', numberType: 'integer' },
-    'bigint unsigned': { type: 'number', numberType: 'bigint' },
+    'float': { type: 'number', numberType: 'float' },
     'double': { type: 'number', numberType: 'float' },
-    'varchar': { type: 'string' },
-    'varbinary': { type: 'string' },
     'char': { type: 'string' },
+    'varchar': { type: 'string' },
+    'tinytext': { type: 'string' },
     'text': { type: 'string' },
+    'mediumtext': { type: 'string' },
+    'longtext': { type: 'string' },
     'date': { type: 'date' },
     'datetime': { type: 'timestamp' },
     'timestamp': { type: 'timestamp' },
     'time': { type: 'string' },
-    'decimal': { type: 'number', numberType: 'float' },
-    // TODO: Check if we need special handling for boolean.
-    'tinyint(1)': { type: 'boolean' },
 };
+/**
+ * Split a reported type into its base name and numeric parameters.
+ *
+ * Only numeric parameter lists are read. ENUM and SET carry quoted value
+ * lists whose quoting rules a regex should not chase, and nothing needs
+ * their values -- both map to `sql native`.
+ */
+function parseMySQLType(sqlType) {
+    var _a, _b;
+    const text = sqlType.trim().toLowerCase();
+    const base = (_b = (_a = text.match(/^\w+/)) === null || _a === void 0 ? void 0 : _a[0]) !== null && _b !== void 0 ? _b : text;
+    const params = text.match(/^\w+\s*\((\d+(?:\s*,\s*\d+)*)\)/);
+    return {
+        base,
+        params: params ? params[1].split(',').map(p => parseInt(p, 10)) : [],
+    };
+}
 function malloyTypeToJSONTableType(malloyType) {
     switch (malloyType.type) {
         case 'number':
-            if (malloyType.numberType === 'integer') {
-                return 'INT';
-            }
-            else if (malloyType.numberType === 'bigint') {
+            // BIGINT for both: JSON_TABLE turns an out-of-range value into NULL
+            // without warning, and `integer` is not bounded at 32 bits.
+            if (malloyType.numberType === 'integer' ||
+                malloyType.numberType === 'bigint') {
                 return 'BIGINT';
             }
             else {
@@ -62,7 +89,8 @@ function malloyTypeToJSONTableType(malloyType) {
         case 'string':
             return 'CHAR(255)'; // JSON_TABLE needs a length
         case 'boolean':
-            return 'INT'; // or TINYINT(1) if you prefer
+            // MySQL has no boolean; TINYINT(1) is only a spelling of TINYINT.
+            return 'INT';
         case 'record':
         case 'array':
             return 'JSON';
@@ -141,11 +169,24 @@ class MySQLDialect extends dialect_1.Dialect {
     }
     sqlTypeToMalloyType(sqlType) {
         var _a, _b;
-        // Remove trailing params
-        const baseSqlType = (_b = (_a = sqlType.match(/^(\w+)/)) === null || _a === void 0 ? void 0 : _a.at(0)) !== null && _b !== void 0 ? _b : sqlType;
-        return (mysqlToMalloyTypes[baseSqlType.toLowerCase()] || {
+        const { base, params } = parseMySQLType(sqlType);
+        if (base === 'decimal') {
+            // DECIMAL is exact, so scale 0 is an integer rather than a float. Above
+            // 15 digits it no longer survives a JS double and must be carried as a
+            // bigint. MySQL reports precision and scale on every decimal; the
+            // defaults match its own when a bare DECIMAL is declared.
+            const [precision, scale] = [(_a = params[0]) !== null && _a !== void 0 ? _a : 10, (_b = params[1]) !== null && _b !== void 0 ? _b : 0];
+            if (scale > 0) {
+                return { type: 'number', numberType: 'float' };
+            }
+            return {
+                type: 'number',
+                numberType: precision <= 15 ? 'integer' : 'bigint',
+            };
+        }
+        return (mysqlToMalloyTypes[base] || {
             type: 'sql native',
-            rawType: baseSqlType,
+            rawType: base,
         });
     }
     sqlGroupSetTable(groupSetCount) {

package/dist/lang/ast/expressions/expr-aggregate-function.js

@@ -45,6 +45,7 @@ const static_space_1 = require("../field-space/static-space");
 const TDU = __importStar(require("../typedesc-utils"));
 const field_references_1 = require("../query-items/field-references");
 const expression_def_1 = require("../types/expression-def");
+const field_space_1 = require("../types/field-space");
 const space_field_1 = require("../types/space-field");
 const expr_id_reference_1 = require("./expr-id-reference");
 class ExprAggregateFunction extends expression_def_1.ExpressionDef {
@@ -224,37 +225,30 @@ function joinPathEq(a1, a2) {
 }
 function getJoinUsage(fs, expr) {
     const result = [];
+    /**
+     * Walk a path from translated IR. Every name in it is known to resolve at
+     * private level, so a failure is an internal error. The names are
+     * unparented; readEntry records no references for them.
+     */
     const lookupWithPath = (fs, path) => {
-        const head = path[0];
-        const rest = path.slice(1);
-        const def = fs.entry(head);
-        if (def === undefined) {
-            throw new Error(`Invalid field lookup ${head}`);
-        }
-        if (def instanceof static_space_1.StructSpaceField && rest.length > 0) {
-            const restDef = lookupWithPath(def.fieldSpace, rest);
-            return {
-                ...restDef,
-                joinPath: [{ ...def.joinPathElement, name: head }, ...restDef.joinPath],
-            };
+        const names = path.map(n => new field_space_1.FieldName(n));
+        const last = names[names.length - 1];
+        const ns = (0, static_space_1.resolveNamespace)(fs, names.slice(0, -1), 'private', last.name);
+        if (ns.error) {
+            throw new Error(ns.error.message);
         }
-        else if (def instanceof space_field_1.SpaceField) {
-            if (rest.length !== 0) {
-                throw new Error(`${head} cannot contain a ${rest.join('.')}`);
-            }
-            const fieldDef = def.fieldDef();
-            if (fieldDef) {
-                return {
-                    fs,
-                    def: fieldDef,
-                    joinPath: [],
-                };
-            }
-            throw new Error('No field def');
+        const read = (0, static_space_1.readEntry)(ns.space, last, 'private');
+        if (read.error) {
+            throw new Error(read.error.message);
         }
-        else {
+        if (!(read.found instanceof space_field_1.SpaceField)) {
             throw new Error('expected a field def or struct');
         }
+        const def = read.found.fieldDef();
+        if (def === undefined) {
+            throw new Error('No field def');
+        }
+        return { fs: ns.space, def, joinPath: ns.joinPath };
     };
     for (const frag of (0, utils_1.exprWalk)(expr)) {
         if (frag.node === 'field') {

package/dist/lang/ast/field-space/def-space.js

@@ -23,6 +23,7 @@ class DefSpace extends passthrough_space_1.PassthroughSpace {
                 error: {
                     message: `Circular reference to '${this.circular.defineName}' in definition`,
                     code: 'circular-reference-in-field-definition',
+                    at: symbol[0],
                 },
                 found: undefined,
             };

package/dist/lang/ast/field-space/index-field-space.js

@@ -12,7 +12,6 @@ const field_references_1 = require("../query-items/field-references");
 const space_field_1 = require("../types/space-field");
 const query_spaces_1 = require("./query-spaces");
 const reference_field_1 = require("./reference-field");
-const static_space_1 = require("./static-space");
 class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
     constructor() {
         super(...arguments);
@@ -81,46 +80,21 @@ class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
     }
     addRefineFromFields(_refineThis) { }
     addWild(wild) {
-        var _a, _b;
-        let current = this.exprSpace;
-        const joinPath = [];
-        if (wild.joinPath) {
-            // walk path to determine namespace for *
-            for (const pathPart of wild.joinPath.list) {
-                const part = pathPart.refString;
-                joinPath.push(part);
-                const ent = current.entry(part);
-                if (ent) {
-                    if (ent instanceof static_space_1.StructSpaceField) {
-                        current = ent.fieldSpace;
-                    }
-                    else {
-                        pathPart.logError('invalid-wildcard-source', `Field '${part}' does not contain rows and cannot be expanded with '*'`);
-                        return;
-                    }
-                }
-                else {
-                    pathPart.logError('wildcard-source-not-found', `No such field as '${part}'`);
-                    return;
-                }
-            }
+        var _a, _b, _c, _d;
+        const entries = this.wildcardExpansion(wild);
+        if (entries === undefined) {
+            return;
         }
+        const joinPath = (_b = (_a = wild.joinPath) === null || _a === void 0 ? void 0 : _a.path) !== null && _b !== void 0 ? _b : [];
         const dialect = this.dialectObj();
         const expandEntries = [];
-        for (const [name, entry] of current.entries()) {
-            if (wild.except.has(name)) {
-                continue;
-            }
-            if (entry.refType === 'parameter') {
-                continue;
-            }
+        for (const [name, entry] of entries) {
             const indexName = field_references_1.IndexFieldReference.indexOutputName([
                 ...joinPath,
                 name,
             ]);
             if (this.entry(indexName)) {
-                const conflict = (_b = (_a = this.expandedWild.get(indexName)) === null || _a === void 0 ? void 0 : _a.path) === null || _b === void 0 ? void 0 : _b.join('.');
-                wild.logError('name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${wild.refString}' because a field with that name already exists${conflict ? ` (conflicts with ${conflict})` : ''}`);
+                this.logWildcardConflict(wild, name, (_d = (_c = this.expandedWild.get(indexName)) === null || _c === void 0 ? void 0 : _c.path) === null || _d === void 0 ? void 0 : _d.join('.'));
             }
             else {
                 const eTypeDesc = entry.typeDesc();

package/dist/lang/ast/field-space/parameter-space.js

@@ -41,6 +41,7 @@ class ParameterSpace {
                 error: {
                     message: `\`${name}\` is not defined`,
                     code: 'parameter-not-found',
+                    at: name,
                 },
                 found: undefined,
             };
@@ -52,6 +53,7 @@ class ParameterSpace {
                         .slice(1)
                         .join('.')}\``,
                     code: 'invalid-parameter-reference',
+                    at: name,
                 },
                 found: undefined,
             };

package/dist/lang/ast/field-space/query-spaces.d.ts

@@ -57,6 +57,16 @@ export declare abstract class QueryOperationSpace extends RefinedSpace implement
     inputSpace(): QueryInputSpace;
     outputSpace(): QueryOperationSpace;
     isQueryOutputSpace(): boolean;
+    protected logWildcardConflict(wild: WildcardFieldReference, name: string, conflict: string | undefined): void;
+    /**
+     * A `*` expands the fields a user of the finished source will see: it
+     * walks its path and reads the namespace at the end as a public reader,
+     * with the same walk and the same per-entry rule as a lookup by name.
+     * A disallowed path is an error, since the user wrote it; a disallowed
+     * field is dropped, since naming it would disclose it. Returns undefined,
+     * having logged, when the path fails or nothing is left to expand.
+     */
+    protected wildcardExpansion(wild: WildcardFieldReference): [string, SpaceEntry][] | undefined;
     protected addWild(wild: WildcardFieldReference): void;
     protected addValidatedCompositeFieldUserFromEntry(name: string, entry: SpaceEntry): void;
     addFieldUserFromFilter(filter: model.FilterCondition): void;

package/dist/lang/ast/field-space/query-spaces.js

@@ -114,43 +114,44 @@ class QueryOperationSpace extends refined_space_1.RefinedSpace {
     isQueryOutputSpace() {
         return true;
     }
+    logWildcardConflict(wild, name, conflict) {
+        wild.logError('name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${wild.refString}' because a field with that name already exists${conflict ? ` (conflicts with ${conflict})` : ''}`);
+    }
+    /**
+     * A `*` expands the fields a user of the finished source will see: it
+     * walks its path and reads the namespace at the end as a public reader,
+     * with the same walk and the same per-entry rule as a lookup by name.
+     * A disallowed path is an error, since the user wrote it; a disallowed
+     * field is dropped, since naming it would disclose it. Returns undefined,
+     * having logged, when the path fails or nothing is left to expand.
+     */
+    wildcardExpansion(wild) {
+        var _a, _b, _c;
+        const ns = (0, static_space_1.resolveNamespace)(this.exprSpace, (_b = (_a = wild.joinPath) === null || _a === void 0 ? void 0 : _a.list) !== null && _b !== void 0 ? _b : [], 'public', '*');
+        if (ns.error) {
+            const at = (_c = ns.error.at) !== null && _c !== void 0 ? _c : wild;
+            at.logError(ns.error.code, ns.error.message);
+            return undefined;
+        }
+        const entries = (0, static_space_1.accessibleEntries)(ns.space, ns.accessLevel).filter(([name, entry]) => !wild.except.has(name) && entry.refType !== 'parameter');
+        if (entries.length === 0) {
+            wild.logError('wildcard-matched-no-fields', `'${wild.refString}' did not match any fields`);
+            return undefined;
+        }
+        return entries;
+    }
     addWild(wild) {
-        var _a;
-        let current = this.exprSpace;
-        const joinPath = [];
-        if (wild.joinPath) {
-            // walk path to determine namespace for *
-            for (const pathPart of wild.joinPath.list) {
-                const part = pathPart.refString;
-                joinPath.push(part);
-                const ent = current.entry(part);
-                if (ent) {
-                    if (ent instanceof static_space_1.StructSpaceField) {
-                        current = ent.fieldSpace;
-                    }
-                    else {
-                        pathPart.logError('invalid-wildcard-source', `Field '${part}' does not contain rows and cannot be expanded with '*'`);
-                        return;
-                    }
-                }
-                else {
-                    pathPart.logError('wildcard-source-not-defined', `No such field as '${part}'`);
-                    return;
-                }
-            }
+        var _a, _b, _c;
+        const entries = this.wildcardExpansion(wild);
+        if (entries === undefined) {
+            return;
         }
+        const joinPath = (_b = (_a = wild.joinPath) === null || _a === void 0 ? void 0 : _a.path) !== null && _b !== void 0 ? _b : [];
         const dialect = this.dialectObj();
         const expandEntries = [];
-        for (const [name, entry] of current.entries()) {
-            if (wild.except.has(name)) {
-                continue;
-            }
-            if (entry.refType === 'parameter') {
-                continue;
-            }
+        for (const [name, entry] of entries) {
             if (this.entry(name)) {
-                const conflict = (_a = this.expandedWild.get(name)) === null || _a === void 0 ? void 0 : _a.path.join('.');
-                wild.logError('name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${wild.refString}' because a field with that name already exists${conflict ? ` (conflicts with ${conflict})` : ''}`);
+                this.logWildcardConflict(wild, name, (_c = this.expandedWild.get(name)) === null || _c === void 0 ? void 0 : _c.path.join('.'));
             }
             else {
                 const eType = entry.typeDesc();

package/dist/lang/ast/field-space/static-space.d.ts

@@ -1,7 +1,7 @@
 import type { Dialect } from '../../../dialect/dialect';
 import type { FieldDef, StructDef, SourceDef, JoinFieldDef, AccessModifierLabel } from '../../../model/malloy_types';
 import type { SpaceEntry } from '../types/space-entry';
-import type { LookupResult } from '../types/lookup-result';
+import type { JoinPath, LookupError, LookupResult } from '../types/lookup-result';
 import type { FieldName, FieldSpace, QueryFieldSpace, SourceFieldSpace } from '../types/field-space';
 import { SpaceField } from '../types/space-field';
 import { StructSpaceFieldBase } from './struct-space-field-base';
@@ -50,3 +50,36 @@ export declare class StaticSourceSpace extends StaticSpace implements SourceFiel
     emptyStructDef(): SourceDef;
     accessProtectionLevel(): AccessModifierLabel;
 }
+/**
+ * A namespace reached by walking a join path, and the access level a reader
+ * who started at `accessLevel` has once they arrive there.
+ */
+export interface NamespaceRead {
+    space: FieldSpace;
+    accessLevel: AccessModifierLabel;
+    joinPath: JoinPath;
+    error: undefined;
+}
+/**
+ * Walk `path` from `from`, one `readEntry` per hop, narrowing the access
+ * level at each join. `lookup()` walks the path before a name with it and
+ * `*` walks the path before a star. `member` is what is about to be read from the namespace at the end
+ * of the path, a field name or `'*'`, named in the error when a hop is not a
+ * namespace.
+ */
+export declare function resolveNamespace(from: FieldSpace, path: FieldName[], accessLevel: AccessModifierLabel, member: string): NamespaceRead | LookupError;
+interface EntryRead {
+    found: SpaceEntry;
+    error: undefined;
+}
+/**
+ * Read one name from a namespace on behalf of a reader at `accessLevel`.
+ * Records the reference, and refuses the entry if the reader may not see it.
+ */
+export declare function readEntry(space: FieldSpace, name: FieldName, accessLevel: AccessModifierLabel): EntryRead | LookupError;
+/**
+ * Every entry of `space` a reader at `accessLevel` may see, by the same
+ * rule `readEntry` applies to one name.
+ */
+export declare function accessibleEntries(space: FieldSpace, accessLevel: AccessModifierLabel): [string, SpaceEntry][];
+export {};

package/dist/lang/ast/field-space/static-space.js

@@ -5,6 +5,9 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StaticSourceSpace = exports.StructSpaceField = exports.StaticSpace = void 0;
+exports.resolveNamespace = resolveNamespace;
+exports.readEntry = readEntry;
+exports.accessibleEntries = accessibleEntries;
 const dialect_map_1 = require("../../../dialect/dialect_map");
 const malloy_types_1 = require("../../../model/malloy_types");
 const field_space_1 = require("../types/field-space");
@@ -107,81 +110,19 @@ class StaticSpace {
     }
     lookup(path, accessLevel) {
         accessLevel !== null && accessLevel !== void 0 ? accessLevel : (accessLevel = this.accessProtectionLevel());
-        const head = path[0];
-        const rest = path.slice(1);
-        let found = this.entry(head.refString);
-        if (!found) {
-            return {
-                error: {
-                    message: `'${head}' is not defined`,
-                    code: 'field-not-found',
-                },
-                found,
-            };
+        const last = path[path.length - 1];
+        const ns = resolveNamespace(this, path.slice(0, -1), accessLevel, last.refString);
+        if (ns.error) {
+            return ns;
         }
-        if (found instanceof space_field_1.SpaceField) {
-            const definition = found.fieldDef();
-            if (definition) {
-                if (!(found instanceof struct_space_field_base_1.StructSpaceFieldBase) && (0, malloy_types_1.isJoined)(definition)) {
-                    // We have looked up a field which is a join, but not a StructSpaceField
-                    // because it is someting like "dimension: joinedArray is arrayComputation"
-                    // which wasn't known to be a join when the fieldspace was constructed.
-                    // TODO don't make one of these every time you do a lookup
-                    found = new StructSpaceField(definition, this.structDialect, this.structConnection);
-                }
-                // cswenson review todo I don't know how to count the reference properly now
-                // i tried only writing it as a join reference if there was more in the path
-                // but that failed because lookup([JOINNAME]) is called when translating JOINNAME.AGGREGATE(...)
-                // with a 1-length-path but that IS a join reference and there is a test
-                head.addReference({
-                    type: found instanceof struct_space_field_base_1.StructSpaceFieldBase
-                        ? 'joinReference'
-                        : 'fieldReference',
-                    definition: {
-                        type: definition.type,
-                        annotations: definition.annotations,
-                        location: definition.location,
-                    },
-                    location: head.location,
-                    text: head.refString,
-                });
-            }
-            if (definition === null || definition === void 0 ? void 0 : definition.accessModifier) {
-                if (!accessAllowed(accessLevel, definition.accessModifier)) {
-                    return {
-                        error: {
-                            message: `'${head}' is ${definition === null || definition === void 0 ? void 0 : definition.accessModifier}`,
-                            code: 'field-not-accessible',
-                        },
-                        found: undefined,
-                    };
-                }
-            }
-        } // cswenson review todo { else this is SpaceEntry not a field which can only be a param and what is going on? }
-        const joinPath = found instanceof struct_space_field_base_1.StructSpaceFieldBase
-            ? [{ ...found.joinPathElement, name: head.refString }]
-            : [];
-        if (rest.length) {
-            if (found instanceof struct_space_field_base_1.StructSpaceFieldBase) {
-                const restResult = found.fieldSpace.lookup(rest, lessPermissiveAccessLevel(accessLevel, found.fieldSpace.accessProtectionLevel()));
-                if (restResult.found) {
-                    return {
-                        ...restResult,
-                        joinPath: [...joinPath, ...restResult.joinPath],
-                    };
-                }
-                else {
-                    return restResult;
-                }
-            }
-            return {
-                error: {
-                    message: `'${head}' cannot contain a '${rest[0]}'`,
-                    code: 'invalid-property-access-in-field-reference',
-                },
-                found: undefined,
-            };
+        const read = readEntry(ns.space, last, ns.accessLevel);
+        if (read.error) {
+            return read;
         }
+        const found = read.found;
+        const joinPath = found instanceof struct_space_field_base_1.StructSpaceFieldBase
+            ? [...ns.joinPath, { ...found.joinPathElement, name: last.refString }]
+            : ns.joinPath;
         return { found, error: undefined, joinPath, isOutputField: false };
     }
     isQueryFieldSpace() {
@@ -225,9 +166,111 @@ class StaticSourceSpace extends StaticSpace {
     }
 }
 exports.StaticSourceSpace = StaticSourceSpace;
+/**
+ * Walk `path` from `from`, one `readEntry` per hop, narrowing the access
+ * level at each join. `lookup()` walks the path before a name with it and
+ * `*` walks the path before a star. `member` is what is about to be read from the namespace at the end
+ * of the path, a field name or `'*'`, named in the error when a hop is not a
+ * namespace.
+ */
+function resolveNamespace(from, path, accessLevel, member) {
+    let space = from;
+    const joinPath = [];
+    for (let i = 0; i < path.length; i++) {
+        const hop = path[i];
+        const read = readEntry(space, hop, accessLevel);
+        if (read.error) {
+            return read;
+        }
+        if (!(read.found instanceof struct_space_field_base_1.StructSpaceFieldBase)) {
+            const next = i + 1 < path.length ? path[i + 1].refString : member;
+            const message = next === '*'
+                ? `'${hop}' does not contain fields and cannot be expanded with '*'`
+                : `'${hop}' cannot contain a '${next}'`;
+            return {
+                error: {
+                    message,
+                    code: 'invalid-property-access-in-field-reference',
+                    at: hop,
+                },
+                found: undefined,
+            };
+        }
+        joinPath.push({ ...read.found.joinPathElement, name: hop.refString });
+        space = read.found.fieldSpace;
+        accessLevel = lessPermissiveAccessLevel(accessLevel, space.accessProtectionLevel());
+    }
+    return { space, accessLevel, joinPath, error: undefined };
+}
+/**
+ * Read one name from a namespace on behalf of a reader at `accessLevel`.
+ * Records the reference, and refuses the entry if the reader may not see it.
+ */
+function readEntry(space, name, accessLevel) {
+    let found = space.entry(name.refString);
+    let restriction;
+    if (!found) {
+        return {
+            error: {
+                message: `'${name}' is not defined`,
+                code: 'field-not-found',
+                at: name,
+            },
+            found: undefined,
+        };
+    }
+    if (found instanceof space_field_1.SpaceField) {
+        const definition = found.fieldDef();
+        restriction = definition === null || definition === void 0 ? void 0 : definition.accessModifier;
+        if (definition) {
+            if (!(found instanceof struct_space_field_base_1.StructSpaceFieldBase) && (0, malloy_types_1.isJoined)(definition)) {
+                // A field which turned out to be a join after the space was built,
+                // e.g. "dimension: joinedArray is arrayComputation", so the entry
+                // is not a StructSpaceField; promote it so the path can continue.
+                found = new StructSpaceField(definition, space.dialectName(), space.connectionName());
+            }
+            // A one-element path to a join is still a join reference:
+            // JOIN.aggregate() looks the join up on its own.
+            name.addReference({
+                type: found instanceof struct_space_field_base_1.StructSpaceFieldBase
+                    ? 'joinReference'
+                    : 'fieldReference',
+                definition: {
+                    type: definition.type,
+                    annotations: definition.annotations,
+                    location: definition.location,
+                },
+                location: name.location,
+                text: name.refString,
+            });
+        }
+    }
+    if (restriction && !accessAllowed(accessLevel, restriction)) {
+        return {
+            error: {
+                message: `'${name}' is ${restriction}`,
+                code: 'field-not-accessible',
+                at: name,
+            },
+            found: undefined,
+        };
+    }
+    return { found, error: undefined };
+}
+/**
+ * Every entry of `space` a reader at `accessLevel` may see, by the same
+ * rule `readEntry` applies to one name.
+ */
+function accessibleEntries(space, accessLevel) {
+    return space.entries().filter(([, entry]) => {
+        var _a;
+        const restriction = entry instanceof space_field_1.SpaceField
+            ? (_a = entry.fieldDef()) === null || _a === void 0 ? void 0 : _a.accessModifier
+            : undefined;
+        return restriction === undefined || accessAllowed(accessLevel, restriction);
+    });
+}
 function accessAllowed(accessLevel, accessModifier) {
-    if (accessModifier === 'public')
-        return true;
     if (accessLevel === 'internal')
         return accessModifier === 'internal';
     if (accessLevel === 'private')

package/dist/lang/ast/types/lookup-result.d.ts

@@ -1,6 +1,7 @@
 import type { JoinElementType, JoinType } from '../../../model';
 import type { MessageCode } from '../../parse-log';
 import type { SpaceEntry } from './space-entry';
+import type { FieldName } from './field-space';
 export interface JoinPathElement {
     name: string;
     joinElementType: JoinElementType;
@@ -17,6 +18,7 @@ export interface LookupError {
     error: {
         message: string;
         code: MessageCode;
+        at?: FieldName | undefined;
     };
     found: undefined;
 }

package/dist/lang/parse-log.d.ts

@@ -154,12 +154,10 @@ type MessageParameterTypes = {
     'top-by-non-aggregate': string;
     'definition-name-conflict': string;
     'invalid-field-in-index-query': string;
-    'invalid-wildcard-source': string;
-    'wildcard-source-not-found': string;
     'name-conflict-in-wildcard-expansion': string;
     'invalid-parameter-reference': string;
     'parameter-not-found': string;
-    'wildcard-source-not-defined': string;
+    'wildcard-matched-no-fields': string;
     'unexpected-index-segment': string;
     'accept-parameter': string;
     'except-parameter': string;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const MALLOY_VERSION = "0.0.431";
+export declare const MALLOY_VERSION = "0.0.433";

package/dist/version.js

@@ -2,5 +2,5 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MALLOY_VERSION = void 0;
 // generated with 'generate-version-file' script; do not edit manually
-exports.MALLOY_VERSION = '0.0.431';
+exports.MALLOY_VERSION = '0.0.433';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.431",
+  "version": "0.0.433",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",
@@ -51,9 +51,9 @@
     "generate-version-file": "VERSION=$(npm pkg get version --workspaces=false | tr -d \\\")\necho \"// generated with 'generate-version-file' script; do not edit manually\\nexport const MALLOY_VERSION = '$VERSION';\" > src/version.ts"
   },
   "dependencies": {
-    "@malloydata/malloy-filter": "0.0.431",
-    "@malloydata/malloy-interfaces": "0.0.431",
-    "@malloydata/malloy-tag": "0.0.431",
+    "@malloydata/malloy-filter": "0.0.433",
+    "@malloydata/malloy-interfaces": "0.0.433",
+    "@malloydata/malloy-tag": "0.0.433",
     "@noble/hashes": "^1.8.0",
     "antlr4ts": "^0.5.0-alpha.4",
     "assert": "^2.0.0",