@malloy-publisher/server 0.11.0 → 0.11.1

package/dist/app/api-doc.yaml

@@ -5365,6 +5365,37 @@ components:
             Fetch one with `GET .../agents/{agentName}`.
           items:
             $ref: "#/components/schemas/AgentSummary"
+        secureGivens:
+          type: array
+          readOnly: true
+          description: >-
+            Server-computed, read-only: every `#(secure)` given the package's
+            compiled models surface, so a host that sets those values learns
+            which ones to set from the load response without compiling the
+            package itself.
+
+
+            Collected from every `.malloy` model and `.malloynb` notebook in
+            the package, files outside `explores` included. A model's givens
+            are the ones its `CompiledModel.givens` reports, so a given
+            declared in one file and imported by another is listed under both;
+            a notebook's are those of the model its cells build up to. One
+            entry per distinct `name` and `type`, sorted by `name` then `type`;
+            a name declared with two types is two entries. It always describes
+            the models the package is serving now, so a reload that changes
+            them changes it.
+
+
+            Present, possibly empty, on a loaded package whose every model and
+            notebook compiled, so `[]` means the package declares no secure
+            given. Absent when the list cannot be known to be complete: when
+            any model or notebook in the package failed to compile (a reload
+            keeps serving the files that did), or when the server does not
+            report it, as on a listing entry for a package that is not loaded.
+            Treat absent as unknown, never as none. It cannot be set: a PATCH
+            body carrying it is ignored.
+          items:
+            $ref: "#/components/schemas/SecureGivenSummary"
 
     AgentSchedule:
       type: object
@@ -6571,6 +6602,33 @@ components:
             Malloy data type of the dimension (e.g. string, number, boolean,
             date, timestamp)
 
+    SecureGivenSummary:
+      type: object
+      readOnly: true
+      description: >-
+        One `#(secure)` given in a package's `secureGivens`: a given whose
+        value the host sets, and the model files that surface it.
+      required:
+        - name
+        - type
+        - modelPaths
+      properties:
+        name:
+          type: string
+          description: Name as declared in the model.
+        type:
+          type: string
+          description: >-
+            The given's type, rendered exactly as `Given.type` renders it
+            (e.g. `string`, `number`, `string[]`, `filter<string>`).
+        modelPaths:
+          type: array
+          description: >-
+            Package-relative paths of the `.malloy` models and `.malloynb`
+            notebooks whose compiled givens include this given, the declaring
+            file and every file that imports it alike. Sorted, no duplicates.
+          items:
+            type: string
     Given:
       type: object
       description: |

package/dist/instrumentation.mjs

@@ -164831,7 +164831,7 @@ var init_telemetry = __esm(() => {
 });
 
 // package.json
-var version = "0.11.0";
+var version = "0.11.1";
 var init_package = () => {};
 
 // src/version.ts

package/dist/package_load_worker.mjs

@@ -14201,6 +14201,11 @@ function withPlaceholderDefaults(givens) {
 }
 var init_model_info = () => {};
 
+// src/compare_text.ts
+function compareText(a, b) {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
 // src/service/package_revision.ts
 import { createHash, randomUUID } from "crypto";
 import fs from "fs";
@@ -14223,7 +14228,7 @@ function computeSourceContentSha(packagePath, contentPaths, extra) {
   return hash.digest("hex");
 }
 function canonicalJson(value) {
-  return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
+  return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => compareText(a, b))) : v);
 }
 function boundedManifestValue(value) {
   try {

package/dist/server.mjs

@@ -164832,7 +164832,7 @@ var init_telemetry = __esm(() => {
 });
 
 // package.json
-var version = "0.11.0";
+var version = "0.11.1";
 var init_package = () => {};
 
 // src/version.ts
@@ -283159,7 +283159,21 @@ function openSshProxy(ssh, target) {
     let localPort = 0;
     let server;
     let settled = false;
+    let opened = false;
+    let dead = false;
+    let closing = false;
+    let lastError;
+    let deathCause;
+    const closeListeners = [];
+    let closePromise;
     const sockets = new Set;
+    function notifyClose(listener) {
+      try {
+        listener(deathCause);
+      } catch (err) {
+        logger.warn("SSH tunnel onClose listener threw", { error: err });
+      }
+    }
     function fail(err) {
       if (settled)
         return;
@@ -283195,34 +283209,64 @@ function openSshProxy(ssh, target) {
         return false;
       }
     };
-    client.on("error", (err) => fail(err));
+    client.on("error", (err) => {
+      lastError = err;
+      fail(err);
+    });
+    function onSessionGone(err) {
+      if (err)
+        lastError ??= err;
+      if (!opened) {
+        fail(lastError ?? new Error(`SSH connection to ${ssh.host} closed before the tunnel was ready`));
+        return;
+      }
+      if (closing || dead)
+        return;
+      dead = true;
+      deathCause = lastError;
+      logger.warn(`SSH tunnel via bastion ${ssh.host} to ${target.host}:${target.port} closed` + (lastError ? `: ${lastError.message}` : "."));
+      for (const listener of closeListeners.splice(0))
+        notifyClose(listener);
+    }
+    client.on("close", () => onSessionGone());
+    client.on("end", () => onSessionGone());
     client.on("ready", () => {
       server = net.createServer((socket) => {
+        if (dead) {
+          socket.destroy();
+          return;
+        }
         sockets.add(socket);
         socket.on("close", () => sockets.delete(socket));
+        socket.on("error", () => {});
         const earlyData = [];
         socket.on("data", (chunk) => earlyData.push(chunk));
-        client.forwardOut("127.0.0.1", localPort, target.host, target.port, (err, channel) => {
-          if (err) {
-            socket.destroy(err);
-            return;
-          }
-          if (socket.destroyed) {
-            channel.destroy();
-            return;
-          }
-          for (const chunk of earlyData) {
-            channel.write(chunk);
-          }
-          earlyData.length = 0;
-          socket.removeAllListeners("data");
-          socket.pipe(channel);
-          channel.pipe(socket);
-          socket.on("error", () => channel.destroy());
-          channel.on("error", () => socket.destroy());
-          socket.on("close", () => channel.destroy());
-          channel.on("close", () => socket.destroy());
-        });
+        try {
+          client.forwardOut("127.0.0.1", localPort, target.host, target.port, (err, channel) => {
+            if (err) {
+              socket.destroy(err);
+              return;
+            }
+            if (socket.destroyed) {
+              channel.destroy();
+              return;
+            }
+            for (const chunk of earlyData) {
+              channel.write(chunk);
+            }
+            earlyData.length = 0;
+            socket.removeAllListeners("data");
+            socket.pipe(channel);
+            channel.pipe(socket);
+            socket.on("error", () => channel.destroy());
+            channel.on("error", () => socket.destroy());
+            socket.on("close", () => channel.destroy());
+            channel.on("close", () => socket.destroy());
+          });
+        } catch (err) {
+          socket.destroy(err);
+          onSessionGone(err);
+        }
       });
       server.listen(0, "127.0.0.1", () => {
         const addr = server.address();
@@ -283231,14 +283275,18 @@ function openSshProxy(ssh, target) {
           return;
         }
         localPort = addr.port;
-        if (settled)
+        if (settled) {
+          server.close();
           return;
+        }
         settled = true;
+        opened = true;
         resolve2({
           host: "127.0.0.1",
           port: localPort,
           close() {
-            return new Promise((res) => {
+            closing = true;
+            closePromise ??= new Promise((res) => {
               for (const s2 of sockets)
                 s2.destroy();
               server.close(() => {
@@ -283246,6 +283294,19 @@ function openSshProxy(ssh, target) {
                 res();
               });
             });
+            return closePromise;
+          },
+          onClose(listener) {
+            if (closing)
+              return;
+            if (dead) {
+              process.nextTick(() => {
+                if (!closing)
+                  notifyClose(listener);
+              });
+              return;
+            }
+            closeListeners.push(listener);
           }
         });
       });
@@ -295931,13 +295992,11 @@ function buildProxiedPostgresConnection(metadata, endpoint) {
   const { query, ssl } = resolveProxiedTls(name, pg.host, pg.sslmode);
   const connectionString = `postgresql://${auth}${endpoint.host}:${endpoint.port}${db}${query}`;
   const setupSQL = statementTimeoutSetupSQL(name, pg);
-  return new PooledPostgresConnection({
+  return new EnvironmentPooledPostgresConnection({
     name,
     connectionString,
     ...ssl ? { ssl } : {},
-    ...setupSQL ? { setupSQL } : {},
-    poolMin: 1,
-    poolMax: 20
+    ...setupSQL ? { setupSQL } : {}
   });
 }
 function buildEnvironmentPostgresConnection(metadata) {
@@ -295983,7 +296042,10 @@ function applyConnectionFingerprint(connection, metadata) {
 function isDuckDBConnection(connection) {
   return connection instanceof DuckDBConnection;
 }
-function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", isUpdateConnectionRequest = false, { origin: origin2 = "connection" } = {}) {
+function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", isUpdateConnectionRequest = false, {
+  origin: origin2 = "connection",
+  openProxy: openTunnel = openProxy
+} = {}) {
   const assembled = assembleEnvironmentConnections(connections, environmentPath, origin2);
   const duckLakeCache = new Map;
   const snowflakeJwtCache = new Map;
@@ -295992,6 +296054,16 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
   const postgresConnectionCache = new Map;
   const proxyEndpoints = new Map;
   const attachPromises = new WeakMap;
+  const closePromises = new WeakMap;
+  const evictions = new Set;
+  function closeOnce(connection) {
+    let closePromise = closePromises.get(connection);
+    if (!closePromise) {
+      closePromise = connection.close();
+      closePromises.set(connection, closePromise);
+    }
+    return closePromise;
+  }
   const malloyConfig = new MalloyConfig(assembled.pojo, {
     config: contextOverlay({ rootDirectory: environmentPath }),
     gcpImpersonation: gcpImpersonationOverlay()
@@ -296045,13 +296117,28 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
             throw new Error(`SSH proxy for connection '${name}' requires explicit host and ` + `port on the postgres connection; the connectionString form ` + `is not supported with a proxy.`);
           }
           connectionPromise = (async () => {
-            const endpoint = await openProxy(metadata.proxy, {
+            const endpoint = await openTunnel(metadata.proxy, {
               host: pgConfig.host,
               port: pgConfig.port
             });
             try {
               const connection2 = buildProxiedPostgresConnection(metadata, endpoint);
               proxyEndpoints.set(name, endpoint);
+              endpoint.onClose((cause) => {
+                if (proxyConnectionCache.get(name) !== connectionPromise) {
+                  return;
+                }
+                proxyConnectionCache.delete(name);
+                proxyEndpoints.delete(name);
+                logger.info("SSH tunnel died; rebuilding on the next query", { connection: name, error: cause?.message });
+                const eviction = Promise.allSettled([
+                  closeOnce(connection2).catch((error) => logger.warn("Failed to close evicted proxied connection", { connection: name, error })),
+                  endpoint.close()
+                ]).then(() => {
+                  evictions.delete(eviction);
+                });
+                evictions.add(eviction);
+              });
               return connection2;
             } catch (err) {
               await endpoint.close().catch(() => {});
@@ -296143,8 +296230,9 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
         malloyConfig.shutdown("close"),
         ...wrapperPromises.map(async (promise) => {
           const connection = await promise;
-          await connection.close();
-        })
+          await closeOnce(connection);
+        }),
+        ...evictions
       ]);
       const endpointResults = await Promise.allSettled([...proxyEndpoints.values()].map((ep) => ep.close()));
       duckLakeCache.clear();
@@ -309149,9 +309237,6 @@ function providerKeyFor(provider) {
 function sanitizeLunrQuery(query) {
   return query.replace(/[~^:*+\-"]/g, " ").trim();
 }
-function compareText(a, b) {
-  return a < b ? -1 : a > b ? 1 : 0;
-}
 function round4(x2) {
   return Math.round(x2 * 1e4) / 1e4;
 }
@@ -316471,6 +316556,11 @@ var init_authorize = __esm(() => {
   WHOLE_BODY_SINGLE_QUOTED_STRING = /^'(?:\\.|[^'\\])*'$/;
 });
 
+// src/compare_text.ts
+function compareText(a, b) {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
 // src/mcp/tools/package_entities.ts
 import path7 from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
@@ -345977,7 +346067,7 @@ function computeSourceContentSha(packagePath, contentPaths, extra) {
   return hash.digest("hex");
 }
 function canonicalJson(value) {
-  return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
+  return JSON.stringify(value, (_key, v) => v !== null && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => compareText(a, b))) : v);
 }
 function boundedManifestValue(value) {
   try {
@@ -346277,7 +346367,7 @@ function readAgentSkills(packagePath, realRoot, raw, paths, warnings) {
   }
   return {
     roots,
-    files: [...served].sort(([a], [b]) => byCodepoint(a, b)).map(([relative_filepath, file_contents]) => ({
+    files: [...served].sort(([a], [b]) => compareText(a, b)).map(([relative_filepath, file_contents]) => ({
       relative_filepath,
       file_contents
     })),
@@ -346378,7 +346468,7 @@ function resolveSkills(bundled, packageSkills) {
     ...packageSkills.filter((s2) => s2.name.includes("/") || !bundledNames.has(s2.name)).map((s2) => ({ ...s2, origin: "package" }))
   ];
 }
-var BUNDLED_SKILLS, PACKAGE_SKILLS_DIR = "skills", PACKAGE_SKILL_FILE_MAX_BYTES, PACKAGE_AGENT_TEXT_MAX_BYTES, PACKAGE_AGENT_SKILLS_MAX_BYTES, AGENT_NAME, AGENT_KEYS, SCHEDULE_KEYS, BUDGET_KEYS, isObject2 = (v) => typeof v === "object" && v !== null && !Array.isArray(v), unknownKeys = (obj, known) => Object.keys(obj).filter((k) => !known.includes(k) && !k.startsWith("x-")), byCodepoint = (a, b) => a < b ? -1 : a > b ? 1 : 0, AgentProblem, emitSkillMd = (name, description2, body) => `---
+var BUNDLED_SKILLS, PACKAGE_SKILLS_DIR = "skills", PACKAGE_SKILL_FILE_MAX_BYTES, PACKAGE_AGENT_TEXT_MAX_BYTES, PACKAGE_AGENT_SKILLS_MAX_BYTES, AGENT_NAME, AGENT_KEYS, SCHEDULE_KEYS, BUDGET_KEYS, isObject2 = (v) => typeof v === "object" && v !== null && !Array.isArray(v), unknownKeys = (obj, known) => Object.keys(obj).filter((k) => !known.includes(k) && !k.startsWith("x-")), AgentProblem, emitSkillMd = (name, description2, body) => `---
 name: ${JSON.stringify(name)}
 description: ${JSON.stringify(description2)}
 ---
@@ -365844,6 +365934,9 @@ ${query}`;
     getType() {
       return this.modelType;
     }
+    getGivens() {
+      return this.givens ?? [];
+    }
     getDeclaredQueryMetadata() {
       if (this.declaredQueryMetadataMemo === undefined) {
         this.declaredQueryMetadataMemo = composeDeclaredQueryMetadata({
@@ -368466,6 +368559,47 @@ var init_notebook_lint = __esm(() => {
   init_report();
 });
 
+// src/service/secure_givens.ts
+function summarizeSecureGivens(models, context = {}) {
+  const byKey = new Map;
+  for (const model of models) {
+    if (!model.compiled)
+      return;
+    for (const given of model.givens) {
+      if (given.secure !== true)
+        continue;
+      if (!given.name || !given.type) {
+        logger.warn("A #(secure) given has no name or type; the package's secureGivens is left unreported", {
+          packageName: context.packageName,
+          modelPath: model.modelPath,
+          name: given.name,
+          type: given.type
+        });
+        return;
+      }
+      const key = `${given.name}\x00${given.type}`;
+      let entry = byKey.get(key);
+      if (!entry) {
+        entry = {
+          name: given.name,
+          type: given.type,
+          modelPaths: new Set
+        };
+        byKey.set(key, entry);
+      }
+      entry.modelPaths.add(model.modelPath);
+    }
+  }
+  return Object.freeze([...byKey.values()].sort((a, b3) => compareText(a.name, b3.name) || compareText(a.type, b3.type)).map(({ name, type, modelPaths }) => Object.freeze({
+    name,
+    type,
+    modelPaths: Object.freeze([...modelPaths].sort(compareText))
+  })));
+}
+var init_secure_givens = __esm(() => {
+  init_logger();
+});
+
 // src/service/persist_annotation_validation.ts
 function assertPersistNamesQuoted(modelSource, modelPath) {
   const unquoted = [];
@@ -368649,6 +368783,7 @@ var init_package2 = __esm(() => {
   init_introspection_sql();
   init_quoting();
   init_model();
+  init_secure_givens();
   init_persist_annotation_validation();
   init_package_load_gates();
   import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
@@ -368662,6 +368797,7 @@ var init_package2 = __esm(() => {
     packageMetadata;
     databases;
     models = new Map;
+    secureGivensMemo;
     packagePath;
     malloyConfig;
     serveDestinationConfig;
@@ -368704,7 +368840,7 @@ var init_package2 = __esm(() => {
       this.environmentName = environmentName;
       this.packageName = packageName;
       this.packagePath = packagePath;
-      this.packageMetadata = packageMetadata;
+      this.packageMetadata = Package.storableMetadata(packageMetadata);
       this.databases = databases;
       this.models = models;
       this.malloyConfig = malloyConfig;
@@ -369244,6 +369380,19 @@ var init_package2 = __esm(() => {
     getRetrievalSettings() {
       return this.retrievalSettings;
     }
+    currentSecureGivens() {
+      const memo = this.secureGivensMemo;
+      if (memo?.forModels === this.models)
+        return memo.value;
+      const forModels = this.models;
+      const value = summarizeSecureGivens([...forModels.entries()].map(([modelPath, model]) => ({
+        modelPath,
+        givens: model.getGivens(),
+        compiled: model.getCompilationError() === undefined
+      })), { packageName: this.packageName });
+      this.secureGivensMemo = { forModels, value };
+      return value;
+    }
     getPackageMetadata() {
       const metadata = {
         ...this.packageMetadata,
@@ -369256,6 +369405,10 @@ var init_package2 = __esm(() => {
         sourceContentSha: this.sourceContentSha,
         agents: this.rawAgents === undefined ? undefined : this.listAgents()
       };
+      const secureGivens = this.currentSecureGivens();
+      if (secureGivens !== undefined) {
+        metadata.secureGivens = secureGivens;
+      }
       const warnings = this.exploreWarnings();
       if (warnings.length > 0) {
         metadata.exploresWarnings = warnings;
@@ -370406,8 +370559,14 @@ var init_package2 = __esm(() => {
     setEnvironmentName(environmentName) {
       this.environmentName = environmentName;
     }
+    static storableMetadata(packageMetadata) {
+      if (!("secureGivens" in packageMetadata))
+        return packageMetadata;
+      const { secureGivens: _computed, ...stored } = packageMetadata;
+      return stored;
+    }
     setPackageMetadata(packageMetadata) {
-      this.packageMetadata = packageMetadata;
+      this.packageMetadata = Package.storableMetadata(packageMetadata);
       this.applyDiscoveryPolicyToModels();
       this.applyQueryBoundaryToModels();
       this.applySiblingModelResolverToModels();

package/package.json

@@ -19,7 +19,7 @@
       "snowflake"
    ],
    "homepage": "https://docs.malloydata.dev/documentation/user_guides/publishing/mcp_agents",
-   "version": "0.11.0",
+   "version": "0.11.1",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"