@malloy-publisher/server 0.0.198-dev2 → 0.0.198-dev4

package/build.ts

@@ -5,14 +5,14 @@ fs.rmSync("./dist", { recursive: true, force: true });
 fs.mkdirSync("./dist");
 
 await build({
+   // compile_worker.ts is bundled as a SEPARATE entrypoint so it can
+   // be loaded by `new Worker(...)` at runtime. It must NOT be
+   // inlined into server.mjs (workers can't share module state with
+   // the parent process — they get their own JS realm).
    entrypoints: [
       "./src/server.ts",
       "./src/instrumentation.ts",
-      // Schema-introspection worker. Loaded at runtime via
-      // `new Worker(new URL("./schema_worker.mjs", import.meta.url))`
-      // from `schema_worker_pool.ts`, so it has to be a separate
-      // bundle output sitting next to server.mjs in dist/.
-      "./src/service/schema_worker.ts",
+      "./src/compile/compile_worker.ts",
    ],
    outdir: "./dist",
    target: "node",
@@ -56,13 +56,23 @@ fs.copyFileSync(
 // Rename ESM outputs to .mjs so both Node and Bun can execute them
 fs.renameSync("./dist/server.js", "./dist/server.mjs");
 fs.renameSync("./dist/instrumentation.js", "./dist/instrumentation.mjs");
-// Bun nests outputs by their path relative to the common entrypoint
-// root (./src). schema_worker.ts is under src/service/, so its output
-// lands at dist/service/schema_worker.js — not dist/.
-fs.renameSync(
-   "./dist/service/schema_worker.js",
-   "./dist/service/schema_worker.mjs",
-);
+// Bun emits compile_worker into its source-relative subdir; flatten
+// so compile_pool.ts's `resolveWorkerScript()` finds it as a sibling
+// of server.mjs. The path layout match is intentional — keep these
+// two in sync or the worker pool falls back to in-process compile.
+if (fs.existsSync("./dist/compile/compile_worker.js")) {
+   fs.renameSync(
+      "./dist/compile/compile_worker.js",
+      "./dist/compile_worker.mjs",
+   );
+   try {
+      fs.rmdirSync("./dist/compile");
+   } catch {
+      /* directory may be non-empty if Bun produced sourcemaps; leave it */
+   }
+} else if (fs.existsSync("./dist/compile_worker.js")) {
+   fs.renameSync("./dist/compile_worker.js", "./dist/compile_worker.mjs");
+}
 
 // Add shebang to server.mjs for npx/bunx compatibility
 const serverJsPath = "./dist/server.mjs";

package/dist/compile_worker.mjs

@@ -0,0 +1,633 @@
+import { createRequire } from "node:module";
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+function __accessProp(key) {
+  return this[key];
+}
+var __toESMCache_node;
+var __toESMCache_esm;
+var __toESM = (mod, isNodeMode, target) => {
+  var canCache = mod != null && typeof mod === "object";
+  if (canCache) {
+    var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
+    var cached = cache.get(mod);
+    if (cached)
+      return cached;
+  }
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: __accessProp.bind(mod, key),
+        enumerable: true
+      });
+  if (canCache)
+    cache.set(mod, to);
+  return to;
+};
+var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
+// src/compile/compile_worker.ts
+import {
+  contextOverlay,
+  MalloyConfig,
+  MalloyError,
+  Runtime,
+  isSourceDef,
+  modelDefToModelInfo
+} from "@malloydata/malloy";
+import * as fs from "fs";
+import { parentPort, threadId } from "node:worker_threads";
+import { fileURLToPath } from "url";
+
+// src/service/filter.ts
+var VALID_FILTER_TYPES = new Set([
+  "equal",
+  "in",
+  "like",
+  "greater_than",
+  "less_than"
+]);
+var ANNOTATION_PREFIX = "#(filter)";
+function parseFilterAnnotation(annotation) {
+  const trimmed = annotation.trim();
+  if (!trimmed.startsWith(ANNOTATION_PREFIX)) {
+    return null;
+  }
+  const body = trimmed.slice(ANNOTATION_PREFIX.length).trim();
+  const tokens = tokenize(body);
+  let name;
+  let dimension;
+  let type;
+  let implicit = false;
+  let required = false;
+  for (const token of tokens) {
+    if (token.includes("=")) {
+      const eqIndex = token.indexOf("=");
+      const key = token.slice(0, eqIndex).toLowerCase();
+      const value = token.slice(eqIndex + 1);
+      switch (key) {
+        case "name":
+          name = value;
+          break;
+        case "dimension":
+          dimension = value;
+          break;
+        case "type":
+          if (!VALID_FILTER_TYPES.has(value)) {
+            throw new Error(`Invalid filter type "${value}". Must be one of: ${[...VALID_FILTER_TYPES].join(", ")}`);
+          }
+          type = value;
+          break;
+        default:
+          throw new Error(`Unknown filter parameter "${key}"`);
+      }
+    } else {
+      const flag = token.toLowerCase();
+      if (flag === "implicit") {
+        implicit = true;
+      } else if (flag === "required") {
+        required = true;
+      } else {
+        throw new Error(`Unknown filter flag "${token}"`);
+      }
+    }
+  }
+  if (!dimension) {
+    throw new Error("filter annotation missing required 'dimension' parameter");
+  }
+  if (!type) {
+    throw new Error("filter annotation missing required 'type' parameter");
+  }
+  return {
+    name: name ?? dimension,
+    dimension,
+    type,
+    implicit,
+    required
+  };
+}
+function parseFilters(annotations) {
+  const byName = new Map;
+  for (const annotation of annotations) {
+    const parsed = parseFilterAnnotation(annotation);
+    if (parsed) {
+      byName.set(parsed.name, parsed);
+    }
+  }
+  return [...byName.values()];
+}
+function escapeMalloyString(value) {
+  return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+}
+function isBooleanLiteral(v) {
+  const lower = v.toLowerCase();
+  return lower === "true" || lower === "false";
+}
+var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
+var ISO_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
+function isDateLiteral(v) {
+  return ISO_DATE_RE.test(v) || ISO_TIMESTAMP_RE.test(v);
+}
+function malloyLiteral(v) {
+  if (isBooleanLiteral(v)) {
+    return v.toLowerCase();
+  }
+  if (isDateLiteral(v)) {
+    return `@${v.slice(0, 10)}`;
+  }
+  return `'${escapeMalloyString(v)}'`;
+}
+function buildPredicate(filter, value) {
+  const dim = `\`${filter.dimension}\``;
+  switch (filter.type) {
+    case "equal": {
+      const v = Array.isArray(value) ? value[0] : value;
+      return `${dim} = ${malloyLiteral(v)}`;
+    }
+    case "in": {
+      const values = Array.isArray(value) ? value : [value];
+      if (values.length === 1) {
+        return `${dim} = ${malloyLiteral(values[0])}`;
+      }
+      const conditions = values.map((v) => `${dim} = ${malloyLiteral(v)}`);
+      return `(${conditions.join(" or ")})`;
+    }
+    case "like": {
+      const v = Array.isArray(value) ? value[0] : value;
+      const escaped = escapeMalloyString(v.toLowerCase());
+      const pattern = escaped.startsWith("%") || escaped.endsWith("%") ? escaped : `%${escaped}%`;
+      return `lower(${dim}) ~ '${pattern}'`;
+    }
+    case "greater_than": {
+      const v = Array.isArray(value) ? value[0] : value;
+      return `${dim} > ${malloyLiteral(v)}`;
+    }
+    case "less_than": {
+      const v = Array.isArray(value) ? value[0] : value;
+      return `${dim} < ${malloyLiteral(v)}`;
+    }
+  }
+}
+function buildFilterClause(filters, params) {
+  const predicates = [];
+  for (const filter of filters) {
+    const value = params[filter.name];
+    const hasValue = value !== undefined && value !== null && (Array.isArray(value) ? value.length > 0 : value !== "");
+    if (!hasValue) {
+      if (filter.required) {
+        throw new FilterValidationError(`Required filter "${filter.name}" (dimension: ${filter.dimension}) was not provided`);
+      }
+      continue;
+    }
+    predicates.push(buildPredicate(filter, value));
+  }
+  if (predicates.length === 0) {
+    return "";
+  }
+  return predicates.join(" and ");
+}
+function injectFilterRefinement(query, filterClause) {
+  if (!filterClause) {
+    return query;
+  }
+  return `${query.trimEnd()} + {where: ${filterClause}}`;
+}
+
+class FilterValidationError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "FilterValidationError";
+  }
+}
+function tokenize(input) {
+  const tokens = [];
+  let current = "";
+  let inQuote = false;
+  let quoteChar = "";
+  for (const ch of input) {
+    if (inQuote) {
+      if (ch === quoteChar) {
+        inQuote = false;
+      } else {
+        current += ch;
+      }
+    } else if (ch === '"' || ch === "'") {
+      inQuote = true;
+      quoteChar = ch;
+    } else if (ch === " " || ch === "\t") {
+      if (current) {
+        tokens.push(current);
+        current = "";
+      }
+    } else {
+      current += ch;
+    }
+  }
+  if (current) {
+    tokens.push(current);
+  }
+  return tokens;
+}
+
+// src/compile/compile_worker.ts
+if (!parentPort) {
+  throw new Error("compile_worker.ts must be loaded inside a worker_threads Worker");
+}
+var port = parentPort;
+var nextRpcId = 0;
+var pendingRpc = new Map;
+function newRpcId() {
+  nextRpcId += 1;
+  return `w${threadId}-rpc-${nextRpcId}`;
+}
+function callMain(send) {
+  const requestId = newRpcId();
+  return new Promise((resolve, reject) => {
+    pendingRpc.set(requestId, {
+      resolve: (value) => resolve(value),
+      reject
+    });
+    send(requestId);
+  });
+}
+function dispatchMainResponse(message) {
+  if (message.type === "schema-for-tables-response" || message.type === "schema-for-sql-response" || message.type === "read-url-response" || message.type === "connection-metadata-response") {
+    const pending = pendingRpc.get(message.requestId);
+    if (!pending)
+      return;
+    pendingRpc.delete(message.requestId);
+    pending.resolve(message);
+    return;
+  }
+  if (message.type === "rpc-error") {
+    const pending = pendingRpc.get(message.requestId);
+    if (!pending)
+      return;
+    pendingRpc.delete(message.requestId);
+    pending.reject(deserializeError(message.error));
+    return;
+  }
+}
+
+class ProxyConnection {
+  name;
+  dialectName;
+  digest;
+  jobId;
+  constructor(metadata, jobId) {
+    this.name = metadata.name;
+    this.dialectName = metadata.dialectName;
+    this.digest = metadata.digest;
+    this.jobId = jobId;
+  }
+  getDigest() {
+    return this.digest;
+  }
+  async fetchSchemaForTables(tables, options) {
+    const response = await callMain((requestId) => {
+      const req = {
+        type: "schema-for-tables",
+        requestId,
+        jobId: this.jobId,
+        connectionName: this.name,
+        tables,
+        options: serializeFetchOptions(options)
+      };
+      port.postMessage(req);
+    });
+    return { schemas: response.schemas, errors: response.errors };
+  }
+  async fetchSchemaForSQLStruct(sentence, options) {
+    const response = await callMain((requestId) => {
+      const req = {
+        type: "schema-for-sql",
+        requestId,
+        jobId: this.jobId,
+        connectionName: this.name,
+        sentence,
+        options: serializeFetchOptions(options)
+      };
+      port.postMessage(req);
+    });
+    if (response.error !== undefined) {
+      return { error: response.error };
+    }
+    if (response.structDef === undefined) {
+      return { error: "Empty SQL schema response from main thread" };
+    }
+    return { structDef: response.structDef };
+  }
+  async runSQL() {
+    throw new Error(`ProxyConnection(${this.name}): runSQL is not available in compile workers`);
+  }
+  isPool() {
+    return false;
+  }
+  canPersist() {
+    return false;
+  }
+  canStream() {
+    return false;
+  }
+  async close() {}
+  async idle() {}
+  async estimateQueryCost() {
+    throw new Error(`ProxyConnection(${this.name}): estimateQueryCost not available in compile workers`);
+  }
+  async fetchMetadata() {
+    return {};
+  }
+  async fetchTableMetadata() {
+    return {};
+  }
+}
+function serializeFetchOptions(options) {
+  const out = {};
+  if (options.refreshTimestamp !== undefined) {
+    out.refreshTimestamp = options.refreshTimestamp;
+  }
+  if (options.modelAnnotation !== undefined) {
+    out.modelAnnotation = options.modelAnnotation;
+  }
+  return out;
+}
+function makeWorkerUrlReader(jobId) {
+  return {
+    readURL: async (url) => {
+      if (url.protocol === "file:") {
+        const filePath = fileURLToPath(url);
+        return fs.promises.readFile(filePath, "utf8");
+      }
+      const response = await callMain((requestId) => {
+        const req = {
+          type: "read-url",
+          requestId,
+          jobId,
+          url: url.toString()
+        };
+        port.postMessage(req);
+      });
+      return response.contents;
+    }
+  };
+}
+function buildWorkerMalloyConfig(job) {
+  const proxies = new Map;
+  const inflight = new Map;
+  const config = new MalloyConfig({ connections: {} }, {
+    config: contextOverlay({ rootDirectory: job.packagePath })
+  });
+  config.wrapConnections((_base) => ({
+    lookupConnection: async (name) => {
+      const effectiveName = name ?? job.defaultConnectionName ?? "duckdb";
+      const cached = proxies.get(effectiveName);
+      if (cached)
+        return cached;
+      let pending = inflight.get(effectiveName);
+      if (!pending) {
+        pending = (async () => {
+          const response = await callMain((requestId) => {
+            const req = {
+              type: "connection-metadata",
+              requestId,
+              jobId: job.requestId,
+              connectionName: effectiveName
+            };
+            port.postMessage(req);
+          });
+          const proxy2 = new ProxyConnection(response.metadata, job.requestId);
+          proxies.set(effectiveName, proxy2);
+          inflight.delete(effectiveName);
+          return proxy2;
+        })();
+        inflight.set(effectiveName, pending);
+      }
+      const proxy = await pending;
+      return proxy;
+    }
+  }));
+  return config;
+}
+async function compile(job) {
+  const compileStart = performance.now();
+  const malloyConfig = buildWorkerMalloyConfig(job);
+  const urlReader = makeWorkerUrlReader(job.requestId);
+  const runtime = new Runtime({
+    urlReader,
+    config: malloyConfig,
+    buildManifest: job.buildManifest !== undefined && job.buildManifest !== null ? {
+      entries: job.buildManifest,
+      strict: false
+    } : undefined
+  });
+  const isInline = typeof job.inlineSource === "string";
+  if (!isInline && typeof job.modelPath !== "string") {
+    throw new Error("CompileJobRequest must supply either inlineSource or modelPath");
+  }
+  const importBaseURL = isInline ? new URL(job.importBaseURL ?? `file://${job.packagePath}/`) : new URL(".", new URL(`file://${job.packagePath}/${job.modelPath}`));
+  const mm = isInline ? runtime.loadModel(job.inlineSource, { importBaseURL }) : runtime.loadModel(new URL(`file://${job.packagePath}/${job.modelPath}`), { importBaseURL });
+  const compiledModel = await mm.getModel();
+  const modelDef = compiledModel._modelDef;
+  const malloyGivens = Array.from(compiledModel.givens.values());
+  const givens = malloyGivens.length > 0 ? malloyGivens.map((g) => malloyGivenToWire(g)) : undefined;
+  const sourceInfos = [];
+  const importedSourceNames = new Set;
+  const imports = modelDef.imports ?? [];
+  for (const importLocation of imports) {
+    try {
+      const modelString = await urlReader.readURL(new URL(importLocation.importURL));
+      const importedModelDef = (await runtime.loadModel(modelString, { importBaseURL }).getModel())._modelDef;
+      const importedInfo = modelDefToModelInfo(importedModelDef);
+      const importedSources = importedInfo.entries.filter((entry) => entry.kind === "source");
+      for (const source of importedSources) {
+        if (!importedSourceNames.has(source.name)) {
+          sourceInfos.push(source);
+          importedSourceNames.add(source.name);
+        }
+      }
+    } catch {}
+  }
+  const localInfo = modelDefToModelInfo(modelDef);
+  const localSources = localInfo.entries.filter((entry) => entry.kind === "source");
+  for (const source of localSources) {
+    if (!importedSourceNames.has(source.name)) {
+      sourceInfos.push(source);
+    }
+  }
+  const modelPathForAnnotations = job.modelPath ?? "";
+  const { sources, filterMap } = extractSources(modelPathForAnnotations, modelDef);
+  const queries = extractQueries(modelPathForAnnotations, modelDef);
+  const filterMapEntries = Array.from(filterMap.entries());
+  return {
+    type: "compile-result",
+    requestId: job.requestId,
+    modelDef,
+    sourceInfos,
+    sources,
+    queries,
+    filterMap: filterMapEntries,
+    givens,
+    dataStyles: {},
+    compileDurationMs: performance.now() - compileStart
+  };
+}
+function malloyGivenToWire(given) {
+  const t = given.type;
+  const renderedType = t.type === "filter expression" ? `filter<${t.filterType}>` : t.type;
+  return {
+    name: given.name,
+    type: renderedType,
+    annotations: given.getTaglines(/^#\(/)
+  };
+}
+function extractSources(modelPath, modelDef) {
+  const filterMap = new Map;
+  const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
+    const sourceName = sourceObj.as || sourceObj.name;
+    const annotations = sourceObj.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text);
+    const collected = [];
+    let cur = sourceObj.annotation;
+    while (cur) {
+      if (cur.blockNotes) {
+        collected.push(cur.blockNotes.map((note) => note.text));
+      }
+      cur = cur.inherits;
+    }
+    const allAnnotations = collected.reverse().flat();
+    let filters;
+    if (allAnnotations.length > 0) {
+      try {
+        const parsed = parseFilters(allAnnotations);
+        if (parsed.length > 0) {
+          filterMap.set(sourceName, parsed);
+          const fields = sourceObj.fields;
+          filters = parsed.map((f) => {
+            const field = fields.find((fd) => (fd.as || fd.name) === f.dimension);
+            return {
+              name: f.name,
+              dimension: f.dimension,
+              type: f.type,
+              implicit: f.implicit,
+              required: f.required,
+              dimensionType: field?.type
+            };
+          });
+        }
+      } catch {}
+    }
+    const views = sourceObj.fields.filter((f) => f.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
+      name: turtle.as || turtle.name,
+      annotations: turtle?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
+    }));
+    return {
+      name: sourceName,
+      annotations,
+      views,
+      filters
+    };
+  });
+  return { sources, filterMap };
+}
+function extractQueries(modelPath, modelDef) {
+  const isNamedQuery = (obj) => obj.type === "query";
+  return Object.values(modelDef.contents).filter(isNamedQuery).map((q) => ({
+    name: q.as || q.name,
+    sourceName: typeof q.structRef === "string" ? q.structRef : undefined,
+    annotations: q?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
+  }));
+}
+function serializeError(error) {
+  if (error instanceof MalloyError) {
+    return {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+      malloyProblems: error.problems,
+      isCompilationError: true
+    };
+  }
+  if (error instanceof Error) {
+    return {
+      name: error.name,
+      message: error.message,
+      stack: error.stack
+    };
+  }
+  return { name: "Error", message: String(error) };
+}
+function deserializeError(serialized) {
+  const err = new Error(serialized.message);
+  err.name = serialized.name;
+  if (serialized.stack)
+    err.stack = serialized.stack;
+  return err;
+}
+var shuttingDown = false;
+var inFlightJobs = new Set;
+port.on("message", (message) => {
+  if (message.type === "shutdown") {
+    shuttingDown = true;
+    maybeExit();
+    return;
+  }
+  if (message.type === "compile") {
+    if (shuttingDown) {
+      const errMsg = {
+        type: "compile-error",
+        requestId: message.requestId,
+        error: {
+          name: "ShuttingDown",
+          message: "Compile worker is shutting down"
+        }
+      };
+      port.postMessage(errMsg);
+      return;
+    }
+    inFlightJobs.add(message.requestId);
+    runJob(message);
+    return;
+  }
+  dispatchMainResponse(message);
+});
+async function runJob(job) {
+  try {
+    const result = await compile(job);
+    port.postMessage(result);
+  } catch (error) {
+    const errMsg = {
+      type: "compile-error",
+      requestId: job.requestId,
+      error: serializeError(error)
+    };
+    port.postMessage(errMsg);
+  } finally {
+    inFlightJobs.delete(job.requestId);
+    maybeExit();
+  }
+}
+function maybeExit() {
+  if (shuttingDown && inFlightJobs.size === 0 && pendingRpc.size === 0) {
+    setImmediate(() => process.exit(0));
+  }
+}
+port.postMessage({ type: "ready" });