houdini 2.0.0-next.9 → 2.0.1

package/build/lib/codegen.js

@@ -0,0 +1,627 @@
+import { spawn } from "node:child_process";
+import { writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createInterface } from "node:readline";
+import { WebSocket } from "ws";
+const wasiRunnerPath = path.join(tmpdir(), "houdini-wasi-runner.mjs");
+writeFileSync(
+  wasiRunnerPath,
+  `import{WASI}from'node:wasi';
+import{readFileSync}from'node:fs';
+import{Worker,isMainThread,workerData,MessageChannel,receiveMessageOnPort}from'worker_threads';
+const[,,w,...r]=process.argv;
+if(isMainThread){
+const{port1:p1,port2:p2}=new MessageChannel();
+const sb=new Int32Array(new SharedArrayBuffer(4));
+process.stdin.on('error',()=>{});
+process.stdin.on('data',d=>{p1.postMessage(d);Atomics.add(sb,0,1);Atomics.notify(sb,0);});
+process.stdin.on('end',()=>{p1.postMessage(null);Atomics.add(sb,0,1);Atomics.notify(sb,0);});
+const wk=new Worker(new URL(import.meta.url),{workerData:{w,r,p:p2,sb},transferList:[p2]});
+wk.on('exit',c=>process.exit(c??0));
+}else{
+const{w:wb,r:args,p:port,sb}=workerData;
+const wasi=new WASI({version:'preview1',args:[wb,...args],env:process.env,preopens:{'/':'/'}});
+let mem=null;
+const io=wasi.getImportObject();
+const rfr=io.wasi_snapshot_preview1.fd_read;
+io.wasi_snapshot_preview1.fd_read=(fd,iovs,il,nr)=>{
+if(fd!==0||!mem)return rfr(fd,iovs,il,nr);
+while(Atomics.load(sb,0)===0)Atomics.wait(sb,0,0);
+const m=receiveMessageOnPort(port);
+Atomics.sub(sb,0,1);
+const v=new DataView(mem.buffer);
+if(!m||m.message===null){v.setUint32(nr,0,true);return 0;}
+const b=Buffer.isBuffer(m.message)?m.message:Buffer.from(m.message);
+let wn=0;
+for(let i=0;i<il;i++){
+const p=v.getUint32(iovs+i*8,true),l=v.getUint32(iovs+i*8+4,true);
+const n=Math.min(l,b.length-wn);
+if(n<=0)break;
+new Uint8Array(mem.buffer,p,n).set(b.subarray(wn,wn+n));
+wn+=n;}
+v.setUint32(nr,wn,true);
+return 0;};
+const mod=new WebAssembly.Module(readFileSync(wb));
+const inst=new WebAssembly.Instance(mod,io);
+mem=inst.exports.memory;
+wasi.start(inst);
+process.exit(0);}
+`
+);
+import * as conventions from "../router/conventions.js";
+import { create_schema, schema_version, write_config } from "./database.js";
+import { openDb } from "./db.js";
+import { format_hook_error } from "./error.js";
+import * as fs from "./fs.js";
+import { Logger } from "./logger.js";
+import { LogLevel } from "./types.js";
+async function connect_db(config) {
+  const filepath = conventions.db_path(config);
+  let db = await openDb(filepath);
+  const stamped = db.get("PRAGMA user_version")?.user_version ?? 0;
+  const existing = db.get(`SELECT 1 FROM sqlite_master WHERE type = 'table' LIMIT 1`);
+  if (existing && stamped !== schema_version) {
+    db.close();
+    try {
+      await fs.remove(filepath);
+    } catch (_e) {
+    }
+    try {
+      await fs.remove(`${filepath}-shm`);
+    } catch (_e) {
+    }
+    try {
+      await fs.remove(`${filepath}-wal`);
+    } catch (_e) {
+    }
+    db = await openDb(filepath);
+  }
+  db.exec(create_schema);
+  db.exec(`PRAGMA user_version = ${schema_version}`);
+  db.flush();
+  return [db, filepath];
+}
+async function init_db(config, preserve) {
+  const db_file = conventions.db_path(config);
+  if (!preserve) {
+    try {
+      await fs.remove(db_file);
+    } catch (_e) {
+    }
+    try {
+      await fs.remove(`${db_file}-shm`);
+    } catch (_e) {
+    }
+    try {
+      await fs.remove(`${db_file}-wal`);
+    } catch (_e) {
+    }
+  }
+  return connect_db(config);
+}
+function plugin_db_key(name) {
+  if (!name.startsWith("./") && !name.startsWith("../") && !path.isAbsolute(name)) {
+    return name;
+  }
+  return name.replace(/\./g, "_").replace(/\//g, "__");
+}
+async function codegen_setup(config, mode, db, db_file) {
+  const _db = db;
+  const logger = new Logger(config.config_file.logLevel ?? LogLevel.ShortSummary);
+  await fs.mkdirpSync(conventions.houdini_root(config));
+  const rawTransport = config.config_file.pluginTransport ?? "websocket";
+  const resolvedTransport = rawTransport.startsWith("env:") ? process.env[rawTransport.slice("env:".length)] ?? "websocket" : rawTransport;
+  const useStdio = resolvedTransport === "stdio";
+  const plugins = {};
+  const plugin_specs = [];
+  const spec_results = {};
+  const stdioStdin = /* @__PURE__ */ new Map();
+  const triggerHookRef = { fn: null };
+  const pendingRequests = /* @__PURE__ */ new Map();
+  const wait_for_plugin_db = async (configKey, dbKey) => {
+    const pollDb = await openDb(db_file);
+    return new Promise((resolve, reject) => {
+      const interval = setInterval(() => {
+        try {
+          pollDb.reload();
+          const row = pollDb.get("SELECT * FROM plugins WHERE name = ?", [dbKey]);
+          if (row) {
+            clearInterval(interval);
+            clearTimeout(timeout);
+            pollDb.run("UPDATE plugins SET config = ? WHERE name = ?", [
+              JSON.stringify(
+                config.plugins.find((p) => p.name === configKey)?.config ?? {}
+              ),
+              dbKey
+            ]);
+            pollDb.flush();
+            pollDb.close();
+            const spec = {
+              name: row.name,
+              port: row.port,
+              hooks: new Set(JSON.parse(row.hooks)),
+              order: row.plugin_order,
+              directory: config.plugins.find((p) => p.name === configKey)?.directory || ""
+            };
+            spec_results[configKey] = spec;
+            if (row.config_module) {
+              import(row.config_module).then((module) => {
+                if (module && typeof module.default === "function") {
+                  config.config_file = module.default(config.config_file);
+                }
+                resolve(spec);
+              });
+            } else {
+              resolve(spec);
+            }
+          }
+        } catch (err) {
+          clearInterval(interval);
+          clearTimeout(timeout);
+          pollDb.close();
+          reject(err);
+        }
+      }, 10);
+      const timeout = setTimeout(() => {
+        clearInterval(interval);
+        pollDb.close();
+        reject(new Error(`Timeout waiting for plugin ${configKey} to register`));
+      }, 1e4);
+    });
+  };
+  const wait_for_plugin_stdio = (name, child) => new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error(`Timeout waiting for plugin ${name} to register`));
+    }, 1e4);
+    const rl = createInterface({ input: child.stdout });
+    let registered = false;
+    rl.on("line", (line) => {
+      try {
+        const msg = JSON.parse(line);
+        if (!registered) {
+          if (msg.type !== "register") {
+            clearTimeout(timeout);
+            reject(new Error(`Plugin ${name} sent ${msg.type} before registering`));
+            return;
+          }
+          registered = true;
+          clearTimeout(timeout);
+          const spec = {
+            name: msg.name ?? plugin_db_key(name),
+            port: msg.port ?? 0,
+            hooks: new Set(msg.hooks ?? []),
+            order: msg.order,
+            directory: config.plugins.find((p) => p.name === name)?.directory || ""
+          };
+          spec_results[name] = spec;
+          _db.run(
+            `INSERT OR IGNORE INTO plugins (name, hooks, port, plugin_order, include_runtime, include_static_runtime, config_module, client_plugins)
+							 VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+            [
+              spec.name,
+              JSON.stringify([...spec.hooks]),
+              spec.port,
+              spec.order,
+              msg.includeRuntime ?? null,
+              msg.includeStaticRuntime ?? null,
+              msg.configModule ?? null,
+              msg.clientPlugins ?? null
+            ]
+          );
+          _db.run("UPDATE plugins SET config = ? WHERE name = ?", [
+            JSON.stringify(
+              config.plugins.find((p) => p.name === name)?.config ?? {}
+            ),
+            spec.name
+          ]);
+          _db.flush();
+          if (msg.configModule) {
+            import(msg.configModule).then((module) => {
+              if (module && typeof module.default === "function") {
+                config.config_file = module.default(config.config_file);
+              }
+              resolve(spec);
+            }).catch((err) => {
+              reject(
+                new Error(
+                  `Failed to load configModule for ${name}: ${err.message}`
+                )
+              );
+            });
+          } else {
+            resolve(spec);
+          }
+          return;
+        }
+        if (msg.type === "response") {
+          const pending = pendingRequests.get(msg.id);
+          if (!pending) return;
+          clearTimeout(pending.timeout);
+          pendingRequests.delete(msg.id);
+          if (msg.error) {
+            const errors = Array.isArray(msg.error) ? msg.error : [msg.error];
+            errors.forEach((error) => {
+              format_hook_error(config.root_dir, error, name, pending.hook);
+            });
+            pending.reject(
+              Object.assign(new Error(`Failed to call ${name}`), {
+                alreadyLogged: true
+              })
+            );
+          } else {
+            pending.resolve(msg.result);
+          }
+        } else if (msg.type === "invoke") {
+          if (!triggerHookRef.fn) {
+            const reply = `${JSON.stringify({
+              id: msg.id,
+              type: "invoke_result",
+              error: { message: "orchestrator not ready" }
+            })}
+`;
+            child.stdin?.write(reply);
+            return;
+          }
+          triggerHookRef.fn(msg.hook, {
+            parallel_safe: msg.parallel,
+            payload: msg.payload,
+            task_id: msg.taskId
+          }).then((result) => {
+            const reply = JSON.stringify({ id: msg.id, type: "invoke_result", result }) + "\n";
+            child.stdin?.write(reply);
+          }).catch((err) => {
+            const reply = `${JSON.stringify({
+              id: msg.id,
+              type: "invoke_result",
+              error: { message: err.message }
+            })}
+`;
+            child.stdin?.write(reply);
+          });
+        }
+      } catch (err) {
+        if (!registered) {
+          reject(err instanceof Error ? err : new Error(String(err)));
+        }
+      }
+    });
+    rl.on("close", () => {
+      if (!registered) {
+        clearTimeout(timeout);
+        reject(new Error(`Plugin ${name} stdout closed before registering`));
+      } else {
+        for (const [id, pending] of pendingRequests.entries()) {
+          if (pending.plugin !== name) continue;
+          clearTimeout(pending.timeout);
+          pendingRequests.delete(id);
+          pending.reject(new Error(`Plugin ${name} closed unexpectedly`));
+        }
+      }
+    });
+  });
+  _db.run("DELETE FROM plugins");
+  _db.flush();
+  logger.time("Start Plugins");
+  await Promise.all(
+    config.plugins.map(async (plugin) => {
+      let executable = plugin.executable;
+      const args = ["--database", db_file];
+      const jsExtensions = [".js", ".mjs", ".cjs"];
+      if (jsExtensions.includes(path.extname(plugin.executable))) {
+        executable = "node";
+        args.unshift(plugin.executable);
+      } else if (path.extname(plugin.executable) === ".wasm") {
+        executable = "node";
+        args.unshift(wasiRunnerPath, plugin.executable);
+      }
+      const dbKey = plugin_db_key(plugin.name);
+      args.push("--plugin-key", dbKey);
+      const pluginUsesStdio = useStdio || path.extname(plugin.executable) === ".wasm";
+      if (pluginUsesStdio) {
+        args.push("--transport", "stdio");
+      }
+      logger.time(`Spawn ${plugin.name}`);
+      const child = spawn(executable, args, {
+        // stdio/WASM plugins carry the protocol over stdin+stdout, so those must be pipes.
+        // websocket plugins talk over TCP, so let stdout/stderr through to the console.
+        stdio: pluginUsesStdio ? ["pipe", "pipe", "inherit"] : ["inherit", "inherit", "inherit"],
+        detached: process.platform !== "win32"
+      });
+      if (pluginUsesStdio) {
+        stdioStdin.set(dbKey, child.stdin);
+        child.stdin.on("error", (err) => {
+          if (err.code !== "EPIPE") {
+            console.error(`[${plugin.name}] stdin error:`, err.message);
+          }
+        });
+      }
+      plugins[plugin.name] = {
+        process: child,
+        ...await (pluginUsesStdio ? wait_for_plugin_stdio(plugin.name, child) : wait_for_plugin_db(plugin.name, dbKey))
+      };
+      logger.timeEnd(`Spawn ${plugin.name}`, LogLevel.Verbose);
+    })
+  );
+  for (const plugin of config.plugins) {
+    plugin_specs.push(spec_results[plugin.name]);
+  }
+  logger.timeEnd("Start Plugins", LogLevel.Summary);
+  const wsConnections = /* @__PURE__ */ new Map();
+  let messageCounter = 0;
+  async function getOrCreateWS(name, port) {
+    const existing = wsConnections.get(name);
+    if (existing && existing.readyState === WebSocket.OPEN) {
+      return existing;
+    }
+    return new Promise((resolve, reject) => {
+      const wsUrl = `ws://localhost:${port}/ws`;
+      const ws = new WebSocket(wsUrl);
+      ws.on("open", () => {
+        wsConnections.set(name, ws);
+        return resolve(ws);
+      });
+      ws.on("message", (data) => {
+        try {
+          const response = JSON.parse(data.toString());
+          const pending = pendingRequests.get(response.id);
+          if (!pending) return;
+          clearTimeout(pending.timeout);
+          pendingRequests.delete(response.id);
+          switch (response.type) {
+            case "error":
+              console.error(`!   [${name}] ${response.error}`);
+              pending.reject(new Error(`${name}: ${response.error}`));
+              break;
+            case "response":
+              if (response.error) {
+                const errors = Array.isArray(response.error) ? response.error : [response.error];
+                errors.forEach((error) => {
+                  format_hook_error(config.root_dir, error, name, pending.hook);
+                });
+                pending.reject(
+                  Object.assign(new Error(`Failed to call ${name}`), {
+                    alreadyLogged: true
+                  })
+                );
+              } else {
+                pending.resolve(response.result);
+              }
+              break;
+            default:
+              console.warn(`[${name}] Unknown message type: ${response.type}`);
+              pending.reject(new Error(`Unknown message type: ${response.type}`));
+          }
+        } catch (err) {
+          console.error(`Error processing WebSocket message for ${name}:`, err);
+        }
+      });
+      ws.on("error", (err) => {
+        console.error(`WebSocket error for ${name}:`, err);
+        wsConnections.delete(name);
+        reject(new Error(`WebSocket error for ${name}: ${err}`));
+      });
+      ws.on("close", () => {
+        wsConnections.delete(name);
+      });
+    });
+  }
+  const invoke_hook = async (name, hook, payload = {}, task_id) => {
+    const plugin = plugin_specs.find((spec) => spec.name === name);
+    if (!plugin) {
+      throw new Error(`unknown plugin: ${name}`);
+    }
+    const { port, directory } = plugin;
+    const messageId = String(++messageCounter);
+    const message = {
+      id: messageId,
+      type: "request",
+      hook,
+      payload,
+      taskId: task_id,
+      pluginDirectory: directory
+    };
+    if (useStdio) {
+      const stdin = stdioStdin.get(name);
+      if (!stdin) {
+        throw new Error(`No stdio channel for plugin ${name}`);
+      }
+      return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          pendingRequests.delete(messageId);
+          reject(new Error(`Request timeout for ${name}/${hook}`));
+        }, 3e4);
+        pendingRequests.set(messageId, { resolve, reject, timeout, hook, plugin: name });
+        stdin.write(`${JSON.stringify(message)}
+`);
+      });
+    } else {
+      const ws = await getOrCreateWS(name, port);
+      return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          pendingRequests.delete(messageId);
+          reject(new Error(`WebSocket request timeout for ${name}/${hook}`));
+        }, 3e4);
+        pendingRequests.set(messageId, { resolve, reject, timeout, hook, plugin: name });
+        ws.send(JSON.stringify(message));
+      });
+    }
+  };
+  _db.reload();
+  const _fireHook = async (hook, {
+    parallel_safe,
+    payload,
+    task_id
+  } = {}) => {
+    const timeName = hook + (task_id ? ` (${task_id})` : "");
+    logger.time(timeName);
+    const plugins2 = plugin_specs.filter(({ hooks }) => hooks.has(hook));
+    const result = {};
+    try {
+      if (parallel_safe) {
+        await Promise.all(
+          plugins2.map(async (plugin) => {
+            result[plugin.name] = await invoke_hook(plugin.name, hook, payload, task_id);
+          })
+        );
+      } else {
+        for (const { name } of plugins2) {
+          result[name] = await invoke_hook(name, hook, payload, task_id);
+        }
+      }
+    } finally {
+      logger.timeEnd(timeName, task_id ? LogLevel.Verbose : LogLevel.Summary);
+    }
+    return result;
+  };
+  const trigger_hook = async (hook, opts) => {
+    _db.flush();
+    const result = await _fireHook(hook, opts);
+    _db.reload();
+    return result;
+  };
+  triggerHookRef.fn = trigger_hook;
+  await write_config(_db, config, invoke_hook, plugin_specs, mode, logger);
+  _db.flush();
+  await trigger_hook("Config");
+  await trigger_hook("AfterLoad");
+  await trigger_hook("Schema");
+  let pipelineQueue = Promise.resolve();
+  return {
+    database_path: db_file,
+    trigger_hook,
+    run_pipeline: (options) => run_pipeline(trigger_hook, options),
+    pipeline_lock: (fn) => {
+      const result = pipelineQueue.then(fn);
+      pipelineQueue = result.then(
+        () => {
+        },
+        () => {
+        }
+      );
+      return result;
+    },
+    close: async () => {
+      for (const [name, ws] of wsConnections.entries()) {
+        try {
+          if (ws.readyState === WebSocket.OPEN) {
+            ws.close(1001, "shutdown");
+          }
+        } catch (err) {
+          console.error(`Error closing WebSocket for ${name}:`, err);
+        }
+      }
+      wsConnections.clear();
+      for (const [, stdin] of stdioStdin.entries()) {
+        try {
+          stdin.end();
+        } catch {
+        }
+      }
+      stdioStdin.clear();
+      for (const [, { timeout, reject }] of pendingRequests.entries()) {
+        clearTimeout(timeout);
+        reject(new Error("codegen closed"));
+      }
+      pendingRequests.clear();
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      try {
+        db.close();
+      } catch {
+      }
+      await Promise.all(
+        Object.entries(plugins).map(async ([, plugin]) => {
+          if (plugin.process.pid) {
+            if (process.platform === "win32") {
+              try {
+                spawn("taskkill", [
+                  "/pid",
+                  plugin.process.pid.toString(),
+                  "/f",
+                  "/t"
+                ]);
+              } catch (_err) {
+              }
+            } else {
+              try {
+                process.kill(plugin.process.pid, 0);
+                process.kill(-plugin.process.pid, "SIGINT");
+              } catch {
+              }
+            }
+          }
+        })
+      );
+    }
+  };
+}
+const PIPELINE_HOOKS = [
+  "Config",
+  "AfterLoad",
+  "Schema",
+  "ExtractDocuments",
+  "AfterExtract",
+  "BeforeValidate",
+  "Validate",
+  "AfterValidate",
+  "BeforeGenerate",
+  "GenerateDocuments",
+  "GenerateRuntime",
+  "AfterGenerate"
+];
+async function run_pipeline(trigger_hook, options = {}) {
+  const { task_id, after, start, through } = options;
+  const results = {};
+  let startIndex = 0;
+  let endIndex = PIPELINE_HOOKS.length - 1;
+  if (after) {
+    const afterIndex = PIPELINE_HOOKS.indexOf(after);
+    if (afterIndex === -1) {
+      throw new Error(`Unknown hook: ${after}`);
+    }
+    startIndex = afterIndex + 1;
+  }
+  if (start) {
+    const index = PIPELINE_HOOKS.indexOf(start);
+    if (index === -1) {
+      throw new Error(`Unknown hook: ${after}`);
+    }
+    startIndex = index;
+  }
+  if (through) {
+    endIndex = PIPELINE_HOOKS.indexOf(through);
+    if (endIndex === -1) {
+      throw new Error(`Unknown hook: ${through}`);
+    }
+  }
+  if (startIndex > endIndex) {
+    return results;
+  }
+  for (let i = startIndex; i <= endIndex; i++) {
+    const hook = PIPELINE_HOOKS[i];
+    const opts = { task_id };
+    if (hook === "Validate" || hook === "GenerateDocuments") {
+      opts.parallel_safe = true;
+    }
+    if (hook === "GenerateDocuments" && i + 1 <= endIndex && PIPELINE_HOOKS[i + 1] === "GenerateRuntime") {
+      const [gdResult, grResult] = await Promise.all([
+        trigger_hook("GenerateDocuments", { task_id, parallel_safe: true }),
+        trigger_hook("GenerateRuntime", { task_id })
+      ]);
+      results.GenerateDocuments = gdResult;
+      results.GenerateRuntime = grResult;
+      i++;
+      continue;
+    }
+    results[hook] = await trigger_hook(hook, opts);
+  }
+  return results;
+}
+export {
+  PIPELINE_HOOKS,
+  codegen_setup,
+  connect_db,
+  init_db,
+  plugin_db_key,
+  run_pipeline
+};