@514labs/moose-lib 0.6.321-ci-5-ga23d35fe → 0.6.322

package/dist/moose-runner.mjs

@@ -37,13 +37,6 @@ var init_dataModelTypes = __esm({
   }
 });
 
-// src/dataModels/types.ts
-var init_types = __esm({
-  "src/dataModels/types.ts"() {
-    "use strict";
-  }
-});
-
 // src/sqlHelpers.ts
 function sql(strings, ...values) {
   return new Sql(strings, values);
@@ -192,6 +185,14 @@ var init_sqlHelpers = __esm({
   }
 });
 
+// src/blocks/helpers.ts
+var init_helpers = __esm({
+  "src/blocks/helpers.ts"() {
+    "use strict";
+    init_sqlHelpers();
+  }
+});
+
 // src/dmv2/sdk/olapTable.ts
 import { Readable } from "stream";
 import { createHash } from "crypto";
@@ -200,7 +201,7 @@ var init_olapTable = __esm({
     "use strict";
     init_typedBase();
     init_dataModelTypes();
-    init_types();
+    init_helpers();
     init_internal();
     init_sqlHelpers();
   }
@@ -251,7 +252,7 @@ var init_ingestPipeline = __esm({
     init_stream();
     init_olapTable();
     init_ingestApi();
-    init_types();
+    init_helpers();
   }
 });
 
@@ -267,7 +268,7 @@ var init_etlPipeline = __esm({
 var init_materializedView = __esm({
   "src/dmv2/sdk/materializedView.ts"() {
     "use strict";
-    init_types();
+    init_helpers();
     init_sqlHelpers();
     init_olapTable();
     init_internal();
@@ -324,7 +325,6 @@ var init_dmv2 = __esm({
   "src/dmv2/index.ts"() {
     "use strict";
     init_olapTable();
-    init_types();
     init_stream();
     init_workflow();
     init_ingestApi();
@@ -340,6 +340,13 @@ var init_dmv2 = __esm({
   }
 });
 
+// src/dataModels/types.ts
+var init_types = __esm({
+  "src/dataModels/types.ts"() {
+    "use strict";
+  }
+});
+
 // src/browserCompatible.ts
 var init_browserCompatible = __esm({
   "src/browserCompatible.ts"() {
@@ -543,7 +550,7 @@ async function getTemporalClient(temporalUrl, namespace, clientCert, clientKey,
   }
 }
 var MooseClient, QueryClient, WorkflowClient;
-var init_helpers = __esm({
+var init_helpers2 = __esm({
   "src/consumption-apis/helpers.ts"() {
     "use strict";
     init_internal();
@@ -666,11 +673,11 @@ var init_helpers = __esm({
       }
       async getWorkflowConfig(name) {
         const workflows = await getWorkflows2();
-        const workflow = workflows.get(name);
-        if (workflow) {
+        const dmv2Workflow = workflows.get(name);
+        if (dmv2Workflow) {
           return {
-            retries: workflow.config.retries || 3,
-            timeout: workflow.config.timeout || "1h"
+            retries: dmv2Workflow.config.retries || 3,
+            timeout: dmv2Workflow.config.timeout || "1h"
           };
         }
         throw new Error(`Workflow config not found for ${name}`);
@@ -916,11 +923,11 @@ var init_cluster_utils = __esm({
 import path from "path";
 import * as toml from "toml";
 async function findConfigFile(startDir = process.cwd()) {
-  const fs4 = await import("fs");
+  const fs5 = await import("fs");
   let currentDir = path.resolve(startDir);
   while (true) {
     const configPath = path.join(currentDir, "moose.config.toml");
-    if (fs4.existsSync(configPath)) {
+    if (fs5.existsSync(configPath)) {
       return configPath;
     }
     const parentDir = path.dirname(currentDir);
@@ -932,7 +939,7 @@ async function findConfigFile(startDir = process.cwd()) {
   return null;
 }
 async function readProjectConfig() {
-  const fs4 = await import("fs");
+  const fs5 = await import("fs");
   const configPath = await findConfigFile();
   if (!configPath) {
     throw new ConfigError(
@@ -940,7 +947,7 @@ async function readProjectConfig() {
     );
   }
   try {
-    const configContent = fs4.readFileSync(configPath, "utf-8");
+    const configContent = fs5.readFileSync(configPath, "utf-8");
     const config = toml.parse(configContent);
     return config;
   } catch (error) {
@@ -1176,7 +1183,7 @@ var standaloneUtils, initPromise, toClientConfig;
 var init_standalone = __esm({
   "src/consumption-apis/standalone.ts"() {
     "use strict";
-    init_helpers();
+    init_helpers2();
     init_commons();
     init_sqlHelpers();
     standaloneUtils = null;
@@ -1191,12 +1198,12 @@ var init_standalone = __esm({
 // src/consumption-apis/runner.ts
 import http2 from "http";
 import * as jose from "jose";
-var toClientConfig2, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
+var toClientConfig2, createPath, httpLogger, modulesCache, apiHandler, createMainRouter, runApis;
 var init_runner = __esm({
   "src/consumption-apis/runner.ts"() {
     "use strict";
     init_commons();
-    init_helpers();
+    init_helpers2();
     init_cluster_utils();
     init_sqlHelpers();
     init_internal();
@@ -1204,14 +1211,15 @@ var init_runner = __esm({
       ...config,
       useSSL: config.useSSL ? "true" : "false"
     });
+    createPath = (apisDir, path5) => `${apisDir}${path5}.ts`;
     httpLogger = (req, res, startMs) => {
       console.log(
         `${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
       );
     };
     modulesCache = /* @__PURE__ */ new Map();
-    apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
-      const apis = await getApis2();
+    apiHandler = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
+      const apis = isDmv2 ? await getApis2() : /* @__PURE__ */ new Map();
       return async (req, res) => {
         const start = Date.now();
         try {
@@ -1248,6 +1256,7 @@ var init_runner = __esm({
             httpLogger(req, res, start);
             return;
           }
+          const pathName = createPath(apisDir, fileName);
           const paramsObject = Array.from(url.searchParams.entries()).reduce(
             (obj, [key, value]) => {
               const existingValue = obj[key];
@@ -1264,44 +1273,53 @@ var init_runner = __esm({
             },
             {}
           );
-          let userFuncModule = modulesCache.get(fileName);
+          let userFuncModule = modulesCache.get(pathName);
           if (userFuncModule === void 0) {
-            let apiName = fileName.replace(/^\/+|\/+$/g, "");
-            let version = null;
-            userFuncModule = apis.get(apiName);
-            if (!userFuncModule) {
-              version = url.searchParams.get("version");
-              if (!version && apiName.includes("/")) {
-                const pathParts = apiName.split("/");
-                if (pathParts.length >= 2) {
-                  userFuncModule = apis.get(apiName);
-                  if (!userFuncModule) {
-                    apiName = pathParts[0];
-                    version = pathParts.slice(1).join("/");
+            if (isDmv2) {
+              let apiName = fileName.replace(/^\/+|\/+$/g, "");
+              let version = null;
+              userFuncModule = apis.get(apiName);
+              if (!userFuncModule) {
+                version = url.searchParams.get("version");
+                if (!version && apiName.includes("/")) {
+                  const pathParts = apiName.split("/");
+                  if (pathParts.length >= 2) {
+                    userFuncModule = apis.get(apiName);
+                    if (!userFuncModule) {
+                      apiName = pathParts[0];
+                      version = pathParts.slice(1).join("/");
+                    }
+                  }
+                }
+                if (!userFuncModule) {
+                  if (version) {
+                    const versionedKey = `${apiName}:${version}`;
+                    userFuncModule = apis.get(versionedKey);
+                  } else {
+                    userFuncModule = apis.get(apiName);
                   }
                 }
               }
               if (!userFuncModule) {
-                if (version) {
-                  const versionedKey = `${apiName}:${version}`;
-                  userFuncModule = apis.get(versionedKey);
-                } else {
-                  userFuncModule = apis.get(apiName);
-                }
+                const availableApis = Array.from(apis.keys()).map(
+                  (key) => key.replace(":", "/")
+                );
+                const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
+                throw new Error(errorMessage);
               }
+              modulesCache.set(pathName, userFuncModule);
+              console.log(`[API] | Executing API: ${apiName}`);
+            } else {
+              userFuncModule = __require(pathName);
+              modulesCache.set(pathName, userFuncModule);
             }
-            if (!userFuncModule) {
-              const availableApis = Array.from(apis.keys()).map(
-                (key) => key.replace(":", "/")
-              );
-              const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
-              throw new Error(errorMessage);
-            }
-            modulesCache.set(fileName, userFuncModule);
-            console.log(`[API] | Executing API: ${apiName}`);
           }
           const queryClient = new QueryClient(clickhouseClient, fileName);
-          let result = await userFuncModule(paramsObject, {
+          let result = isDmv2 ? await userFuncModule(paramsObject, {
+            client: new MooseClient(queryClient, temporalClient),
+            sql,
+            jwt: jwtPayload
+          }) : await userFuncModule.default(paramsObject, {
             client: new MooseClient(queryClient, temporalClient),
             sql,
             jwt: jwtPayload
@@ -1345,15 +1363,17 @@ var init_runner = __esm({
         }
       };
     };
-    createMainRouter = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
+    createMainRouter = async (publicKey, clickhouseClient, temporalClient, apisDir, enforceAuth, isDmv2, jwtConfig) => {
       const apiRequestHandler = await apiHandler(
         publicKey,
         clickhouseClient,
         temporalClient,
+        apisDir,
         enforceAuth,
+        isDmv2,
         jwtConfig
       );
-      const webApps = await getWebApps2();
+      const webApps = isDmv2 ? await getWebApps2() : /* @__PURE__ */ new Map();
       const sortedWebApps = Array.from(webApps.values()).sort((a, b) => {
         const pathA = a.config.mountPath || "/";
         const pathB = b.config.mountPath || "/";
@@ -1475,7 +1495,9 @@ var init_runner = __esm({
               publicKey,
               clickhouseClient,
               temporalClient,
+              config.apisDir,
               config.enforceAuth,
+              config.isDmv2,
               config.jwtConfig
             )
           );
@@ -1673,14 +1695,15 @@ var init_index = __esm({
   "src/index.ts"() {
     "use strict";
     init_browserCompatible();
+    init_helpers();
     init_commons();
     init_secrets();
-    init_helpers();
+    init_helpers2();
     init_webAppHelpers();
     init_task();
     init_runner();
     init_redisClient();
-    init_helpers();
+    init_helpers2();
     init_standalone();
     init_sqlHelpers();
     init_utilities();
@@ -2395,9 +2418,120 @@ var init_internal = __esm({
 
 // src/moose-runner.ts
 init_internal();
-init_runner();
 import { register } from "ts-node";
 
+// src/blocks/runner.ts
+init_commons();
+import fastq from "fastq";
+import fs3 from "fs";
+import path3 from "path";
+var walkDir = (dir, fileExtension, fileList) => {
+  const files = fs3.readdirSync(dir);
+  files.forEach((file) => {
+    if (fs3.statSync(path3.join(dir, file)).isDirectory()) {
+      fileList = walkDir(path3.join(dir, file), fileExtension, fileList);
+    } else if (file.endsWith(fileExtension)) {
+      fileList.push(path3.join(dir, file));
+    }
+  });
+  return fileList;
+};
+var DependencyError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "DependencyError";
+  }
+};
+var toClientConfig3 = (config) => ({
+  ...config,
+  useSSL: config.useSSL ? "true" : "false"
+});
+var createBlocks = async (chClient, blocks) => {
+  for (const query of blocks.setup) {
+    try {
+      console.log(`Creating block using query ${query}`);
+      await chClient.command({
+        query,
+        clickhouse_settings: {
+          wait_end_of_query: 1
+          // Ensure at least once delivery and DDL acknowledgment
+        }
+      });
+    } catch (err) {
+      cliLog({
+        action: "Blocks",
+        message: `Failed to create blocks: ${err}`,
+        message_type: "Error"
+      });
+      if (err && JSON.stringify(err).includes(`UNKNOWN_TABLE`)) {
+        throw new DependencyError(err.toString());
+      }
+    }
+  }
+};
+var deleteBlocks = async (chClient, blocks) => {
+  for (const query of blocks.teardown) {
+    try {
+      console.log(`Deleting block using query ${query}`);
+      await chClient.command({
+        query,
+        clickhouse_settings: {
+          wait_end_of_query: 1
+          // Ensure at least once delivery and DDL acknowledgment
+        }
+      });
+    } catch (err) {
+      cliLog({
+        action: "Blocks",
+        message: `Failed to delete blocks: ${err}`,
+        message_type: "Error"
+      });
+    }
+  }
+};
+var asyncWorker = async (task) => {
+  await deleteBlocks(task.chClient, task.blocks);
+  await createBlocks(task.chClient, task.blocks);
+};
+var runBlocks = async (config) => {
+  const chClient = getClickhouseClient(toClientConfig3(config.clickhouseConfig));
+  console.log(`Connected`);
+  const blocksFiles = walkDir(config.blocksDir, ".ts", []);
+  const numOfBlockFiles = blocksFiles.length;
+  console.log(`Found ${numOfBlockFiles} blocks files`);
+  const queue = fastq.promise(asyncWorker, 1);
+  queue.error((err, task) => {
+    if (err && task.retries > 0) {
+      if (err instanceof DependencyError) {
+        queue.push({ ...task, retries: task.retries - 1 });
+      }
+    }
+  });
+  for (const path5 of blocksFiles) {
+    console.log(`Adding to queue: ${path5}`);
+    try {
+      const blocks = __require(path5).default;
+      queue.push({
+        chClient,
+        blocks,
+        retries: numOfBlockFiles
+      });
+    } catch (err) {
+      cliLog({
+        action: "Blocks",
+        message: `Failed to import blocks from ${path5}: ${err}`,
+        message_type: "Error"
+      });
+    }
+  }
+  while (!queue.idle()) {
+    await new Promise((resolve3) => setTimeout(resolve3, 1e3));
+  }
+};
+
+// src/moose-runner.ts
+init_runner();
+
 // src/streaming-functions/runner.ts
 init_commons();
 init_cluster_utils();
@@ -2680,7 +2814,17 @@ var sendMessageMetrics = (logger2, metrics) => {
   metrics.count_out = 0;
   setTimeout(() => sendMessageMetrics(logger2, metrics), 1e3);
 };
-async function loadStreamingFunction(sourceTopic, targetTopic) {
+function loadStreamingFunction(functionFilePath) {
+  let streamingFunctionImport;
+  try {
+    streamingFunctionImport = __require(functionFilePath.substring(0, functionFilePath.length - 3));
+  } catch (e) {
+    cliLog({ action: "Function", message: `${e}`, message_type: "Error" });
+    throw e;
+  }
+  return streamingFunctionImport.default;
+}
+async function loadStreamingFunctionV2(sourceTopic, targetTopic) {
   const transformFunctions = await getStreamingFunctions();
   const transformFunctionKey = `${topicNameToStreamName(sourceTopic)}_${targetTopic ? topicNameToStreamName(targetTopic) : "<no-target>"}`;
   const matchingEntries = Array.from(transformFunctions.entries()).filter(
@@ -2725,12 +2869,17 @@ var startConsumer = async (args, logger2, metrics, _parallelism, consumer, produ
   );
   let streamingFunctions;
   let fieldMutations;
-  const result = await loadStreamingFunction(
-    args.sourceTopic,
-    args.targetTopic
-  );
-  streamingFunctions = result.functions;
-  fieldMutations = result.fieldMutations;
+  if (args.isDmv2) {
+    const result = await loadStreamingFunctionV2(
+      args.sourceTopic,
+      args.targetTopic
+    );
+    streamingFunctions = result.functions;
+    fieldMutations = result.fieldMutations;
+  } else {
+    streamingFunctions = [[loadStreamingFunction(args.functionFilePath), {}]];
+    fieldMutations = void 0;
+  }
   await consumer.subscribe({
     topics: [args.sourceTopic.name]
     // Use full topic name for Kafka operations
@@ -2969,8 +3118,8 @@ import {
   Worker,
   bundleWorkflowCode
 } from "@temporalio/worker";
-import * as path3 from "path";
-import * as fs3 from "fs";
+import * as path4 from "path";
+import * as fs4 from "fs";
 
 // src/scripts/activity.ts
 init_internal();
@@ -2978,7 +3127,7 @@ init_json();
 import { log as logger, Context } from "@temporalio/activity";
 import { isCancellation } from "@temporalio/workflow";
 var activities = {
-  async hasWorkflow(name) {
+  async hasDmv2Workflow(name) {
     try {
       const workflows = await getWorkflows2();
       const hasWorkflow = workflows.has(name);
@@ -2989,7 +3138,7 @@ var activities = {
       return false;
     }
   },
-  async getWorkflowByName(name) {
+  async getDmv2Workflow(name) {
     try {
       logger.info(`Getting workflow ${name}`);
       const workflows = await getWorkflows2();
@@ -3034,7 +3183,7 @@ var activities = {
       throw new Error(errorMsg);
     }
   },
-  async executeTask(workflow, task, inputData) {
+  async executeDmv2Task(workflow, task, inputData) {
     const context = Context.current();
     const taskState = {};
     let heartbeatInterval = null;
@@ -3095,7 +3244,7 @@ var activities = {
 };
 function createActivityForScript(scriptName) {
   return {
-    [scriptName]: activities.executeTask
+    [scriptName]: activities.executeDmv2Task
   };
 }
 
@@ -3137,12 +3286,12 @@ var initializeLogger = LoggerSingleton.initializeLogger;
 
 // src/scripts/runner.ts
 var ALREADY_REGISTERED = /* @__PURE__ */ new Set();
-function collectActivities(logger2, workflows) {
-  logger2.info(`<WF> Collecting tasks from workflows`);
+function collectActivitiesDmv2(logger2, workflows) {
+  logger2.info(`<DMV2WF> Collecting tasks from dmv2 workflows`);
   const scriptNames = [];
   for (const [name, workflow] of workflows.entries()) {
     logger2.info(
-      `<WF> Registering workflow: ${name} with starting task: ${workflow.config.startingTask.name}`
+      `<DMV2WF> Registering dmv2 workflow: ${name} with starting task: ${workflow.config.startingTask.name}`
     );
     scriptNames.push(`${name}/${workflow.config.startingTask.name}`);
   }
@@ -3157,8 +3306,8 @@ async function createTemporalConnection(logger2, temporalConfig) {
   };
   if (temporalConfig.clientCert && temporalConfig.clientKey) {
     logger2.info("Using TLS for secure Temporal");
-    const cert = await fs3.readFileSync(temporalConfig.clientCert);
-    const key = await fs3.readFileSync(temporalConfig.clientKey);
+    const cert = await fs4.readFileSync(temporalConfig.clientCert);
+    const key = await fs4.readFileSync(temporalConfig.clientKey);
     connectionOptions.tls = {
       clientCertPair: {
         crt: cert,
@@ -3205,27 +3354,29 @@ async function registerWorkflows(logger2, config) {
   try {
     const workflows = await getWorkflows2();
     if (workflows.size > 0) {
-      logger2.info(`<WF> Found ${workflows.size} workflows`);
-      allScriptPaths.push(...collectActivities(logger2, workflows));
+      logger2.info(`<DMV2WF> Found ${workflows.size} dmv2 workflows`);
+      allScriptPaths.push(...collectActivitiesDmv2(logger2, workflows));
       if (allScriptPaths.length === 0) {
-        logger2.info(`<WF> No tasks found in workflows`);
+        logger2.info(`<DMV2WF> No tasks found in dmv2 workflows`);
         return null;
       }
-      logger2.info(`<WF> Found ${allScriptPaths.length} tasks in workflows`);
+      logger2.info(
+        `<DMV2WF> Found ${allScriptPaths.length} tasks in dmv2 workflows`
+      );
       for (const activityName of allScriptPaths) {
         if (!ALREADY_REGISTERED.has(activityName)) {
           const activity = await createActivityForScript(activityName);
           dynamicActivities.push(activity);
           ALREADY_REGISTERED.add(activityName);
-          logger2.info(`<WF> Registered task ${activityName}`);
+          logger2.info(`<DMV2WF> Registered task ${activityName}`);
         }
       }
       if (dynamicActivities.length === 0) {
-        logger2.info(`<WF> No dynamic activities found in workflows`);
+        logger2.info(`<DMV2WF> No dynamic activities found in dmv2 workflows`);
         return null;
       }
       logger2.info(
-        `<WF> Found ${dynamicActivities.length} dynamic activities in workflows`
+        `<DMV2WF> Found ${dynamicActivities.length} dynamic activities in dmv2 workflows`
       );
     }
     if (allScriptPaths.length === 0) {
@@ -3263,7 +3414,7 @@ async function registerWorkflows(logger2, config) {
       }
     };
     const workflowBundle = await bundleWorkflowCode({
-      workflowsPath: path3.resolve(__dirname, "scripts/workflow.js"),
+      workflowsPath: path4.resolve(__dirname, "scripts/workflow.js"),
       logger: silentLogger
     });
     const worker = await Worker.create({
@@ -3378,17 +3529,33 @@ program.command("dmv2-serializer").description("Load DMv2 index").action(() => {
 program.command("export-serializer").description("Run export serializer").argument("<target-model>", "Target model to serialize").action((targetModel) => {
   runExportSerializer(targetModel);
 });
-program.command("consumption-apis").description("Run consumption APIs").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).option("--jwt-secret <secret>", "JWT public key for verification").option("--jwt-issuer <issuer>", "Expected JWT issuer").option("--jwt-audience <audience>", "Expected JWT audience").option(
+program.command("blocks").description("Run blocks").argument("<blocks-dir>", "Directory containing blocks").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).action(
+  (blocksDir, clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
+    runBlocks({
+      blocksDir,
+      clickhouseConfig: {
+        database: clickhouseDb,
+        host: clickhouseHost,
+        port: clickhousePort,
+        username: clickhouseUsername,
+        password: clickhousePassword,
+        useSSL: options.clickhouseUseSsl
+      }
+    });
+  }
+);
+program.command("consumption-apis").description("Run consumption APIs").argument("<consumption-dir>", "Directory containing consumption APIs").argument("<clickhouse-db>", "Clickhouse database name").argument("<clickhouse-host>", "Clickhouse host").argument("<clickhouse-port>", "Clickhouse port").argument("<clickhouse-username>", "Clickhouse username").argument("<clickhouse-password>", "Clickhouse password").option("--clickhouse-use-ssl", "Use SSL for Clickhouse connection", false).option("--jwt-secret <secret>", "JWT public key for verification").option("--jwt-issuer <issuer>", "Expected JWT issuer").option("--jwt-audience <audience>", "Expected JWT audience").option(
   "--enforce-auth",
   "Enforce authentication on all consumption APIs",
   false
-).option("--temporal-url <url>", "Temporal server URL").option("--temporal-namespace <namespace>", "Temporal namespace").option("--client-cert <path>", "Path to client certificate").option("--client-key <path>", "Path to client key").option("--api-key <key>", "API key for authentication").option("--proxy-port <port>", "Port to run the proxy server on", parseInt).option(
+).option("--temporal-url <url>", "Temporal server URL").option("--temporal-namespace <namespace>", "Temporal namespace").option("--client-cert <path>", "Path to client certificate").option("--client-key <path>", "Path to client key").option("--api-key <key>", "API key for authentication").option("--is-dmv2", "Whether this is a DMv2 consumption", false).option("--proxy-port <port>", "Port to run the proxy server on", parseInt).option(
   "--worker-count <count>",
   "Number of worker processes for the consumption API cluster",
   parseInt
 ).action(
-  (clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
+  (apisDir, clickhouseDb, clickhouseHost, clickhousePort, clickhouseUsername, clickhousePassword, options) => {
     runApis({
+      apisDir,
       clickhouseConfig: {
         database: clickhouseDb,
         host: clickhouseHost,
@@ -3410,6 +3577,7 @@ program.command("consumption-apis").description("Run consumption APIs").argument
         apiKey: options.apiKey
       },
       enforceAuth: options.enforceAuth,
+      isDmv2: options.isDmv2,
       proxyPort: options.proxyPort,
       workerCount: options.workerCount
     });
@@ -3418,7 +3586,7 @@ program.command("consumption-apis").description("Run consumption APIs").argument
 program.command("streaming-functions").description("Run streaming functions").argument("<source-topic>", "Source topic configuration as JSON").argument("<function-file-path>", "Path to the function file").argument(
   "<broker>",
   "Kafka broker address(es) - comma-separated for multiple brokers (e.g., 'broker1:9092, broker2:9092'). Whitespace around commas is automatically trimmed."
-).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--log-payloads", "Log payloads for debugging", false).action(
+).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--is-dmv2", "Whether this is a DMv2 function", false).option("--log-payloads", "Log payloads for debugging", false).action(
   (sourceTopic, functionFilePath, broker, maxSubscriberCount, options) => {
     const config = {
       sourceTopic: JSON.parse(sourceTopic),
@@ -3426,6 +3594,7 @@ program.command("streaming-functions").description("Run streaming functions").ar
       functionFilePath,
       broker,
       maxSubscriberCount: parseInt(maxSubscriberCount),
+      isDmv2: options.isDmv2,
       logPayloads: options.logPayloads,
       saslUsername: options.saslUsername,
       saslPassword: options.saslPassword,