@pd4castr/cli 0.0.1 → 0.0.2

package/README.md

@@ -20,6 +20,7 @@ yarn link
 yarn dev
 
 # from a model project, link the module
+# ex. https://github.com/pipelabs/pd4castr-model-examples/tree/main/examples/python-demo
 yarn link @pd4castr/cli
 
 # from that project, execute a command

package/dist/index.js

@@ -1,4 +1,49 @@
 #!/usr/bin/env node
+var __knownSymbol = (name, symbol) => (symbol = Symbol[name]) ? symbol : Symbol.for("Symbol." + name);
+var __typeError = (msg) => {
+  throw TypeError(msg);
+};
+var __using = (stack, value, async) => {
+  if (value != null) {
+    if (typeof value !== "object" && typeof value !== "function") __typeError("Object expected");
+    var dispose, inner;
+    if (async) dispose = value[__knownSymbol("asyncDispose")];
+    if (dispose === void 0) {
+      dispose = value[__knownSymbol("dispose")];
+      if (async) inner = dispose;
+    }
+    if (typeof dispose !== "function") __typeError("Object not disposable");
+    if (inner) dispose = function() {
+      try {
+        inner.call(this);
+      } catch (e) {
+        return Promise.reject(e);
+      }
+    };
+    stack.push([async, dispose, value]);
+  } else if (async) {
+    stack.push([async]);
+  }
+  return value;
+};
+var __callDispose = (stack, error, hasError) => {
+  var E = typeof SuppressedError === "function" ? SuppressedError : function(e, s, m, _) {
+    return _ = Error(m), _.name = "SuppressedError", _.error = e, _.suppressed = s, _;
+  };
+  var fail = (e) => error = hasError ? new E(e, error, "An error was suppressed during disposal") : (hasError = true, e);
+  var next = (it) => {
+    while (it = stack.pop()) {
+      try {
+        var result = it[1] && it[1].call(it[2]);
+        if (it[0]) return Promise.resolve(result).then(next, (e) => (fail(e), next()));
+      } catch (e) {
+        fail(e);
+      }
+    }
+    if (hasError) throw error;
+  };
+  return next();
+};
 
 // src/program.ts
 import { Command } from "commander";
@@ -6,7 +51,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "@pd4castr/cli",
-  version: "0.0.1",
+  version: "0.0.2",
   description: "CLI tool for creating, testing, and publishing pd4castr models",
   main: "dist/index.js",
   type: "module",
@@ -72,6 +117,7 @@ var package_default = {
     ky: "1.8.2",
     lilconfig: "3.1.3",
     ora: "8.2.0",
+    slugify: "1.6.6",
     tiged: "2.12.7",
     "tiny-invariant": "1.3.3",
     zod: "4.0.14"
@@ -112,9 +158,9 @@ function getTemplatePath(template) {
 
 // src/utils/is-existing-path.ts
 import fs from "fs/promises";
-async function isExistingPath(path8) {
+async function isExistingPath(path12) {
   try {
-    await fs.access(path8);
+    await fs.access(path12);
     return true;
   } catch {
     return false;
@@ -148,8 +194,11 @@ async function handleAction() {
   try {
     await fetchTemplate(template, projectName);
     spinner.succeed("Template fetched successfully");
-  } catch {
+  } catch (error) {
     spinner.fail("Error fetching template");
+    if (error instanceof Error) {
+      console.error(error.message);
+    }
     process.exit(1);
   }
 }
@@ -184,7 +233,9 @@ var AUTH0_AUDIENCE = "https://api.pd4castr.com.au";
 var GLOBAL_CONFIG_FILE = ".pd4castr";
 var PROJECT_CONFIG_FILE = ".pd4castrrc.json";
 var API_URL = "https://api.pd4castr.com.au";
-var TEST_INPUT_DATA_DIR = "test_data";
+var TEST_INPUT_DATA_DIR = "test_input";
+var TEST_OUTPUT_DATA_DIR = "test_output";
+var TEST_OUTPUT_FILENAME = `output-${Date.now()}.json`;
 
 // src/schemas/global-config-schema.ts
 import { z } from "zod";
@@ -335,19 +386,20 @@ function registerLoginCommand(program2) {
 // src/commands/test/handle-action.ts
 import ora3 from "ora";
 import express from "express";
-import path5 from "path";
+import path8 from "path";
+import slugify from "slugify";
 import { ExecaError } from "execa";
 import { ZodError as ZodError2 } from "zod";
 
 // src/config/load-project-config.ts
+import path4 from "path";
 import { lilconfig } from "lilconfig";
 
 // src/schemas/project-config-schema.ts
 import { z as z2 } from "zod";
 var aemoDataFetcherSchema = z2.object({
-  key: z2.string(),
   type: z2.literal("AEMO_MMS"),
-  checkInterval: z2.int().positive(),
+  checkInterval: z2.number().int().min(60),
   config: z2.object({
     checkQuery: z2.string(),
     fetchQuery: z2.string()
@@ -356,13 +408,26 @@ var aemoDataFetcherSchema = z2.object({
 var dataFetcherSchema = z2.discriminatedUnion("type", [aemoDataFetcherSchema]);
 var modelInputSchema = z2.object({
   key: z2.string(),
-  inputSource: z2.string(),
-  dataFetcher: z2.string(),
-  trigger: z2.string()
+  inputSource: z2.string().optional(),
+  trigger: z2.enum(["WAIT_FOR_LATEST_FILE", "USE_MOST_RECENT_FILE"]),
+  fetcher: dataFetcherSchema.optional()
+});
+var modelOutputSchema = z2.object({
+  name: z2.string(),
+  seriesKey: z2.boolean(),
+  colour: z2.string().regex(/^#[0-9A-Fa-f]{6}$/).optional()
 });
 var projectConfigSchema = z2.object({
-  dataFetchers: z2.array(dataFetcherSchema).default([]),
-  modelInputs: z2.array(modelInputSchema).default([])
+  $$id: z2.string().nullable(),
+  $$modelGroupID: z2.string().nullable(),
+  $$revision: z2.number().int().min(0).default(0),
+  $$dockerImage: z2.string().nullable(),
+  name: z2.string(),
+  metadata: z2.record(z2.string(), z2.any()).optional(),
+  forecastVariable: z2.enum(["PRICE"]),
+  timeHorizon: z2.enum(["ACTUAL", "DAY_AHEAD", "WEEK_AHEAD"]),
+  inputs: z2.array(modelInputSchema),
+  outputs: z2.array(modelOutputSchema)
 });
 
 // src/config/parse-project-config.ts
@@ -380,23 +445,30 @@ async function loadProjectConfig() {
       "No config found (docs: https://github.com/pipelabs/pd4castr-model-examples/blob/main/docs/005-config.md)."
     );
   }
-  return parseProjectConfig(result.config);
+  const config = parseProjectConfig(result.config);
+  const projectRoot = path4.dirname(result.filepath);
+  return {
+    config,
+    projectRoot
+  };
 }
 
-// src/utils/get-cwd.ts
-function getCWD() {
-  if (!process.env.INIT_CWD) {
-    throw new Error("INIT_CWD environment variable is not set");
-  }
-  return process.env.INIT_CWD;
+// src/utils/create-link.ts
+var ESC = "\x1B";
+var OSC = `${ESC}]`;
+var SEP = ";";
+function createLink(text, url) {
+  const start = `${OSC}8${SEP}${SEP}${url}${ESC}\\`;
+  const end = `${OSC}8${SEP}${SEP}${ESC}\\`;
+  return `${start}${text}${end}`;
 }
 
 // src/commands/test/utils/build-docker-image.ts
 import { execa } from "execa";
-async function buildDockerImage(dockerImage) {
+async function buildDockerImage(dockerImage, ctx) {
   try {
     await execa("docker", ["build", "-t", dockerImage, "."], {
-      cwd: getCWD(),
+      cwd: ctx.projectRoot,
       stdio: "pipe"
     });
   } catch (error) {
@@ -405,37 +477,60 @@ async function buildDockerImage(dockerImage) {
 }
 
 // src/commands/test/utils/create-input-handler.ts
-import path4 from "path";
-function createInputHandler(inputFilesPath, modelIOChecks) {
+import path5 from "path";
+function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
   return (req, res) => {
     if (!modelIOChecks.isValidInput(req.params.filename)) {
       return res.status(404).json({ error: "File not found" });
     }
     modelIOChecks.trackInputHandled(req.params.filename);
-    const filePath = path4.join(getCWD(), inputFilesPath, req.params.filename);
+    const filePath = path5.join(
+      ctx.projectRoot,
+      inputFilesPath,
+      req.params.filename
+    );
     return res.sendFile(filePath);
   };
 }
 
 // src/commands/test/utils/create-output-handler.ts
-function createOutputHandler(modelIOChecks) {
-  return (_, res) => {
+import path6 from "path";
+import fs4 from "fs/promises";
+function createOutputHandler(modelIOChecks, ctx) {
+  return async (req, res) => {
     modelIOChecks.trackOutputHandled();
-    return res.status(200).json({ message: "Output received" });
+    const outputPath = path6.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
+    await fs4.mkdir(outputPath, { recursive: true });
+    const outputFilePath = path6.join(outputPath, TEST_OUTPUT_FILENAME);
+    const outputData = JSON.stringify(req.body, null, 2);
+    await fs4.writeFile(outputFilePath, outputData, "utf8");
+    return res.status(200).json({ success: true });
   };
 }
 
-// src/utils/get-test-data-filename.ts
-function getTestDataFilename(modelInput) {
-  return `${modelInput.inputSource}-${modelInput.key}.json`;
+// src/commands/test/utils/check-input-files.ts
+import path7 from "path";
+async function checkInputFiles(inputFiles, inputDataPath, ctx) {
+  for (const inputFile of inputFiles) {
+    const filePath = path7.join(ctx.projectRoot, inputDataPath, inputFile);
+    const exists = await isExistingPath(filePath);
+    if (!exists) {
+      throw new Error(
+        `Input data not found (${inputFile}) - did you need to run \`pd4castr fetch\`?`
+      );
+    }
+  }
+}
+
+// src/utils/get-input-filename.ts
+function getInputFilename(modelInput) {
+  return `${modelInput.key}.json`;
 }
 
-// src/commands/test/utils/get-test-data-filenames.ts
-function getTestDataFilenames(config) {
-  const testDataFilenames = config.modelInputs.map(
-    (modelInput) => getTestDataFilename(modelInput)
-  );
-  return testDataFilenames;
+// src/commands/test/utils/get-input-files.ts
+function getInputFiles(config) {
+  const inputFiles = config.inputs.map((input2) => getInputFilename(input2));
+  return inputFiles;
 }
 
 // src/commands/test/utils/model-io-checks.ts
@@ -477,15 +572,15 @@ import { execa as execa2 } from "execa";
 // src/commands/test/utils/get-input-env.ts
 function getInputEnv(modelInput, webserverURL) {
   const variableName = modelInput.key.toUpperCase();
-  const filename = getTestDataFilename(modelInput);
+  const filename = getInputFilename(modelInput);
   const inputFileURL = `${webserverURL}/input/${filename}`;
   return `INPUT_${variableName}_URL=${inputFileURL}`;
 }
 
 // src/commands/test/utils/run-model-container.ts
-async function runModelContainer(dockerImage, config, webserverURL) {
-  const inputEnvs = config.modelInputs.map(
-    (modelInput) => getInputEnv(modelInput, webserverURL)
+async function runModelContainer(dockerImage, webserverURL, ctx) {
+  const inputEnvs = ctx.config.inputs.map(
+    (input2) => getInputEnv(input2, webserverURL)
   );
   const outputEnv = `OUTPUT_URL=${webserverURL}/output`;
   const envs = [...inputEnvs, outputEnv];
@@ -497,7 +592,10 @@ async function runModelContainer(dockerImage, config, webserverURL) {
       ...envs.flatMap((env) => ["--env", env]),
       dockerImage
     ];
-    await execa2("docker", args, { cwd: getCWD(), stdio: "pipe" });
+    await execa2("docker", args, {
+      cwd: ctx.projectRoot,
+      stdio: "pipe"
+    });
   } catch (error) {
     throw new Error("Failed to run model container", { cause: error });
   }
@@ -507,57 +605,87 @@ async function runModelContainer(dockerImage, config, webserverURL) {
 async function startWebServer(app, port) {
   return new Promise((resolve) => {
     const server = app.listen(port, () => {
-      resolve(server);
+      resolve({
+        server,
+        [Symbol.dispose]: () => server.close()
+      });
     });
   });
 }
 
 // src/commands/test/handle-action.ts
 async function handleAction3(options) {
-  const spinner = ora3("Starting model tests...").start();
-  const app = express();
-  const server = await startWebServer(app, options.port);
+  var _stack = [];
   try {
-    const projectConfig = await loadProjectConfig();
-    spinner.start("Building docker image");
-    await buildDockerImage(options.dockerImage);
-    spinner.succeed(`Built docker image (${options.dockerImage})`);
-    const inputFiles = getTestDataFilenames(projectConfig);
-    const modelIOChecks = new ModelIOChecks({ inputFiles });
-    spinner.succeed(`Found ${inputFiles.length} input data files`);
-    const handleInput = createInputHandler(options.inputData, modelIOChecks);
-    const handleOutput = createOutputHandler(modelIOChecks);
-    app.use("/data", express.static(path5.join(getCWD(), options.inputData)));
-    app.get("/input/:filename", handleInput);
-    app.put("/output", handleOutput);
-    spinner.start("Running model container");
-    const webserverURL = `http://localhost:${options.port}`;
-    await runModelContainer(options.dockerImage, projectConfig, webserverURL);
-    spinner.succeed("Model run complete");
-    for (const file of inputFiles) {
-      const status = modelIOChecks.isInputHandled(file) ? "\u2714" : "\u2718";
-      console.log(`    ${status} Input Fetched - ${file}`);
-    }
-    const outputStatus = modelIOChecks.isOutputHandled() ? "\u2714" : "\u2718";
-    console.log(`    ${outputStatus} Output Uploaded`);
-    if (modelIOChecks.isInputsHandled() && modelIOChecks.isOutputHandled()) {
-      spinner.succeed("Model I/O test passed");
-    } else {
-      spinner.fail("Model I/O test failed");
-    }
-  } catch (error) {
-    if (error instanceof ZodError2) {
-      spinner.fail("Config validation failed");
-      logZodIssues(error);
-    } else if (error instanceof Error) {
-      spinner.fail(error.message);
-    }
-    if (error instanceof Error && error.cause instanceof ExecaError) {
-      console.error(error.cause.stderr);
+    const spinner = ora3("Starting model tests...").info();
+    const app = express();
+    const webServer = __using(_stack, await startWebServer(app, options.port));
+    try {
+      const ctx = await loadProjectConfig();
+      const inputFiles = getInputFiles(ctx.config);
+      spinner.start("Checking test input data files");
+      await checkInputFiles(inputFiles, options.inputDir, ctx);
+      spinner.succeed(`Found ${inputFiles.length} test input data files`);
+      spinner.start("Building docker image");
+      const sluggedName = slugify(ctx.config.name, { lower: true });
+      const dockerImage = `pd4castr/${sluggedName}-local:${Date.now()}`;
+      await buildDockerImage(dockerImage, ctx);
+      spinner.succeed(`Built docker image (${dockerImage})`);
+      const modelIOChecks = new ModelIOChecks({ inputFiles });
+      const handleInput = createInputHandler(
+        options.inputDir,
+        modelIOChecks,
+        ctx
+      );
+      const handleOutput = createOutputHandler(modelIOChecks, ctx);
+      const inputPath = path8.join(ctx.projectRoot, options.inputDir);
+      app.use(express.json());
+      app.use("/data", express.static(inputPath));
+      app.get("/input/:filename", handleInput);
+      app.put("/output", handleOutput);
+      spinner.start("Running model container");
+      const webserverURL = `http://localhost:${options.port}`;
+      await runModelContainer(dockerImage, webserverURL, ctx);
+      spinner.succeed("Model run complete");
+      for (const inputFile of inputFiles) {
+        const status = modelIOChecks.isInputHandled(inputFile) ? "\u2714" : "\u2718";
+        console.log(`	${status} Input Fetched - ${inputFile}`);
+      }
+      const outputStatus = modelIOChecks.isOutputHandled() ? "\u2714" : "\u2718";
+      console.log(`	${outputStatus} Output Uploaded`);
+      if (modelIOChecks.isInputsHandled() && modelIOChecks.isOutputHandled()) {
+        spinner.succeed("Model I/O test passed");
+      } else {
+        spinner.fail("Model I/O test failed");
+      }
+      if (modelIOChecks.isOutputHandled()) {
+        const outputPath = path8.join(
+          ctx.projectRoot,
+          TEST_OUTPUT_DATA_DIR,
+          TEST_OUTPUT_FILENAME
+        );
+        const clickHereLink = createLink("Click here", `file://${outputPath}`);
+        const fileLink = createLink(TEST_OUTPUT_FILENAME, `file://${outputPath}`);
+        console.log(`
+${clickHereLink} to view output (${fileLink})
+`);
+      }
+    } catch (error) {
+      if (error instanceof ZodError2) {
+        spinner.fail("Config validation failed");
+        logZodIssues(error);
+      } else if (error instanceof Error) {
+        spinner.fail(error.message);
+      }
+      if (error instanceof Error && error.cause instanceof ExecaError) {
+        console.error(error.cause.stderr);
+      }
+      process.exit(1);
     }
-    process.exit(1);
+  } catch (_) {
+    var _error = _, _hasError = true;
   } finally {
-    server.close();
+    __callDispose(_stack, _error, _hasError);
   }
 }
 
@@ -566,20 +694,15 @@ function registerTestCommand(program2) {
   program2.command("test").description(
     "Test a model by verifying input and output is handled correctly."
   ).option(
-    "-i, --input-data <path>",
-    "The path to the input data directory",
+    "-i, --input-dir <path>",
+    "The input test data directory",
     TEST_INPUT_DATA_DIR
-  ).option(
-    "-d, --docker-image <image>",
-    "The Docker image to execute for testing",
-    // TODO: determine a default value here intelligently when we
-    // implement our image tagging strategy
-    `pd4castr/my-model:${Date.now()}`
   ).option("-p, --port <port>", "The port to run the webserver on", "9800").action(handleAction3);
 }
 
 // src/commands/fetch/handle-action.ts
 import ora4 from "ora";
+import path11 from "path";
 import { ZodError as ZodError3 } from "zod";
 
 // src/utils/get-auth.ts
@@ -598,8 +721,8 @@ async function getAuth() {
 }
 
 // src/commands/fetch/utils/fetch-aemo-data.ts
-import path6 from "path";
-import fs4 from "fs/promises";
+import path9 from "path";
+import fs5 from "fs/promises";
 
 // src/api.ts
 import ky2 from "ky";
@@ -608,12 +731,15 @@ var api = ky2.create({
 });
 
 // src/commands/fetch/utils/fetch-aemo-data.ts
-async function fetchAEMOData(dataFetcher, authCtx) {
-  const queryPath = path6.resolve(getCWD(), dataFetcher.config.fetchQuery);
-  const querySQL = await fs4.readFile(queryPath, "utf8");
+async function fetchAEMOData(dataFetcher, authCtx, ctx) {
+  const queryPath = path9.resolve(
+    ctx.projectRoot,
+    dataFetcher.config.fetchQuery
+  );
+  const querySQL = await fs5.readFile(queryPath, "utf8");
   const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
   const payload2 = { query: querySQL, type: "AEMO_MMS" };
-  const result = await api.post("data-fetcher/run-query", { json: payload2, headers }).json();
+  const result = await api.post("data-fetcher/query", { json: payload2, headers }).json();
   return result;
 }
 
@@ -630,47 +756,45 @@ function getFetcher(type) {
 }
 
 // src/commands/fetch/utils/write-test-data.ts
-import path7 from "path";
-import fs5 from "fs/promises";
-async function writeTestData(output, modelInput) {
-  const outputDir = path7.resolve(getCWD(), TEST_INPUT_DATA_DIR);
-  await fs5.mkdir(outputDir, { recursive: true });
-  const testDataFilename = getTestDataFilename(modelInput);
-  const outputPath = path7.resolve(outputDir, testDataFilename);
-  await fs5.writeFile(outputPath, JSON.stringify(output, void 0, 2));
+import path10 from "path";
+import fs6 from "fs/promises";
+async function writeTestData(inputData, modelInput, inputDataDir, ctx) {
+  const inputDir = path10.resolve(ctx.projectRoot, inputDataDir);
+  await fs6.mkdir(inputDir, { recursive: true });
+  const inputFilename = getInputFilename(modelInput);
+  const inputPath = path10.resolve(inputDir, inputFilename);
+  await fs6.writeFile(inputPath, JSON.stringify(inputData, void 0, 2));
 }
 
 // src/commands/fetch/handle-action.ts
 var FETCHABLE_DATA_FETCHER_TYPES = /* @__PURE__ */ new Set(["AEMO_MMS"]);
-async function handleAction4() {
+async function handleAction4(options) {
   const spinner = ora4("Starting data fetch...").start();
   try {
     const authCtx = await getAuth();
-    const projectConfig = await loadProjectConfig();
-    for (const modelInput of projectConfig.modelInputs) {
-      const dataFetcher = projectConfig.dataFetchers.find(
-        (dataFetcher2) => dataFetcher2.key === modelInput.dataFetcher
-      );
-      if (!dataFetcher) {
-        spinner.warn(
-          `\`${modelInput.key}\` - input has no data fetcher, skipping`
-        );
+    const ctx = await loadProjectConfig();
+    for (const input2 of ctx.config.inputs) {
+      if (!input2.fetcher) {
+        spinner.warn(`\`${input2.key}\` - no data fetcher defined, skipping`);
         continue;
       }
-      if (!FETCHABLE_DATA_FETCHER_TYPES.has(dataFetcher.type)) {
+      if (!FETCHABLE_DATA_FETCHER_TYPES.has(input2.fetcher.type)) {
         spinner.warn(
-          `\`${dataFetcher.key}\` (${dataFetcher.type}) - unsupported, skipping`
+          `\`${input2.key}\` (${input2.fetcher.type}) - unsupported, skipping`
         );
         continue;
       }
-      spinner.start(
-        `\`${dataFetcher.key}\` (${dataFetcher.type}) - fetching...`
-      );
-      const fetchData = getFetcher(dataFetcher.type);
-      const output = await fetchData(dataFetcher, authCtx);
-      await writeTestData(output, modelInput);
-      spinner.succeed(`\`${dataFetcher.key}\` (${dataFetcher.type}) - fetched`);
+      spinner.start(`\`${input2.key}\` (${input2.fetcher.type}) - fetching...`);
+      const fetchData = getFetcher(input2.fetcher.type);
+      const output = await fetchData(input2.fetcher, authCtx, ctx);
+      await writeTestData(output, input2, options.inputDir, ctx);
+      spinner.succeed(`\`${input2.key}\` (${input2.fetcher.type}) - fetched`);
     }
+    const inputPath = path11.resolve(ctx.projectRoot, options.inputDir);
+    const link = createLink("Click here", `file://${inputPath}`);
+    console.log(`
+${link} to view fetched data
+`);
   } catch (error) {
     if (error instanceof ZodError3) {
       spinner.fail("Config validation failed");
@@ -685,8 +809,8 @@ async function handleAction4() {
 // src/commands/fetch/index.ts
 function registerFetchCommand(program2) {
   program2.command("fetch").description("Fetches test data from configured data fetchers.").option(
-    "-i, --input-data <path>",
-    "The path to the input data directory",
+    "-i, --input-dir <path>",
+    "The input test data directory",
     TEST_INPUT_DATA_DIR
   ).action(handleAction4);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pd4castr/cli",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "CLI tool for creating, testing, and publishing pd4castr models",
   "main": "dist/index.js",
   "type": "module",
@@ -66,6 +66,7 @@
     "ky": "1.8.2",
     "lilconfig": "3.1.3",
     "ora": "8.2.0",
+    "slugify": "1.6.6",
     "tiged": "2.12.7",
     "tiny-invariant": "1.3.3",
     "zod": "4.0.14"