@pd4castr/cli 1.9.0 → 1.10.0

package/dist/index.js

@@ -60,7 +60,7 @@ var aemoDataFetcherSchema = z.object({
 var dataFetcherSchema = z.discriminatedUnion("type", [
   aemoDataFetcherSchema
 ]);
-var modelInputSchema = z.object({
+var staticModelInputSchema = z.object({
   key: z.string(),
   inputSource: z.string().optional().default(DEFAULT_INPUT_SOURCE_ID),
   trigger: z.enum([
@@ -69,8 +69,25 @@ var modelInputSchema = z.object({
   ]),
   uploadFileFormat: fileFormatSchema.optional().default("json"),
   targetFileFormat: fileFormatSchema.optional().default("json"),
-  fetcher: dataFetcherSchema.optional().nullable()
-});
+  // Explicitly forbid `fetcher` so invalid data-fetcher inputs can't fall through
+  // and be accepted as static inputs.
+  fetcher: z.never().optional()
+}).strict();
+var dataFetcherModelInputSchema = z.object({
+  key: z.string(),
+  inputSource: z.string().optional().default(DEFAULT_INPUT_SOURCE_ID),
+  fetcher: dataFetcherSchema,
+  trigger: z.enum([
+    "WAIT_FOR_LATEST_FILE",
+    "USE_MOST_RECENT_FILE"
+  ]),
+  uploadFileFormat: fileFormatSchema.optional().default("json"),
+  targetFileFormat: fileFormatSchema.optional().default("json")
+}).strict();
+var modelInputSchema = z.union([
+  dataFetcherModelInputSchema,
+  staticModelInputSchema
+]);
 var modelOutputSchema = z.object({
   name: z.string(),
   type: z.enum([
@@ -123,9 +140,9 @@ var projectConfigSchema = z.object({
 
 // src/utils/is-existing-path.ts
 import fs from "fs/promises";
-async function isExistingPath(path15) {
+async function isExistingPath(path17) {
   try {
-    await fs.access(path15);
+    await fs.access(path17);
     return true;
   } catch {
     return false;
@@ -237,6 +254,18 @@ async function getAuth() {
 }
 __name(getAuth, "getAuth");
 
+// src/utils/is-data-fetcher-input.ts
+function isDataFetcherInput(input2) {
+  return "fetcher" in input2;
+}
+__name(isDataFetcherInput, "isDataFetcherInput");
+
+// src/utils/is-static-input.ts
+function isStaticInput(input2) {
+  return !("fetcher" in input2);
+}
+__name(isStaticInput, "isStaticInput");
+
 // src/utils/log-zod-issues.ts
 function logZodIssues(error) {
   for (const issue of error.issues) {
@@ -344,18 +373,18 @@ async function handleAction(options) {
   try {
     const authCtx = await getAuth();
     const ctx = await loadProjectContext(options.config);
-    const inputsWithFetchers = ctx.config.inputs.filter((input2) => input2.fetcher);
-    if (inputsWithFetchers.length === 0) {
+    const dataFetcherInputs = ctx.config.inputs.filter((input2) => isDataFetcherInput(input2));
+    if (dataFetcherInputs.length === 0) {
       spinner.info("No inputs with data fetchers found, skipping");
       return;
     }
     for (const input2 of ctx.config.inputs) {
-      if (!input2.fetcher) {
-        spinner.info(`\`${input2.key}\` - no data fetcher defined, skipping`);
+      if (isStaticInput(input2)) {
+        spinner.info(`\`${input2.key}\` - static input, skipping`);
         continue;
       }
       if (!FETCHABLE_DATA_FETCHER_TYPES.has(input2.fetcher.type)) {
-        spinner.warn(`\`${input2.key}\` (${input2.fetcher.type}) - unsupported, skipping`);
+        spinner.warn(`\`${input2.key}\` (${input2.fetcher.type}) - unsupported data fetcher type, skipping`);
         continue;
       }
       spinner.start(`\`${input2.key}\` (${input2.fetcher.type}) - fetching...`);
@@ -689,11 +718,11 @@ async function createModel(config, authCtx) {
 __name(createModel, "createModel");
 
 // src/api/get-registry-push-credentials.ts
-async function getRegistryPushCredentials(modelID, authCtx) {
+async function getRegistryPushCredentials(modelId, authCtx) {
   const headers = {
     Authorization: `Bearer ${authCtx.accessToken}`
   };
-  const searchParams = new URLSearchParams(`modelId=${modelID}`);
+  const searchParams = new URLSearchParams(`modelId=${modelId}`);
   const result = await api.get("registry/push-credentials", {
     headers,
     searchParams
@@ -912,13 +941,53 @@ function logEmptyLine() {
 }
 __name(logEmptyLine, "logEmptyLine");
 
+// src/utils/validate-data-fetcher-input-files.ts
+import path10 from "path";
+async function validateDataFetcherInputFiles(options, ctx) {
+  const dataFetcherInputs = ctx.config.inputs.filter((input2) => isDataFetcherInput(input2));
+  for (const input2 of dataFetcherInputs) {
+    const expectedFilename = `${input2.key}.${input2.uploadFileFormat}`;
+    const targetFilename = `${input2.key}.${input2.targetFileFormat}`;
+    const isConversionRequired = targetFilename !== expectedFilename;
+    if (isConversionRequired) {
+      throw new Error(`Data fetcher input (${input2.key}) must not require conversion (${input2.uploadFileFormat} -> ${input2.targetFileFormat}).`);
+    }
+    const expectedFilePath = path10.join(ctx.projectRoot, options.inputDir, expectedFilename);
+    const isFileOnDisk = await isExistingPath(expectedFilePath);
+    if (!isFileOnDisk) {
+      throw new Error(`Data fetcher input (${input2.key}) data not found.
+
+Did you need to run \`pd4castr fetch\`?`);
+    }
+  }
+}
+__name(validateDataFetcherInputFiles, "validateDataFetcherInputFiles");
+
+// src/utils/validate-static-input-files.ts
+import path11 from "path";
+async function validateStaticInputFiles(options, ctx) {
+  const staticInputs = ctx.config.inputs.filter((input2) => isStaticInput(input2));
+  for (const input2 of staticInputs) {
+    const expectedFilename = `${input2.key}.${input2.uploadFileFormat}`;
+    const targetFilename = `${input2.key}.${input2.targetFileFormat}`;
+    const isConversionRequired = targetFilename !== expectedFilename;
+    if (isConversionRequired) {
+      throw new Error(`Static input (${input2.key}) requires conversion (${input2.uploadFileFormat} -> ${input2.targetFileFormat}), which is not supported via the CLI.`);
+    }
+    const expectedFilePath = path11.join(ctx.projectRoot, options.inputDir, expectedFilename);
+    const isFileOnDisk = await isExistingPath(expectedFilePath);
+    if (!isFileOnDisk) {
+      throw new Error(`Static input (${input2.key}) data not found
+
+Ensure your data is in the the input directory (${options.inputDir})`);
+    }
+  }
+}
+__name(validateStaticInputFiles, "validateStaticInputFiles");
+
 // src/commands/publish/constants.ts
 import chalk from "chalk";
-var MODEL_RUN_TRIGGER_MESSAGE = `${chalk.whiteBright.bold("NOTE!")} If you do not see your model output in the pd4castr UI:
-
-\u2022 If you are using static inputs - check you have uploaded your input data to your input bucket
-\u2022 If you are using inputs with data fetchers - wait a few minutes and check again
-`;
+var MODEL_RUN_TRIGGER_MESSAGE = `${chalk.whiteBright.bold("NOTE!")} It may take a few minutes for your model run to appear in the pd4castr UI.`;
 
 // src/commands/publish/utils/get-model-summary-lines.ts
 import chalk2 from "chalk";
@@ -1016,7 +1085,7 @@ function getWSLMachineIP() {
 __name(getWSLMachineIP, "getWSLMachineIP");
 
 // src/model-io-checks/setup-model-io-checks.ts
-import path12 from "path";
+import path14 from "path";
 import express from "express";
 
 // src/model-io-checks/model-io-checks.ts
@@ -1056,7 +1125,7 @@ var ModelIOChecks = class {
 };
 
 // src/model-io-checks/utils/create-input-handler.ts
-import path10 from "path";
+import path12 from "path";
 function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
   return (req, res) => {
     if (!modelIOChecks.isValidInput(req.params.filename)) {
@@ -1065,7 +1134,7 @@ function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
       });
     }
     modelIOChecks.trackInputHandled(req.params.filename);
-    const filePath = path10.join(ctx.projectRoot, inputFilesPath, req.params.filename);
+    const filePath = path12.join(ctx.projectRoot, inputFilesPath, req.params.filename);
     return res.sendFile(filePath);
   };
 }
@@ -1073,15 +1142,15 @@ __name(createInputHandler, "createInputHandler");
 
 // src/model-io-checks/utils/create-output-handler.ts
 import fs9 from "fs/promises";
-import path11 from "path";
+import path13 from "path";
 function createOutputHandler(modelIOChecks, ctx) {
   return async (req, res) => {
     modelIOChecks.trackOutputHandled();
-    const outputPath = path11.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
+    const outputPath = path13.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
     await fs9.mkdir(outputPath, {
       recursive: true
     });
-    const outputFilePath = path11.join(outputPath, TEST_OUTPUT_FILENAME);
+    const outputFilePath = path13.join(outputPath, TEST_OUTPUT_FILENAME);
     const outputData = JSON.stringify(req.body, null, 2);
     await fs9.writeFile(outputFilePath, outputData, "utf8");
     return res.status(200).json({
@@ -1098,7 +1167,7 @@ function setupModelIOChecks(app, inputDir, inputFiles, ctx) {
   });
   const handleInput = createInputHandler(inputDir, modelIOChecks, ctx);
   const handleOutput = createOutputHandler(modelIOChecks, ctx);
-  const inputPath = path12.join(ctx.projectRoot, inputDir);
+  const inputPath = path14.join(ctx.projectRoot, inputDir);
   app.use(express.json());
   app.use("/data", express.static(inputPath));
   app.get("/input/:filename", handleInput);
@@ -1107,19 +1176,6 @@ function setupModelIOChecks(app, inputDir, inputFiles, ctx) {
 }
 __name(setupModelIOChecks, "setupModelIOChecks");
 
-// src/utils/check-input-files.ts
-import path13 from "path";
-async function checkInputFiles(inputFiles, inputDataPath, ctx) {
-  for (const inputFile of inputFiles) {
-    const filePath = path13.join(ctx.projectRoot, inputDataPath, inputFile);
-    const exists = await isExistingPath(filePath);
-    if (!exists) {
-      throw new Error(`Input data not found (${inputFile}) - did you need to run \`pd4castr fetch\`?`);
-    }
-  }
-}
-__name(checkInputFiles, "checkInputFiles");
-
 // src/utils/get-input-files.ts
 function getInputFiles(config) {
   const inputFiles = config.inputs.map((input2) => getInputFilename(input2));
@@ -1130,7 +1186,6 @@ __name(getInputFiles, "getInputFiles");
 // src/commands/publish/utils/run-model-io-tests.ts
 async function runModelIOTests(dockerImage, options, app, ctx) {
   const inputFiles = getInputFiles(ctx.config);
-  await checkInputFiles(inputFiles, options.inputDir, ctx);
   await buildDockerImage(dockerImage, ctx);
   const modelIOChecks = setupModelIOChecks(app, options.inputDir, inputFiles, ctx);
   await runModelContainer(dockerImage, options.port, ctx);
@@ -1140,6 +1195,141 @@ async function runModelIOTests(dockerImage, options, app, ctx) {
 }
 __name(runModelIOTests, "runModelIOTests");
 
+// src/commands/publish/utils/upload-static-inputs.ts
+import fs10 from "fs";
+import path15 from "path";
+import ky3, { isHTTPError } from "ky";
+import promiseRetry from "promise-retry";
+
+// src/api/get-model-input-upload-url.ts
+async function getModelInputUploadURL(modelId, modelInputId, authCtx) {
+  const headers = {
+    Authorization: `Bearer ${authCtx.accessToken}`
+  };
+  const result = await api.post(`model/${modelId}/input/${modelInputId}/get-upload-url`, {
+    headers
+  }).json();
+  return result.signedUploadURL;
+}
+__name(getModelInputUploadURL, "getModelInputUploadURL");
+
+// src/commands/publish/utils/upload-static-inputs.ts
+async function uploadStaticInputFiles(model, options, ctx, authCtx) {
+  const staticInputs = ctx.config.inputs.filter((input2) => isStaticInput(input2));
+  if (staticInputs.length === 0) {
+    return;
+  }
+  if (!model.inputs) {
+    throw new Error("Model inputs were not found in the published model");
+  }
+  const modelInputsByKey = new Map(model.inputs.map((input2) => [
+    input2.key,
+    input2.id
+  ]));
+  for (const input2 of staticInputs) {
+    const modelInputId = modelInputsByKey.get(input2.key);
+    if (!modelInputId) {
+      throw new Error(`Model input not found for key (${input2.key})`);
+    }
+    const filename = `${input2.key}.${input2.uploadFileFormat}`;
+    const filePath = path15.join(ctx.projectRoot, options.inputDir, filename);
+    const exists = await isExistingPath(filePath);
+    if (!exists) {
+      throw new Error(`Static input data not found (${filename}) in --input-dir ${options.inputDir}`);
+    }
+    const uploadURL = await getModelInputUploadURL(model.id, modelInputId, authCtx);
+    try {
+      await uploadWithRetry(filePath, uploadURL);
+    } catch (error) {
+      throw new Error(`Failed to upload static input (${input2.key})
+`, {
+        cause: getErrorMessage(error)
+      });
+    }
+  }
+}
+__name(uploadStaticInputFiles, "uploadStaticInputFiles");
+function getErrorMessage(error) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return "Unknown error";
+}
+__name(getErrorMessage, "getErrorMessage");
+var RETRY_LIMIT = 3;
+var RETRY_DELAY_MS = 250;
+async function uploadWithRetry(filePath, signedUploadURL) {
+  const upload = /* @__PURE__ */ __name(async (retry) => {
+    const { size } = fs10.statSync(filePath);
+    const body = fs10.createReadStream(filePath);
+    try {
+      const response = await ky3.put(signedUploadURL, {
+        body,
+        // streaming request bodies require duplex for node fetch
+        duplex: "half",
+        // we handle retries ourselves to ensure our request body stream is rerecreated per attempt
+        retry: {
+          limit: 0
+        },
+        headers: {
+          "Content-Length": size.toString()
+        }
+      });
+      await safeConsumeResponseBody(response);
+    } catch (error) {
+      body.destroy();
+      if (isHTTPError(error) && isRetriableStatus(error.response.status)) {
+        await safeConsumeResponseBody(error.response);
+        return retry(error);
+      }
+      if (isFetchError(error) && isTransientFetchError(error)) {
+        return retry(error);
+      }
+      throw error;
+    }
+  }, "upload");
+  return promiseRetry(upload, {
+    retries: RETRY_LIMIT,
+    minTimeout: RETRY_DELAY_MS,
+    factor: 2
+  });
+}
+__name(uploadWithRetry, "uploadWithRetry");
+async function safeConsumeResponseBody(response) {
+  try {
+    await response.arrayBuffer();
+  } catch {
+  }
+}
+__name(safeConsumeResponseBody, "safeConsumeResponseBody");
+function isRetriableStatus(status) {
+  const isTimeout = status === 408;
+  const isServerError = status >= 500 && status <= 599;
+  return isTimeout || isServerError;
+}
+__name(isRetriableStatus, "isRetriableStatus");
+function isFetchError(error) {
+  if (!(error instanceof Error)) {
+    return false;
+  }
+  if (!("code" in error)) {
+    return false;
+  }
+  return typeof error.code === "string";
+}
+__name(isFetchError, "isFetchError");
+var TRANSIENT_FETCH_ERROR_CODES = /* @__PURE__ */ new Set([
+  "ECONNRESET",
+  "ETIMEDOUT",
+  "EPIPE",
+  "ENOTFOUND",
+  "EAI_AGAIN"
+]);
+function isTransientFetchError(error) {
+  return TRANSIENT_FETCH_ERROR_CODES.has(error.code);
+}
+__name(isTransientFetchError, "isTransientFetchError");
+
 // src/commands/publish/handle-create-model-flow.ts
 async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
   spinner.info(`You are publishing a ${chalk3.bold("new")} model:
@@ -1156,6 +1346,8 @@ async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
   const dockerImage = getDockerImage(ctx);
   if (!options.skipChecks) {
     spinner.start("Performing model I/O test...");
+    await validateStaticInputFiles(options, ctx);
+    await validateDataFetcherInputFiles(options, ctx);
     await runModelIOTests(dockerImage, options, app, ctx);
     spinner.succeed("Model I/O test passed");
   }
@@ -1175,7 +1367,13 @@ async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
   await loginToDockerRegistry(pushCredentials);
   await buildDockerImage(dockerImage, ctx);
   await pushDockerImage(dockerImage, model.dockerImage);
-  spinner.succeed("Model image pushed to reogistry successfully");
+  spinner.succeed("Model image pushed to registry successfully");
+  const isStaticInputUploadRequired = ctx.config.inputs.some((input2) => isStaticInput(input2));
+  if (isStaticInputUploadRequired) {
+    spinner.start("Uploading static input files...");
+    await uploadStaticInputFiles(model, options, ctx, authCtx);
+    spinner.succeed("Static input files uploaded successfully");
+  }
   spinner.start("Triggering model run...");
   let modelRunTriggered = false;
   if (!options.skipTrigger) {
@@ -1211,11 +1409,11 @@ import chalk4 from "chalk";
 import invariant2 from "tiny-invariant";
 
 // src/api/get-model.ts
-async function getModel(id, authCtx) {
+async function getModel(modelId, authCtx) {
   const headers = {
     Authorization: `Bearer ${authCtx.accessToken}`
   };
-  const result = await api.get(`model/${id}`, {
+  const result = await api.get(`model/${modelId}`, {
     headers
   }).json();
   return result;
@@ -1256,6 +1454,8 @@ async function handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx
   const dockerImage = getDockerImage(ctx);
   if (!options.skipChecks) {
     spinner.start("Performing model I/O test...");
+    await validateStaticInputFiles(options, ctx);
+    await validateDataFetcherInputFiles(options, ctx);
     await runModelIOTests(dockerImage, options, app, ctx);
     spinner.succeed("Model I/O test passed");
   }
@@ -1275,6 +1475,12 @@ async function handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx
   await buildDockerImage(dockerImage, ctx);
   await pushDockerImage(dockerImage, model.dockerImage);
   spinner.succeed("New model revision image pushed to registry successfully");
+  const isStaticInputUploadRequired = ctx.config.inputs.some((input2) => isStaticInput(input2));
+  if (isStaticInputUploadRequired) {
+    spinner.start("Uploading static input files...");
+    await uploadStaticInputFiles(model, options, ctx, authCtx);
+    spinner.succeed("Static input files uploaded successfully");
+  }
   spinner.start("Triggering model run...");
   let modelRunTriggered = false;
   if (!options.skipTrigger) {
@@ -1336,6 +1542,8 @@ async function handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx
   const dockerImage = getDockerImage(ctx);
   if (!options.skipChecks) {
     spinner.start("Performing model I/O test...");
+    await validateStaticInputFiles(options, ctx);
+    await validateDataFetcherInputFiles(options, ctx);
     await runModelIOTests(dockerImage, options, app, ctx);
     spinner.succeed("Model I/O test passed");
   }
@@ -1356,6 +1564,12 @@ async function handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx
   await buildDockerImage(dockerImage, ctx);
   await pushDockerImage(dockerImage, model.dockerImage);
   spinner.succeed("Updated model image pushed to registry successfully");
+  const isStaticInputUploadRequired = ctx.config.inputs.some((input2) => isStaticInput(input2));
+  if (isStaticInputUploadRequired) {
+    spinner.start("Uploading static input files...");
+    await uploadStaticInputFiles(model, options, ctx, authCtx);
+    spinner.succeed("Static input files uploaded successfully");
+  }
   spinner.start("Triggering model run...");
   let modelRunTriggered = false;
   if (!options.skipTrigger) {
@@ -1533,7 +1747,7 @@ function registerPublishCommand(program2) {
 __name(registerPublishCommand, "registerPublishCommand");
 
 // src/commands/test/handle-action.ts
-import path14 from "path";
+import path16 from "path";
 import { ExecaError as ExecaError6 } from "execa";
 import express3 from "express";
 import ora6 from "ora";
@@ -1628,7 +1842,8 @@ async function handleAction6(options) {
       const ctx = await loadProjectContext(options.config);
       const inputFiles = getInputFiles(ctx.config);
       spinner.start("Checking test input data files");
-      await checkInputFiles(inputFiles, options.inputDir, ctx);
+      await validateDataFetcherInputFiles(options, ctx);
+      await validateStaticInputFiles(options, ctx);
       spinner.succeed(`Found ${inputFiles.length} test input data files`);
       spinner.start("Building docker image");
       const dockerImage = getDockerImage(ctx);
@@ -1650,7 +1865,7 @@ async function handleAction6(options) {
         throw new Error("Model I/O test failed");
       }
       if (modelIOChecks.isOutputHandled()) {
-        const outputPath = path14.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR, TEST_OUTPUT_FILENAME);
+        const outputPath = path16.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR, TEST_OUTPUT_FILENAME);
         const clickHereLink = createLink("Click here", `file://${outputPath}`);
         const fileLink = createLink(TEST_OUTPUT_FILENAME, `file://${outputPath}`);
         console.log(`
@@ -1692,7 +1907,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "@pd4castr/cli",
-  version: "1.9.0",
+  version: "1.10.0",
   description: "CLI tool for creating, testing, and publishing pd4castr models",
   license: "MIT",
   main: "dist/index.js",
@@ -1713,7 +1928,6 @@ var package_default = {
     build: "tsup",
     dev: "tsup --watch",
     pd4castr: "node dist/index.js",
-    release: "semantic-release -e semantic-release-monorepo",
     format: "prettier --write .",
     lint: "eslint .",
     typecheck: "tsc --noEmit",
@@ -1727,8 +1941,9 @@ var package_default = {
     execa: "9.6.0",
     express: "4.21.2",
     immer: "10.1.1",
-    ky: "1.8.2",
+    ky: "1.14.2",
     ora: "8.2.0",
+    "promise-retry": "2.0.1",
     slugify: "1.6.6",
     tiged: "2.12.7",
     "tiny-invariant": "1.3.3",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pd4castr/cli",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "CLI tool for creating, testing, and publishing pd4castr models",
   "license": "MIT",
   "main": "dist/index.js",
@@ -21,7 +21,6 @@
     "build": "tsup",
     "dev": "tsup --watch",
     "pd4castr": "node dist/index.js",
-    "release": "semantic-release -e semantic-release-monorepo",
     "format": "prettier --write .",
     "lint": "eslint .",
     "typecheck": "tsc --noEmit",
@@ -35,8 +34,9 @@
     "execa": "9.6.0",
     "express": "4.21.2",
     "immer": "10.1.1",
-    "ky": "1.8.2",
+    "ky": "1.14.2",
     "ora": "8.2.0",
+    "promise-retry": "2.0.1",
     "slugify": "1.6.6",
     "tiged": "2.12.7",
     "tiny-invariant": "1.3.3",