@pd4castr/cli 1.8.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +21 -0
- package/README.md +25 -13
- package/dist/index.js +266 -49
- package/docs/assets/logo.png +0 -0
- package/package.json +7 -5
package/LICENSE.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Endgame Economics Pty Ltd t/as Endgame Analytics
|
|
4
|
+
<contact@endgameanalytics.com.au> https://endgameanalytics.com.au
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
7
|
+
this software and associated documentation files (the “Software”), to deal in
|
|
8
|
+
the Software without restriction, including without limitation the rights to
|
|
9
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
10
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
|
11
|
+
subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
18
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
19
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
20
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
21
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,38 +1,49 @@
|
|
|
1
|
-
|
|
1
|
+
<div align="center">
|
|
2
|
+
<img src="./docs/assets/logo.png" alt="pdView logo" width="92">
|
|
2
3
|
|
|
3
|
-
CLI
|
|
4
|
-
[pd4castr](https://pdview.com.au/services/pd4castr/) models.
|
|
4
|
+
<h1>pd4castr CLI</h1>
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
<p>
|
|
7
|
+
CLI tool for creating, testing, and publishing
|
|
8
|
+
<a href="https://pdview.com.au/services/pd4castr/">pd4castr</a> models.
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<p>
|
|
12
|
+
<a href="#installation">Installation</a> •
|
|
13
|
+
<a href="#quick-start">Quick Start</a> •
|
|
14
|
+
<a href="https://github.com/pipelabs/pd4castr-model-examples/" target="_blank">Full Documentation</a>
|
|
15
|
+
</p>
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
|
|
20
|
+
> Requires Node.js >= 20.
|
|
7
21
|
|
|
8
22
|
```bash
|
|
9
23
|
npm install -g @pd4castr/cli
|
|
10
24
|
```
|
|
11
25
|
|
|
12
|
-
|
|
13
|
-
[full documentation here](https://github.com/pipelabs/pd4castr-model-examples/)
|
|
14
|
-
|
|
15
|
-
## Quick Usage
|
|
26
|
+
## Quick Start
|
|
16
27
|
|
|
17
|
-
Authenticate with the pd4castr API
|
|
28
|
+
Authenticate with the pd4castr API:
|
|
18
29
|
|
|
19
30
|
```sh
|
|
20
31
|
pd4castr login
|
|
21
32
|
```
|
|
22
33
|
|
|
23
|
-
Run model input data fetchers and generate test input data
|
|
34
|
+
Run model input data fetchers and generate test input data:
|
|
24
35
|
|
|
25
36
|
```sh
|
|
26
37
|
pd4castr fetch
|
|
27
38
|
```
|
|
28
39
|
|
|
29
|
-
Run your model locally and verify it reads inputs & uploads output as expected
|
|
40
|
+
Run your model locally and verify it reads inputs & uploads output as expected:
|
|
30
41
|
|
|
31
42
|
```sh
|
|
32
43
|
pd4castr test
|
|
33
44
|
```
|
|
34
45
|
|
|
35
|
-
Publish your model to the pd4castr platform
|
|
46
|
+
Publish your model to the pd4castr platform:
|
|
36
47
|
|
|
37
48
|
```sh
|
|
38
49
|
pd4castr publish
|
|
@@ -40,4 +51,5 @@ pd4castr publish
|
|
|
40
51
|
|
|
41
52
|
## Contributing
|
|
42
53
|
|
|
43
|
-
[
|
|
54
|
+
See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup, testing, and
|
|
55
|
+
contribution guidelines.
|
package/dist/index.js
CHANGED
|
@@ -60,7 +60,7 @@ var aemoDataFetcherSchema = z.object({
|
|
|
60
60
|
var dataFetcherSchema = z.discriminatedUnion("type", [
|
|
61
61
|
aemoDataFetcherSchema
|
|
62
62
|
]);
|
|
63
|
-
var
|
|
63
|
+
var staticModelInputSchema = z.object({
|
|
64
64
|
key: z.string(),
|
|
65
65
|
inputSource: z.string().optional().default(DEFAULT_INPUT_SOURCE_ID),
|
|
66
66
|
trigger: z.enum([
|
|
@@ -69,8 +69,25 @@ var modelInputSchema = z.object({
|
|
|
69
69
|
]),
|
|
70
70
|
uploadFileFormat: fileFormatSchema.optional().default("json"),
|
|
71
71
|
targetFileFormat: fileFormatSchema.optional().default("json"),
|
|
72
|
-
fetcher
|
|
73
|
-
|
|
72
|
+
// Explicitly forbid `fetcher` so invalid data-fetcher inputs can't fall through
|
|
73
|
+
// and be accepted as static inputs.
|
|
74
|
+
fetcher: z.never().optional()
|
|
75
|
+
}).strict();
|
|
76
|
+
var dataFetcherModelInputSchema = z.object({
|
|
77
|
+
key: z.string(),
|
|
78
|
+
inputSource: z.string().optional().default(DEFAULT_INPUT_SOURCE_ID),
|
|
79
|
+
fetcher: dataFetcherSchema,
|
|
80
|
+
trigger: z.enum([
|
|
81
|
+
"WAIT_FOR_LATEST_FILE",
|
|
82
|
+
"USE_MOST_RECENT_FILE"
|
|
83
|
+
]),
|
|
84
|
+
uploadFileFormat: fileFormatSchema.optional().default("json"),
|
|
85
|
+
targetFileFormat: fileFormatSchema.optional().default("json")
|
|
86
|
+
}).strict();
|
|
87
|
+
var modelInputSchema = z.union([
|
|
88
|
+
dataFetcherModelInputSchema,
|
|
89
|
+
staticModelInputSchema
|
|
90
|
+
]);
|
|
74
91
|
var modelOutputSchema = z.object({
|
|
75
92
|
name: z.string(),
|
|
76
93
|
type: z.enum([
|
|
@@ -123,9 +140,9 @@ var projectConfigSchema = z.object({
|
|
|
123
140
|
|
|
124
141
|
// src/utils/is-existing-path.ts
|
|
125
142
|
import fs from "fs/promises";
|
|
126
|
-
async function isExistingPath(
|
|
143
|
+
async function isExistingPath(path17) {
|
|
127
144
|
try {
|
|
128
|
-
await fs.access(
|
|
145
|
+
await fs.access(path17);
|
|
129
146
|
return true;
|
|
130
147
|
} catch {
|
|
131
148
|
return false;
|
|
@@ -237,6 +254,18 @@ async function getAuth() {
|
|
|
237
254
|
}
|
|
238
255
|
__name(getAuth, "getAuth");
|
|
239
256
|
|
|
257
|
+
// src/utils/is-data-fetcher-input.ts
|
|
258
|
+
function isDataFetcherInput(input2) {
|
|
259
|
+
return "fetcher" in input2;
|
|
260
|
+
}
|
|
261
|
+
__name(isDataFetcherInput, "isDataFetcherInput");
|
|
262
|
+
|
|
263
|
+
// src/utils/is-static-input.ts
|
|
264
|
+
function isStaticInput(input2) {
|
|
265
|
+
return !("fetcher" in input2);
|
|
266
|
+
}
|
|
267
|
+
__name(isStaticInput, "isStaticInput");
|
|
268
|
+
|
|
240
269
|
// src/utils/log-zod-issues.ts
|
|
241
270
|
function logZodIssues(error) {
|
|
242
271
|
for (const issue of error.issues) {
|
|
@@ -344,18 +373,18 @@ async function handleAction(options) {
|
|
|
344
373
|
try {
|
|
345
374
|
const authCtx = await getAuth();
|
|
346
375
|
const ctx = await loadProjectContext(options.config);
|
|
347
|
-
const
|
|
348
|
-
if (
|
|
376
|
+
const dataFetcherInputs = ctx.config.inputs.filter((input2) => isDataFetcherInput(input2));
|
|
377
|
+
if (dataFetcherInputs.length === 0) {
|
|
349
378
|
spinner.info("No inputs with data fetchers found, skipping");
|
|
350
379
|
return;
|
|
351
380
|
}
|
|
352
381
|
for (const input2 of ctx.config.inputs) {
|
|
353
|
-
if (
|
|
354
|
-
spinner.info(`\`${input2.key}\` -
|
|
382
|
+
if (isStaticInput(input2)) {
|
|
383
|
+
spinner.info(`\`${input2.key}\` - static input, skipping`);
|
|
355
384
|
continue;
|
|
356
385
|
}
|
|
357
386
|
if (!FETCHABLE_DATA_FETCHER_TYPES.has(input2.fetcher.type)) {
|
|
358
|
-
spinner.warn(`\`${input2.key}\` (${input2.fetcher.type}) - unsupported, skipping`);
|
|
387
|
+
spinner.warn(`\`${input2.key}\` (${input2.fetcher.type}) - unsupported data fetcher type, skipping`);
|
|
359
388
|
continue;
|
|
360
389
|
}
|
|
361
390
|
spinner.start(`\`${input2.key}\` (${input2.fetcher.type}) - fetching...`);
|
|
@@ -689,11 +718,11 @@ async function createModel(config, authCtx) {
|
|
|
689
718
|
__name(createModel, "createModel");
|
|
690
719
|
|
|
691
720
|
// src/api/get-registry-push-credentials.ts
|
|
692
|
-
async function getRegistryPushCredentials(
|
|
721
|
+
async function getRegistryPushCredentials(modelId, authCtx) {
|
|
693
722
|
const headers = {
|
|
694
723
|
Authorization: `Bearer ${authCtx.accessToken}`
|
|
695
724
|
};
|
|
696
|
-
const searchParams = new URLSearchParams(`modelId=${
|
|
725
|
+
const searchParams = new URLSearchParams(`modelId=${modelId}`);
|
|
697
726
|
const result = await api.get("registry/push-credentials", {
|
|
698
727
|
headers,
|
|
699
728
|
searchParams
|
|
@@ -912,13 +941,53 @@ function logEmptyLine() {
|
|
|
912
941
|
}
|
|
913
942
|
__name(logEmptyLine, "logEmptyLine");
|
|
914
943
|
|
|
944
|
+
// src/utils/validate-data-fetcher-input-files.ts
|
|
945
|
+
import path10 from "path";
|
|
946
|
+
async function validateDataFetcherInputFiles(options, ctx) {
|
|
947
|
+
const dataFetcherInputs = ctx.config.inputs.filter((input2) => isDataFetcherInput(input2));
|
|
948
|
+
for (const input2 of dataFetcherInputs) {
|
|
949
|
+
const expectedFilename = `${input2.key}.${input2.uploadFileFormat}`;
|
|
950
|
+
const targetFilename = `${input2.key}.${input2.targetFileFormat}`;
|
|
951
|
+
const isConversionRequired = targetFilename !== expectedFilename;
|
|
952
|
+
if (isConversionRequired) {
|
|
953
|
+
throw new Error(`Data fetcher input (${input2.key}) must not require conversion (${input2.uploadFileFormat} -> ${input2.targetFileFormat}).`);
|
|
954
|
+
}
|
|
955
|
+
const expectedFilePath = path10.join(ctx.projectRoot, options.inputDir, expectedFilename);
|
|
956
|
+
const isFileOnDisk = await isExistingPath(expectedFilePath);
|
|
957
|
+
if (!isFileOnDisk) {
|
|
958
|
+
throw new Error(`Data fetcher input (${input2.key}) data not found.
|
|
959
|
+
|
|
960
|
+
Did you need to run \`pd4castr fetch\`?`);
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
__name(validateDataFetcherInputFiles, "validateDataFetcherInputFiles");
|
|
965
|
+
|
|
966
|
+
// src/utils/validate-static-input-files.ts
|
|
967
|
+
import path11 from "path";
|
|
968
|
+
async function validateStaticInputFiles(options, ctx) {
|
|
969
|
+
const staticInputs = ctx.config.inputs.filter((input2) => isStaticInput(input2));
|
|
970
|
+
for (const input2 of staticInputs) {
|
|
971
|
+
const expectedFilename = `${input2.key}.${input2.uploadFileFormat}`;
|
|
972
|
+
const targetFilename = `${input2.key}.${input2.targetFileFormat}`;
|
|
973
|
+
const isConversionRequired = targetFilename !== expectedFilename;
|
|
974
|
+
if (isConversionRequired) {
|
|
975
|
+
throw new Error(`Static input (${input2.key}) requires conversion (${input2.uploadFileFormat} -> ${input2.targetFileFormat}), which is not supported via the CLI.`);
|
|
976
|
+
}
|
|
977
|
+
const expectedFilePath = path11.join(ctx.projectRoot, options.inputDir, expectedFilename);
|
|
978
|
+
const isFileOnDisk = await isExistingPath(expectedFilePath);
|
|
979
|
+
if (!isFileOnDisk) {
|
|
980
|
+
throw new Error(`Static input (${input2.key}) data not found
|
|
981
|
+
|
|
982
|
+
Ensure your data is in the the input directory (${options.inputDir})`);
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
__name(validateStaticInputFiles, "validateStaticInputFiles");
|
|
987
|
+
|
|
915
988
|
// src/commands/publish/constants.ts
|
|
916
989
|
import chalk from "chalk";
|
|
917
|
-
var MODEL_RUN_TRIGGER_MESSAGE = `${chalk.whiteBright.bold("NOTE!")}
|
|
918
|
-
|
|
919
|
-
\u2022 If you are using static inputs - check you have uploaded your input data to your input bucket
|
|
920
|
-
\u2022 If you are using inputs with data fetchers - wait a few minutes and check again
|
|
921
|
-
`;
|
|
990
|
+
var MODEL_RUN_TRIGGER_MESSAGE = `${chalk.whiteBright.bold("NOTE!")} It may take a few minutes for your model run to appear in the pd4castr UI.`;
|
|
922
991
|
|
|
923
992
|
// src/commands/publish/utils/get-model-summary-lines.ts
|
|
924
993
|
import chalk2 from "chalk";
|
|
@@ -1016,7 +1085,7 @@ function getWSLMachineIP() {
|
|
|
1016
1085
|
__name(getWSLMachineIP, "getWSLMachineIP");
|
|
1017
1086
|
|
|
1018
1087
|
// src/model-io-checks/setup-model-io-checks.ts
|
|
1019
|
-
import
|
|
1088
|
+
import path14 from "path";
|
|
1020
1089
|
import express from "express";
|
|
1021
1090
|
|
|
1022
1091
|
// src/model-io-checks/model-io-checks.ts
|
|
@@ -1056,7 +1125,7 @@ var ModelIOChecks = class {
|
|
|
1056
1125
|
};
|
|
1057
1126
|
|
|
1058
1127
|
// src/model-io-checks/utils/create-input-handler.ts
|
|
1059
|
-
import
|
|
1128
|
+
import path12 from "path";
|
|
1060
1129
|
function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
|
|
1061
1130
|
return (req, res) => {
|
|
1062
1131
|
if (!modelIOChecks.isValidInput(req.params.filename)) {
|
|
@@ -1065,7 +1134,7 @@ function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
|
|
|
1065
1134
|
});
|
|
1066
1135
|
}
|
|
1067
1136
|
modelIOChecks.trackInputHandled(req.params.filename);
|
|
1068
|
-
const filePath =
|
|
1137
|
+
const filePath = path12.join(ctx.projectRoot, inputFilesPath, req.params.filename);
|
|
1069
1138
|
return res.sendFile(filePath);
|
|
1070
1139
|
};
|
|
1071
1140
|
}
|
|
@@ -1073,15 +1142,15 @@ __name(createInputHandler, "createInputHandler");
|
|
|
1073
1142
|
|
|
1074
1143
|
// src/model-io-checks/utils/create-output-handler.ts
|
|
1075
1144
|
import fs9 from "fs/promises";
|
|
1076
|
-
import
|
|
1145
|
+
import path13 from "path";
|
|
1077
1146
|
function createOutputHandler(modelIOChecks, ctx) {
|
|
1078
1147
|
return async (req, res) => {
|
|
1079
1148
|
modelIOChecks.trackOutputHandled();
|
|
1080
|
-
const outputPath =
|
|
1149
|
+
const outputPath = path13.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
|
|
1081
1150
|
await fs9.mkdir(outputPath, {
|
|
1082
1151
|
recursive: true
|
|
1083
1152
|
});
|
|
1084
|
-
const outputFilePath =
|
|
1153
|
+
const outputFilePath = path13.join(outputPath, TEST_OUTPUT_FILENAME);
|
|
1085
1154
|
const outputData = JSON.stringify(req.body, null, 2);
|
|
1086
1155
|
await fs9.writeFile(outputFilePath, outputData, "utf8");
|
|
1087
1156
|
return res.status(200).json({
|
|
@@ -1098,7 +1167,7 @@ function setupModelIOChecks(app, inputDir, inputFiles, ctx) {
|
|
|
1098
1167
|
});
|
|
1099
1168
|
const handleInput = createInputHandler(inputDir, modelIOChecks, ctx);
|
|
1100
1169
|
const handleOutput = createOutputHandler(modelIOChecks, ctx);
|
|
1101
|
-
const inputPath =
|
|
1170
|
+
const inputPath = path14.join(ctx.projectRoot, inputDir);
|
|
1102
1171
|
app.use(express.json());
|
|
1103
1172
|
app.use("/data", express.static(inputPath));
|
|
1104
1173
|
app.get("/input/:filename", handleInput);
|
|
@@ -1107,19 +1176,6 @@ function setupModelIOChecks(app, inputDir, inputFiles, ctx) {
|
|
|
1107
1176
|
}
|
|
1108
1177
|
__name(setupModelIOChecks, "setupModelIOChecks");
|
|
1109
1178
|
|
|
1110
|
-
// src/utils/check-input-files.ts
|
|
1111
|
-
import path13 from "path";
|
|
1112
|
-
async function checkInputFiles(inputFiles, inputDataPath, ctx) {
|
|
1113
|
-
for (const inputFile of inputFiles) {
|
|
1114
|
-
const filePath = path13.join(ctx.projectRoot, inputDataPath, inputFile);
|
|
1115
|
-
const exists = await isExistingPath(filePath);
|
|
1116
|
-
if (!exists) {
|
|
1117
|
-
throw new Error(`Input data not found (${inputFile}) - did you need to run \`pd4castr fetch\`?`);
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
}
|
|
1121
|
-
__name(checkInputFiles, "checkInputFiles");
|
|
1122
|
-
|
|
1123
1179
|
// src/utils/get-input-files.ts
|
|
1124
1180
|
function getInputFiles(config) {
|
|
1125
1181
|
const inputFiles = config.inputs.map((input2) => getInputFilename(input2));
|
|
@@ -1130,7 +1186,6 @@ __name(getInputFiles, "getInputFiles");
|
|
|
1130
1186
|
// src/commands/publish/utils/run-model-io-tests.ts
|
|
1131
1187
|
async function runModelIOTests(dockerImage, options, app, ctx) {
|
|
1132
1188
|
const inputFiles = getInputFiles(ctx.config);
|
|
1133
|
-
await checkInputFiles(inputFiles, options.inputDir, ctx);
|
|
1134
1189
|
await buildDockerImage(dockerImage, ctx);
|
|
1135
1190
|
const modelIOChecks = setupModelIOChecks(app, options.inputDir, inputFiles, ctx);
|
|
1136
1191
|
await runModelContainer(dockerImage, options.port, ctx);
|
|
@@ -1140,6 +1195,141 @@ async function runModelIOTests(dockerImage, options, app, ctx) {
|
|
|
1140
1195
|
}
|
|
1141
1196
|
__name(runModelIOTests, "runModelIOTests");
|
|
1142
1197
|
|
|
1198
|
+
// src/commands/publish/utils/upload-static-inputs.ts
|
|
1199
|
+
import fs10 from "fs";
|
|
1200
|
+
import path15 from "path";
|
|
1201
|
+
import ky3, { isHTTPError } from "ky";
|
|
1202
|
+
import promiseRetry from "promise-retry";
|
|
1203
|
+
|
|
1204
|
+
// src/api/get-model-input-upload-url.ts
|
|
1205
|
+
async function getModelInputUploadURL(modelId, modelInputId, authCtx) {
|
|
1206
|
+
const headers = {
|
|
1207
|
+
Authorization: `Bearer ${authCtx.accessToken}`
|
|
1208
|
+
};
|
|
1209
|
+
const result = await api.post(`model/${modelId}/input/${modelInputId}/get-upload-url`, {
|
|
1210
|
+
headers
|
|
1211
|
+
}).json();
|
|
1212
|
+
return result.signedUploadURL;
|
|
1213
|
+
}
|
|
1214
|
+
__name(getModelInputUploadURL, "getModelInputUploadURL");
|
|
1215
|
+
|
|
1216
|
+
// src/commands/publish/utils/upload-static-inputs.ts
|
|
1217
|
+
async function uploadStaticInputFiles(model, options, ctx, authCtx) {
|
|
1218
|
+
const staticInputs = ctx.config.inputs.filter((input2) => isStaticInput(input2));
|
|
1219
|
+
if (staticInputs.length === 0) {
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
if (!model.inputs) {
|
|
1223
|
+
throw new Error("Model inputs were not found in the published model");
|
|
1224
|
+
}
|
|
1225
|
+
const modelInputsByKey = new Map(model.inputs.map((input2) => [
|
|
1226
|
+
input2.key,
|
|
1227
|
+
input2.id
|
|
1228
|
+
]));
|
|
1229
|
+
for (const input2 of staticInputs) {
|
|
1230
|
+
const modelInputId = modelInputsByKey.get(input2.key);
|
|
1231
|
+
if (!modelInputId) {
|
|
1232
|
+
throw new Error(`Model input not found for key (${input2.key})`);
|
|
1233
|
+
}
|
|
1234
|
+
const filename = `${input2.key}.${input2.uploadFileFormat}`;
|
|
1235
|
+
const filePath = path15.join(ctx.projectRoot, options.inputDir, filename);
|
|
1236
|
+
const exists = await isExistingPath(filePath);
|
|
1237
|
+
if (!exists) {
|
|
1238
|
+
throw new Error(`Static input data not found (${filename}) in --input-dir ${options.inputDir}`);
|
|
1239
|
+
}
|
|
1240
|
+
const uploadURL = await getModelInputUploadURL(model.id, modelInputId, authCtx);
|
|
1241
|
+
try {
|
|
1242
|
+
await uploadWithRetry(filePath, uploadURL);
|
|
1243
|
+
} catch (error) {
|
|
1244
|
+
throw new Error(`Failed to upload static input (${input2.key})
|
|
1245
|
+
`, {
|
|
1246
|
+
cause: getErrorMessage(error)
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
__name(uploadStaticInputFiles, "uploadStaticInputFiles");
|
|
1252
|
+
function getErrorMessage(error) {
|
|
1253
|
+
if (error instanceof Error) {
|
|
1254
|
+
return error.message;
|
|
1255
|
+
}
|
|
1256
|
+
return "Unknown error";
|
|
1257
|
+
}
|
|
1258
|
+
__name(getErrorMessage, "getErrorMessage");
|
|
1259
|
+
var RETRY_LIMIT = 3;
|
|
1260
|
+
var RETRY_DELAY_MS = 250;
|
|
1261
|
+
async function uploadWithRetry(filePath, signedUploadURL) {
|
|
1262
|
+
const upload = /* @__PURE__ */ __name(async (retry) => {
|
|
1263
|
+
const { size } = fs10.statSync(filePath);
|
|
1264
|
+
const body = fs10.createReadStream(filePath);
|
|
1265
|
+
try {
|
|
1266
|
+
const response = await ky3.put(signedUploadURL, {
|
|
1267
|
+
body,
|
|
1268
|
+
// streaming request bodies require duplex for node fetch
|
|
1269
|
+
duplex: "half",
|
|
1270
|
+
// we handle retries ourselves to ensure our request body stream is rerecreated per attempt
|
|
1271
|
+
retry: {
|
|
1272
|
+
limit: 0
|
|
1273
|
+
},
|
|
1274
|
+
headers: {
|
|
1275
|
+
"Content-Length": size.toString()
|
|
1276
|
+
}
|
|
1277
|
+
});
|
|
1278
|
+
await safeConsumeResponseBody(response);
|
|
1279
|
+
} catch (error) {
|
|
1280
|
+
body.destroy();
|
|
1281
|
+
if (isHTTPError(error) && isRetriableStatus(error.response.status)) {
|
|
1282
|
+
await safeConsumeResponseBody(error.response);
|
|
1283
|
+
return retry(error);
|
|
1284
|
+
}
|
|
1285
|
+
if (isFetchError(error) && isTransientFetchError(error)) {
|
|
1286
|
+
return retry(error);
|
|
1287
|
+
}
|
|
1288
|
+
throw error;
|
|
1289
|
+
}
|
|
1290
|
+
}, "upload");
|
|
1291
|
+
return promiseRetry(upload, {
|
|
1292
|
+
retries: RETRY_LIMIT,
|
|
1293
|
+
minTimeout: RETRY_DELAY_MS,
|
|
1294
|
+
factor: 2
|
|
1295
|
+
});
|
|
1296
|
+
}
|
|
1297
|
+
__name(uploadWithRetry, "uploadWithRetry");
|
|
1298
|
+
async function safeConsumeResponseBody(response) {
|
|
1299
|
+
try {
|
|
1300
|
+
await response.arrayBuffer();
|
|
1301
|
+
} catch {
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
__name(safeConsumeResponseBody, "safeConsumeResponseBody");
|
|
1305
|
+
function isRetriableStatus(status) {
|
|
1306
|
+
const isTimeout = status === 408;
|
|
1307
|
+
const isServerError = status >= 500 && status <= 599;
|
|
1308
|
+
return isTimeout || isServerError;
|
|
1309
|
+
}
|
|
1310
|
+
__name(isRetriableStatus, "isRetriableStatus");
|
|
1311
|
+
function isFetchError(error) {
|
|
1312
|
+
if (!(error instanceof Error)) {
|
|
1313
|
+
return false;
|
|
1314
|
+
}
|
|
1315
|
+
if (!("code" in error)) {
|
|
1316
|
+
return false;
|
|
1317
|
+
}
|
|
1318
|
+
return typeof error.code === "string";
|
|
1319
|
+
}
|
|
1320
|
+
__name(isFetchError, "isFetchError");
|
|
1321
|
+
var TRANSIENT_FETCH_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
1322
|
+
"ECONNRESET",
|
|
1323
|
+
"ETIMEDOUT",
|
|
1324
|
+
"EPIPE",
|
|
1325
|
+
"ENOTFOUND",
|
|
1326
|
+
"EAI_AGAIN"
|
|
1327
|
+
]);
|
|
1328
|
+
function isTransientFetchError(error) {
|
|
1329
|
+
return TRANSIENT_FETCH_ERROR_CODES.has(error.code);
|
|
1330
|
+
}
|
|
1331
|
+
__name(isTransientFetchError, "isTransientFetchError");
|
|
1332
|
+
|
|
1143
1333
|
// src/commands/publish/handle-create-model-flow.ts
|
|
1144
1334
|
async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
|
|
1145
1335
|
spinner.info(`You are publishing a ${chalk3.bold("new")} model:
|
|
@@ -1156,6 +1346,8 @@ async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
|
|
|
1156
1346
|
const dockerImage = getDockerImage(ctx);
|
|
1157
1347
|
if (!options.skipChecks) {
|
|
1158
1348
|
spinner.start("Performing model I/O test...");
|
|
1349
|
+
await validateStaticInputFiles(options, ctx);
|
|
1350
|
+
await validateDataFetcherInputFiles(options, ctx);
|
|
1159
1351
|
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1160
1352
|
spinner.succeed("Model I/O test passed");
|
|
1161
1353
|
}
|
|
@@ -1175,7 +1367,13 @@ async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
|
|
|
1175
1367
|
await loginToDockerRegistry(pushCredentials);
|
|
1176
1368
|
await buildDockerImage(dockerImage, ctx);
|
|
1177
1369
|
await pushDockerImage(dockerImage, model.dockerImage);
|
|
1178
|
-
spinner.succeed("Model image pushed to
|
|
1370
|
+
spinner.succeed("Model image pushed to registry successfully");
|
|
1371
|
+
const isStaticInputUploadRequired = ctx.config.inputs.some((input2) => isStaticInput(input2));
|
|
1372
|
+
if (isStaticInputUploadRequired) {
|
|
1373
|
+
spinner.start("Uploading static input files...");
|
|
1374
|
+
await uploadStaticInputFiles(model, options, ctx, authCtx);
|
|
1375
|
+
spinner.succeed("Static input files uploaded successfully");
|
|
1376
|
+
}
|
|
1179
1377
|
spinner.start("Triggering model run...");
|
|
1180
1378
|
let modelRunTriggered = false;
|
|
1181
1379
|
if (!options.skipTrigger) {
|
|
@@ -1211,11 +1409,11 @@ import chalk4 from "chalk";
|
|
|
1211
1409
|
import invariant2 from "tiny-invariant";
|
|
1212
1410
|
|
|
1213
1411
|
// src/api/get-model.ts
|
|
1214
|
-
async function getModel(
|
|
1412
|
+
async function getModel(modelId, authCtx) {
|
|
1215
1413
|
const headers = {
|
|
1216
1414
|
Authorization: `Bearer ${authCtx.accessToken}`
|
|
1217
1415
|
};
|
|
1218
|
-
const result = await api.get(`model/${
|
|
1416
|
+
const result = await api.get(`model/${modelId}`, {
|
|
1219
1417
|
headers
|
|
1220
1418
|
}).json();
|
|
1221
1419
|
return result;
|
|
@@ -1256,6 +1454,8 @@ async function handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx
|
|
|
1256
1454
|
const dockerImage = getDockerImage(ctx);
|
|
1257
1455
|
if (!options.skipChecks) {
|
|
1258
1456
|
spinner.start("Performing model I/O test...");
|
|
1457
|
+
await validateStaticInputFiles(options, ctx);
|
|
1458
|
+
await validateDataFetcherInputFiles(options, ctx);
|
|
1259
1459
|
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1260
1460
|
spinner.succeed("Model I/O test passed");
|
|
1261
1461
|
}
|
|
@@ -1275,6 +1475,12 @@ async function handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx
|
|
|
1275
1475
|
await buildDockerImage(dockerImage, ctx);
|
|
1276
1476
|
await pushDockerImage(dockerImage, model.dockerImage);
|
|
1277
1477
|
spinner.succeed("New model revision image pushed to registry successfully");
|
|
1478
|
+
const isStaticInputUploadRequired = ctx.config.inputs.some((input2) => isStaticInput(input2));
|
|
1479
|
+
if (isStaticInputUploadRequired) {
|
|
1480
|
+
spinner.start("Uploading static input files...");
|
|
1481
|
+
await uploadStaticInputFiles(model, options, ctx, authCtx);
|
|
1482
|
+
spinner.succeed("Static input files uploaded successfully");
|
|
1483
|
+
}
|
|
1278
1484
|
spinner.start("Triggering model run...");
|
|
1279
1485
|
let modelRunTriggered = false;
|
|
1280
1486
|
if (!options.skipTrigger) {
|
|
@@ -1336,6 +1542,8 @@ async function handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx
|
|
|
1336
1542
|
const dockerImage = getDockerImage(ctx);
|
|
1337
1543
|
if (!options.skipChecks) {
|
|
1338
1544
|
spinner.start("Performing model I/O test...");
|
|
1545
|
+
await validateStaticInputFiles(options, ctx);
|
|
1546
|
+
await validateDataFetcherInputFiles(options, ctx);
|
|
1339
1547
|
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1340
1548
|
spinner.succeed("Model I/O test passed");
|
|
1341
1549
|
}
|
|
@@ -1356,6 +1564,12 @@ async function handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx
|
|
|
1356
1564
|
await buildDockerImage(dockerImage, ctx);
|
|
1357
1565
|
await pushDockerImage(dockerImage, model.dockerImage);
|
|
1358
1566
|
spinner.succeed("Updated model image pushed to registry successfully");
|
|
1567
|
+
const isStaticInputUploadRequired = ctx.config.inputs.some((input2) => isStaticInput(input2));
|
|
1568
|
+
if (isStaticInputUploadRequired) {
|
|
1569
|
+
spinner.start("Uploading static input files...");
|
|
1570
|
+
await uploadStaticInputFiles(model, options, ctx, authCtx);
|
|
1571
|
+
spinner.succeed("Static input files uploaded successfully");
|
|
1572
|
+
}
|
|
1359
1573
|
spinner.start("Triggering model run...");
|
|
1360
1574
|
let modelRunTriggered = false;
|
|
1361
1575
|
if (!options.skipTrigger) {
|
|
@@ -1533,7 +1747,7 @@ function registerPublishCommand(program2) {
|
|
|
1533
1747
|
__name(registerPublishCommand, "registerPublishCommand");
|
|
1534
1748
|
|
|
1535
1749
|
// src/commands/test/handle-action.ts
|
|
1536
|
-
import
|
|
1750
|
+
import path16 from "path";
|
|
1537
1751
|
import { ExecaError as ExecaError6 } from "execa";
|
|
1538
1752
|
import express3 from "express";
|
|
1539
1753
|
import ora6 from "ora";
|
|
@@ -1628,7 +1842,8 @@ async function handleAction6(options) {
|
|
|
1628
1842
|
const ctx = await loadProjectContext(options.config);
|
|
1629
1843
|
const inputFiles = getInputFiles(ctx.config);
|
|
1630
1844
|
spinner.start("Checking test input data files");
|
|
1631
|
-
await
|
|
1845
|
+
await validateDataFetcherInputFiles(options, ctx);
|
|
1846
|
+
await validateStaticInputFiles(options, ctx);
|
|
1632
1847
|
spinner.succeed(`Found ${inputFiles.length} test input data files`);
|
|
1633
1848
|
spinner.start("Building docker image");
|
|
1634
1849
|
const dockerImage = getDockerImage(ctx);
|
|
@@ -1650,7 +1865,7 @@ async function handleAction6(options) {
|
|
|
1650
1865
|
throw new Error("Model I/O test failed");
|
|
1651
1866
|
}
|
|
1652
1867
|
if (modelIOChecks.isOutputHandled()) {
|
|
1653
|
-
const outputPath =
|
|
1868
|
+
const outputPath = path16.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR, TEST_OUTPUT_FILENAME);
|
|
1654
1869
|
const clickHereLink = createLink("Click here", `file://${outputPath}`);
|
|
1655
1870
|
const fileLink = createLink(TEST_OUTPUT_FILENAME, `file://${outputPath}`);
|
|
1656
1871
|
console.log(`
|
|
@@ -1692,9 +1907,9 @@ import { Command } from "commander";
|
|
|
1692
1907
|
// package.json
|
|
1693
1908
|
var package_default = {
|
|
1694
1909
|
name: "@pd4castr/cli",
|
|
1695
|
-
version: "1.
|
|
1910
|
+
version: "1.10.0",
|
|
1696
1911
|
description: "CLI tool for creating, testing, and publishing pd4castr models",
|
|
1697
|
-
license: "
|
|
1912
|
+
license: "MIT",
|
|
1698
1913
|
main: "dist/index.js",
|
|
1699
1914
|
private: false,
|
|
1700
1915
|
type: "module",
|
|
@@ -1702,7 +1917,9 @@ var package_default = {
|
|
|
1702
1917
|
pd4castr: "dist/index.js"
|
|
1703
1918
|
},
|
|
1704
1919
|
files: [
|
|
1705
|
-
"dist/**/*"
|
|
1920
|
+
"dist/**/*",
|
|
1921
|
+
"LICENSE.md",
|
|
1922
|
+
"docs/assets/logo.png"
|
|
1706
1923
|
],
|
|
1707
1924
|
engines: {
|
|
1708
1925
|
node: ">=20.0.0"
|
|
@@ -1711,7 +1928,6 @@ var package_default = {
|
|
|
1711
1928
|
build: "tsup",
|
|
1712
1929
|
dev: "tsup --watch",
|
|
1713
1930
|
pd4castr: "node dist/index.js",
|
|
1714
|
-
release: "semantic-release -e semantic-release-monorepo",
|
|
1715
1931
|
format: "prettier --write .",
|
|
1716
1932
|
lint: "eslint .",
|
|
1717
1933
|
typecheck: "tsc --noEmit",
|
|
@@ -1725,8 +1941,9 @@ var package_default = {
|
|
|
1725
1941
|
execa: "9.6.0",
|
|
1726
1942
|
express: "4.21.2",
|
|
1727
1943
|
immer: "10.1.1",
|
|
1728
|
-
ky: "1.
|
|
1944
|
+
ky: "1.14.2",
|
|
1729
1945
|
ora: "8.2.0",
|
|
1946
|
+
"promise-retry": "2.0.1",
|
|
1730
1947
|
slugify: "1.6.6",
|
|
1731
1948
|
tiged: "2.12.7",
|
|
1732
1949
|
"tiny-invariant": "1.3.3",
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pd4castr/cli",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"description": "CLI tool for creating, testing, and publishing pd4castr models",
|
|
5
|
-
"license": "
|
|
5
|
+
"license": "MIT",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"private": false,
|
|
8
8
|
"type": "module",
|
|
@@ -10,7 +10,9 @@
|
|
|
10
10
|
"pd4castr": "dist/index.js"
|
|
11
11
|
},
|
|
12
12
|
"files": [
|
|
13
|
-
"dist/**/*"
|
|
13
|
+
"dist/**/*",
|
|
14
|
+
"LICENSE.md",
|
|
15
|
+
"docs/assets/logo.png"
|
|
14
16
|
],
|
|
15
17
|
"engines": {
|
|
16
18
|
"node": ">=20.0.0"
|
|
@@ -19,7 +21,6 @@
|
|
|
19
21
|
"build": "tsup",
|
|
20
22
|
"dev": "tsup --watch",
|
|
21
23
|
"pd4castr": "node dist/index.js",
|
|
22
|
-
"release": "semantic-release -e semantic-release-monorepo",
|
|
23
24
|
"format": "prettier --write .",
|
|
24
25
|
"lint": "eslint .",
|
|
25
26
|
"typecheck": "tsc --noEmit",
|
|
@@ -33,8 +34,9 @@
|
|
|
33
34
|
"execa": "9.6.0",
|
|
34
35
|
"express": "4.21.2",
|
|
35
36
|
"immer": "10.1.1",
|
|
36
|
-
"ky": "1.
|
|
37
|
+
"ky": "1.14.2",
|
|
37
38
|
"ora": "8.2.0",
|
|
39
|
+
"promise-retry": "2.0.1",
|
|
38
40
|
"slugify": "1.6.6",
|
|
39
41
|
"tiged": "2.12.7",
|
|
40
42
|
"tiny-invariant": "1.3.3",
|