@pd4castr/cli 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +756 -190
- package/package.json +9 -3
package/dist/index.js
CHANGED
|
@@ -51,22 +51,30 @@ var AUTH0_CLIENT_ID = "Q5tQNF57cQlVXnVsqnU0hhgy92rVb03W";
|
|
|
51
51
|
var AUTH0_AUDIENCE = "https://api.pd4castr.com.au";
|
|
52
52
|
var GLOBAL_CONFIG_FILE = ".pd4castr";
|
|
53
53
|
var PROJECT_CONFIG_FILE = ".pd4castrrc.json";
|
|
54
|
-
var
|
|
54
|
+
var DEFAULT_API_URL = "https://pd4castr-api.pipelabs.app";
|
|
55
|
+
var DEFAULT_INPUT_SOURCE_ID = "0bdfd52b-efaa-455e-9a3b-1a6d2b879b73";
|
|
55
56
|
var TEST_INPUT_DATA_DIR = "test_input";
|
|
56
57
|
var TEST_OUTPUT_DATA_DIR = "test_output";
|
|
58
|
+
var TEST_WEBSERVER_PORT = 9800;
|
|
57
59
|
var TEST_OUTPUT_FILENAME = `output-${Date.now()}.json`;
|
|
60
|
+
var DOCKER_HOSTNAME_DEFAULT = "host.docker.internal";
|
|
61
|
+
var DOCKER_HOSTNAME_WSL = "wsl.host";
|
|
62
|
+
var WSL_NETWORK_INTERFACE_DEFAULT = "eth0";
|
|
58
63
|
|
|
59
64
|
// src/commands/fetch/handle-action.ts
|
|
60
65
|
import path5 from "path";
|
|
66
|
+
import { HTTPError } from "ky";
|
|
61
67
|
import ora from "ora";
|
|
62
|
-
import { ZodError } from "zod";
|
|
68
|
+
import { ZodError as ZodError2 } from "zod";
|
|
63
69
|
|
|
64
|
-
// src/config/load-project-
|
|
70
|
+
// src/config/load-project-context.ts
|
|
71
|
+
import fs2 from "fs/promises";
|
|
65
72
|
import path from "path";
|
|
66
|
-
import {
|
|
73
|
+
import { ZodError } from "zod";
|
|
67
74
|
|
|
68
75
|
// src/schemas/project-config-schema.ts
|
|
69
76
|
import { z } from "zod";
|
|
77
|
+
var fileFormatSchema = z.enum(["csv", "json", "parquet"]);
|
|
70
78
|
var aemoDataFetcherSchema = z.object({
|
|
71
79
|
type: z.literal("AEMO_MMS"),
|
|
72
80
|
checkInterval: z.number().int().min(60),
|
|
@@ -78,49 +86,70 @@ var aemoDataFetcherSchema = z.object({
|
|
|
78
86
|
var dataFetcherSchema = z.discriminatedUnion("type", [aemoDataFetcherSchema]);
|
|
79
87
|
var modelInputSchema = z.object({
|
|
80
88
|
key: z.string(),
|
|
81
|
-
inputSource: z.string().optional(),
|
|
89
|
+
inputSource: z.string().optional().default(DEFAULT_INPUT_SOURCE_ID),
|
|
82
90
|
trigger: z.enum(["WAIT_FOR_LATEST_FILE", "USE_MOST_RECENT_FILE"]),
|
|
83
|
-
|
|
91
|
+
uploadFileFormat: fileFormatSchema.optional().default("json"),
|
|
92
|
+
targetFileFormat: fileFormatSchema.optional().default("json"),
|
|
93
|
+
fetcher: dataFetcherSchema.optional().nullable()
|
|
84
94
|
});
|
|
85
95
|
var modelOutputSchema = z.object({
|
|
86
96
|
name: z.string(),
|
|
97
|
+
type: z.enum(["float", "integer", "string", "date", "boolean", "unknown"]),
|
|
87
98
|
seriesKey: z.boolean(),
|
|
88
99
|
colour: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional()
|
|
89
100
|
});
|
|
101
|
+
var CONFIG_WARNING_KEY = "// WARNING: DO NOT MODIFY THESE SYSTEM MANAGED VALUES";
|
|
90
102
|
var projectConfigSchema = z.object({
|
|
91
|
-
$$id: z.string().nullable(),
|
|
92
|
-
$$modelGroupID: z.string().nullable(),
|
|
93
|
-
$$revision: z.number().int().min(0).default(0),
|
|
94
|
-
$$dockerImage: z.string().nullable(),
|
|
95
103
|
name: z.string(),
|
|
104
|
+
forecastVariable: z.enum(["price"]),
|
|
105
|
+
timeHorizon: z.enum(["actual", "day_ahead", "week_ahead", "quarterly"]),
|
|
96
106
|
metadata: z.record(z.string(), z.any()).optional(),
|
|
97
|
-
forecastVariable: z.enum(["PRICE"]),
|
|
98
|
-
timeHorizon: z.enum(["ACTUAL", "DAY_AHEAD", "WEEK_AHEAD"]),
|
|
99
107
|
inputs: z.array(modelInputSchema),
|
|
100
|
-
outputs: z.array(modelOutputSchema)
|
|
108
|
+
outputs: z.array(modelOutputSchema),
|
|
109
|
+
[CONFIG_WARNING_KEY]: z.string().optional().default(""),
|
|
110
|
+
$$id: z.string().nullable().optional().default(null),
|
|
111
|
+
$$modelGroupID: z.string().nullable().optional().default(null),
|
|
112
|
+
$$revision: z.number().int().min(0).nullable().optional().default(null),
|
|
113
|
+
$$dockerImage: z.string().nullable().optional().default(null)
|
|
101
114
|
});
|
|
102
115
|
|
|
103
|
-
// src/
|
|
104
|
-
|
|
105
|
-
|
|
116
|
+
// src/utils/is-existing-path.ts
|
|
117
|
+
import fs from "fs/promises";
|
|
118
|
+
async function isExistingPath(path15) {
|
|
119
|
+
try {
|
|
120
|
+
await fs.access(path15);
|
|
121
|
+
return true;
|
|
122
|
+
} catch {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
106
125
|
}
|
|
107
126
|
|
|
108
|
-
// src/config/load-project-
|
|
109
|
-
async function
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
if (!
|
|
127
|
+
// src/config/load-project-context.ts
|
|
128
|
+
async function loadProjectContext() {
|
|
129
|
+
const projectRoot = process.cwd();
|
|
130
|
+
const configPath = path.join(projectRoot, PROJECT_CONFIG_FILE);
|
|
131
|
+
const configExists = await isExistingPath(configPath);
|
|
132
|
+
if (!configExists) {
|
|
114
133
|
throw new Error(
|
|
115
134
|
"No config found (docs: https://github.com/pipelabs/pd4castr-model-examples/blob/main/docs/005-config.md)."
|
|
116
135
|
);
|
|
117
136
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
config
|
|
122
|
-
|
|
123
|
-
|
|
137
|
+
try {
|
|
138
|
+
const configFileContents = await fs2.readFile(configPath, "utf8");
|
|
139
|
+
const rawConfig = JSON.parse(configFileContents);
|
|
140
|
+
const config = projectConfigSchema.parse(rawConfig);
|
|
141
|
+
return {
|
|
142
|
+
config,
|
|
143
|
+
projectRoot
|
|
144
|
+
};
|
|
145
|
+
} catch (error) {
|
|
146
|
+
if (error instanceof ZodError) {
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
throw new Error(
|
|
150
|
+
"Failed to parse project config (docs: https://github.com/pipelabs/pd4castr-model-examples/blob/main/docs/005-config.md)."
|
|
151
|
+
);
|
|
152
|
+
}
|
|
124
153
|
}
|
|
125
154
|
|
|
126
155
|
// src/utils/create-link.ts
|
|
@@ -133,54 +162,54 @@ function createLink(text, url) {
|
|
|
133
162
|
return `${start}${text}${end}`;
|
|
134
163
|
}
|
|
135
164
|
|
|
165
|
+
// src/utils/format-nest-error-message.ts
|
|
166
|
+
function formatNestErrorMessage(error) {
|
|
167
|
+
return `[${error.error?.toUpperCase() ?? "UNKNOWN"}] ${error.message}`;
|
|
168
|
+
}
|
|
169
|
+
|
|
136
170
|
// src/utils/get-auth.ts
|
|
137
171
|
import invariant from "tiny-invariant";
|
|
138
172
|
|
|
139
|
-
// src/commands/login/utils/is-authed.ts
|
|
140
|
-
function isAuthed(config) {
|
|
141
|
-
const isTokenExpired = config.accessTokenExpiresAt && config.accessTokenExpiresAt <= Date.now();
|
|
142
|
-
return Boolean(config.accessToken) && !isTokenExpired;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
173
|
// src/config/load-global-config.ts
|
|
146
|
-
import
|
|
174
|
+
import fs3 from "fs/promises";
|
|
147
175
|
import os from "os";
|
|
148
176
|
import path2 from "path";
|
|
149
177
|
|
|
150
178
|
// src/schemas/global-config-schema.ts
|
|
151
179
|
import { z as z2 } from "zod";
|
|
152
180
|
var globalConfigSchema = z2.object({
|
|
153
|
-
accessToken: z2.string().
|
|
154
|
-
accessTokenExpiresAt: z2.number().
|
|
181
|
+
accessToken: z2.string().nullable(),
|
|
182
|
+
accessTokenExpiresAt: z2.number().nullable()
|
|
155
183
|
});
|
|
156
184
|
|
|
157
|
-
// src/utils/is-existing-path.ts
|
|
158
|
-
import fs from "fs/promises";
|
|
159
|
-
async function isExistingPath(path12) {
|
|
160
|
-
try {
|
|
161
|
-
await fs.access(path12);
|
|
162
|
-
return true;
|
|
163
|
-
} catch {
|
|
164
|
-
return false;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
185
|
// src/config/load-global-config.ts
|
|
169
186
|
async function loadGlobalConfig() {
|
|
170
187
|
const configPath = path2.join(os.homedir(), GLOBAL_CONFIG_FILE);
|
|
171
188
|
const configExists = await isExistingPath(configPath);
|
|
172
189
|
if (!configExists) {
|
|
173
|
-
await
|
|
174
|
-
return
|
|
190
|
+
await fs3.writeFile(configPath, JSON.stringify({}));
|
|
191
|
+
return getDefaultConfig();
|
|
175
192
|
}
|
|
176
193
|
try {
|
|
177
|
-
const configFileContents = await
|
|
194
|
+
const configFileContents = await fs3.readFile(configPath, "utf8");
|
|
178
195
|
const config = JSON.parse(configFileContents);
|
|
179
196
|
return globalConfigSchema.parse(config);
|
|
180
197
|
} catch {
|
|
181
|
-
return
|
|
198
|
+
return getDefaultConfig();
|
|
182
199
|
}
|
|
183
200
|
}
|
|
201
|
+
function getDefaultConfig() {
|
|
202
|
+
return {
|
|
203
|
+
accessToken: null,
|
|
204
|
+
accessTokenExpiresAt: null
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// src/utils/is-authed.ts
|
|
209
|
+
function isAuthed(config) {
|
|
210
|
+
const isTokenExpired = config.accessTokenExpiresAt && config.accessTokenExpiresAt <= Date.now();
|
|
211
|
+
return Boolean(config.accessToken) && !isTokenExpired;
|
|
212
|
+
}
|
|
184
213
|
|
|
185
214
|
// src/utils/get-auth.ts
|
|
186
215
|
async function getAuth() {
|
|
@@ -204,25 +233,49 @@ function logZodIssues(error) {
|
|
|
204
233
|
}
|
|
205
234
|
|
|
206
235
|
// src/commands/fetch/utils/fetch-aemo-data.ts
|
|
207
|
-
import
|
|
236
|
+
import fs4 from "fs/promises";
|
|
208
237
|
import path3 from "path";
|
|
209
238
|
|
|
210
|
-
// src/api.ts
|
|
239
|
+
// src/api/api.ts
|
|
211
240
|
import ky from "ky";
|
|
241
|
+
|
|
242
|
+
// src/schemas/env-schema.ts
|
|
243
|
+
import { z as z3 } from "zod";
|
|
244
|
+
var envSchema = z3.object({
|
|
245
|
+
// wsl sets this environment variable on all distros that i've checked
|
|
246
|
+
isWSL: z3.boolean().optional().default(() => Boolean(process.env.WSL_DISTRO_NAME)),
|
|
247
|
+
apiURL: z3.string().optional().default(() => process.env.PD4CASTR_API_URL ?? DEFAULT_API_URL),
|
|
248
|
+
wslNetworkInterface: z3.string().optional().default(
|
|
249
|
+
() => process.env.PD4CASTR_WSL_NETWORK_INTERFACE ?? WSL_NETWORK_INTERFACE_DEFAULT
|
|
250
|
+
)
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
// src/utils/get-env.ts
|
|
254
|
+
function getEnv() {
|
|
255
|
+
return envSchema.parse(process.env);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// src/api/api.ts
|
|
212
259
|
var api = ky.create({
|
|
213
|
-
prefixUrl:
|
|
260
|
+
prefixUrl: getEnv().apiURL
|
|
214
261
|
});
|
|
215
262
|
|
|
263
|
+
// src/api/query-data-fetcher.ts
|
|
264
|
+
async function queryDataFetcher(querySQL, authCtx) {
|
|
265
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
266
|
+
const payload2 = { query: querySQL, type: "AEMO_MMS" };
|
|
267
|
+
const result = await api.post("data-fetcher/query", { json: payload2, headers }).json();
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
|
|
216
271
|
// src/commands/fetch/utils/fetch-aemo-data.ts
|
|
217
272
|
async function fetchAEMOData(dataFetcher, authCtx, ctx) {
|
|
218
273
|
const queryPath = path3.resolve(
|
|
219
274
|
ctx.projectRoot,
|
|
220
275
|
dataFetcher.config.fetchQuery
|
|
221
276
|
);
|
|
222
|
-
const querySQL = await
|
|
223
|
-
const
|
|
224
|
-
const payload2 = { query: querySQL, type: "AEMO_MMS" };
|
|
225
|
-
const result = await api.post("data-fetcher/query", { json: payload2, headers }).json();
|
|
277
|
+
const querySQL = await fs4.readFile(queryPath, "utf8");
|
|
278
|
+
const result = await queryDataFetcher(querySQL, authCtx);
|
|
226
279
|
return result;
|
|
227
280
|
}
|
|
228
281
|
|
|
@@ -239,21 +292,21 @@ function getFetcher(type) {
|
|
|
239
292
|
}
|
|
240
293
|
|
|
241
294
|
// src/commands/fetch/utils/write-test-data.ts
|
|
242
|
-
import
|
|
295
|
+
import fs5 from "fs/promises";
|
|
243
296
|
import path4 from "path";
|
|
244
297
|
|
|
245
298
|
// src/utils/get-input-filename.ts
|
|
246
299
|
function getInputFilename(modelInput) {
|
|
247
|
-
return `${modelInput.key}.
|
|
300
|
+
return `${modelInput.key}.${modelInput.targetFileFormat}`;
|
|
248
301
|
}
|
|
249
302
|
|
|
250
303
|
// src/commands/fetch/utils/write-test-data.ts
|
|
251
304
|
async function writeTestData(inputData, modelInput, inputDataDir, ctx) {
|
|
252
305
|
const inputDir = path4.resolve(ctx.projectRoot, inputDataDir);
|
|
253
|
-
await
|
|
306
|
+
await fs5.mkdir(inputDir, { recursive: true });
|
|
254
307
|
const inputFilename = getInputFilename(modelInput);
|
|
255
308
|
const inputPath = path4.resolve(inputDir, inputFilename);
|
|
256
|
-
await
|
|
309
|
+
await fs5.writeFile(inputPath, JSON.stringify(inputData, void 0, 2));
|
|
257
310
|
}
|
|
258
311
|
|
|
259
312
|
// src/commands/fetch/handle-action.ts
|
|
@@ -262,7 +315,7 @@ async function handleAction(options) {
|
|
|
262
315
|
const spinner = ora("Starting data fetch...").start();
|
|
263
316
|
try {
|
|
264
317
|
const authCtx = await getAuth();
|
|
265
|
-
const ctx = await
|
|
318
|
+
const ctx = await loadProjectContext();
|
|
266
319
|
for (const input2 of ctx.config.inputs) {
|
|
267
320
|
if (!input2.fetcher) {
|
|
268
321
|
spinner.warn(`\`${input2.key}\` - no data fetcher defined, skipping`);
|
|
@@ -286,11 +339,17 @@ async function handleAction(options) {
|
|
|
286
339
|
${link} to view fetched data
|
|
287
340
|
`);
|
|
288
341
|
} catch (error) {
|
|
289
|
-
if (error instanceof
|
|
342
|
+
if (error instanceof ZodError2) {
|
|
290
343
|
spinner.fail("Config validation failed");
|
|
291
344
|
logZodIssues(error);
|
|
345
|
+
} else if (error instanceof HTTPError) {
|
|
346
|
+
const errorResponse = await error.response.json();
|
|
347
|
+
spinner.fail(formatNestErrorMessage(errorResponse));
|
|
292
348
|
} else if (error instanceof Error) {
|
|
293
349
|
spinner.fail(error.message);
|
|
350
|
+
if (error.cause) {
|
|
351
|
+
console.error(error.cause);
|
|
352
|
+
}
|
|
294
353
|
}
|
|
295
354
|
process.exit(1);
|
|
296
355
|
}
|
|
@@ -359,8 +418,8 @@ async function handleAction2() {
|
|
|
359
418
|
spinner.succeed("Template fetched successfully");
|
|
360
419
|
} catch (error) {
|
|
361
420
|
spinner.fail("Error fetching template");
|
|
362
|
-
if (error instanceof Error) {
|
|
363
|
-
console.error(error.
|
|
421
|
+
if (error instanceof Error && error.cause) {
|
|
422
|
+
console.error(error.cause);
|
|
364
423
|
}
|
|
365
424
|
process.exit(1);
|
|
366
425
|
}
|
|
@@ -382,19 +441,22 @@ function registerInitCommand(program2) {
|
|
|
382
441
|
|
|
383
442
|
// src/commands/login/handle-action.ts
|
|
384
443
|
import ora3 from "ora";
|
|
385
|
-
import { ZodError as
|
|
444
|
+
import { ZodError as ZodError3 } from "zod";
|
|
386
445
|
|
|
387
446
|
// src/config/update-global-config.ts
|
|
388
|
-
import
|
|
447
|
+
import fs6 from "fs/promises";
|
|
389
448
|
import os2 from "os";
|
|
390
449
|
import path7 from "path";
|
|
391
|
-
|
|
450
|
+
import { produce } from "immer";
|
|
451
|
+
async function updateGlobalConfig(updateFn) {
|
|
452
|
+
const globalConfig = await loadGlobalConfig();
|
|
453
|
+
const updatedConfig = produce(globalConfig, updateFn);
|
|
392
454
|
const configPath = path7.join(os2.homedir(), GLOBAL_CONFIG_FILE);
|
|
393
|
-
await
|
|
455
|
+
await fs6.writeFile(configPath, JSON.stringify(updatedConfig, void 0, 2));
|
|
394
456
|
}
|
|
395
457
|
|
|
396
458
|
// src/commands/login/utils/complete-auth-flow.ts
|
|
397
|
-
import { HTTPError } from "ky";
|
|
459
|
+
import { HTTPError as HTTPError2 } from "ky";
|
|
398
460
|
|
|
399
461
|
// src/commands/login/auth0-api.ts
|
|
400
462
|
import ky2 from "ky";
|
|
@@ -417,7 +479,7 @@ async function completeAuthFlow(authCtx) {
|
|
|
417
479
|
};
|
|
418
480
|
return authPayload;
|
|
419
481
|
} catch (error) {
|
|
420
|
-
if (!(error instanceof
|
|
482
|
+
if (!(error instanceof HTTPError2)) {
|
|
421
483
|
throw error;
|
|
422
484
|
}
|
|
423
485
|
const errorResponse = await error.response.json();
|
|
@@ -472,19 +534,20 @@ async function handleAction3() {
|
|
|
472
534
|
`);
|
|
473
535
|
spinner.start("Waiting for login to complete...");
|
|
474
536
|
const authPayload = await completeAuthFlow(authCtx);
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
};
|
|
480
|
-
await updateGlobalConfig(updatedGlobalConfig);
|
|
537
|
+
await updateGlobalConfig((config) => {
|
|
538
|
+
config.accessToken = authPayload.accessToken;
|
|
539
|
+
config.accessTokenExpiresAt = authPayload.expiresAt;
|
|
540
|
+
});
|
|
481
541
|
spinner.succeed("Successfully logged in to the pd4castr API");
|
|
482
542
|
} catch (error) {
|
|
483
|
-
if (error instanceof
|
|
543
|
+
if (error instanceof ZodError3) {
|
|
484
544
|
spinner.fail("Config validation failed");
|
|
485
545
|
logZodIssues(error);
|
|
486
546
|
} else if (error instanceof Error) {
|
|
487
547
|
spinner.fail(error.message);
|
|
548
|
+
if (error.cause) {
|
|
549
|
+
console.error(error.cause);
|
|
550
|
+
}
|
|
488
551
|
}
|
|
489
552
|
process.exit(1);
|
|
490
553
|
}
|
|
@@ -495,15 +558,92 @@ function registerLoginCommand(program2) {
|
|
|
495
558
|
program2.command("login").description("Logs in to the pd4castr API.").action(handleAction3);
|
|
496
559
|
}
|
|
497
560
|
|
|
498
|
-
// src/commands/
|
|
499
|
-
import path11 from "path";
|
|
500
|
-
import { ExecaError } from "execa";
|
|
501
|
-
import express from "express";
|
|
561
|
+
// src/commands/logout/handle-action.ts
|
|
502
562
|
import ora4 from "ora";
|
|
503
|
-
import
|
|
504
|
-
|
|
563
|
+
import { ZodError as ZodError4 } from "zod";
|
|
564
|
+
async function handleAction4() {
|
|
565
|
+
const spinner = ora4("Logging out of the pd4castr API...").start();
|
|
566
|
+
try {
|
|
567
|
+
const globalConfig = await loadGlobalConfig();
|
|
568
|
+
if (!isAuthed(globalConfig)) {
|
|
569
|
+
spinner.succeed("Already logged out!");
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
await updateGlobalConfig((config) => {
|
|
573
|
+
config.accessToken = null;
|
|
574
|
+
config.accessTokenExpiresAt = null;
|
|
575
|
+
});
|
|
576
|
+
spinner.succeed("Successfully logged out of the pd4castr API");
|
|
577
|
+
} catch (error) {
|
|
578
|
+
if (error instanceof ZodError4) {
|
|
579
|
+
spinner.fail("Config validation failed");
|
|
580
|
+
logZodIssues(error);
|
|
581
|
+
} else if (error instanceof Error) {
|
|
582
|
+
spinner.fail(error.message);
|
|
583
|
+
if (error.cause) {
|
|
584
|
+
console.error(error.cause);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
process.exit(1);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// src/commands/logout/index.ts
|
|
592
|
+
function registerLogoutCommand(program2) {
|
|
593
|
+
program2.command("logout").description("Logs out of the pd4castr API.").action(handleAction4);
|
|
594
|
+
}
|
|
505
595
|
|
|
506
|
-
// src/commands/
|
|
596
|
+
// src/commands/publish/handle-action.ts
|
|
597
|
+
import express2 from "express";
|
|
598
|
+
import { HTTPError as HTTPError3 } from "ky";
|
|
599
|
+
import ora5 from "ora";
|
|
600
|
+
import { ZodError as ZodError5 } from "zod";
|
|
601
|
+
|
|
602
|
+
// src/utils/start-web-server.ts
|
|
603
|
+
async function startWebServer(app, port) {
|
|
604
|
+
return new Promise((resolve) => {
|
|
605
|
+
const server = app.listen(port, () => {
|
|
606
|
+
resolve({
|
|
607
|
+
server,
|
|
608
|
+
[Symbol.dispose]: () => server.close()
|
|
609
|
+
});
|
|
610
|
+
});
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
// src/commands/publish/handle-create-model-flow.ts
|
|
615
|
+
import * as inquirer2 from "@inquirer/prompts";
|
|
616
|
+
import chalk2 from "chalk";
|
|
617
|
+
|
|
618
|
+
// src/api/create-model.ts
|
|
619
|
+
async function createModel(config, authCtx) {
|
|
620
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
621
|
+
const result = await api.post("model", { headers, json: config }).json();
|
|
622
|
+
return result;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// src/api/get-registry-push-credentials.ts
|
|
626
|
+
async function getRegistryPushCredentials(modelID, authCtx) {
|
|
627
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
628
|
+
const searchParams = new URLSearchParams(`modelId=${modelID}`);
|
|
629
|
+
const result = await api.get("registry/push-credentials", { headers, searchParams }).json();
|
|
630
|
+
return result;
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// src/config/update-project-config.ts
|
|
634
|
+
import fs7 from "fs/promises";
|
|
635
|
+
import path8 from "path";
|
|
636
|
+
import { produce as produce2 } from "immer";
|
|
637
|
+
async function updateProjectConfig(updateFn) {
|
|
638
|
+
const projectConfig = await loadProjectContext();
|
|
639
|
+
const updatedConfig = produce2(projectConfig.config, updateFn);
|
|
640
|
+
await fs7.writeFile(
|
|
641
|
+
path8.join(projectConfig.projectRoot, PROJECT_CONFIG_FILE),
|
|
642
|
+
JSON.stringify(updatedConfig, void 0, 2)
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// src/docker/build-docker-image.ts
|
|
507
647
|
import { execa } from "execa";
|
|
508
648
|
async function buildDockerImage(dockerImage, ctx) {
|
|
509
649
|
try {
|
|
@@ -516,59 +656,185 @@ async function buildDockerImage(dockerImage, ctx) {
|
|
|
516
656
|
}
|
|
517
657
|
}
|
|
518
658
|
|
|
519
|
-
// src/
|
|
520
|
-
import
|
|
521
|
-
async function
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
659
|
+
// src/docker/login-to-docker-registry.ts
|
|
660
|
+
import { execa as execa2 } from "execa";
|
|
661
|
+
async function loginToDockerRegistry(authConfig) {
|
|
662
|
+
try {
|
|
663
|
+
await execa2(
|
|
664
|
+
"docker",
|
|
665
|
+
[
|
|
666
|
+
"login",
|
|
667
|
+
authConfig.registry,
|
|
668
|
+
"--username",
|
|
669
|
+
authConfig.username,
|
|
670
|
+
"--password-stdin"
|
|
671
|
+
],
|
|
672
|
+
{ input: authConfig.password }
|
|
673
|
+
);
|
|
674
|
+
} catch (error) {
|
|
675
|
+
throw new Error("Failed to login to docker registry", { cause: error });
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// src/docker/push-docker-image.ts
|
|
680
|
+
import { execa as execa3 } from "execa";
|
|
681
|
+
async function pushDockerImage(dockerImage, pushRef) {
|
|
682
|
+
try {
|
|
683
|
+
await execa3("docker", ["tag", dockerImage, pushRef]);
|
|
684
|
+
await execa3("docker", ["push", pushRef]);
|
|
685
|
+
} catch (error) {
|
|
686
|
+
throw new Error("Failed to push docker image", { cause: error });
|
|
530
687
|
}
|
|
531
688
|
}
|
|
532
689
|
|
|
533
|
-
// src/
|
|
690
|
+
// src/utils/get-docker-image.ts
|
|
691
|
+
import slugify from "slugify";
|
|
692
|
+
function getDockerImage(ctx) {
|
|
693
|
+
const sluggedName = slugify(ctx.config.name, { lower: true });
|
|
694
|
+
const dockerImage = `pd4castr/${sluggedName}-local:${Date.now()}`;
|
|
695
|
+
return dockerImage;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// src/utils/get-model-config-from-project-config.ts
|
|
699
|
+
import fs8 from "fs/promises";
|
|
534
700
|
import path9 from "path";
|
|
535
|
-
function
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
701
|
+
async function getModelConfigFromProjectConfig(ctx) {
|
|
702
|
+
const inputs = await getInputsWithInlinedSQL(ctx);
|
|
703
|
+
const { $$id, $$modelGroupID, $$revision, $$dockerImage, ...config } = ctx.config;
|
|
704
|
+
return {
|
|
705
|
+
...config,
|
|
706
|
+
id: $$id,
|
|
707
|
+
modelGroupId: $$modelGroupID,
|
|
708
|
+
revision: $$revision ?? 0,
|
|
709
|
+
dockerImage: $$dockerImage,
|
|
710
|
+
inputs
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
var FETCHERS_WITH_SQL = /* @__PURE__ */ new Set(["AEMO_MMS"]);
|
|
714
|
+
async function getInputsWithInlinedSQL(ctx) {
|
|
715
|
+
const inputsWithSQL = [];
|
|
716
|
+
for (const input2 of ctx.config.inputs) {
|
|
717
|
+
if (!input2.fetcher || !FETCHERS_WITH_SQL.has(input2.fetcher.type)) {
|
|
718
|
+
inputsWithSQL.push(input2);
|
|
719
|
+
continue;
|
|
539
720
|
}
|
|
540
|
-
|
|
541
|
-
const filePath = path9.join(
|
|
721
|
+
const fetchQueryPath = path9.resolve(
|
|
542
722
|
ctx.projectRoot,
|
|
543
|
-
|
|
544
|
-
req.params.filename
|
|
723
|
+
input2.fetcher.config.fetchQuery
|
|
545
724
|
);
|
|
546
|
-
|
|
547
|
-
|
|
725
|
+
const checkQueryPath = path9.resolve(
|
|
726
|
+
ctx.projectRoot,
|
|
727
|
+
input2.fetcher.config.checkQuery
|
|
728
|
+
);
|
|
729
|
+
const [fetchQuerySQL, checkQuerySQL] = await Promise.all([
|
|
730
|
+
fs8.readFile(fetchQueryPath, "utf8"),
|
|
731
|
+
fs8.readFile(checkQueryPath, "utf8")
|
|
732
|
+
]);
|
|
733
|
+
const inputWithSQL = {
|
|
734
|
+
...input2,
|
|
735
|
+
fetcher: {
|
|
736
|
+
...input2.fetcher,
|
|
737
|
+
config: {
|
|
738
|
+
...input2.fetcher.config,
|
|
739
|
+
fetchQuery: fetchQuerySQL,
|
|
740
|
+
checkQuery: checkQuerySQL
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
inputsWithSQL.push(inputWithSQL);
|
|
745
|
+
}
|
|
746
|
+
return inputsWithSQL;
|
|
548
747
|
}
|
|
549
748
|
|
|
550
|
-
// src/
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
function createOutputHandler(modelIOChecks, ctx) {
|
|
554
|
-
return async (req, res) => {
|
|
555
|
-
modelIOChecks.trackOutputHandled();
|
|
556
|
-
const outputPath = path10.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
|
|
557
|
-
await fs6.mkdir(outputPath, { recursive: true });
|
|
558
|
-
const outputFilePath = path10.join(outputPath, TEST_OUTPUT_FILENAME);
|
|
559
|
-
const outputData = JSON.stringify(req.body, null, 2);
|
|
560
|
-
await fs6.writeFile(outputFilePath, outputData, "utf8");
|
|
561
|
-
return res.status(200).json({ success: true });
|
|
562
|
-
};
|
|
749
|
+
// src/utils/log-empty-line.ts
|
|
750
|
+
function logEmptyLine() {
|
|
751
|
+
console.log("");
|
|
563
752
|
}
|
|
564
753
|
|
|
565
|
-
// src/commands/
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
return
|
|
754
|
+
// src/commands/publish/utils/get-model-summary-lines.ts
|
|
755
|
+
import chalk from "chalk";
|
|
756
|
+
function getModelSummaryLines(ctx) {
|
|
757
|
+
return [
|
|
758
|
+
` ${chalk.bold("Model name:")} ${ctx.config.name}`,
|
|
759
|
+
` ${chalk.bold("Revision:")} ${ctx.config.$$revision}`,
|
|
760
|
+
` ${chalk.bold("Forecast variable:")} ${ctx.config.forecastVariable}`,
|
|
761
|
+
` ${chalk.bold("Time horizon:")} ${ctx.config.timeHorizon}`,
|
|
762
|
+
` ${chalk.bold("Inputs:")}`,
|
|
763
|
+
...ctx.config.inputs.map(
|
|
764
|
+
(input2) => ` \u2022 ${input2.key} - ${getInputType(input2)}`
|
|
765
|
+
),
|
|
766
|
+
` ${chalk.bold("Outputs:")}`,
|
|
767
|
+
...ctx.config.outputs.map((output) => ` \u2022 ${output.name} - ${output.type}`),
|
|
768
|
+
""
|
|
769
|
+
];
|
|
770
|
+
}
|
|
771
|
+
function getInputType(input2) {
|
|
772
|
+
if (input2.fetcher) {
|
|
773
|
+
return input2.fetcher.type;
|
|
774
|
+
}
|
|
775
|
+
return "static";
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// src/docker/run-model-container.ts
|
|
779
|
+
import os3 from "os";
|
|
780
|
+
import { execa as execa4 } from "execa";
|
|
781
|
+
|
|
782
|
+
// src/docker/utils/get-input-env.ts
|
|
783
|
+
function getInputEnv(modelInput, webserverURL) {
|
|
784
|
+
const variableName = modelInput.key.toUpperCase();
|
|
785
|
+
const filename = getInputFilename(modelInput);
|
|
786
|
+
const inputFileURL = `${webserverURL}/input/${filename}`;
|
|
787
|
+
return `INPUT_${variableName}_URL=${inputFileURL}`;
|
|
569
788
|
}
|
|
570
789
|
|
|
571
|
-
// src/
|
|
790
|
+
// src/docker/run-model-container.ts
|
|
791
|
+
async function runModelContainer(dockerImage, webserverPort, ctx) {
|
|
792
|
+
const env = getEnv();
|
|
793
|
+
const webserverHostname = env.isWSL ? DOCKER_HOSTNAME_WSL : DOCKER_HOSTNAME_DEFAULT;
|
|
794
|
+
const webserverURL = `http://${webserverHostname}:${webserverPort}`;
|
|
795
|
+
const inputEnvs = ctx.config.inputs.map(
|
|
796
|
+
(input2) => getInputEnv(input2, webserverURL)
|
|
797
|
+
);
|
|
798
|
+
const outputEnv = `OUTPUT_URL=${webserverURL}/output`;
|
|
799
|
+
const envs = [...inputEnvs, outputEnv];
|
|
800
|
+
try {
|
|
801
|
+
const extraRunArgs = [];
|
|
802
|
+
if (env.isWSL) {
|
|
803
|
+
const ip = getWSLMachineIP();
|
|
804
|
+
extraRunArgs.push("--add-host", `${DOCKER_HOSTNAME_WSL}:${ip}`);
|
|
805
|
+
}
|
|
806
|
+
const args = [
|
|
807
|
+
"run",
|
|
808
|
+
"--rm",
|
|
809
|
+
...extraRunArgs,
|
|
810
|
+
...envs.flatMap((env2) => ["--env", env2]),
|
|
811
|
+
dockerImage
|
|
812
|
+
];
|
|
813
|
+
await execa4("docker", args, {
|
|
814
|
+
cwd: ctx.projectRoot,
|
|
815
|
+
stdio: "pipe"
|
|
816
|
+
});
|
|
817
|
+
} catch (error) {
|
|
818
|
+
throw new Error("Failed to run model container", { cause: error });
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
function getWSLMachineIP() {
|
|
822
|
+
const env = getEnv();
|
|
823
|
+
const interfaces = os3.networkInterfaces();
|
|
824
|
+
const interfaceInfo = interfaces[env.wslNetworkInterface]?.[0];
|
|
825
|
+
if (!interfaceInfo) {
|
|
826
|
+
throw new Error(
|
|
827
|
+
`WSL machine IP not found for interface \`${env.wslNetworkInterface}\``
|
|
828
|
+
);
|
|
829
|
+
}
|
|
830
|
+
return interfaceInfo.address;
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// src/model-io-checks/setup-model-io-checks.ts
|
|
834
|
+
import path12 from "path";
|
|
835
|
+
import express from "express";
|
|
836
|
+
|
|
837
|
+
// src/model-io-checks/model-io-checks.ts
|
|
572
838
|
var ModelIOChecks = class {
|
|
573
839
|
inputsToDownload;
|
|
574
840
|
outputUploaded;
|
|
@@ -601,85 +867,373 @@ var ModelIOChecks = class {
|
|
|
601
867
|
}
|
|
602
868
|
};
|
|
603
869
|
|
|
604
|
-
// src/
|
|
605
|
-
import
|
|
870
|
+
// src/model-io-checks/utils/create-input-handler.ts
|
|
871
|
+
import path10 from "path";
|
|
872
|
+
function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
|
|
873
|
+
return (req, res) => {
|
|
874
|
+
if (!modelIOChecks.isValidInput(req.params.filename)) {
|
|
875
|
+
return res.status(404).json({ error: "File not found" });
|
|
876
|
+
}
|
|
877
|
+
modelIOChecks.trackInputHandled(req.params.filename);
|
|
878
|
+
const filePath = path10.join(
|
|
879
|
+
ctx.projectRoot,
|
|
880
|
+
inputFilesPath,
|
|
881
|
+
req.params.filename
|
|
882
|
+
);
|
|
883
|
+
return res.sendFile(filePath);
|
|
884
|
+
};
|
|
885
|
+
}
|
|
606
886
|
|
|
607
|
-
// src/
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
887
|
+
// src/model-io-checks/utils/create-output-handler.ts
|
|
888
|
+
import fs9 from "fs/promises";
|
|
889
|
+
import path11 from "path";
|
|
890
|
+
function createOutputHandler(modelIOChecks, ctx) {
|
|
891
|
+
return async (req, res) => {
|
|
892
|
+
modelIOChecks.trackOutputHandled();
|
|
893
|
+
const outputPath = path11.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
|
|
894
|
+
await fs9.mkdir(outputPath, { recursive: true });
|
|
895
|
+
const outputFilePath = path11.join(outputPath, TEST_OUTPUT_FILENAME);
|
|
896
|
+
const outputData = JSON.stringify(req.body, null, 2);
|
|
897
|
+
await fs9.writeFile(outputFilePath, outputData, "utf8");
|
|
898
|
+
return res.status(200).json({ success: true });
|
|
899
|
+
};
|
|
613
900
|
}
|
|
614
901
|
|
|
615
|
-
// src/
|
|
616
|
-
|
|
617
|
-
const
|
|
618
|
-
|
|
902
|
+
// src/model-io-checks/setup-model-io-checks.ts
|
|
903
|
+
function setupModelIOChecks(app, inputDir, inputFiles, ctx) {
|
|
904
|
+
const modelIOChecks = new ModelIOChecks({ inputFiles });
|
|
905
|
+
const handleInput = createInputHandler(inputDir, modelIOChecks, ctx);
|
|
906
|
+
const handleOutput = createOutputHandler(modelIOChecks, ctx);
|
|
907
|
+
const inputPath = path12.join(ctx.projectRoot, inputDir);
|
|
908
|
+
app.use(express.json());
|
|
909
|
+
app.use("/data", express.static(inputPath));
|
|
910
|
+
app.get("/input/:filename", handleInput);
|
|
911
|
+
app.put("/output", handleOutput);
|
|
912
|
+
return modelIOChecks;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// src/utils/check-input-files.ts
|
|
916
|
+
import path13 from "path";
|
|
917
|
+
async function checkInputFiles(inputFiles, inputDataPath, ctx) {
|
|
918
|
+
for (const inputFile of inputFiles) {
|
|
919
|
+
const filePath = path13.join(ctx.projectRoot, inputDataPath, inputFile);
|
|
920
|
+
const exists = await isExistingPath(filePath);
|
|
921
|
+
if (!exists) {
|
|
922
|
+
throw new Error(
|
|
923
|
+
`Input data not found (${inputFile}) - did you need to run \`pd4castr fetch\`?`
|
|
924
|
+
);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
// src/utils/get-input-files.ts
|
|
930
|
+
function getInputFiles(config) {
|
|
931
|
+
const inputFiles = config.inputs.map((input2) => getInputFilename(input2));
|
|
932
|
+
return inputFiles;
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
// src/commands/publish/utils/run-model-io-tests.ts
|
|
936
|
+
async function runModelIOTests(dockerImage, options, app, ctx) {
|
|
937
|
+
const inputFiles = getInputFiles(ctx.config);
|
|
938
|
+
await checkInputFiles(inputFiles, options.inputDir, ctx);
|
|
939
|
+
await buildDockerImage(dockerImage, ctx);
|
|
940
|
+
const modelIOChecks = setupModelIOChecks(
|
|
941
|
+
app,
|
|
942
|
+
options.inputDir,
|
|
943
|
+
inputFiles,
|
|
944
|
+
ctx
|
|
619
945
|
);
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
"--rm",
|
|
626
|
-
...envs.flatMap((env) => ["--env", env]),
|
|
627
|
-
dockerImage
|
|
628
|
-
];
|
|
629
|
-
await execa2("docker", args, {
|
|
630
|
-
cwd: ctx.projectRoot,
|
|
631
|
-
stdio: "pipe"
|
|
632
|
-
});
|
|
633
|
-
} catch (error) {
|
|
634
|
-
throw new Error("Failed to run model container", { cause: error });
|
|
946
|
+
await runModelContainer(dockerImage, options.port, ctx);
|
|
947
|
+
if (!modelIOChecks.isInputsHandled() || !modelIOChecks.isOutputHandled()) {
|
|
948
|
+
throw new Error(
|
|
949
|
+
"Model I/O test failed. Please run `pd4castr test` to debug the issue."
|
|
950
|
+
);
|
|
635
951
|
}
|
|
636
952
|
}
|
|
637
953
|
|
|
638
|
-
// src/commands/
|
|
639
|
-
async function
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
954
|
+
// src/commands/publish/handle-create-model-flow.ts
|
|
955
|
+
async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
|
|
956
|
+
spinner.info(`You are publishing a ${chalk2.bold("new")} model:
|
|
957
|
+
`);
|
|
958
|
+
getModelSummaryLines(ctx).map((line) => console.log(line));
|
|
959
|
+
const confirm4 = await inquirer2.confirm({
|
|
960
|
+
message: "Do you want to continue?",
|
|
961
|
+
default: false
|
|
962
|
+
});
|
|
963
|
+
logEmptyLine();
|
|
964
|
+
if (!confirm4) {
|
|
965
|
+
throw new Error("Aborted");
|
|
966
|
+
}
|
|
967
|
+
const dockerImage = getDockerImage(ctx);
|
|
968
|
+
if (!options.skipChecks) {
|
|
969
|
+
spinner.start("Performing model I/O test...");
|
|
970
|
+
await runModelIOTests(dockerImage, options, app, ctx);
|
|
971
|
+
spinner.succeed("Model I/O test passed");
|
|
972
|
+
}
|
|
973
|
+
spinner.start("Publishing model...");
|
|
974
|
+
const modelConfig = await getModelConfigFromProjectConfig(ctx);
|
|
975
|
+
const model = await createModel(modelConfig, authCtx);
|
|
976
|
+
await updateProjectConfig((config) => {
|
|
977
|
+
config.$$id = model.id;
|
|
978
|
+
config.$$modelGroupID = model.modelGroupId;
|
|
979
|
+
config.$$revision = model.revision;
|
|
980
|
+
config.$$dockerImage = model.dockerImage;
|
|
981
|
+
});
|
|
982
|
+
const pushCredentials = await getRegistryPushCredentials(model.id, authCtx);
|
|
983
|
+
await loginToDockerRegistry(pushCredentials);
|
|
984
|
+
await buildDockerImage(dockerImage, ctx);
|
|
985
|
+
await pushDockerImage(dockerImage, pushCredentials.ref);
|
|
986
|
+
spinner.stopAndPersist({
|
|
987
|
+
symbol: "\u{1F680} ",
|
|
988
|
+
prefixText: "\n",
|
|
989
|
+
suffixText: "\n",
|
|
990
|
+
text: chalk2.bold("Model published successfully")
|
|
647
991
|
});
|
|
648
992
|
}
|
|
649
993
|
|
|
994
|
+
// src/commands/publish/handle-update-existing-model-flow.ts
|
|
995
|
+
import * as inquirer5 from "@inquirer/prompts";
|
|
996
|
+
import chalk5 from "chalk";
|
|
997
|
+
|
|
998
|
+
// src/commands/publish/handle-model-revision-create-flow.ts
|
|
999
|
+
import * as inquirer3 from "@inquirer/prompts";
|
|
1000
|
+
import chalk3 from "chalk";
|
|
1001
|
+
|
|
1002
|
+
// src/commands/publish/utils/validate-local-model-state.ts
|
|
1003
|
+
import invariant2 from "tiny-invariant";
|
|
1004
|
+
|
|
1005
|
+
// src/api/get-model.ts
|
|
1006
|
+
async function getModel(id, authCtx) {
|
|
1007
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
1008
|
+
const result = await api.get(`model/${id}`, { headers }).json();
|
|
1009
|
+
return result;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
// src/commands/publish/utils/validate-local-model-state.ts
|
|
1013
|
+
async function validateLocalModelState(ctx, authCtx) {
|
|
1014
|
+
invariant2(ctx.config.$$id, "model ID is required to fetch published model");
|
|
1015
|
+
const currentModel = await getModel(ctx.config.$$id, authCtx);
|
|
1016
|
+
if (currentModel.revision !== ctx.config.$$revision) {
|
|
1017
|
+
throw new Error(
|
|
1018
|
+
`OUT OF SYNC: Local revision (${ctx.config.$$revision}) does not match the current published revision (${currentModel.revision})`
|
|
1019
|
+
);
|
|
1020
|
+
}
|
|
1021
|
+
if (currentModel.modelGroupId !== ctx.config.$$modelGroupID) {
|
|
1022
|
+
throw new Error(
|
|
1023
|
+
`OUT OF SYNC: Local model group ID (${ctx.config.$$modelGroupID}) does not match the current published model group ID (${currentModel.modelGroupId})`
|
|
1024
|
+
);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
// src/commands/publish/handle-model-revision-create-flow.ts
|
|
1029
|
+
var WARNING_LABEL = chalk3.yellowBright.bold("WARNING!");
|
|
1030
|
+
var CONFIRMATION_MESSAGE = `${WARNING_LABEL} Creating a new revision will preserve existing revisions.
|
|
1031
|
+
Previous revisions will still be available in the pd4castr UI.
|
|
1032
|
+
`;
|
|
1033
|
+
async function handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx) {
|
|
1034
|
+
console.log(CONFIRMATION_MESSAGE);
|
|
1035
|
+
const confirmed = await inquirer3.confirm({
|
|
1036
|
+
message: "Are you sure you want to continue?",
|
|
1037
|
+
default: false
|
|
1038
|
+
});
|
|
1039
|
+
logEmptyLine();
|
|
1040
|
+
if (!confirmed) {
|
|
1041
|
+
throw new Error("Model revision update cancelled");
|
|
1042
|
+
}
|
|
1043
|
+
spinner.start("Validating local model state...");
|
|
1044
|
+
await validateLocalModelState(ctx, authCtx);
|
|
1045
|
+
spinner.succeed("Local model is synced with published model");
|
|
1046
|
+
const dockerImage = getDockerImage(ctx);
|
|
1047
|
+
if (!options.skipChecks) {
|
|
1048
|
+
spinner.start("Performing model I/O test...");
|
|
1049
|
+
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1050
|
+
spinner.succeed("Model I/O test passed");
|
|
1051
|
+
}
|
|
1052
|
+
const modelConfig = await getModelConfigFromProjectConfig(ctx);
|
|
1053
|
+
const model = await createModel(modelConfig, authCtx);
|
|
1054
|
+
await updateProjectConfig((config) => {
|
|
1055
|
+
config.$$id = model.id;
|
|
1056
|
+
config.$$modelGroupID = model.modelGroupId;
|
|
1057
|
+
config.$$revision = model.revision;
|
|
1058
|
+
config.$$dockerImage = model.dockerImage;
|
|
1059
|
+
});
|
|
1060
|
+
const pushCredentials = await getRegistryPushCredentials(model.id, authCtx);
|
|
1061
|
+
await loginToDockerRegistry(pushCredentials);
|
|
1062
|
+
await buildDockerImage(dockerImage, ctx);
|
|
1063
|
+
await pushDockerImage(dockerImage, pushCredentials.ref);
|
|
1064
|
+
spinner.stopAndPersist({
|
|
1065
|
+
symbol: "\u{1F680} ",
|
|
1066
|
+
prefixText: "\n",
|
|
1067
|
+
suffixText: "\n",
|
|
1068
|
+
text: chalk3.bold(
|
|
1069
|
+
`New model revision (r${model.revision}) published successfully`
|
|
1070
|
+
)
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
// src/commands/publish/handle-model-revision-update-flow.ts
|
|
1075
|
+
import * as inquirer4 from "@inquirer/prompts";
|
|
1076
|
+
import chalk4 from "chalk";
|
|
1077
|
+
|
|
1078
|
+
// src/api/update-model.ts
|
|
1079
|
+
async function updateModel(config, authCtx) {
|
|
1080
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
1081
|
+
const result = await api.patch(`model/${config.id}`, { headers, json: config }).json();
|
|
1082
|
+
return result;
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
// src/commands/publish/handle-model-revision-update-flow.ts
|
|
1086
|
+
var WARNING_LABEL2 = chalk4.yellowBright.bold("WARNING!");
|
|
1087
|
+
var CONFIRMATION_MESSAGE2 = `${WARNING_LABEL2} Updating a model revision recreates the associated inputs and outputs.
|
|
1088
|
+
Historical data is preserved, but it will no longer be displayed in the pd4castr UI.
|
|
1089
|
+
`;
|
|
1090
|
+
async function handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx) {
|
|
1091
|
+
console.log(CONFIRMATION_MESSAGE2);
|
|
1092
|
+
const confirmed = await inquirer4.confirm({
|
|
1093
|
+
message: "Are you sure you want to continue?",
|
|
1094
|
+
default: false
|
|
1095
|
+
});
|
|
1096
|
+
logEmptyLine();
|
|
1097
|
+
if (!confirmed) {
|
|
1098
|
+
throw new Error("Model revision update cancelled");
|
|
1099
|
+
}
|
|
1100
|
+
spinner.start("Validating local model state...");
|
|
1101
|
+
await validateLocalModelState(ctx, authCtx);
|
|
1102
|
+
spinner.succeed("Local model is synced with published model");
|
|
1103
|
+
const dockerImage = getDockerImage(ctx);
|
|
1104
|
+
if (!options.skipChecks) {
|
|
1105
|
+
spinner.start("Performing model I/O test...");
|
|
1106
|
+
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1107
|
+
spinner.succeed("Model I/O test passed");
|
|
1108
|
+
}
|
|
1109
|
+
const modelConfig = await getModelConfigFromProjectConfig(ctx);
|
|
1110
|
+
const model = await updateModel(modelConfig, authCtx);
|
|
1111
|
+
await updateProjectConfig((config) => {
|
|
1112
|
+
config.$$id = model.id;
|
|
1113
|
+
config.$$modelGroupID = model.modelGroupId;
|
|
1114
|
+
config.$$revision = model.revision;
|
|
1115
|
+
config.$$dockerImage = model.dockerImage;
|
|
1116
|
+
});
|
|
1117
|
+
const pushCredentials = await getRegistryPushCredentials(model.id, authCtx);
|
|
1118
|
+
await loginToDockerRegistry(pushCredentials);
|
|
1119
|
+
await buildDockerImage(dockerImage, ctx);
|
|
1120
|
+
await pushDockerImage(dockerImage, pushCredentials.ref);
|
|
1121
|
+
spinner.stopAndPersist({
|
|
1122
|
+
symbol: "\u{1F680} ",
|
|
1123
|
+
prefixText: "\n",
|
|
1124
|
+
suffixText: "\n",
|
|
1125
|
+
text: chalk4.bold(`${model.name} (r${model.revision}) updated successfully`)
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// src/commands/publish/handle-update-existing-model-flow.ts
|
|
1130
|
+
async function handleUpdateExistingModelFlow(options, app, spinner, ctx, authCtx) {
|
|
1131
|
+
spinner.info(`You are publishing an ${chalk5.bold("existing")} model:
|
|
1132
|
+
`);
|
|
1133
|
+
getModelSummaryLines(ctx).map((line) => console.log(line));
|
|
1134
|
+
const revision = ctx.config.$$revision ?? 0;
|
|
1135
|
+
const action = await inquirer5.select({
|
|
1136
|
+
message: "Do you want to update the existing revision or create a new one?",
|
|
1137
|
+
choices: [
|
|
1138
|
+
{
|
|
1139
|
+
value: "new" /* NewRevision */,
|
|
1140
|
+
name: `New Revision (r${revision} \u2192 r${revision + 1})`
|
|
1141
|
+
},
|
|
1142
|
+
{
|
|
1143
|
+
value: "update" /* UpdateExisting */,
|
|
1144
|
+
name: `Update Existing Revision (r${revision})`
|
|
1145
|
+
}
|
|
1146
|
+
]
|
|
1147
|
+
});
|
|
1148
|
+
logEmptyLine();
|
|
1149
|
+
if (action === "new" /* NewRevision */) {
|
|
1150
|
+
await handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx);
|
|
1151
|
+
} else if (action === "update" /* UpdateExisting */) {
|
|
1152
|
+
await handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx);
|
|
1153
|
+
} else {
|
|
1154
|
+
throw new Error("Invalid CLI state");
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
// src/commands/publish/handle-action.ts
|
|
1159
|
+
async function handleAction5(options) {
|
|
1160
|
+
var _stack = [];
|
|
1161
|
+
try {
|
|
1162
|
+
const spinner = ora5("Starting model publish...").start();
|
|
1163
|
+
const app = express2();
|
|
1164
|
+
app.use(express2.json({ limit: "100mb" }));
|
|
1165
|
+
app.use(express2.urlencoded({ limit: "100mb", extended: true }));
|
|
1166
|
+
const webServer = __using(_stack, await startWebServer(app, options.port));
|
|
1167
|
+
try {
|
|
1168
|
+
const ctx = await loadProjectContext();
|
|
1169
|
+
const authCtx = await getAuth();
|
|
1170
|
+
await (ctx.config.$$id ? handleUpdateExistingModelFlow(options, app, spinner, ctx, authCtx) : handleCreateModelFlow(options, app, spinner, ctx, authCtx));
|
|
1171
|
+
} catch (error) {
|
|
1172
|
+
if (error instanceof ZodError5) {
|
|
1173
|
+
spinner.fail("Config validation failed");
|
|
1174
|
+
logZodIssues(error);
|
|
1175
|
+
} else if (error instanceof HTTPError3) {
|
|
1176
|
+
const errorResponse = await error.response.json();
|
|
1177
|
+
spinner.fail(formatNestErrorMessage(errorResponse));
|
|
1178
|
+
} else if (error instanceof Error) {
|
|
1179
|
+
spinner.fail(error.message);
|
|
1180
|
+
if (error.cause) {
|
|
1181
|
+
console.error(error.cause);
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
process.exit(1);
|
|
1185
|
+
}
|
|
1186
|
+
} catch (_) {
|
|
1187
|
+
var _error = _, _hasError = true;
|
|
1188
|
+
} finally {
|
|
1189
|
+
__callDispose(_stack, _error, _hasError);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
// src/commands/publish/index.ts
|
|
1194
|
+
function registerPublishCommand(program2) {
|
|
1195
|
+
program2.command("publish").description("Publishes a pd4castr model.").option(
|
|
1196
|
+
"-i, --input-dir <path>",
|
|
1197
|
+
"The input test data directory",
|
|
1198
|
+
TEST_INPUT_DATA_DIR
|
|
1199
|
+
).option(
|
|
1200
|
+
"-p, --port <port>",
|
|
1201
|
+
"The port to run the IO testing webserver on",
|
|
1202
|
+
TEST_WEBSERVER_PORT.toString()
|
|
1203
|
+
).option("-s, --skip-checks", "Skip the model I/O checks", false).action(handleAction5);
|
|
1204
|
+
}
|
|
1205
|
+
|
|
650
1206
|
// src/commands/test/handle-action.ts
|
|
651
|
-
|
|
1207
|
+
import path14 from "path";
|
|
1208
|
+
import express3 from "express";
|
|
1209
|
+
import ora6 from "ora";
|
|
1210
|
+
import { ZodError as ZodError6 } from "zod";
|
|
1211
|
+
async function handleAction6(options) {
|
|
652
1212
|
var _stack = [];
|
|
653
1213
|
try {
|
|
654
|
-
const spinner =
|
|
655
|
-
const app =
|
|
1214
|
+
const spinner = ora6("Starting model tests...").info();
|
|
1215
|
+
const app = express3();
|
|
1216
|
+
app.use(express3.json({ limit: "100mb" }));
|
|
1217
|
+
app.use(express3.urlencoded({ limit: "100mb", extended: true }));
|
|
656
1218
|
const webServer = __using(_stack, await startWebServer(app, options.port));
|
|
657
1219
|
try {
|
|
658
|
-
const ctx = await
|
|
1220
|
+
const ctx = await loadProjectContext();
|
|
659
1221
|
const inputFiles = getInputFiles(ctx.config);
|
|
660
1222
|
spinner.start("Checking test input data files");
|
|
661
1223
|
await checkInputFiles(inputFiles, options.inputDir, ctx);
|
|
662
1224
|
spinner.succeed(`Found ${inputFiles.length} test input data files`);
|
|
663
1225
|
spinner.start("Building docker image");
|
|
664
|
-
const
|
|
665
|
-
const dockerImage = `pd4castr/${sluggedName}-local:${Date.now()}`;
|
|
1226
|
+
const dockerImage = getDockerImage(ctx);
|
|
666
1227
|
await buildDockerImage(dockerImage, ctx);
|
|
667
1228
|
spinner.succeed(`Built docker image (${dockerImage})`);
|
|
668
|
-
const modelIOChecks =
|
|
669
|
-
|
|
1229
|
+
const modelIOChecks = setupModelIOChecks(
|
|
1230
|
+
app,
|
|
670
1231
|
options.inputDir,
|
|
671
|
-
|
|
1232
|
+
inputFiles,
|
|
672
1233
|
ctx
|
|
673
1234
|
);
|
|
674
|
-
const handleOutput = createOutputHandler(modelIOChecks, ctx);
|
|
675
|
-
const inputPath = path11.join(ctx.projectRoot, options.inputDir);
|
|
676
|
-
app.use(express.json());
|
|
677
|
-
app.use("/data", express.static(inputPath));
|
|
678
|
-
app.get("/input/:filename", handleInput);
|
|
679
|
-
app.put("/output", handleOutput);
|
|
680
1235
|
spinner.start("Running model container");
|
|
681
|
-
|
|
682
|
-
await runModelContainer(dockerImage, webserverURL, ctx);
|
|
1236
|
+
await runModelContainer(dockerImage, options.port, ctx);
|
|
683
1237
|
spinner.succeed("Model run complete");
|
|
684
1238
|
for (const inputFile of inputFiles) {
|
|
685
1239
|
const status = modelIOChecks.isInputHandled(inputFile) ? "\u2714" : "\u2718";
|
|
@@ -690,10 +1244,10 @@ async function handleAction4(options) {
|
|
|
690
1244
|
if (modelIOChecks.isInputsHandled() && modelIOChecks.isOutputHandled()) {
|
|
691
1245
|
spinner.succeed("Model I/O test passed");
|
|
692
1246
|
} else {
|
|
693
|
-
|
|
1247
|
+
throw new Error("Model I/O test failed");
|
|
694
1248
|
}
|
|
695
1249
|
if (modelIOChecks.isOutputHandled()) {
|
|
696
|
-
const outputPath =
|
|
1250
|
+
const outputPath = path14.join(
|
|
697
1251
|
ctx.projectRoot,
|
|
698
1252
|
TEST_OUTPUT_DATA_DIR,
|
|
699
1253
|
TEST_OUTPUT_FILENAME
|
|
@@ -705,14 +1259,14 @@ ${clickHereLink} to view output (${fileLink})
|
|
|
705
1259
|
`);
|
|
706
1260
|
}
|
|
707
1261
|
} catch (error) {
|
|
708
|
-
if (error instanceof
|
|
1262
|
+
if (error instanceof ZodError6) {
|
|
709
1263
|
spinner.fail("Config validation failed");
|
|
710
1264
|
logZodIssues(error);
|
|
711
1265
|
} else if (error instanceof Error) {
|
|
712
1266
|
spinner.fail(error.message);
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
1267
|
+
if (error.cause) {
|
|
1268
|
+
console.error(error.cause);
|
|
1269
|
+
}
|
|
716
1270
|
}
|
|
717
1271
|
process.exit(1);
|
|
718
1272
|
}
|
|
@@ -731,7 +1285,11 @@ function registerTestCommand(program2) {
|
|
|
731
1285
|
"-i, --input-dir <path>",
|
|
732
1286
|
"The input test data directory",
|
|
733
1287
|
TEST_INPUT_DATA_DIR
|
|
734
|
-
).option(
|
|
1288
|
+
).option(
|
|
1289
|
+
"-p, --port <port>",
|
|
1290
|
+
"The port to run the IO testing webserver on",
|
|
1291
|
+
TEST_WEBSERVER_PORT.toString()
|
|
1292
|
+
).action(handleAction6);
|
|
735
1293
|
}
|
|
736
1294
|
|
|
737
1295
|
// src/program.ts
|
|
@@ -740,7 +1298,7 @@ import { Command } from "commander";
|
|
|
740
1298
|
// package.json
|
|
741
1299
|
var package_default = {
|
|
742
1300
|
name: "@pd4castr/cli",
|
|
743
|
-
version: "0.0.
|
|
1301
|
+
version: "0.0.7",
|
|
744
1302
|
description: "CLI tool for creating, testing, and publishing pd4castr models",
|
|
745
1303
|
main: "dist/index.js",
|
|
746
1304
|
type: "module",
|
|
@@ -761,7 +1319,7 @@ var package_default = {
|
|
|
761
1319
|
"lint:fix": "eslint . --fix",
|
|
762
1320
|
format: "prettier --write .",
|
|
763
1321
|
"format:check": "prettier --check .",
|
|
764
|
-
|
|
1322
|
+
typecheck: "tsc --noEmit",
|
|
765
1323
|
prepublishOnly: "yarn build"
|
|
766
1324
|
},
|
|
767
1325
|
keywords: [
|
|
@@ -778,6 +1336,8 @@ var package_default = {
|
|
|
778
1336
|
},
|
|
779
1337
|
homepage: "https://github.com/pipelabs/pd4castr-cli#readme",
|
|
780
1338
|
devDependencies: {
|
|
1339
|
+
"@faker-js/faker": "10.0.0",
|
|
1340
|
+
"@mswjs/data": "0.16.2",
|
|
781
1341
|
"@types/express": "4.17.21",
|
|
782
1342
|
"@types/node": "24.1.0",
|
|
783
1343
|
"@types/supertest": "6.0.3",
|
|
@@ -788,12 +1348,15 @@ var package_default = {
|
|
|
788
1348
|
"eslint-plugin-simple-import-sort": "12.1.1",
|
|
789
1349
|
"eslint-plugin-unicorn": "60.0.0",
|
|
790
1350
|
"eslint-plugin-vitest": "0.5.4",
|
|
1351
|
+
"hook-std": "3.0.0",
|
|
791
1352
|
"jest-extended": "6.0.0",
|
|
792
1353
|
memfs: "4.23.0",
|
|
793
1354
|
msw: "2.10.4",
|
|
794
1355
|
prettier: "3.6.2",
|
|
1356
|
+
"strip-ansi": "7.1.0",
|
|
795
1357
|
supertest: "7.1.4",
|
|
796
1358
|
tsup: "8.5.0",
|
|
1359
|
+
"type-fest": "4.41.0",
|
|
797
1360
|
typescript: "5.8.3",
|
|
798
1361
|
"typescript-eslint": "8.38.0",
|
|
799
1362
|
vitest: "3.2.4"
|
|
@@ -801,11 +1364,12 @@ var package_default = {
|
|
|
801
1364
|
dependencies: {
|
|
802
1365
|
"@inquirer/prompts": "7.7.1",
|
|
803
1366
|
auth0: "4.27.0",
|
|
1367
|
+
chalk: "5.6.0",
|
|
804
1368
|
commander: "14.0.0",
|
|
805
1369
|
execa: "9.6.0",
|
|
806
1370
|
express: "4.21.2",
|
|
1371
|
+
immer: "10.1.1",
|
|
807
1372
|
ky: "1.8.2",
|
|
808
|
-
lilconfig: "3.1.3",
|
|
809
1373
|
ora: "8.2.0",
|
|
810
1374
|
slugify: "1.6.6",
|
|
811
1375
|
tiged: "2.12.7",
|
|
@@ -824,6 +1388,8 @@ program.name("pd4castr").description("CLI tool for pd4castr").version(package_de
|
|
|
824
1388
|
// src/index.ts
|
|
825
1389
|
registerInitCommand(program);
|
|
826
1390
|
registerLoginCommand(program);
|
|
1391
|
+
registerLogoutCommand(program);
|
|
827
1392
|
registerTestCommand(program);
|
|
828
1393
|
registerFetchCommand(program);
|
|
1394
|
+
registerPublishCommand(program);
|
|
829
1395
|
await program.parseAsync(process.argv);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pd4castr/cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.7",
|
|
4
4
|
"description": "CLI tool for creating, testing, and publishing pd4castr models",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"lint:fix": "eslint . --fix",
|
|
22
22
|
"format": "prettier --write .",
|
|
23
23
|
"format:check": "prettier --check .",
|
|
24
|
-
"
|
|
24
|
+
"typecheck": "tsc --noEmit",
|
|
25
25
|
"prepublishOnly": "yarn build"
|
|
26
26
|
},
|
|
27
27
|
"keywords": [
|
|
@@ -38,6 +38,8 @@
|
|
|
38
38
|
},
|
|
39
39
|
"homepage": "https://github.com/pipelabs/pd4castr-cli#readme",
|
|
40
40
|
"devDependencies": {
|
|
41
|
+
"@faker-js/faker": "10.0.0",
|
|
42
|
+
"@mswjs/data": "0.16.2",
|
|
41
43
|
"@types/express": "4.17.21",
|
|
42
44
|
"@types/node": "24.1.0",
|
|
43
45
|
"@types/supertest": "6.0.3",
|
|
@@ -48,12 +50,15 @@
|
|
|
48
50
|
"eslint-plugin-simple-import-sort": "12.1.1",
|
|
49
51
|
"eslint-plugin-unicorn": "60.0.0",
|
|
50
52
|
"eslint-plugin-vitest": "0.5.4",
|
|
53
|
+
"hook-std": "3.0.0",
|
|
51
54
|
"jest-extended": "6.0.0",
|
|
52
55
|
"memfs": "4.23.0",
|
|
53
56
|
"msw": "2.10.4",
|
|
54
57
|
"prettier": "3.6.2",
|
|
58
|
+
"strip-ansi": "7.1.0",
|
|
55
59
|
"supertest": "7.1.4",
|
|
56
60
|
"tsup": "8.5.0",
|
|
61
|
+
"type-fest": "4.41.0",
|
|
57
62
|
"typescript": "5.8.3",
|
|
58
63
|
"typescript-eslint": "8.38.0",
|
|
59
64
|
"vitest": "3.2.4"
|
|
@@ -61,11 +66,12 @@
|
|
|
61
66
|
"dependencies": {
|
|
62
67
|
"@inquirer/prompts": "7.7.1",
|
|
63
68
|
"auth0": "4.27.0",
|
|
69
|
+
"chalk": "5.6.0",
|
|
64
70
|
"commander": "14.0.0",
|
|
65
71
|
"execa": "9.6.0",
|
|
66
72
|
"express": "4.21.2",
|
|
73
|
+
"immer": "10.1.1",
|
|
67
74
|
"ky": "1.8.2",
|
|
68
|
-
"lilconfig": "3.1.3",
|
|
69
75
|
"ora": "8.2.0",
|
|
70
76
|
"slugify": "1.6.6",
|
|
71
77
|
"tiged": "2.12.7",
|