@pd4castr/cli 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1048 -478
- package/package.json +10 -3
package/dist/index.js
CHANGED
|
@@ -45,97 +45,330 @@ var __callDispose = (stack, error, hasError) => {
|
|
|
45
45
|
return next();
|
|
46
46
|
};
|
|
47
47
|
|
|
48
|
-
// src/
|
|
49
|
-
|
|
48
|
+
// src/constants.ts
|
|
49
|
+
var AUTH0_DOMAIN = "pdview.au.auth0.com";
|
|
50
|
+
var AUTH0_CLIENT_ID = "Q5tQNF57cQlVXnVsqnU0hhgy92rVb03W";
|
|
51
|
+
var AUTH0_AUDIENCE = "https://api.pd4castr.com.au";
|
|
52
|
+
var GLOBAL_CONFIG_FILE = ".pd4castr";
|
|
53
|
+
var PROJECT_CONFIG_FILE = ".pd4castrrc.json";
|
|
54
|
+
var DEFAULT_API_URL = "https://pd4castr-api.pipelabs.app";
|
|
55
|
+
var DEFAULT_INPUT_SOURCE_ID = "0bdfd52b-efaa-455e-9a3b-1a6d2b879b73";
|
|
56
|
+
var TEST_INPUT_DATA_DIR = "test_input";
|
|
57
|
+
var TEST_OUTPUT_DATA_DIR = "test_output";
|
|
58
|
+
var TEST_WEBSERVER_PORT = 9800;
|
|
59
|
+
var TEST_OUTPUT_FILENAME = `output-${Date.now()}.json`;
|
|
60
|
+
var DOCKER_HOSTNAME_DEFAULT = "host.docker.internal";
|
|
61
|
+
var DOCKER_HOSTNAME_WSL = "wsl.host";
|
|
62
|
+
var WSL_NETWORK_INTERFACE_DEFAULT = "eth0";
|
|
50
63
|
|
|
51
|
-
//
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
64
|
+
// src/commands/fetch/handle-action.ts
|
|
65
|
+
import path5 from "path";
|
|
66
|
+
import { HTTPError } from "ky";
|
|
67
|
+
import ora from "ora";
|
|
68
|
+
import { ZodError as ZodError2 } from "zod";
|
|
69
|
+
|
|
70
|
+
// src/config/load-project-context.ts
|
|
71
|
+
import fs2 from "fs/promises";
|
|
72
|
+
import path from "path";
|
|
73
|
+
import { ZodError } from "zod";
|
|
74
|
+
|
|
75
|
+
// src/schemas/project-config-schema.ts
|
|
76
|
+
import { z } from "zod";
|
|
77
|
+
var fileFormatSchema = z.enum(["csv", "json", "parquet"]);
|
|
78
|
+
var aemoDataFetcherSchema = z.object({
|
|
79
|
+
type: z.literal("AEMO_MMS"),
|
|
80
|
+
checkInterval: z.number().int().min(60),
|
|
81
|
+
config: z.object({
|
|
82
|
+
checkQuery: z.string(),
|
|
83
|
+
fetchQuery: z.string()
|
|
84
|
+
})
|
|
85
|
+
});
|
|
86
|
+
var dataFetcherSchema = z.discriminatedUnion("type", [aemoDataFetcherSchema]);
|
|
87
|
+
var modelInputSchema = z.object({
|
|
88
|
+
key: z.string(),
|
|
89
|
+
inputSource: z.string().optional().default(DEFAULT_INPUT_SOURCE_ID),
|
|
90
|
+
trigger: z.enum(["WAIT_FOR_LATEST_FILE", "USE_MOST_RECENT_FILE"]),
|
|
91
|
+
uploadFileFormat: fileFormatSchema.optional().default("json"),
|
|
92
|
+
targetFileFormat: fileFormatSchema.optional().default("json"),
|
|
93
|
+
fetcher: dataFetcherSchema.optional().nullable()
|
|
94
|
+
});
|
|
95
|
+
var modelOutputSchema = z.object({
|
|
96
|
+
name: z.string(),
|
|
97
|
+
type: z.enum(["float", "integer", "string", "date", "boolean", "unknown"]),
|
|
98
|
+
seriesKey: z.boolean(),
|
|
99
|
+
colour: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional()
|
|
100
|
+
});
|
|
101
|
+
var CONFIG_WARNING_KEY = "// WARNING: DO NOT MODIFY THESE SYSTEM MANAGED VALUES";
|
|
102
|
+
var projectConfigSchema = z.object({
|
|
103
|
+
name: z.string(),
|
|
104
|
+
forecastVariable: z.enum(["price"]),
|
|
105
|
+
timeHorizon: z.enum(["actual", "day_ahead", "week_ahead", "quarterly"]),
|
|
106
|
+
metadata: z.record(z.string(), z.any()).optional(),
|
|
107
|
+
inputs: z.array(modelInputSchema),
|
|
108
|
+
outputs: z.array(modelOutputSchema),
|
|
109
|
+
[CONFIG_WARNING_KEY]: z.string().optional().default(""),
|
|
110
|
+
$$id: z.string().nullable().optional().default(null),
|
|
111
|
+
$$modelGroupID: z.string().nullable().optional().default(null),
|
|
112
|
+
$$revision: z.number().int().min(0).nullable().optional().default(null),
|
|
113
|
+
$$dockerImage: z.string().nullable().optional().default(null)
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
// src/utils/is-existing-path.ts
|
|
117
|
+
import fs from "fs/promises";
|
|
118
|
+
async function isExistingPath(path15) {
|
|
119
|
+
try {
|
|
120
|
+
await fs.access(path15);
|
|
121
|
+
return true;
|
|
122
|
+
} catch {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// src/config/load-project-context.ts
|
|
128
|
+
async function loadProjectContext() {
|
|
129
|
+
const projectRoot = process.cwd();
|
|
130
|
+
const configPath = path.join(projectRoot, PROJECT_CONFIG_FILE);
|
|
131
|
+
const configExists = await isExistingPath(configPath);
|
|
132
|
+
if (!configExists) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
"No config found (docs: https://github.com/pipelabs/pd4castr-model-examples/blob/main/docs/005-config.md)."
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
try {
|
|
138
|
+
const configFileContents = await fs2.readFile(configPath, "utf8");
|
|
139
|
+
const rawConfig = JSON.parse(configFileContents);
|
|
140
|
+
const config = projectConfigSchema.parse(rawConfig);
|
|
141
|
+
return {
|
|
142
|
+
config,
|
|
143
|
+
projectRoot
|
|
144
|
+
};
|
|
145
|
+
} catch (error) {
|
|
146
|
+
if (error instanceof ZodError) {
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
throw new Error(
|
|
150
|
+
"Failed to parse project config (docs: https://github.com/pipelabs/pd4castr-model-examples/blob/main/docs/005-config.md)."
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// src/utils/create-link.ts
|
|
156
|
+
var ESC = "\x1B";
|
|
157
|
+
var OSC = `${ESC}]`;
|
|
158
|
+
var SEP = ";";
|
|
159
|
+
function createLink(text, url) {
|
|
160
|
+
const start = `${OSC}8${SEP}${SEP}${url}${ESC}\\`;
|
|
161
|
+
const end = `${OSC}8${SEP}${SEP}${ESC}\\`;
|
|
162
|
+
return `${start}${text}${end}`;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// src/utils/format-nest-error-message.ts
|
|
166
|
+
function formatNestErrorMessage(error) {
|
|
167
|
+
return `[${error.error?.toUpperCase() ?? "UNKNOWN"}] ${error.message}`;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// src/utils/get-auth.ts
|
|
171
|
+
import invariant from "tiny-invariant";
|
|
172
|
+
|
|
173
|
+
// src/config/load-global-config.ts
|
|
174
|
+
import fs3 from "fs/promises";
|
|
175
|
+
import os from "os";
|
|
176
|
+
import path2 from "path";
|
|
177
|
+
|
|
178
|
+
// src/schemas/global-config-schema.ts
|
|
179
|
+
import { z as z2 } from "zod";
|
|
180
|
+
var globalConfigSchema = z2.object({
|
|
181
|
+
accessToken: z2.string().nullable(),
|
|
182
|
+
accessTokenExpiresAt: z2.number().nullable()
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
// src/config/load-global-config.ts
|
|
186
|
+
async function loadGlobalConfig() {
|
|
187
|
+
const configPath = path2.join(os.homedir(), GLOBAL_CONFIG_FILE);
|
|
188
|
+
const configExists = await isExistingPath(configPath);
|
|
189
|
+
if (!configExists) {
|
|
190
|
+
await fs3.writeFile(configPath, JSON.stringify({}));
|
|
191
|
+
return getDefaultConfig();
|
|
192
|
+
}
|
|
193
|
+
try {
|
|
194
|
+
const configFileContents = await fs3.readFile(configPath, "utf8");
|
|
195
|
+
const config = JSON.parse(configFileContents);
|
|
196
|
+
return globalConfigSchema.parse(config);
|
|
197
|
+
} catch {
|
|
198
|
+
return getDefaultConfig();
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
function getDefaultConfig() {
|
|
202
|
+
return {
|
|
203
|
+
accessToken: null,
|
|
204
|
+
accessTokenExpiresAt: null
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// src/utils/is-authed.ts
|
|
209
|
+
function isAuthed(config) {
|
|
210
|
+
const isTokenExpired = config.accessTokenExpiresAt && config.accessTokenExpiresAt <= Date.now();
|
|
211
|
+
return Boolean(config.accessToken) && !isTokenExpired;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// src/utils/get-auth.ts
|
|
215
|
+
async function getAuth() {
|
|
216
|
+
const config = await loadGlobalConfig();
|
|
217
|
+
if (!isAuthed(config)) {
|
|
218
|
+
throw new Error("Not authenticated. Please run `pd4castr login` to login.");
|
|
127
219
|
}
|
|
220
|
+
invariant(config.accessToken, "Access token is required");
|
|
221
|
+
invariant(config.accessTokenExpiresAt, "Access token expiry is required");
|
|
222
|
+
return {
|
|
223
|
+
accessToken: config.accessToken,
|
|
224
|
+
expiresAt: config.accessTokenExpiresAt
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// src/utils/log-zod-issues.ts
|
|
229
|
+
function logZodIssues(error) {
|
|
230
|
+
for (const issue of error.issues) {
|
|
231
|
+
console.log(` \u2718 ${issue.path.join(".")} - ${issue.message}`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// src/commands/fetch/utils/fetch-aemo-data.ts
|
|
236
|
+
import fs4 from "fs/promises";
|
|
237
|
+
import path3 from "path";
|
|
238
|
+
|
|
239
|
+
// src/api/api.ts
|
|
240
|
+
import ky from "ky";
|
|
241
|
+
|
|
242
|
+
// src/schemas/env-schema.ts
|
|
243
|
+
import { z as z3 } from "zod";
|
|
244
|
+
var envSchema = z3.object({
|
|
245
|
+
// wsl sets this environment variable on all distros that i've checked
|
|
246
|
+
isWSL: z3.boolean().optional().default(() => Boolean(process.env.WSL_DISTRO_NAME)),
|
|
247
|
+
apiURL: z3.string().optional().default(() => process.env.PD4CASTR_API_URL ?? DEFAULT_API_URL),
|
|
248
|
+
wslNetworkInterface: z3.string().optional().default(
|
|
249
|
+
() => process.env.PD4CASTR_WSL_NETWORK_INTERFACE ?? WSL_NETWORK_INTERFACE_DEFAULT
|
|
250
|
+
)
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
// src/utils/get-env.ts
|
|
254
|
+
function getEnv() {
|
|
255
|
+
return envSchema.parse(process.env);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// src/api/api.ts
|
|
259
|
+
var api = ky.create({
|
|
260
|
+
prefixUrl: getEnv().apiURL
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
// src/api/query-data-fetcher.ts
|
|
264
|
+
async function queryDataFetcher(querySQL, authCtx) {
|
|
265
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
266
|
+
const payload2 = { query: querySQL, type: "AEMO_MMS" };
|
|
267
|
+
const result = await api.post("data-fetcher/query", { json: payload2, headers }).json();
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// src/commands/fetch/utils/fetch-aemo-data.ts
|
|
272
|
+
async function fetchAEMOData(dataFetcher, authCtx, ctx) {
|
|
273
|
+
const queryPath = path3.resolve(
|
|
274
|
+
ctx.projectRoot,
|
|
275
|
+
dataFetcher.config.fetchQuery
|
|
276
|
+
);
|
|
277
|
+
const querySQL = await fs4.readFile(queryPath, "utf8");
|
|
278
|
+
const result = await queryDataFetcher(querySQL, authCtx);
|
|
279
|
+
return result;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// src/commands/fetch/utils/get-fetcher.ts
|
|
283
|
+
var DATA_FETCHER_FNS = {
|
|
284
|
+
AEMO_MMS: fetchAEMOData
|
|
128
285
|
};
|
|
286
|
+
function getFetcher(type) {
|
|
287
|
+
const fetcher = DATA_FETCHER_FNS[type];
|
|
288
|
+
if (!fetcher) {
|
|
289
|
+
throw new Error(`Unsupported data fetcher type: ${type}`);
|
|
290
|
+
}
|
|
291
|
+
return fetcher;
|
|
292
|
+
}
|
|
129
293
|
|
|
130
|
-
// src/
|
|
131
|
-
|
|
132
|
-
|
|
294
|
+
// src/commands/fetch/utils/write-test-data.ts
|
|
295
|
+
import fs5 from "fs/promises";
|
|
296
|
+
import path4 from "path";
|
|
297
|
+
|
|
298
|
+
// src/utils/get-input-filename.ts
|
|
299
|
+
function getInputFilename(modelInput) {
|
|
300
|
+
return `${modelInput.key}.${modelInput.targetFileFormat}`;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// src/commands/fetch/utils/write-test-data.ts
|
|
304
|
+
async function writeTestData(inputData, modelInput, inputDataDir, ctx) {
|
|
305
|
+
const inputDir = path4.resolve(ctx.projectRoot, inputDataDir);
|
|
306
|
+
await fs5.mkdir(inputDir, { recursive: true });
|
|
307
|
+
const inputFilename = getInputFilename(modelInput);
|
|
308
|
+
const inputPath = path4.resolve(inputDir, inputFilename);
|
|
309
|
+
await fs5.writeFile(inputPath, JSON.stringify(inputData, void 0, 2));
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// src/commands/fetch/handle-action.ts
|
|
313
|
+
var FETCHABLE_DATA_FETCHER_TYPES = /* @__PURE__ */ new Set(["AEMO_MMS"]);
|
|
314
|
+
async function handleAction(options) {
|
|
315
|
+
const spinner = ora("Starting data fetch...").start();
|
|
316
|
+
try {
|
|
317
|
+
const authCtx = await getAuth();
|
|
318
|
+
const ctx = await loadProjectContext();
|
|
319
|
+
for (const input2 of ctx.config.inputs) {
|
|
320
|
+
if (!input2.fetcher) {
|
|
321
|
+
spinner.warn(`\`${input2.key}\` - no data fetcher defined, skipping`);
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
if (!FETCHABLE_DATA_FETCHER_TYPES.has(input2.fetcher.type)) {
|
|
325
|
+
spinner.warn(
|
|
326
|
+
`\`${input2.key}\` (${input2.fetcher.type}) - unsupported, skipping`
|
|
327
|
+
);
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
spinner.start(`\`${input2.key}\` (${input2.fetcher.type}) - fetching...`);
|
|
331
|
+
const fetchData = getFetcher(input2.fetcher.type);
|
|
332
|
+
const output = await fetchData(input2.fetcher, authCtx, ctx);
|
|
333
|
+
await writeTestData(output, input2, options.inputDir, ctx);
|
|
334
|
+
spinner.succeed(`\`${input2.key}\` (${input2.fetcher.type}) - fetched`);
|
|
335
|
+
}
|
|
336
|
+
const inputPath = path5.resolve(ctx.projectRoot, options.inputDir);
|
|
337
|
+
const link = createLink("Click here", `file://${inputPath}`);
|
|
338
|
+
console.log(`
|
|
339
|
+
${link} to view fetched data
|
|
340
|
+
`);
|
|
341
|
+
} catch (error) {
|
|
342
|
+
if (error instanceof ZodError2) {
|
|
343
|
+
spinner.fail("Config validation failed");
|
|
344
|
+
logZodIssues(error);
|
|
345
|
+
} else if (error instanceof HTTPError) {
|
|
346
|
+
const errorResponse = await error.response.json();
|
|
347
|
+
spinner.fail(formatNestErrorMessage(errorResponse));
|
|
348
|
+
} else if (error instanceof Error) {
|
|
349
|
+
spinner.fail(error.message);
|
|
350
|
+
if (error.cause) {
|
|
351
|
+
console.error(error.cause);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
process.exit(1);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// src/commands/fetch/index.ts
|
|
359
|
+
function registerFetchCommand(program2) {
|
|
360
|
+
program2.command("fetch").description("Fetches test data from configured data fetchers.").option(
|
|
361
|
+
"-i, --input-dir <path>",
|
|
362
|
+
"The input test data directory",
|
|
363
|
+
TEST_INPUT_DATA_DIR
|
|
364
|
+
).action(handleAction);
|
|
365
|
+
}
|
|
133
366
|
|
|
134
367
|
// src/commands/init/handle-action.ts
|
|
135
|
-
import
|
|
368
|
+
import path6 from "path";
|
|
136
369
|
import * as inquirer from "@inquirer/prompts";
|
|
370
|
+
import ora2 from "ora";
|
|
137
371
|
import tiged from "tiged";
|
|
138
|
-
import ora from "ora";
|
|
139
372
|
|
|
140
373
|
// src/commands/init/constants.ts
|
|
141
374
|
var templates = {
|
|
@@ -156,17 +389,6 @@ function getTemplatePath(template) {
|
|
|
156
389
|
return `https://github.com/${template.repo}/${template.path}`;
|
|
157
390
|
}
|
|
158
391
|
|
|
159
|
-
// src/utils/is-existing-path.ts
|
|
160
|
-
import fs from "fs/promises";
|
|
161
|
-
async function isExistingPath(path12) {
|
|
162
|
-
try {
|
|
163
|
-
await fs.access(path12);
|
|
164
|
-
return true;
|
|
165
|
-
} catch {
|
|
166
|
-
return false;
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
392
|
// src/commands/init/utils/validate-name.ts
|
|
171
393
|
async function validateName(value) {
|
|
172
394
|
const exists = await isExistingPath(`./${value}`);
|
|
@@ -177,7 +399,7 @@ async function validateName(value) {
|
|
|
177
399
|
}
|
|
178
400
|
|
|
179
401
|
// src/commands/init/handle-action.ts
|
|
180
|
-
async function
|
|
402
|
+
async function handleAction2() {
|
|
181
403
|
const projectName = await inquirer.input({
|
|
182
404
|
message: "Name your new model project",
|
|
183
405
|
default: "my-model",
|
|
@@ -190,14 +412,14 @@ async function handleAction() {
|
|
|
190
412
|
value: template2.name
|
|
191
413
|
}))
|
|
192
414
|
});
|
|
193
|
-
const spinner =
|
|
415
|
+
const spinner = ora2("Fetching template...").start();
|
|
194
416
|
try {
|
|
195
417
|
await fetchTemplate(template, projectName);
|
|
196
418
|
spinner.succeed("Template fetched successfully");
|
|
197
419
|
} catch (error) {
|
|
198
420
|
spinner.fail("Error fetching template");
|
|
199
|
-
if (error instanceof Error) {
|
|
200
|
-
console.error(error.
|
|
421
|
+
if (error instanceof Error && error.cause) {
|
|
422
|
+
console.error(error.cause);
|
|
201
423
|
}
|
|
202
424
|
process.exit(1);
|
|
203
425
|
}
|
|
@@ -208,104 +430,39 @@ async function fetchTemplate(template, projectName) {
|
|
|
208
430
|
disableCache: true,
|
|
209
431
|
force: true
|
|
210
432
|
});
|
|
211
|
-
const destination =
|
|
433
|
+
const destination = path6.join(process.cwd(), projectName);
|
|
212
434
|
await fetcher.clone(destination);
|
|
213
435
|
}
|
|
214
436
|
|
|
215
437
|
// src/commands/init/index.ts
|
|
216
438
|
function registerInitCommand(program2) {
|
|
217
|
-
program2.command("init").description("Initialize a new model using a template.").action(
|
|
439
|
+
program2.command("init").description("Initialize a new model using a template.").action(handleAction2);
|
|
218
440
|
}
|
|
219
441
|
|
|
220
442
|
// src/commands/login/handle-action.ts
|
|
221
|
-
import
|
|
222
|
-
import { ZodError } from "zod";
|
|
223
|
-
|
|
224
|
-
// src/config/load-global-config.ts
|
|
225
|
-
import fs2 from "fs/promises";
|
|
226
|
-
import path2 from "path";
|
|
227
|
-
import os from "os";
|
|
228
|
-
|
|
229
|
-
// src/constants.ts
|
|
230
|
-
var AUTH0_DOMAIN = "pdview.au.auth0.com";
|
|
231
|
-
var AUTH0_CLIENT_ID = "Q5tQNF57cQlVXnVsqnU0hhgy92rVb03W";
|
|
232
|
-
var AUTH0_AUDIENCE = "https://api.pd4castr.com.au";
|
|
233
|
-
var GLOBAL_CONFIG_FILE = ".pd4castr";
|
|
234
|
-
var PROJECT_CONFIG_FILE = ".pd4castrrc.json";
|
|
235
|
-
var API_URL = "https://pd4castr-api.pipelabs.app";
|
|
236
|
-
var TEST_INPUT_DATA_DIR = "test_input";
|
|
237
|
-
var TEST_OUTPUT_DATA_DIR = "test_output";
|
|
238
|
-
var TEST_OUTPUT_FILENAME = `output-${Date.now()}.json`;
|
|
239
|
-
|
|
240
|
-
// src/schemas/global-config-schema.ts
|
|
241
|
-
import { z } from "zod";
|
|
242
|
-
var globalConfigSchema = z.object({
|
|
243
|
-
accessToken: z.string().optional(),
|
|
244
|
-
accessTokenExpiresAt: z.number().optional()
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
// src/config/load-global-config.ts
|
|
248
|
-
async function loadGlobalConfig() {
|
|
249
|
-
const configPath = path2.join(os.homedir(), GLOBAL_CONFIG_FILE);
|
|
250
|
-
const configExists = await isExistingPath(configPath);
|
|
251
|
-
if (!configExists) {
|
|
252
|
-
await fs2.writeFile(configPath, JSON.stringify({}));
|
|
253
|
-
return {};
|
|
254
|
-
}
|
|
255
|
-
try {
|
|
256
|
-
const configFileContents = await fs2.readFile(configPath, "utf8");
|
|
257
|
-
const config = JSON.parse(configFileContents);
|
|
258
|
-
return globalConfigSchema.parse(config);
|
|
259
|
-
} catch {
|
|
260
|
-
return {};
|
|
261
|
-
}
|
|
262
|
-
}
|
|
443
|
+
import ora3 from "ora";
|
|
444
|
+
import { ZodError as ZodError3 } from "zod";
|
|
263
445
|
|
|
264
446
|
// src/config/update-global-config.ts
|
|
265
|
-
import
|
|
266
|
-
import path3 from "path";
|
|
447
|
+
import fs6 from "fs/promises";
|
|
267
448
|
import os2 from "os";
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
for (const issue of error.issues) {
|
|
276
|
-
console.log(` \u2718 ${issue.path.join(".")} - ${issue.message}`);
|
|
277
|
-
}
|
|
449
|
+
import path7 from "path";
|
|
450
|
+
import { produce } from "immer";
|
|
451
|
+
async function updateGlobalConfig(updateFn) {
|
|
452
|
+
const globalConfig = await loadGlobalConfig();
|
|
453
|
+
const updatedConfig = produce(globalConfig, updateFn);
|
|
454
|
+
const configPath = path7.join(os2.homedir(), GLOBAL_CONFIG_FILE);
|
|
455
|
+
await fs6.writeFile(configPath, JSON.stringify(updatedConfig, void 0, 2));
|
|
278
456
|
}
|
|
279
457
|
|
|
280
|
-
// src/commands/login/utils/
|
|
281
|
-
|
|
282
|
-
const isTokenExpired = config.accessTokenExpiresAt && config.accessTokenExpiresAt <= Date.now();
|
|
283
|
-
return Boolean(config.accessToken) && !isTokenExpired;
|
|
284
|
-
}
|
|
458
|
+
// src/commands/login/utils/complete-auth-flow.ts
|
|
459
|
+
import { HTTPError as HTTPError2 } from "ky";
|
|
285
460
|
|
|
286
461
|
// src/commands/login/auth0-api.ts
|
|
287
|
-
import
|
|
288
|
-
var auth0API =
|
|
289
|
-
|
|
290
|
-
// src/commands/login/utils/start-auth-flow.ts
|
|
291
|
-
var payload = {
|
|
292
|
-
client_id: AUTH0_CLIENT_ID,
|
|
293
|
-
audience: AUTH0_AUDIENCE,
|
|
294
|
-
scope: "openid email"
|
|
295
|
-
};
|
|
296
|
-
async function startAuthFlow() {
|
|
297
|
-
const codeResponse = await auth0API.post("oauth/device/code", { json: payload }).json();
|
|
298
|
-
const authContext = {
|
|
299
|
-
deviceCode: codeResponse.device_code,
|
|
300
|
-
verificationURL: codeResponse.verification_uri_complete,
|
|
301
|
-
userCode: codeResponse.user_code,
|
|
302
|
-
checkInterval: codeResponse.interval
|
|
303
|
-
};
|
|
304
|
-
return authContext;
|
|
305
|
-
}
|
|
462
|
+
import ky2 from "ky";
|
|
463
|
+
var auth0API = ky2.create({ prefixUrl: `https://${AUTH0_DOMAIN}` });
|
|
306
464
|
|
|
307
465
|
// src/commands/login/utils/complete-auth-flow.ts
|
|
308
|
-
import { HTTPError } from "ky";
|
|
309
466
|
var FAILED_AUTH_ERRORS = /* @__PURE__ */ new Set(["expired_token", "access_denied"]);
|
|
310
467
|
async function completeAuthFlow(authCtx) {
|
|
311
468
|
const payload2 = {
|
|
@@ -322,7 +479,7 @@ async function completeAuthFlow(authCtx) {
|
|
|
322
479
|
};
|
|
323
480
|
return authPayload;
|
|
324
481
|
} catch (error) {
|
|
325
|
-
if (!(error instanceof
|
|
482
|
+
if (!(error instanceof HTTPError2)) {
|
|
326
483
|
throw error;
|
|
327
484
|
}
|
|
328
485
|
const errorResponse = await error.response.json();
|
|
@@ -341,9 +498,26 @@ async function completeAuthFlow(authCtx) {
|
|
|
341
498
|
return fetchAuthResponse();
|
|
342
499
|
}
|
|
343
500
|
|
|
501
|
+
// src/commands/login/utils/start-auth-flow.ts
|
|
502
|
+
var payload = {
|
|
503
|
+
client_id: AUTH0_CLIENT_ID,
|
|
504
|
+
audience: AUTH0_AUDIENCE,
|
|
505
|
+
scope: "openid email"
|
|
506
|
+
};
|
|
507
|
+
async function startAuthFlow() {
|
|
508
|
+
const codeResponse = await auth0API.post("oauth/device/code", { json: payload }).json();
|
|
509
|
+
const authContext = {
|
|
510
|
+
deviceCode: codeResponse.device_code,
|
|
511
|
+
verificationURL: codeResponse.verification_uri_complete,
|
|
512
|
+
userCode: codeResponse.user_code,
|
|
513
|
+
checkInterval: codeResponse.interval
|
|
514
|
+
};
|
|
515
|
+
return authContext;
|
|
516
|
+
}
|
|
517
|
+
|
|
344
518
|
// src/commands/login/handle-action.ts
|
|
345
|
-
async function
|
|
346
|
-
const spinner =
|
|
519
|
+
async function handleAction3() {
|
|
520
|
+
const spinner = ora3("Logging in to the pd4castr API...").start();
|
|
347
521
|
try {
|
|
348
522
|
const globalConfig = await loadGlobalConfig();
|
|
349
523
|
if (isAuthed(globalConfig)) {
|
|
@@ -360,19 +534,20 @@ async function handleAction2() {
|
|
|
360
534
|
`);
|
|
361
535
|
spinner.start("Waiting for login to complete...");
|
|
362
536
|
const authPayload = await completeAuthFlow(authCtx);
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
};
|
|
368
|
-
await updateGlobalConfig(updatedGlobalConfig);
|
|
537
|
+
await updateGlobalConfig((config) => {
|
|
538
|
+
config.accessToken = authPayload.accessToken;
|
|
539
|
+
config.accessTokenExpiresAt = authPayload.expiresAt;
|
|
540
|
+
});
|
|
369
541
|
spinner.succeed("Successfully logged in to the pd4castr API");
|
|
370
542
|
} catch (error) {
|
|
371
|
-
if (error instanceof
|
|
543
|
+
if (error instanceof ZodError3) {
|
|
372
544
|
spinner.fail("Config validation failed");
|
|
373
545
|
logZodIssues(error);
|
|
374
546
|
} else if (error instanceof Error) {
|
|
375
547
|
spinner.fail(error.message);
|
|
548
|
+
if (error.cause) {
|
|
549
|
+
console.error(error.cause);
|
|
550
|
+
}
|
|
376
551
|
}
|
|
377
552
|
process.exit(1);
|
|
378
553
|
}
|
|
@@ -380,90 +555,95 @@ async function handleAction2() {
|
|
|
380
555
|
|
|
381
556
|
// src/commands/login/index.ts
|
|
382
557
|
function registerLoginCommand(program2) {
|
|
383
|
-
program2.command("login").description("Logs in to the pd4castr API.").action(
|
|
558
|
+
program2.command("login").description("Logs in to the pd4castr API.").action(handleAction3);
|
|
384
559
|
}
|
|
385
560
|
|
|
386
|
-
// src/commands/
|
|
387
|
-
import
|
|
388
|
-
import
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
561
|
+
// src/commands/logout/handle-action.ts
|
|
562
|
+
import ora4 from "ora";
|
|
563
|
+
import { ZodError as ZodError4 } from "zod";
|
|
564
|
+
async function handleAction4() {
|
|
565
|
+
const spinner = ora4("Logging out of the pd4castr API...").start();
|
|
566
|
+
try {
|
|
567
|
+
const globalConfig = await loadGlobalConfig();
|
|
568
|
+
if (!isAuthed(globalConfig)) {
|
|
569
|
+
spinner.succeed("Already logged out!");
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
await updateGlobalConfig((config) => {
|
|
573
|
+
config.accessToken = null;
|
|
574
|
+
config.accessTokenExpiresAt = null;
|
|
575
|
+
});
|
|
576
|
+
spinner.succeed("Successfully logged out of the pd4castr API");
|
|
577
|
+
} catch (error) {
|
|
578
|
+
if (error instanceof ZodError4) {
|
|
579
|
+
spinner.fail("Config validation failed");
|
|
580
|
+
logZodIssues(error);
|
|
581
|
+
} else if (error instanceof Error) {
|
|
582
|
+
spinner.fail(error.message);
|
|
583
|
+
if (error.cause) {
|
|
584
|
+
console.error(error.cause);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
process.exit(1);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
393
590
|
|
|
394
|
-
// src/
|
|
395
|
-
|
|
396
|
-
|
|
591
|
+
// src/commands/logout/index.ts
|
|
592
|
+
function registerLogoutCommand(program2) {
|
|
593
|
+
program2.command("logout").description("Logs out of the pd4castr API.").action(handleAction4);
|
|
594
|
+
}
|
|
397
595
|
|
|
398
|
-
// src/
|
|
399
|
-
import
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
config: z2.object({
|
|
404
|
-
checkQuery: z2.string(),
|
|
405
|
-
fetchQuery: z2.string()
|
|
406
|
-
})
|
|
407
|
-
});
|
|
408
|
-
var dataFetcherSchema = z2.discriminatedUnion("type", [aemoDataFetcherSchema]);
|
|
409
|
-
var modelInputSchema = z2.object({
|
|
410
|
-
key: z2.string(),
|
|
411
|
-
inputSource: z2.string().optional(),
|
|
412
|
-
trigger: z2.enum(["WAIT_FOR_LATEST_FILE", "USE_MOST_RECENT_FILE"]),
|
|
413
|
-
fetcher: dataFetcherSchema.optional()
|
|
414
|
-
});
|
|
415
|
-
var modelOutputSchema = z2.object({
|
|
416
|
-
name: z2.string(),
|
|
417
|
-
seriesKey: z2.boolean(),
|
|
418
|
-
colour: z2.string().regex(/^#[0-9A-Fa-f]{6}$/).optional()
|
|
419
|
-
});
|
|
420
|
-
var projectConfigSchema = z2.object({
|
|
421
|
-
$$id: z2.string().nullable(),
|
|
422
|
-
$$modelGroupID: z2.string().nullable(),
|
|
423
|
-
$$revision: z2.number().int().min(0).default(0),
|
|
424
|
-
$$dockerImage: z2.string().nullable(),
|
|
425
|
-
name: z2.string(),
|
|
426
|
-
metadata: z2.record(z2.string(), z2.any()).optional(),
|
|
427
|
-
forecastVariable: z2.enum(["PRICE"]),
|
|
428
|
-
timeHorizon: z2.enum(["ACTUAL", "DAY_AHEAD", "WEEK_AHEAD"]),
|
|
429
|
-
inputs: z2.array(modelInputSchema),
|
|
430
|
-
outputs: z2.array(modelOutputSchema)
|
|
431
|
-
});
|
|
596
|
+
// src/commands/publish/handle-action.ts
|
|
597
|
+
import express2 from "express";
|
|
598
|
+
import { HTTPError as HTTPError3 } from "ky";
|
|
599
|
+
import ora5 from "ora";
|
|
600
|
+
import { ZodError as ZodError5 } from "zod";
|
|
432
601
|
|
|
433
|
-
// src/
|
|
434
|
-
function
|
|
435
|
-
return
|
|
602
|
+
// src/utils/start-web-server.ts
|
|
603
|
+
async function startWebServer(app, port) {
|
|
604
|
+
return new Promise((resolve) => {
|
|
605
|
+
const server = app.listen(port, () => {
|
|
606
|
+
resolve({
|
|
607
|
+
server,
|
|
608
|
+
[Symbol.dispose]: () => server.close()
|
|
609
|
+
});
|
|
610
|
+
});
|
|
611
|
+
});
|
|
436
612
|
}
|
|
437
613
|
|
|
438
|
-
// src/
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
}
|
|
448
|
-
const config = parseProjectConfig(result.config);
|
|
449
|
-
const projectRoot = path4.dirname(result.filepath);
|
|
450
|
-
return {
|
|
451
|
-
config,
|
|
452
|
-
projectRoot
|
|
453
|
-
};
|
|
614
|
+
// src/commands/publish/handle-create-model-flow.ts
|
|
615
|
+
import * as inquirer2 from "@inquirer/prompts";
|
|
616
|
+
import chalk2 from "chalk";
|
|
617
|
+
|
|
618
|
+
// src/api/create-model.ts
|
|
619
|
+
async function createModel(config, authCtx) {
|
|
620
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
621
|
+
const result = await api.post("model", { headers, json: config }).json();
|
|
622
|
+
return result;
|
|
454
623
|
}
|
|
455
624
|
|
|
456
|
-
// src/
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
625
|
+
// src/api/get-registry-push-credentials.ts
|
|
626
|
+
async function getRegistryPushCredentials(modelID, authCtx) {
|
|
627
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
628
|
+
const searchParams = new URLSearchParams(`modelId=${modelID}`);
|
|
629
|
+
const result = await api.get("registry/push-credentials", { headers, searchParams }).json();
|
|
630
|
+
return result;
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// src/config/update-project-config.ts
|
|
634
|
+
import fs7 from "fs/promises";
|
|
635
|
+
import path8 from "path";
|
|
636
|
+
import { produce as produce2 } from "immer";
|
|
637
|
+
async function updateProjectConfig(updateFn) {
|
|
638
|
+
const projectConfig = await loadProjectContext();
|
|
639
|
+
const updatedConfig = produce2(projectConfig.config, updateFn);
|
|
640
|
+
await fs7.writeFile(
|
|
641
|
+
path8.join(projectConfig.projectRoot, PROJECT_CONFIG_FILE),
|
|
642
|
+
JSON.stringify(updatedConfig, void 0, 2)
|
|
643
|
+
);
|
|
464
644
|
}
|
|
465
645
|
|
|
466
|
-
// src/
|
|
646
|
+
// src/docker/build-docker-image.ts
|
|
467
647
|
import { execa } from "execa";
|
|
468
648
|
async function buildDockerImage(dockerImage, ctx) {
|
|
469
649
|
try {
|
|
@@ -476,15 +656,224 @@ async function buildDockerImage(dockerImage, ctx) {
|
|
|
476
656
|
}
|
|
477
657
|
}
|
|
478
658
|
|
|
479
|
-
// src/
|
|
480
|
-
import
|
|
659
|
+
// src/docker/login-to-docker-registry.ts
|
|
660
|
+
import { execa as execa2 } from "execa";
|
|
661
|
+
async function loginToDockerRegistry(authConfig) {
|
|
662
|
+
try {
|
|
663
|
+
await execa2(
|
|
664
|
+
"docker",
|
|
665
|
+
[
|
|
666
|
+
"login",
|
|
667
|
+
authConfig.registry,
|
|
668
|
+
"--username",
|
|
669
|
+
authConfig.username,
|
|
670
|
+
"--password-stdin"
|
|
671
|
+
],
|
|
672
|
+
{ input: authConfig.password }
|
|
673
|
+
);
|
|
674
|
+
} catch (error) {
|
|
675
|
+
throw new Error("Failed to login to docker registry", { cause: error });
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// src/docker/push-docker-image.ts
|
|
680
|
+
import { execa as execa3 } from "execa";
|
|
681
|
+
async function pushDockerImage(dockerImage, pushRef) {
|
|
682
|
+
try {
|
|
683
|
+
await execa3("docker", ["tag", dockerImage, pushRef]);
|
|
684
|
+
await execa3("docker", ["push", pushRef]);
|
|
685
|
+
} catch (error) {
|
|
686
|
+
throw new Error("Failed to push docker image", { cause: error });
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
// src/utils/get-docker-image.ts
|
|
691
|
+
import slugify from "slugify";
|
|
692
|
+
function getDockerImage(ctx) {
|
|
693
|
+
const sluggedName = slugify(ctx.config.name, { lower: true });
|
|
694
|
+
const dockerImage = `pd4castr/${sluggedName}-local:${Date.now()}`;
|
|
695
|
+
return dockerImage;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// src/utils/get-model-config-from-project-config.ts
|
|
699
|
+
import fs8 from "fs/promises";
|
|
700
|
+
import path9 from "path";
|
|
701
|
+
async function getModelConfigFromProjectConfig(ctx) {
|
|
702
|
+
const inputs = await getInputsWithInlinedSQL(ctx);
|
|
703
|
+
const { $$id, $$modelGroupID, $$revision, $$dockerImage, ...config } = ctx.config;
|
|
704
|
+
return {
|
|
705
|
+
...config,
|
|
706
|
+
id: $$id,
|
|
707
|
+
modelGroupId: $$modelGroupID,
|
|
708
|
+
revision: $$revision ?? 0,
|
|
709
|
+
dockerImage: $$dockerImage,
|
|
710
|
+
inputs
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
var FETCHERS_WITH_SQL = /* @__PURE__ */ new Set(["AEMO_MMS"]);
|
|
714
|
+
async function getInputsWithInlinedSQL(ctx) {
|
|
715
|
+
const inputsWithSQL = [];
|
|
716
|
+
for (const input2 of ctx.config.inputs) {
|
|
717
|
+
if (!input2.fetcher || !FETCHERS_WITH_SQL.has(input2.fetcher.type)) {
|
|
718
|
+
inputsWithSQL.push(input2);
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
const fetchQueryPath = path9.resolve(
|
|
722
|
+
ctx.projectRoot,
|
|
723
|
+
input2.fetcher.config.fetchQuery
|
|
724
|
+
);
|
|
725
|
+
const checkQueryPath = path9.resolve(
|
|
726
|
+
ctx.projectRoot,
|
|
727
|
+
input2.fetcher.config.checkQuery
|
|
728
|
+
);
|
|
729
|
+
const [fetchQuerySQL, checkQuerySQL] = await Promise.all([
|
|
730
|
+
fs8.readFile(fetchQueryPath, "utf8"),
|
|
731
|
+
fs8.readFile(checkQueryPath, "utf8")
|
|
732
|
+
]);
|
|
733
|
+
const inputWithSQL = {
|
|
734
|
+
...input2,
|
|
735
|
+
fetcher: {
|
|
736
|
+
...input2.fetcher,
|
|
737
|
+
config: {
|
|
738
|
+
...input2.fetcher.config,
|
|
739
|
+
fetchQuery: fetchQuerySQL,
|
|
740
|
+
checkQuery: checkQuerySQL
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
inputsWithSQL.push(inputWithSQL);
|
|
745
|
+
}
|
|
746
|
+
return inputsWithSQL;
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
// src/utils/log-empty-line.ts
|
|
750
|
+
function logEmptyLine() {
|
|
751
|
+
console.log("");
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// src/commands/publish/utils/get-model-summary-lines.ts
|
|
755
|
+
import chalk from "chalk";
|
|
756
|
+
function getModelSummaryLines(ctx) {
|
|
757
|
+
return [
|
|
758
|
+
` ${chalk.bold("Model name:")} ${ctx.config.name}`,
|
|
759
|
+
` ${chalk.bold("Revision:")} ${ctx.config.$$revision}`,
|
|
760
|
+
` ${chalk.bold("Forecast variable:")} ${ctx.config.forecastVariable}`,
|
|
761
|
+
` ${chalk.bold("Time horizon:")} ${ctx.config.timeHorizon}`,
|
|
762
|
+
` ${chalk.bold("Inputs:")}`,
|
|
763
|
+
...ctx.config.inputs.map(
|
|
764
|
+
(input2) => ` \u2022 ${input2.key} - ${getInputType(input2)}`
|
|
765
|
+
),
|
|
766
|
+
` ${chalk.bold("Outputs:")}`,
|
|
767
|
+
...ctx.config.outputs.map((output) => ` \u2022 ${output.name} - ${output.type}`),
|
|
768
|
+
""
|
|
769
|
+
];
|
|
770
|
+
}
|
|
771
|
+
function getInputType(input2) {
|
|
772
|
+
if (input2.fetcher) {
|
|
773
|
+
return input2.fetcher.type;
|
|
774
|
+
}
|
|
775
|
+
return "static";
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// src/docker/run-model-container.ts
|
|
779
|
+
import os3 from "os";
|
|
780
|
+
import { execa as execa4 } from "execa";
|
|
781
|
+
|
|
782
|
+
// src/docker/utils/get-input-env.ts
|
|
783
|
+
function getInputEnv(modelInput, webserverURL) {
|
|
784
|
+
const variableName = modelInput.key.toUpperCase();
|
|
785
|
+
const filename = getInputFilename(modelInput);
|
|
786
|
+
const inputFileURL = `${webserverURL}/input/${filename}`;
|
|
787
|
+
return `INPUT_${variableName}_URL=${inputFileURL}`;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
// src/docker/run-model-container.ts
|
|
791
|
+
async function runModelContainer(dockerImage, webserverPort, ctx) {
|
|
792
|
+
const env = getEnv();
|
|
793
|
+
const webserverHostname = env.isWSL ? DOCKER_HOSTNAME_WSL : DOCKER_HOSTNAME_DEFAULT;
|
|
794
|
+
const webserverURL = `http://${webserverHostname}:${webserverPort}`;
|
|
795
|
+
const inputEnvs = ctx.config.inputs.map(
|
|
796
|
+
(input2) => getInputEnv(input2, webserverURL)
|
|
797
|
+
);
|
|
798
|
+
const outputEnv = `OUTPUT_URL=${webserverURL}/output`;
|
|
799
|
+
const envs = [...inputEnvs, outputEnv];
|
|
800
|
+
try {
|
|
801
|
+
const args = [
|
|
802
|
+
"run",
|
|
803
|
+
"--rm",
|
|
804
|
+
...envs.flatMap((env2) => ["--env", env2]),
|
|
805
|
+
dockerImage
|
|
806
|
+
];
|
|
807
|
+
if (env.isWSL) {
|
|
808
|
+
const ip = getWSLMachineIP();
|
|
809
|
+
args.push("--add-host", `${DOCKER_HOSTNAME_WSL}:${ip}`);
|
|
810
|
+
}
|
|
811
|
+
await execa4("docker", args, {
|
|
812
|
+
cwd: ctx.projectRoot,
|
|
813
|
+
stdio: "pipe"
|
|
814
|
+
});
|
|
815
|
+
} catch (error) {
|
|
816
|
+
throw new Error("Failed to run model container", { cause: error });
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
function getWSLMachineIP() {
|
|
820
|
+
const env = getEnv();
|
|
821
|
+
const interfaces = os3.networkInterfaces();
|
|
822
|
+
const interfaceInfo = interfaces[env.wslNetworkInterface]?.[0];
|
|
823
|
+
if (!interfaceInfo) {
|
|
824
|
+
throw new Error(
|
|
825
|
+
`WSL machine IP not found for interface \`${env.wslNetworkInterface}\``
|
|
826
|
+
);
|
|
827
|
+
}
|
|
828
|
+
return interfaceInfo.address;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// src/model-io-checks/setup-model-io-checks.ts
|
|
832
|
+
import path12 from "path";
|
|
833
|
+
import express from "express";
|
|
834
|
+
|
|
835
|
+
// src/model-io-checks/model-io-checks.ts
|
|
836
|
+
var ModelIOChecks = class {
|
|
837
|
+
inputsToDownload;
|
|
838
|
+
outputUploaded;
|
|
839
|
+
constructor(data) {
|
|
840
|
+
this.inputsToDownload = {};
|
|
841
|
+
this.outputUploaded = false;
|
|
842
|
+
for (const file of data.inputFiles) {
|
|
843
|
+
this.inputsToDownload[file] = false;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
trackInputHandled(filename) {
|
|
847
|
+
if (this.inputsToDownload[filename] !== void 0) {
|
|
848
|
+
this.inputsToDownload[filename] = true;
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
trackOutputHandled() {
|
|
852
|
+
this.outputUploaded = true;
|
|
853
|
+
}
|
|
854
|
+
isOutputHandled() {
|
|
855
|
+
return this.outputUploaded;
|
|
856
|
+
}
|
|
857
|
+
isInputHandled(filename) {
|
|
858
|
+
return this.inputsToDownload[filename];
|
|
859
|
+
}
|
|
860
|
+
isInputsHandled() {
|
|
861
|
+
return Object.values(this.inputsToDownload).every(Boolean);
|
|
862
|
+
}
|
|
863
|
+
isValidInput(filename) {
|
|
864
|
+
return this.inputsToDownload[filename] !== void 0;
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
|
|
868
|
+
// src/model-io-checks/utils/create-input-handler.ts
|
|
869
|
+
import path10 from "path";
|
|
481
870
|
function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
|
|
482
871
|
return (req, res) => {
|
|
483
872
|
if (!modelIOChecks.isValidInput(req.params.filename)) {
|
|
484
873
|
return res.status(404).json({ error: "File not found" });
|
|
485
874
|
}
|
|
486
875
|
modelIOChecks.trackInputHandled(req.params.filename);
|
|
487
|
-
const filePath =
|
|
876
|
+
const filePath = path10.join(
|
|
488
877
|
ctx.projectRoot,
|
|
489
878
|
inputFilesPath,
|
|
490
879
|
req.params.filename
|
|
@@ -493,26 +882,39 @@ function createInputHandler(inputFilesPath, modelIOChecks, ctx) {
|
|
|
493
882
|
};
|
|
494
883
|
}
|
|
495
884
|
|
|
496
|
-
// src/
|
|
497
|
-
import
|
|
498
|
-
import
|
|
885
|
+
// src/model-io-checks/utils/create-output-handler.ts
|
|
886
|
+
import fs9 from "fs/promises";
|
|
887
|
+
import path11 from "path";
|
|
499
888
|
function createOutputHandler(modelIOChecks, ctx) {
|
|
500
889
|
return async (req, res) => {
|
|
501
890
|
modelIOChecks.trackOutputHandled();
|
|
502
|
-
const outputPath =
|
|
503
|
-
await
|
|
504
|
-
const outputFilePath =
|
|
891
|
+
const outputPath = path11.join(ctx.projectRoot, TEST_OUTPUT_DATA_DIR);
|
|
892
|
+
await fs9.mkdir(outputPath, { recursive: true });
|
|
893
|
+
const outputFilePath = path11.join(outputPath, TEST_OUTPUT_FILENAME);
|
|
505
894
|
const outputData = JSON.stringify(req.body, null, 2);
|
|
506
|
-
await
|
|
895
|
+
await fs9.writeFile(outputFilePath, outputData, "utf8");
|
|
507
896
|
return res.status(200).json({ success: true });
|
|
508
897
|
};
|
|
509
898
|
}
|
|
510
899
|
|
|
511
|
-
// src/
|
|
512
|
-
|
|
900
|
+
// src/model-io-checks/setup-model-io-checks.ts
|
|
901
|
+
function setupModelIOChecks(app, inputDir, inputFiles, ctx) {
|
|
902
|
+
const modelIOChecks = new ModelIOChecks({ inputFiles });
|
|
903
|
+
const handleInput = createInputHandler(inputDir, modelIOChecks, ctx);
|
|
904
|
+
const handleOutput = createOutputHandler(modelIOChecks, ctx);
|
|
905
|
+
const inputPath = path12.join(ctx.projectRoot, inputDir);
|
|
906
|
+
app.use(express.json());
|
|
907
|
+
app.use("/data", express.static(inputPath));
|
|
908
|
+
app.get("/input/:filename", handleInput);
|
|
909
|
+
app.put("/output", handleOutput);
|
|
910
|
+
return modelIOChecks;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
// src/utils/check-input-files.ts
|
|
914
|
+
import path13 from "path";
|
|
513
915
|
async function checkInputFiles(inputFiles, inputDataPath, ctx) {
|
|
514
916
|
for (const inputFile of inputFiles) {
|
|
515
|
-
const filePath =
|
|
917
|
+
const filePath = path13.join(ctx.projectRoot, inputDataPath, inputFile);
|
|
516
918
|
const exists = await isExistingPath(filePath);
|
|
517
919
|
if (!exists) {
|
|
518
920
|
throw new Error(
|
|
@@ -522,130 +924,314 @@ async function checkInputFiles(inputFiles, inputDataPath, ctx) {
|
|
|
522
924
|
}
|
|
523
925
|
}
|
|
524
926
|
|
|
525
|
-
// src/utils/get-input-
|
|
526
|
-
function getInputFilename(modelInput) {
|
|
527
|
-
return `${modelInput.key}.json`;
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
// src/commands/test/utils/get-input-files.ts
|
|
927
|
+
// src/utils/get-input-files.ts
|
|
531
928
|
function getInputFiles(config) {
|
|
532
929
|
const inputFiles = config.inputs.map((input2) => getInputFilename(input2));
|
|
533
930
|
return inputFiles;
|
|
534
931
|
}
|
|
535
932
|
|
|
536
|
-
// src/commands/
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
933
|
+
// src/commands/publish/utils/run-model-io-tests.ts
|
|
934
|
+
async function runModelIOTests(dockerImage, options, app, ctx) {
|
|
935
|
+
const inputFiles = getInputFiles(ctx.config);
|
|
936
|
+
await checkInputFiles(inputFiles, options.inputDir, ctx);
|
|
937
|
+
await buildDockerImage(dockerImage, ctx);
|
|
938
|
+
const modelIOChecks = setupModelIOChecks(
|
|
939
|
+
app,
|
|
940
|
+
options.inputDir,
|
|
941
|
+
inputFiles,
|
|
942
|
+
ctx
|
|
943
|
+
);
|
|
944
|
+
await runModelContainer(dockerImage, options.port, ctx);
|
|
945
|
+
if (!modelIOChecks.isInputsHandled() || !modelIOChecks.isOutputHandled()) {
|
|
946
|
+
throw new Error(
|
|
947
|
+
"Model I/O test failed. Please run `pd4castr test` to debug the issue."
|
|
948
|
+
);
|
|
546
949
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// src/commands/publish/handle-create-model-flow.ts
|
|
953
|
+
async function handleCreateModelFlow(options, app, spinner, ctx, authCtx) {
|
|
954
|
+
spinner.info(`You are publishing a ${chalk2.bold("new")} model:
|
|
955
|
+
`);
|
|
956
|
+
getModelSummaryLines(ctx).map((line) => console.log(line));
|
|
957
|
+
const confirm4 = await inquirer2.confirm({
|
|
958
|
+
message: "Do you want to continue?",
|
|
959
|
+
default: false
|
|
960
|
+
});
|
|
961
|
+
logEmptyLine();
|
|
962
|
+
if (!confirm4) {
|
|
963
|
+
throw new Error("Aborted");
|
|
551
964
|
}
|
|
552
|
-
|
|
553
|
-
|
|
965
|
+
const dockerImage = getDockerImage(ctx);
|
|
966
|
+
if (!options.skipChecks) {
|
|
967
|
+
spinner.start("Performing model I/O test...");
|
|
968
|
+
await runModelIOTests(dockerImage, options, app, ctx);
|
|
969
|
+
spinner.succeed("Model I/O test passed");
|
|
554
970
|
}
|
|
555
|
-
|
|
556
|
-
|
|
971
|
+
spinner.start("Publishing model...");
|
|
972
|
+
const modelConfig = await getModelConfigFromProjectConfig(ctx);
|
|
973
|
+
const model = await createModel(modelConfig, authCtx);
|
|
974
|
+
await updateProjectConfig((config) => {
|
|
975
|
+
config.$$id = model.id;
|
|
976
|
+
config.$$modelGroupID = model.modelGroupId;
|
|
977
|
+
config.$$revision = model.revision;
|
|
978
|
+
config.$$dockerImage = model.dockerImage;
|
|
979
|
+
});
|
|
980
|
+
const pushCredentials = await getRegistryPushCredentials(model.id, authCtx);
|
|
981
|
+
await loginToDockerRegistry(pushCredentials);
|
|
982
|
+
await buildDockerImage(dockerImage, ctx);
|
|
983
|
+
await pushDockerImage(dockerImage, pushCredentials.ref);
|
|
984
|
+
spinner.stopAndPersist({
|
|
985
|
+
symbol: "\u{1F680} ",
|
|
986
|
+
prefixText: "\n",
|
|
987
|
+
suffixText: "\n",
|
|
988
|
+
text: chalk2.bold("Model published successfully")
|
|
989
|
+
});
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
// src/commands/publish/handle-update-existing-model-flow.ts
|
|
993
|
+
import * as inquirer5 from "@inquirer/prompts";
|
|
994
|
+
import chalk5 from "chalk";
|
|
995
|
+
|
|
996
|
+
// src/commands/publish/handle-model-revision-create-flow.ts
|
|
997
|
+
import * as inquirer3 from "@inquirer/prompts";
|
|
998
|
+
import chalk3 from "chalk";
|
|
999
|
+
|
|
1000
|
+
// src/commands/publish/utils/validate-local-model-state.ts
|
|
1001
|
+
import invariant2 from "tiny-invariant";
|
|
1002
|
+
|
|
1003
|
+
// src/api/get-model.ts
|
|
1004
|
+
async function getModel(id, authCtx) {
|
|
1005
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
1006
|
+
const result = await api.get(`model/${id}`, { headers }).json();
|
|
1007
|
+
return result;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
// src/commands/publish/utils/validate-local-model-state.ts
|
|
1011
|
+
async function validateLocalModelState(ctx, authCtx) {
|
|
1012
|
+
invariant2(ctx.config.$$id, "model ID is required to fetch published model");
|
|
1013
|
+
const currentModel = await getModel(ctx.config.$$id, authCtx);
|
|
1014
|
+
if (currentModel.revision !== ctx.config.$$revision) {
|
|
1015
|
+
throw new Error(
|
|
1016
|
+
`OUT OF SYNC: Local revision (${ctx.config.$$revision}) does not match the current published revision (${currentModel.revision})`
|
|
1017
|
+
);
|
|
557
1018
|
}
|
|
558
|
-
|
|
559
|
-
|
|
1019
|
+
if (currentModel.modelGroupId !== ctx.config.$$modelGroupID) {
|
|
1020
|
+
throw new Error(
|
|
1021
|
+
`OUT OF SYNC: Local model group ID (${ctx.config.$$modelGroupID}) does not match the current published model group ID (${currentModel.modelGroupId})`
|
|
1022
|
+
);
|
|
560
1023
|
}
|
|
561
|
-
|
|
562
|
-
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
// src/commands/publish/handle-model-revision-create-flow.ts
|
|
1027
|
+
var WARNING_LABEL = chalk3.yellowBright.bold("WARNING!");
|
|
1028
|
+
var CONFIRMATION_MESSAGE = `${WARNING_LABEL} Creating a new revision will preserve existing revisions.
|
|
1029
|
+
Previous revisions will still be available in the pd4castr UI.
|
|
1030
|
+
`;
|
|
1031
|
+
async function handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx) {
|
|
1032
|
+
console.log(CONFIRMATION_MESSAGE);
|
|
1033
|
+
const confirmed = await inquirer3.confirm({
|
|
1034
|
+
message: "Are you sure you want to continue?",
|
|
1035
|
+
default: false
|
|
1036
|
+
});
|
|
1037
|
+
logEmptyLine();
|
|
1038
|
+
if (!confirmed) {
|
|
1039
|
+
throw new Error("Model revision update cancelled");
|
|
563
1040
|
}
|
|
564
|
-
|
|
565
|
-
|
|
1041
|
+
spinner.start("Validating local model state...");
|
|
1042
|
+
await validateLocalModelState(ctx, authCtx);
|
|
1043
|
+
spinner.succeed("Local model is synced with published model");
|
|
1044
|
+
const dockerImage = getDockerImage(ctx);
|
|
1045
|
+
if (!options.skipChecks) {
|
|
1046
|
+
spinner.start("Performing model I/O test...");
|
|
1047
|
+
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1048
|
+
spinner.succeed("Model I/O test passed");
|
|
566
1049
|
}
|
|
567
|
-
|
|
1050
|
+
const modelConfig = await getModelConfigFromProjectConfig(ctx);
|
|
1051
|
+
const model = await createModel(modelConfig, authCtx);
|
|
1052
|
+
await updateProjectConfig((config) => {
|
|
1053
|
+
config.$$id = model.id;
|
|
1054
|
+
config.$$modelGroupID = model.modelGroupId;
|
|
1055
|
+
config.$$revision = model.revision;
|
|
1056
|
+
config.$$dockerImage = model.dockerImage;
|
|
1057
|
+
});
|
|
1058
|
+
const pushCredentials = await getRegistryPushCredentials(model.id, authCtx);
|
|
1059
|
+
await loginToDockerRegistry(pushCredentials);
|
|
1060
|
+
await buildDockerImage(dockerImage, ctx);
|
|
1061
|
+
await pushDockerImage(dockerImage, pushCredentials.ref);
|
|
1062
|
+
spinner.stopAndPersist({
|
|
1063
|
+
symbol: "\u{1F680} ",
|
|
1064
|
+
prefixText: "\n",
|
|
1065
|
+
suffixText: "\n",
|
|
1066
|
+
text: chalk3.bold(
|
|
1067
|
+
`New model revision (r${model.revision}) published successfully`
|
|
1068
|
+
)
|
|
1069
|
+
});
|
|
1070
|
+
}
|
|
568
1071
|
|
|
569
|
-
// src/commands/
|
|
570
|
-
import
|
|
1072
|
+
// src/commands/publish/handle-model-revision-update-flow.ts
|
|
1073
|
+
import * as inquirer4 from "@inquirer/prompts";
|
|
1074
|
+
import chalk4 from "chalk";
|
|
571
1075
|
|
|
572
|
-
// src/
|
|
573
|
-
function
|
|
574
|
-
const
|
|
575
|
-
const
|
|
576
|
-
|
|
577
|
-
return `INPUT_${variableName}_URL=${inputFileURL}`;
|
|
1076
|
+
// src/api/update-model.ts
|
|
1077
|
+
async function updateModel(config, authCtx) {
|
|
1078
|
+
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
1079
|
+
const result = await api.patch(`model/${config.id}`, { headers, json: config }).json();
|
|
1080
|
+
return result;
|
|
578
1081
|
}
|
|
579
1082
|
|
|
580
|
-
// src/commands/
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
1083
|
+
// src/commands/publish/handle-model-revision-update-flow.ts
|
|
1084
|
+
var WARNING_LABEL2 = chalk4.yellowBright.bold("WARNING!");
|
|
1085
|
+
var CONFIRMATION_MESSAGE2 = `${WARNING_LABEL2} Updating a model revision recreates the associated inputs and outputs.
|
|
1086
|
+
Historical data is preserved, but it will no longer be displayed in the pd4castr UI.
|
|
1087
|
+
`;
|
|
1088
|
+
async function handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx) {
|
|
1089
|
+
console.log(CONFIRMATION_MESSAGE2);
|
|
1090
|
+
const confirmed = await inquirer4.confirm({
|
|
1091
|
+
message: "Are you sure you want to continue?",
|
|
1092
|
+
default: false
|
|
1093
|
+
});
|
|
1094
|
+
logEmptyLine();
|
|
1095
|
+
if (!confirmed) {
|
|
1096
|
+
throw new Error("Model revision update cancelled");
|
|
1097
|
+
}
|
|
1098
|
+
spinner.start("Validating local model state...");
|
|
1099
|
+
await validateLocalModelState(ctx, authCtx);
|
|
1100
|
+
spinner.succeed("Local model is synced with published model");
|
|
1101
|
+
const dockerImage = getDockerImage(ctx);
|
|
1102
|
+
if (!options.skipChecks) {
|
|
1103
|
+
spinner.start("Performing model I/O test...");
|
|
1104
|
+
await runModelIOTests(dockerImage, options, app, ctx);
|
|
1105
|
+
spinner.succeed("Model I/O test passed");
|
|
601
1106
|
}
|
|
1107
|
+
const modelConfig = await getModelConfigFromProjectConfig(ctx);
|
|
1108
|
+
const model = await updateModel(modelConfig, authCtx);
|
|
1109
|
+
await updateProjectConfig((config) => {
|
|
1110
|
+
config.$$id = model.id;
|
|
1111
|
+
config.$$modelGroupID = model.modelGroupId;
|
|
1112
|
+
config.$$revision = model.revision;
|
|
1113
|
+
config.$$dockerImage = model.dockerImage;
|
|
1114
|
+
});
|
|
1115
|
+
const pushCredentials = await getRegistryPushCredentials(model.id, authCtx);
|
|
1116
|
+
await loginToDockerRegistry(pushCredentials);
|
|
1117
|
+
await buildDockerImage(dockerImage, ctx);
|
|
1118
|
+
await pushDockerImage(dockerImage, pushCredentials.ref);
|
|
1119
|
+
spinner.stopAndPersist({
|
|
1120
|
+
symbol: "\u{1F680} ",
|
|
1121
|
+
prefixText: "\n",
|
|
1122
|
+
suffixText: "\n",
|
|
1123
|
+
text: chalk4.bold(`${model.name} (r${model.revision}) updated successfully`)
|
|
1124
|
+
});
|
|
602
1125
|
}
|
|
603
1126
|
|
|
604
|
-
// src/commands/
|
|
605
|
-
async function
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
1127
|
+
// src/commands/publish/handle-update-existing-model-flow.ts
|
|
1128
|
+
async function handleUpdateExistingModelFlow(options, app, spinner, ctx, authCtx) {
|
|
1129
|
+
spinner.info(`You are publishing an ${chalk5.bold("existing")} model:
|
|
1130
|
+
`);
|
|
1131
|
+
getModelSummaryLines(ctx).map((line) => console.log(line));
|
|
1132
|
+
const revision = ctx.config.$$revision ?? 0;
|
|
1133
|
+
const action = await inquirer5.select({
|
|
1134
|
+
message: "Do you want to update the existing revision or create a new one?",
|
|
1135
|
+
choices: [
|
|
1136
|
+
{
|
|
1137
|
+
value: "new" /* NewRevision */,
|
|
1138
|
+
name: `New Revision (r${revision} \u2192 r${revision + 1})`
|
|
1139
|
+
},
|
|
1140
|
+
{
|
|
1141
|
+
value: "update" /* UpdateExisting */,
|
|
1142
|
+
name: `Update Existing Revision (r${revision})`
|
|
1143
|
+
}
|
|
1144
|
+
]
|
|
613
1145
|
});
|
|
1146
|
+
logEmptyLine();
|
|
1147
|
+
if (action === "new" /* NewRevision */) {
|
|
1148
|
+
await handleModelRevisionCreateFlow(options, app, spinner, ctx, authCtx);
|
|
1149
|
+
} else if (action === "update" /* UpdateExisting */) {
|
|
1150
|
+
await handleModelRevisionUpdateFlow(options, app, spinner, ctx, authCtx);
|
|
1151
|
+
} else {
|
|
1152
|
+
throw new Error("Invalid CLI state");
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
// src/commands/publish/handle-action.ts
|
|
1157
|
+
async function handleAction5(options) {
|
|
1158
|
+
var _stack = [];
|
|
1159
|
+
try {
|
|
1160
|
+
const spinner = ora5("Starting model publish...").start();
|
|
1161
|
+
const app = express2();
|
|
1162
|
+
app.use(express2.json({ limit: "100mb" }));
|
|
1163
|
+
app.use(express2.urlencoded({ limit: "100mb", extended: true }));
|
|
1164
|
+
const webServer = __using(_stack, await startWebServer(app, options.port));
|
|
1165
|
+
try {
|
|
1166
|
+
const ctx = await loadProjectContext();
|
|
1167
|
+
const authCtx = await getAuth();
|
|
1168
|
+
await (ctx.config.$$id ? handleUpdateExistingModelFlow(options, app, spinner, ctx, authCtx) : handleCreateModelFlow(options, app, spinner, ctx, authCtx));
|
|
1169
|
+
} catch (error) {
|
|
1170
|
+
if (error instanceof ZodError5) {
|
|
1171
|
+
spinner.fail("Config validation failed");
|
|
1172
|
+
logZodIssues(error);
|
|
1173
|
+
} else if (error instanceof HTTPError3) {
|
|
1174
|
+
const errorResponse = await error.response.json();
|
|
1175
|
+
spinner.fail(formatNestErrorMessage(errorResponse));
|
|
1176
|
+
} else if (error instanceof Error) {
|
|
1177
|
+
spinner.fail(error.message);
|
|
1178
|
+
if (error.cause) {
|
|
1179
|
+
console.error(error.cause);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
process.exit(1);
|
|
1183
|
+
}
|
|
1184
|
+
} catch (_) {
|
|
1185
|
+
var _error = _, _hasError = true;
|
|
1186
|
+
} finally {
|
|
1187
|
+
__callDispose(_stack, _error, _hasError);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
// src/commands/publish/index.ts
|
|
1192
|
+
function registerPublishCommand(program2) {
|
|
1193
|
+
program2.command("publish").description("Publishes a pd4castr model.").option(
|
|
1194
|
+
"-i, --input-dir <path>",
|
|
1195
|
+
"The input test data directory",
|
|
1196
|
+
TEST_INPUT_DATA_DIR
|
|
1197
|
+
).option(
|
|
1198
|
+
"-p, --port <port>",
|
|
1199
|
+
"The port to run the IO testing webserver on",
|
|
1200
|
+
TEST_WEBSERVER_PORT.toString()
|
|
1201
|
+
).option("-s, --skip-checks", "Skip the model I/O checks", false).action(handleAction5);
|
|
614
1202
|
}
|
|
615
1203
|
|
|
616
1204
|
// src/commands/test/handle-action.ts
|
|
617
|
-
|
|
1205
|
+
import path14 from "path";
|
|
1206
|
+
import express3 from "express";
|
|
1207
|
+
import ora6 from "ora";
|
|
1208
|
+
import { ZodError as ZodError6 } from "zod";
|
|
1209
|
+
async function handleAction6(options) {
|
|
618
1210
|
var _stack = [];
|
|
619
1211
|
try {
|
|
620
|
-
const spinner =
|
|
621
|
-
const app =
|
|
1212
|
+
const spinner = ora6("Starting model tests...").info();
|
|
1213
|
+
const app = express3();
|
|
1214
|
+
app.use(express3.json({ limit: "100mb" }));
|
|
1215
|
+
app.use(express3.urlencoded({ limit: "100mb", extended: true }));
|
|
622
1216
|
const webServer = __using(_stack, await startWebServer(app, options.port));
|
|
623
1217
|
try {
|
|
624
|
-
const ctx = await
|
|
1218
|
+
const ctx = await loadProjectContext();
|
|
625
1219
|
const inputFiles = getInputFiles(ctx.config);
|
|
626
1220
|
spinner.start("Checking test input data files");
|
|
627
1221
|
await checkInputFiles(inputFiles, options.inputDir, ctx);
|
|
628
1222
|
spinner.succeed(`Found ${inputFiles.length} test input data files`);
|
|
629
1223
|
spinner.start("Building docker image");
|
|
630
|
-
const
|
|
631
|
-
const dockerImage = `pd4castr/${sluggedName}-local:${Date.now()}`;
|
|
1224
|
+
const dockerImage = getDockerImage(ctx);
|
|
632
1225
|
await buildDockerImage(dockerImage, ctx);
|
|
633
1226
|
spinner.succeed(`Built docker image (${dockerImage})`);
|
|
634
|
-
const modelIOChecks =
|
|
635
|
-
|
|
1227
|
+
const modelIOChecks = setupModelIOChecks(
|
|
1228
|
+
app,
|
|
636
1229
|
options.inputDir,
|
|
637
|
-
|
|
1230
|
+
inputFiles,
|
|
638
1231
|
ctx
|
|
639
1232
|
);
|
|
640
|
-
const handleOutput = createOutputHandler(modelIOChecks, ctx);
|
|
641
|
-
const inputPath = path8.join(ctx.projectRoot, options.inputDir);
|
|
642
|
-
app.use(express.json());
|
|
643
|
-
app.use("/data", express.static(inputPath));
|
|
644
|
-
app.get("/input/:filename", handleInput);
|
|
645
|
-
app.put("/output", handleOutput);
|
|
646
1233
|
spinner.start("Running model container");
|
|
647
|
-
|
|
648
|
-
await runModelContainer(dockerImage, webserverURL, ctx);
|
|
1234
|
+
await runModelContainer(dockerImage, options.port, ctx);
|
|
649
1235
|
spinner.succeed("Model run complete");
|
|
650
1236
|
for (const inputFile of inputFiles) {
|
|
651
1237
|
const status = modelIOChecks.isInputHandled(inputFile) ? "\u2714" : "\u2718";
|
|
@@ -656,10 +1242,10 @@ async function handleAction3(options) {
|
|
|
656
1242
|
if (modelIOChecks.isInputsHandled() && modelIOChecks.isOutputHandled()) {
|
|
657
1243
|
spinner.succeed("Model I/O test passed");
|
|
658
1244
|
} else {
|
|
659
|
-
|
|
1245
|
+
throw new Error("Model I/O test failed");
|
|
660
1246
|
}
|
|
661
1247
|
if (modelIOChecks.isOutputHandled()) {
|
|
662
|
-
const outputPath =
|
|
1248
|
+
const outputPath = path14.join(
|
|
663
1249
|
ctx.projectRoot,
|
|
664
1250
|
TEST_OUTPUT_DATA_DIR,
|
|
665
1251
|
TEST_OUTPUT_FILENAME
|
|
@@ -671,14 +1257,14 @@ ${clickHereLink} to view output (${fileLink})
|
|
|
671
1257
|
`);
|
|
672
1258
|
}
|
|
673
1259
|
} catch (error) {
|
|
674
|
-
if (error instanceof
|
|
1260
|
+
if (error instanceof ZodError6) {
|
|
675
1261
|
spinner.fail("Config validation failed");
|
|
676
1262
|
logZodIssues(error);
|
|
677
1263
|
} else if (error instanceof Error) {
|
|
678
1264
|
spinner.fail(error.message);
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
1265
|
+
if (error.cause) {
|
|
1266
|
+
console.error(error.cause);
|
|
1267
|
+
}
|
|
682
1268
|
}
|
|
683
1269
|
process.exit(1);
|
|
684
1270
|
}
|
|
@@ -697,127 +1283,111 @@ function registerTestCommand(program2) {
|
|
|
697
1283
|
"-i, --input-dir <path>",
|
|
698
1284
|
"The input test data directory",
|
|
699
1285
|
TEST_INPUT_DATA_DIR
|
|
700
|
-
).option(
|
|
1286
|
+
).option(
|
|
1287
|
+
"-p, --port <port>",
|
|
1288
|
+
"The port to run the IO testing webserver on",
|
|
1289
|
+
TEST_WEBSERVER_PORT.toString()
|
|
1290
|
+
).action(handleAction6);
|
|
701
1291
|
}
|
|
702
1292
|
|
|
703
|
-
// src/
|
|
704
|
-
import
|
|
705
|
-
import path11 from "path";
|
|
706
|
-
import { ZodError as ZodError3 } from "zod";
|
|
1293
|
+
// src/program.ts
|
|
1294
|
+
import { Command } from "commander";
|
|
707
1295
|
|
|
708
|
-
//
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
1296
|
+
// package.json
|
|
1297
|
+
var package_default = {
|
|
1298
|
+
name: "@pd4castr/cli",
|
|
1299
|
+
version: "0.0.6",
|
|
1300
|
+
description: "CLI tool for creating, testing, and publishing pd4castr models",
|
|
1301
|
+
main: "dist/index.js",
|
|
1302
|
+
type: "module",
|
|
1303
|
+
bin: {
|
|
1304
|
+
pd4castr: "dist/index.js"
|
|
1305
|
+
},
|
|
1306
|
+
files: [
|
|
1307
|
+
"dist/**/*"
|
|
1308
|
+
],
|
|
1309
|
+
scripts: {
|
|
1310
|
+
build: "tsup",
|
|
1311
|
+
dev: "tsup --watch",
|
|
1312
|
+
cli: "node dist/index.js",
|
|
1313
|
+
test: "vitest run",
|
|
1314
|
+
"test:watch": "vitest",
|
|
1315
|
+
"test:coverage": "vitest run --coverage",
|
|
1316
|
+
lint: "eslint .",
|
|
1317
|
+
"lint:fix": "eslint . --fix",
|
|
1318
|
+
format: "prettier --write .",
|
|
1319
|
+
"format:check": "prettier --check .",
|
|
1320
|
+
typecheck: "tsc --noEmit",
|
|
1321
|
+
prepublishOnly: "yarn build"
|
|
1322
|
+
},
|
|
1323
|
+
keywords: [
|
|
1324
|
+
"cli",
|
|
1325
|
+
"pd4castr"
|
|
1326
|
+
],
|
|
1327
|
+
license: "UNLICENSED",
|
|
1328
|
+
repository: {
|
|
1329
|
+
type: "git",
|
|
1330
|
+
url: "git+https://github.com/pipelabs/pd4castr-cli.git"
|
|
1331
|
+
},
|
|
1332
|
+
bugs: {
|
|
1333
|
+
url: "https://github.com/pipelabs/pd4castr-cli/issues"
|
|
1334
|
+
},
|
|
1335
|
+
homepage: "https://github.com/pipelabs/pd4castr-cli#readme",
|
|
1336
|
+
devDependencies: {
|
|
1337
|
+
"@faker-js/faker": "10.0.0",
|
|
1338
|
+
"@mswjs/data": "0.16.2",
|
|
1339
|
+
"@types/express": "4.17.21",
|
|
1340
|
+
"@types/node": "24.1.0",
|
|
1341
|
+
"@types/supertest": "6.0.3",
|
|
1342
|
+
"@typescript-eslint/eslint-plugin": "8.38.0",
|
|
1343
|
+
"@typescript-eslint/parser": "8.38.0",
|
|
1344
|
+
eslint: "9.32.0",
|
|
1345
|
+
"eslint-config-prettier": "10.1.8",
|
|
1346
|
+
"eslint-plugin-simple-import-sort": "12.1.1",
|
|
1347
|
+
"eslint-plugin-unicorn": "60.0.0",
|
|
1348
|
+
"eslint-plugin-vitest": "0.5.4",
|
|
1349
|
+
"hook-std": "3.0.0",
|
|
1350
|
+
"jest-extended": "6.0.0",
|
|
1351
|
+
memfs: "4.23.0",
|
|
1352
|
+
msw: "2.10.4",
|
|
1353
|
+
prettier: "3.6.2",
|
|
1354
|
+
"strip-ansi": "7.1.0",
|
|
1355
|
+
supertest: "7.1.4",
|
|
1356
|
+
tsup: "8.5.0",
|
|
1357
|
+
"type-fest": "4.41.0",
|
|
1358
|
+
typescript: "5.8.3",
|
|
1359
|
+
"typescript-eslint": "8.38.0",
|
|
1360
|
+
vitest: "3.2.4"
|
|
1361
|
+
},
|
|
1362
|
+
dependencies: {
|
|
1363
|
+
"@inquirer/prompts": "7.7.1",
|
|
1364
|
+
auth0: "4.27.0",
|
|
1365
|
+
chalk: "5.6.0",
|
|
1366
|
+
commander: "14.0.0",
|
|
1367
|
+
execa: "9.6.0",
|
|
1368
|
+
express: "4.21.2",
|
|
1369
|
+
immer: "10.1.1",
|
|
1370
|
+
ky: "1.8.2",
|
|
1371
|
+
ora: "8.2.0",
|
|
1372
|
+
slugify: "1.6.6",
|
|
1373
|
+
tiged: "2.12.7",
|
|
1374
|
+
"tiny-invariant": "1.3.3",
|
|
1375
|
+
zod: "4.0.14"
|
|
1376
|
+
},
|
|
1377
|
+
engines: {
|
|
1378
|
+
node: ">=20.0.0"
|
|
714
1379
|
}
|
|
715
|
-
invariant(config.accessToken, "Access token is required");
|
|
716
|
-
invariant(config.accessTokenExpiresAt, "Access token expiry is required");
|
|
717
|
-
return {
|
|
718
|
-
accessToken: config.accessToken,
|
|
719
|
-
expiresAt: config.accessTokenExpiresAt
|
|
720
|
-
};
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
// src/commands/fetch/utils/fetch-aemo-data.ts
|
|
724
|
-
import path9 from "path";
|
|
725
|
-
import fs5 from "fs/promises";
|
|
726
|
-
|
|
727
|
-
// src/api.ts
|
|
728
|
-
import ky2 from "ky";
|
|
729
|
-
var api = ky2.create({
|
|
730
|
-
prefixUrl: process.env.PD4CASTR_API_URL ?? API_URL
|
|
731
|
-
});
|
|
732
|
-
|
|
733
|
-
// src/commands/fetch/utils/fetch-aemo-data.ts
|
|
734
|
-
async function fetchAEMOData(dataFetcher, authCtx, ctx) {
|
|
735
|
-
const queryPath = path9.resolve(
|
|
736
|
-
ctx.projectRoot,
|
|
737
|
-
dataFetcher.config.fetchQuery
|
|
738
|
-
);
|
|
739
|
-
const querySQL = await fs5.readFile(queryPath, "utf8");
|
|
740
|
-
const headers = { Authorization: `Bearer ${authCtx.accessToken}` };
|
|
741
|
-
const payload2 = { query: querySQL, type: "AEMO_MMS" };
|
|
742
|
-
const result = await api.post("data-fetcher/query", { json: payload2, headers }).json();
|
|
743
|
-
return result;
|
|
744
|
-
}
|
|
745
|
-
|
|
746
|
-
// src/commands/fetch/utils/get-fetcher.ts
|
|
747
|
-
var DATA_FETCHER_FNS = {
|
|
748
|
-
AEMO_MMS: fetchAEMOData
|
|
749
1380
|
};
|
|
750
|
-
function getFetcher(type) {
|
|
751
|
-
const fetcher = DATA_FETCHER_FNS[type];
|
|
752
|
-
if (!fetcher) {
|
|
753
|
-
throw new Error(`Unsupported data fetcher type: ${type}`);
|
|
754
|
-
}
|
|
755
|
-
return fetcher;
|
|
756
|
-
}
|
|
757
|
-
|
|
758
|
-
// src/commands/fetch/utils/write-test-data.ts
|
|
759
|
-
import path10 from "path";
|
|
760
|
-
import fs6 from "fs/promises";
|
|
761
|
-
async function writeTestData(inputData, modelInput, inputDataDir, ctx) {
|
|
762
|
-
const inputDir = path10.resolve(ctx.projectRoot, inputDataDir);
|
|
763
|
-
await fs6.mkdir(inputDir, { recursive: true });
|
|
764
|
-
const inputFilename = getInputFilename(modelInput);
|
|
765
|
-
const inputPath = path10.resolve(inputDir, inputFilename);
|
|
766
|
-
await fs6.writeFile(inputPath, JSON.stringify(inputData, void 0, 2));
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
// src/commands/fetch/handle-action.ts
|
|
770
|
-
var FETCHABLE_DATA_FETCHER_TYPES = /* @__PURE__ */ new Set(["AEMO_MMS"]);
|
|
771
|
-
async function handleAction4(options) {
|
|
772
|
-
const spinner = ora4("Starting data fetch...").start();
|
|
773
|
-
try {
|
|
774
|
-
const authCtx = await getAuth();
|
|
775
|
-
const ctx = await loadProjectConfig();
|
|
776
|
-
for (const input2 of ctx.config.inputs) {
|
|
777
|
-
if (!input2.fetcher) {
|
|
778
|
-
spinner.warn(`\`${input2.key}\` - no data fetcher defined, skipping`);
|
|
779
|
-
continue;
|
|
780
|
-
}
|
|
781
|
-
if (!FETCHABLE_DATA_FETCHER_TYPES.has(input2.fetcher.type)) {
|
|
782
|
-
spinner.warn(
|
|
783
|
-
`\`${input2.key}\` (${input2.fetcher.type}) - unsupported, skipping`
|
|
784
|
-
);
|
|
785
|
-
continue;
|
|
786
|
-
}
|
|
787
|
-
spinner.start(`\`${input2.key}\` (${input2.fetcher.type}) - fetching...`);
|
|
788
|
-
const fetchData = getFetcher(input2.fetcher.type);
|
|
789
|
-
const output = await fetchData(input2.fetcher, authCtx, ctx);
|
|
790
|
-
await writeTestData(output, input2, options.inputDir, ctx);
|
|
791
|
-
spinner.succeed(`\`${input2.key}\` (${input2.fetcher.type}) - fetched`);
|
|
792
|
-
}
|
|
793
|
-
const inputPath = path11.resolve(ctx.projectRoot, options.inputDir);
|
|
794
|
-
const link = createLink("Click here", `file://${inputPath}`);
|
|
795
|
-
console.log(`
|
|
796
|
-
${link} to view fetched data
|
|
797
|
-
`);
|
|
798
|
-
} catch (error) {
|
|
799
|
-
if (error instanceof ZodError3) {
|
|
800
|
-
spinner.fail("Config validation failed");
|
|
801
|
-
logZodIssues(error);
|
|
802
|
-
} else if (error instanceof Error) {
|
|
803
|
-
spinner.fail(error.message);
|
|
804
|
-
}
|
|
805
|
-
process.exit(1);
|
|
806
|
-
}
|
|
807
|
-
}
|
|
808
1381
|
|
|
809
|
-
// src/
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
"-i, --input-dir <path>",
|
|
813
|
-
"The input test data directory",
|
|
814
|
-
TEST_INPUT_DATA_DIR
|
|
815
|
-
).action(handleAction4);
|
|
816
|
-
}
|
|
1382
|
+
// src/program.ts
|
|
1383
|
+
var program = new Command();
|
|
1384
|
+
program.name("pd4castr").description("CLI tool for pd4castr").version(package_default.version);
|
|
817
1385
|
|
|
818
1386
|
// src/index.ts
|
|
819
1387
|
registerInitCommand(program);
|
|
820
1388
|
registerLoginCommand(program);
|
|
1389
|
+
registerLogoutCommand(program);
|
|
821
1390
|
registerTestCommand(program);
|
|
822
1391
|
registerFetchCommand(program);
|
|
1392
|
+
registerPublishCommand(program);
|
|
823
1393
|
await program.parseAsync(process.argv);
|