@dreamlake/dreamlake-cli 0.122.0 → 0.123.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/cict.d.ts +323 -0
- package/dist/api/cict.js +104 -0
- package/dist/api/index.d.ts +173 -2
- package/dist/api/index.js +6 -0
- package/dist/cli/cict/index.d.ts +2 -0
- package/dist/cli/cict/index.js +298 -0
- package/dist/cli/help-examples.generated.js +24 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/vault/sshTransfer.d.ts +1 -1
- package/dist/cli/version.d.ts +1 -1
- package/dist/cli/version.js +1 -1
- package/package.json +9 -9
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
import { type Transport } from "./client.js";
|
|
2
|
+
/** Where a matched job's runner starts. Only Slurm targets start runners today. */
|
|
3
|
+
export type CictPoolTarget = {
|
|
4
|
+
kind: "slurm";
|
|
5
|
+
providerId: string;
|
|
6
|
+
} | {
|
|
7
|
+
kind: "host";
|
|
8
|
+
hostId: string;
|
|
9
|
+
};
|
|
10
|
+
/** A runner pool: the `runs-on` labels it serves, where runners start and where evidence lands. */
|
|
11
|
+
export interface CictPool {
|
|
12
|
+
id: string;
|
|
13
|
+
namespaceId: string;
|
|
14
|
+
name: string;
|
|
15
|
+
labels: string[];
|
|
16
|
+
target: CictPoolTarget;
|
|
17
|
+
evidence: {
|
|
18
|
+
sourceId: string;
|
|
19
|
+
prefix?: string;
|
|
20
|
+
} | null;
|
|
21
|
+
env?: Record<string, string>;
|
|
22
|
+
createdByUserId: string;
|
|
23
|
+
createdAt: string;
|
|
24
|
+
}
|
|
25
|
+
export interface CictPoolInput {
|
|
26
|
+
/** Lowercase letters, digits and hyphens; the default label is `dreamlake-<name>`. */
|
|
27
|
+
name: string;
|
|
28
|
+
/** Must contain `dreamlake` or a `dreamlake-…` label. */
|
|
29
|
+
labels?: string[];
|
|
30
|
+
target: CictPoolTarget;
|
|
31
|
+
/** Source that receives each run's declared evidence (Slurm pools). */
|
|
32
|
+
evidence?: {
|
|
33
|
+
sourceId: string;
|
|
34
|
+
prefix?: string;
|
|
35
|
+
};
|
|
36
|
+
/** Machine facts every job on this pool sees, such as cache paths. Not secrets. */
|
|
37
|
+
env?: Record<string, string>;
|
|
38
|
+
}
|
|
39
|
+
export interface CictInstallation {
|
|
40
|
+
installationId: number;
|
|
41
|
+
namespaceId: string;
|
|
42
|
+
accountLogin: string;
|
|
43
|
+
accountType: string;
|
|
44
|
+
boundByUserId: string;
|
|
45
|
+
boundAt: string;
|
|
46
|
+
state: "active" | "suspended" | "deleted";
|
|
47
|
+
}
|
|
48
|
+
/** One GitHub Actions job that asked for a DreamLake runner, with the launch that served it. */
|
|
49
|
+
export interface CictRun {
|
|
50
|
+
/** `<installationId>-<jobId>`. */
|
|
51
|
+
id: string;
|
|
52
|
+
repository: {
|
|
53
|
+
fullName: string;
|
|
54
|
+
private: boolean;
|
|
55
|
+
};
|
|
56
|
+
workflowName: string | null;
|
|
57
|
+
name: string | null;
|
|
58
|
+
githubUrl: string;
|
|
59
|
+
runId: number;
|
|
60
|
+
runAttempt: number;
|
|
61
|
+
jobId: number;
|
|
62
|
+
headSha: string | null;
|
|
63
|
+
headBranch: string | null;
|
|
64
|
+
labels: string[];
|
|
65
|
+
status: "queued" | "in_progress" | "completed" | "waiting";
|
|
66
|
+
conclusion: string | null;
|
|
67
|
+
createdAt: string | null;
|
|
68
|
+
startedAt: string | null;
|
|
69
|
+
completedAt: string | null;
|
|
70
|
+
updatedAt: string;
|
|
71
|
+
routing: {
|
|
72
|
+
kind: "matched";
|
|
73
|
+
pool: {
|
|
74
|
+
id: string;
|
|
75
|
+
name: string;
|
|
76
|
+
};
|
|
77
|
+
} | {
|
|
78
|
+
kind: "platform";
|
|
79
|
+
machine: {
|
|
80
|
+
id: string;
|
|
81
|
+
label: string;
|
|
82
|
+
};
|
|
83
|
+
} | {
|
|
84
|
+
kind: "unmatched" | "unbound";
|
|
85
|
+
} | {
|
|
86
|
+
kind: "rejected";
|
|
87
|
+
reason: string;
|
|
88
|
+
} | {
|
|
89
|
+
kind: "refused";
|
|
90
|
+
label: string;
|
|
91
|
+
reason: string;
|
|
92
|
+
};
|
|
93
|
+
/** A DreamLake GPU job waiting for a slot: 1 is next. */
|
|
94
|
+
queue?: {
|
|
95
|
+
position: number;
|
|
96
|
+
} | null;
|
|
97
|
+
/** GPU time a finished DreamLake GPU run used. */
|
|
98
|
+
usage?: {
|
|
99
|
+
gpuSeconds: number;
|
|
100
|
+
} | null;
|
|
101
|
+
/** Who triggered it on GitHub. */
|
|
102
|
+
sender?: string | null;
|
|
103
|
+
/** Where baselines come from. */
|
|
104
|
+
defaultBranch?: string;
|
|
105
|
+
/** Regressions and fixes against the baseline, once compared. */
|
|
106
|
+
comparison?: {
|
|
107
|
+
baseline: CictBaseline | null;
|
|
108
|
+
regressions: number;
|
|
109
|
+
fixes: number;
|
|
110
|
+
} | null;
|
|
111
|
+
runner: {
|
|
112
|
+
name: string;
|
|
113
|
+
state: "waiting" | "starting" | "started" | "failed" | "completed" | "cancelled";
|
|
114
|
+
slurmJobId: string | null;
|
|
115
|
+
providerId: string | null;
|
|
116
|
+
error: string | null;
|
|
117
|
+
} | null;
|
|
118
|
+
evidence: {
|
|
119
|
+
state: "none" | "pending" | "written" | "failed";
|
|
120
|
+
sourceId: string;
|
|
121
|
+
prefix: string;
|
|
122
|
+
} | null;
|
|
123
|
+
}
|
|
124
|
+
export interface CictBaseline {
|
|
125
|
+
runId: string;
|
|
126
|
+
branch: string;
|
|
127
|
+
headSha: string | null;
|
|
128
|
+
completedAt: string | null;
|
|
129
|
+
}
|
|
130
|
+
/** A run compared case by case with its baseline (the latest earlier default-branch run of the same job). */
|
|
131
|
+
export interface CictComparison {
|
|
132
|
+
baseline: CictBaseline | null;
|
|
133
|
+
regressions: number;
|
|
134
|
+
fixes: number;
|
|
135
|
+
passed: number;
|
|
136
|
+
total: number;
|
|
137
|
+
cases: Array<{
|
|
138
|
+
id: string;
|
|
139
|
+
change: "regressed" | "fixed" | "still_failing" | "unchanged" | "added" | "removed";
|
|
140
|
+
score?: {
|
|
141
|
+
name: string;
|
|
142
|
+
now?: number;
|
|
143
|
+
base?: number;
|
|
144
|
+
delta?: number;
|
|
145
|
+
};
|
|
146
|
+
checks: Array<{
|
|
147
|
+
name: string;
|
|
148
|
+
change: string;
|
|
149
|
+
now?: Record<string, unknown>;
|
|
150
|
+
base?: Record<string, unknown>;
|
|
151
|
+
}>;
|
|
152
|
+
}>;
|
|
153
|
+
}
|
|
154
|
+
export interface CictPoolPatch {
|
|
155
|
+
labels?: string[];
|
|
156
|
+
target?: CictPoolTarget;
|
|
157
|
+
/** `null` stops keeping evidence. */
|
|
158
|
+
evidence?: {
|
|
159
|
+
sourceId: string;
|
|
160
|
+
prefix?: string;
|
|
161
|
+
} | null;
|
|
162
|
+
/** Replaces every variable; `null` removes them. */
|
|
163
|
+
env?: Record<string, string> | null;
|
|
164
|
+
}
|
|
165
|
+
/** A Slurm provider a pool can target, and why runners cannot start there if not. */
|
|
166
|
+
export interface CictRunnerTarget {
|
|
167
|
+
id: string;
|
|
168
|
+
name: string;
|
|
169
|
+
kind: "slurm";
|
|
170
|
+
usable: boolean;
|
|
171
|
+
reason: "no_association" | "no_launch_layer" | "legacy_provider" | null;
|
|
172
|
+
}
|
|
173
|
+
export interface CictPlatformMachine {
|
|
174
|
+
id: string;
|
|
175
|
+
label: string;
|
|
176
|
+
title: string;
|
|
177
|
+
spec: {
|
|
178
|
+
gpus: number;
|
|
179
|
+
gpu: string;
|
|
180
|
+
cpus: number;
|
|
181
|
+
memGiB: number;
|
|
182
|
+
timeLimit: string;
|
|
183
|
+
};
|
|
184
|
+
enabled: boolean;
|
|
185
|
+
}
|
|
186
|
+
export interface CictPlatformAccess {
|
|
187
|
+
state: "allowed" | "active" | "suspended";
|
|
188
|
+
maxConcurrent: number;
|
|
189
|
+
evidence: {
|
|
190
|
+
sourceId: string;
|
|
191
|
+
prefix: string;
|
|
192
|
+
} | null;
|
|
193
|
+
authorizedByUserId: string | null;
|
|
194
|
+
}
|
|
195
|
+
export interface CictPlatformStatus {
|
|
196
|
+
available: boolean;
|
|
197
|
+
machines: CictPlatformMachine[];
|
|
198
|
+
access: CictPlatformAccess | null;
|
|
199
|
+
running: number;
|
|
200
|
+
waiting: number;
|
|
201
|
+
monthGpuSeconds: number;
|
|
202
|
+
}
|
|
203
|
+
export interface CictUsage {
|
|
204
|
+
month: string;
|
|
205
|
+
gpuSeconds: number;
|
|
206
|
+
jobs: number;
|
|
207
|
+
byWorkflow: Array<{
|
|
208
|
+
repository: string;
|
|
209
|
+
workflowName: string | null;
|
|
210
|
+
gpuSeconds: number;
|
|
211
|
+
jobs: number;
|
|
212
|
+
}>;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* CI/CT through GitHub Actions (channel A): connect a GitHub App installation,
|
|
216
|
+
* route `runs-on` labels to runner pools, and read the runs they served.
|
|
217
|
+
* The suite result is evidence in the pool's Source; `runs.result` reads the
|
|
218
|
+
* run's `result.json` through a short-lived Source read capability.
|
|
219
|
+
*/
|
|
220
|
+
export declare function cictApi(t: Transport, namespace: string, fetchImpl?: typeof fetch): {
|
|
221
|
+
runs: {
|
|
222
|
+
list: (query?: {
|
|
223
|
+
limit?: number;
|
|
224
|
+
}) => Promise<CictRun[]>;
|
|
225
|
+
get: (id: string) => Promise<CictRun>;
|
|
226
|
+
/** The run's `result.json`, or `null` when the run has no evidence location or wrote no result. */
|
|
227
|
+
result: (id: string, options?: {
|
|
228
|
+
file?: string;
|
|
229
|
+
}) => Promise<unknown>;
|
|
230
|
+
} & {
|
|
231
|
+
comparison: (id: string) => Promise<CictComparison | null>;
|
|
232
|
+
};
|
|
233
|
+
/** Raw recorded GitHub job events, including jobs that matched no pool. */
|
|
234
|
+
jobs: (query?: {
|
|
235
|
+
limit?: number;
|
|
236
|
+
}) => Promise<unknown[]>;
|
|
237
|
+
pools: {
|
|
238
|
+
list: () => Promise<CictPool[]>;
|
|
239
|
+
create: (input: CictPoolInput) => Promise<CictPool>;
|
|
240
|
+
/** Edits apply to the next job; running jobs keep what they started with. */
|
|
241
|
+
update: (poolId: string, patch: CictPoolPatch) => Promise<CictPool>;
|
|
242
|
+
/** Refused with `runner_pool_busy` while the pool still serves running jobs. */
|
|
243
|
+
delete: (poolId: string) => Promise<{
|
|
244
|
+
deleted: true;
|
|
245
|
+
}>;
|
|
246
|
+
/** Slurm providers a pool can target, with why runners cannot start on the others. */
|
|
247
|
+
targets: () => Promise<CictRunnerTarget[]>;
|
|
248
|
+
};
|
|
249
|
+
/** DreamLake GPU: machines DreamLake provides, chosen by label. */
|
|
250
|
+
platform: {
|
|
251
|
+
status: () => Promise<CictPlatformStatus>;
|
|
252
|
+
/** Start using DreamLake GPU: uses `sourceId`, or creates `cict-evidence`. The caller authorizes evidence writes. */
|
|
253
|
+
start: (sourceId?: string) => Promise<{
|
|
254
|
+
access: CictPlatformAccess;
|
|
255
|
+
source: {
|
|
256
|
+
sourceId: string;
|
|
257
|
+
name: string;
|
|
258
|
+
};
|
|
259
|
+
}>;
|
|
260
|
+
/** Change where DreamLake GPU runs save results. */
|
|
261
|
+
setSource: (sourceId: string) => Promise<{
|
|
262
|
+
access: CictPlatformAccess;
|
|
263
|
+
source: {
|
|
264
|
+
sourceId: string;
|
|
265
|
+
name: string;
|
|
266
|
+
};
|
|
267
|
+
}>;
|
|
268
|
+
};
|
|
269
|
+
/** DreamLake GPU time in a month (default: this month), by repository and workflow. */
|
|
270
|
+
usage: (month?: string) => Promise<CictUsage>;
|
|
271
|
+
github: {
|
|
272
|
+
/** GitHub App installations bound to this namespace. */
|
|
273
|
+
installations: () => Promise<CictInstallation[]>;
|
|
274
|
+
/**
|
|
275
|
+
* Start binding a GitHub App installation: open `installUrl` in a browser
|
|
276
|
+
* signed in to DreamLake as the same user within 15 minutes. GitHub then
|
|
277
|
+
* redirects to the DreamLake setup page, which completes the binding.
|
|
278
|
+
*/
|
|
279
|
+
connect: () => Promise<{
|
|
280
|
+
installUrl: string;
|
|
281
|
+
}>;
|
|
282
|
+
};
|
|
283
|
+
};
|
|
284
|
+
/**
|
|
285
|
+
* DreamLake GPU administration, for admins of the platform namespace:
|
|
286
|
+
* which namespaces may use it, their limits, and the machine types.
|
|
287
|
+
*/
|
|
288
|
+
export declare function cictPlatformAdminApi(t: Transport): {
|
|
289
|
+
overview: () => Promise<{
|
|
290
|
+
machines: Array<CictPlatformMachine & {
|
|
291
|
+
providerId: string;
|
|
292
|
+
env: Record<string, string>;
|
|
293
|
+
maxConcurrent: number;
|
|
294
|
+
}>;
|
|
295
|
+
namespaces: Array<CictPlatformAccess & {
|
|
296
|
+
namespaceId: string;
|
|
297
|
+
slug: string | null;
|
|
298
|
+
running: number;
|
|
299
|
+
waiting: number;
|
|
300
|
+
monthGpuSeconds: number;
|
|
301
|
+
}>;
|
|
302
|
+
}>;
|
|
303
|
+
/** Allow (`allowed`), pause (`suspended`) or set the concurrency limit of a namespace. */
|
|
304
|
+
setNamespace: (slug: string, body: {
|
|
305
|
+
state?: "allowed" | "suspended";
|
|
306
|
+
maxConcurrent?: number;
|
|
307
|
+
}) => Promise<CictPlatformAccess & {
|
|
308
|
+
namespaceId: string;
|
|
309
|
+
}>;
|
|
310
|
+
/** Create or change a machine type; the provider must be in the platform namespace. */
|
|
311
|
+
putMachine: (id: string, body: {
|
|
312
|
+
label?: string;
|
|
313
|
+
title?: string;
|
|
314
|
+
providerId?: string;
|
|
315
|
+
enabled?: boolean;
|
|
316
|
+
maxConcurrent?: number;
|
|
317
|
+
env?: Record<string, string>;
|
|
318
|
+
spec?: CictPlatformMachine["spec"];
|
|
319
|
+
}) => Promise<CictPlatformMachine & {
|
|
320
|
+
providerId: string;
|
|
321
|
+
maxConcurrent: number;
|
|
322
|
+
}>;
|
|
323
|
+
};
|
package/dist/api/cict.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { seg } from "./client.js";
|
|
2
|
+
import { call, namespacePath } from "./common.js";
|
|
3
|
+
const RUN_ID = /^\d{1,12}-\d{1,20}$/;
|
|
4
|
+
const MONTH = /^\d{4}-(0[1-9]|1[0-2])$/;
|
|
5
|
+
const runPath = (base, id) => {
|
|
6
|
+
if (!RUN_ID.test(id))
|
|
7
|
+
throw new Error("A CI/CT run ID is <installationId>-<jobId>");
|
|
8
|
+
return `${base}/cict/runs/${id}`;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* CI/CT through GitHub Actions (channel A): connect a GitHub App installation,
|
|
12
|
+
* route `runs-on` labels to runner pools, and read the runs they served.
|
|
13
|
+
* The suite result is evidence in the pool's Source; `runs.result` reads the
|
|
14
|
+
* run's `result.json` through a short-lived Source read capability.
|
|
15
|
+
*/
|
|
16
|
+
export function cictApi(t, namespace, fetchImpl) {
|
|
17
|
+
const ns = namespacePath(namespace);
|
|
18
|
+
const runs = {
|
|
19
|
+
list: async (query = {}) => (await call(t, "GET", ns + "/cict/runs", undefined, query)).runs,
|
|
20
|
+
get: async (id) => (await call(t, "GET", runPath(ns, id))).run,
|
|
21
|
+
/** The run's `result.json`, or `null` when the run has no evidence location or wrote no result. */
|
|
22
|
+
result: async (id, options = {}) => {
|
|
23
|
+
const run = await runs.get(id);
|
|
24
|
+
if (!run.evidence)
|
|
25
|
+
return null;
|
|
26
|
+
const file = options.file ?? "result.json";
|
|
27
|
+
if (!/^[A-Za-z0-9_.][A-Za-z0-9_./-]{0,1023}$/.test(file) || file.split("/").some((s) => s === "" || s === "." || s === ".."))
|
|
28
|
+
throw new Error("Use a relative evidence file path");
|
|
29
|
+
let capability;
|
|
30
|
+
try {
|
|
31
|
+
capability = await call(t, "POST", `${ns}/sources/${seg(run.evidence.sourceId)}/read-capabilities`, { path: `${run.evidence.prefix}/${file}`, intent: "preview" });
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (error?.status === 404)
|
|
35
|
+
return null;
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
const response = await (fetchImpl ?? fetch)(capability.url, {
|
|
39
|
+
headers: capability.allowedHeaders ?? {}, redirect: "error", signal: AbortSignal.timeout(30_000),
|
|
40
|
+
});
|
|
41
|
+
if (response.status === 404)
|
|
42
|
+
return null;
|
|
43
|
+
if (!response.ok)
|
|
44
|
+
throw new Error(`Reading ${file} failed (HTTP ${response.status})`);
|
|
45
|
+
return (await response.json());
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
Object.assign(runs, {
|
|
49
|
+
/** The run compared with its baseline; `null` until its evidence is written. */
|
|
50
|
+
comparison: async (id) => (await call(t, "GET", runPath(ns, id) + "/comparison")).comparison,
|
|
51
|
+
});
|
|
52
|
+
return {
|
|
53
|
+
runs: runs,
|
|
54
|
+
/** Raw recorded GitHub job events, including jobs that matched no pool. */
|
|
55
|
+
jobs: async (query = {}) => (await call(t, "GET", ns + "/github/jobs", undefined, query)).jobs,
|
|
56
|
+
pools: {
|
|
57
|
+
list: async () => (await call(t, "GET", ns + "/github/runner-pools")).pools,
|
|
58
|
+
create: (input) => call(t, "POST", ns + "/github/runner-pools", input),
|
|
59
|
+
/** Edits apply to the next job; running jobs keep what they started with. */
|
|
60
|
+
update: (poolId, patch) => call(t, "PATCH", ns + "/github/runner-pools/" + seg(poolId), patch),
|
|
61
|
+
/** Refused with `runner_pool_busy` while the pool still serves running jobs. */
|
|
62
|
+
delete: (poolId) => call(t, "DELETE", ns + "/github/runner-pools/" + seg(poolId)),
|
|
63
|
+
/** Slurm providers a pool can target, with why runners cannot start on the others. */
|
|
64
|
+
targets: async () => (await call(t, "GET", ns + "/github/runner-targets")).targets,
|
|
65
|
+
},
|
|
66
|
+
/** DreamLake GPU: machines DreamLake provides, chosen by label. */
|
|
67
|
+
platform: {
|
|
68
|
+
status: () => call(t, "GET", ns + "/cict/platform"),
|
|
69
|
+
/** Start using DreamLake GPU: uses `sourceId`, or creates `cict-evidence`. The caller authorizes evidence writes. */
|
|
70
|
+
start: (sourceId) => call(t, "POST", ns + "/cict/platform/activate", sourceId ? { sourceId } : {}),
|
|
71
|
+
/** Change where DreamLake GPU runs save results. */
|
|
72
|
+
setSource: (sourceId) => call(t, "PATCH", ns + "/cict/platform", { sourceId }),
|
|
73
|
+
},
|
|
74
|
+
/** DreamLake GPU time in a month (default: this month), by repository and workflow. */
|
|
75
|
+
usage: (month) => {
|
|
76
|
+
if (month !== undefined && !MONTH.test(month))
|
|
77
|
+
throw new Error("Use a month as YYYY-MM");
|
|
78
|
+
return call(t, "GET", ns + "/cict/usage", undefined, month ? { month } : {});
|
|
79
|
+
},
|
|
80
|
+
github: {
|
|
81
|
+
/** GitHub App installations bound to this namespace. */
|
|
82
|
+
installations: async () => (await call(t, "GET", ns + "/github/installations")).installations,
|
|
83
|
+
/**
|
|
84
|
+
* Start binding a GitHub App installation: open `installUrl` in a browser
|
|
85
|
+
* signed in to DreamLake as the same user within 15 minutes. GitHub then
|
|
86
|
+
* redirects to the DreamLake setup page, which completes the binding.
|
|
87
|
+
*/
|
|
88
|
+
connect: () => call(t, "POST", ns + "/github/installations/start", {}),
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* DreamLake GPU administration, for admins of the platform namespace:
|
|
94
|
+
* which namespaces may use it, their limits, and the machine types.
|
|
95
|
+
*/
|
|
96
|
+
export function cictPlatformAdminApi(t) {
|
|
97
|
+
return {
|
|
98
|
+
overview: () => call(t, "GET", "/cict/platform/admin"),
|
|
99
|
+
/** Allow (`allowed`), pause (`suspended`) or set the concurrency limit of a namespace. */
|
|
100
|
+
setNamespace: (slug, body) => call(t, "PUT", "/cict/platform/admin/namespaces/" + seg(slug), body),
|
|
101
|
+
/** Create or change a machine type; the provider must be in the platform namespace. */
|
|
102
|
+
putMachine: (id, body) => call(t, "PUT", "/cict/platform/admin/machines/" + seg(id), body),
|
|
103
|
+
};
|
|
104
|
+
}
|
package/dist/api/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ export * from './annotations.js';
|
|
|
2
2
|
import { FileWorkspace } from "./file-workspace.js";
|
|
3
3
|
export * from "./file-workspace.js";
|
|
4
4
|
export * from "./file-source.js";
|
|
5
|
+
export * from './cict.js';
|
|
5
6
|
export * from './apps.js';
|
|
6
7
|
export * from './lakeshore-connections.js';
|
|
7
8
|
export * from "./open-resources.js";
|
|
@@ -469,7 +470,7 @@ export declare function createApi(transport: Transport, storageFetch?: typeof fe
|
|
|
469
470
|
generate: (input: {
|
|
470
471
|
name: string;
|
|
471
472
|
entryId?: string;
|
|
472
|
-
expectedRevision
|
|
473
|
+
expectedRevision? /** @deprecated Use memos. */: number;
|
|
473
474
|
requestId?: string;
|
|
474
475
|
}) => Promise<unknown>;
|
|
475
476
|
};
|
|
@@ -1269,6 +1270,91 @@ export declare function createApi(transport: Transport, storageFetch?: typeof fe
|
|
|
1269
1270
|
}) => Promise<unknown>;
|
|
1270
1271
|
disconnect: (id: string) => Promise<unknown>;
|
|
1271
1272
|
};
|
|
1273
|
+
/** CI/CT through GitHub Actions: runner pools, GitHub App installations and runs. */
|
|
1274
|
+
cict: (namespace: string) => {
|
|
1275
|
+
runs: {
|
|
1276
|
+
list: (query?: {
|
|
1277
|
+
limit?: number;
|
|
1278
|
+
}) => Promise<import("./cict.js").CictRun[]>;
|
|
1279
|
+
get: (id: string) => Promise<import("./cict.js").CictRun>;
|
|
1280
|
+
result: (id: string, options?: {
|
|
1281
|
+
file?: string;
|
|
1282
|
+
}) => Promise<unknown>;
|
|
1283
|
+
} & {
|
|
1284
|
+
comparison: (id: string) => Promise<import("./cict.js").CictComparison | null>;
|
|
1285
|
+
};
|
|
1286
|
+
jobs: (query?: {
|
|
1287
|
+
limit?: number;
|
|
1288
|
+
}) => Promise<unknown[]>;
|
|
1289
|
+
pools: {
|
|
1290
|
+
list: () => Promise<import("./cict.js").CictPool[]>;
|
|
1291
|
+
create: (input: import("./cict.js").CictPoolInput) => Promise<import("./cict.js").CictPool>;
|
|
1292
|
+
update: (poolId: string, patch: import("./cict.js").CictPoolPatch) => Promise<import("./cict.js").CictPool>;
|
|
1293
|
+
delete: (poolId: string) => Promise<{
|
|
1294
|
+
deleted: true;
|
|
1295
|
+
}>;
|
|
1296
|
+
targets: () => Promise<import("./cict.js").CictRunnerTarget[]>;
|
|
1297
|
+
};
|
|
1298
|
+
platform: {
|
|
1299
|
+
status: () => Promise<import("./cict.js").CictPlatformStatus>;
|
|
1300
|
+
start: (sourceId?: string) => Promise<{
|
|
1301
|
+
access: import("./cict.js").CictPlatformAccess;
|
|
1302
|
+
source: {
|
|
1303
|
+
sourceId: string;
|
|
1304
|
+
name: string;
|
|
1305
|
+
};
|
|
1306
|
+
}>;
|
|
1307
|
+
setSource: (sourceId: string) => Promise<{
|
|
1308
|
+
access: import("./cict.js").CictPlatformAccess;
|
|
1309
|
+
source: {
|
|
1310
|
+
sourceId: string;
|
|
1311
|
+
name: string;
|
|
1312
|
+
};
|
|
1313
|
+
}>;
|
|
1314
|
+
};
|
|
1315
|
+
usage: (month?: string) => Promise<import("./cict.js").CictUsage>;
|
|
1316
|
+
github: {
|
|
1317
|
+
installations: () => Promise<import("./cict.js").CictInstallation[]>;
|
|
1318
|
+
connect: () => Promise<{
|
|
1319
|
+
installUrl: string;
|
|
1320
|
+
}>;
|
|
1321
|
+
};
|
|
1322
|
+
};
|
|
1323
|
+
/** DreamLake GPU administration (admins of the platform namespace). */
|
|
1324
|
+
cictPlatformAdmin: {
|
|
1325
|
+
overview: () => Promise<{
|
|
1326
|
+
machines: Array<import("./cict.js").CictPlatformMachine & {
|
|
1327
|
+
providerId: string;
|
|
1328
|
+
env: Record<string, string>;
|
|
1329
|
+
maxConcurrent: number;
|
|
1330
|
+
}>;
|
|
1331
|
+
namespaces: Array<import("./cict.js").CictPlatformAccess & {
|
|
1332
|
+
namespaceId: string;
|
|
1333
|
+
slug: string | null;
|
|
1334
|
+
running: number;
|
|
1335
|
+
waiting: number;
|
|
1336
|
+
monthGpuSeconds: number;
|
|
1337
|
+
}>;
|
|
1338
|
+
}>;
|
|
1339
|
+
setNamespace: (slug: string, body: {
|
|
1340
|
+
state?: "allowed" | "suspended";
|
|
1341
|
+
maxConcurrent?: number;
|
|
1342
|
+
}) => Promise<import("./cict.js").CictPlatformAccess & {
|
|
1343
|
+
namespaceId: string;
|
|
1344
|
+
}>;
|
|
1345
|
+
putMachine: (id: string, body: {
|
|
1346
|
+
label?: string;
|
|
1347
|
+
title?: string;
|
|
1348
|
+
providerId?: string;
|
|
1349
|
+
enabled?: boolean;
|
|
1350
|
+
maxConcurrent?: number;
|
|
1351
|
+
env?: Record<string, string>;
|
|
1352
|
+
spec?: import("./cict.js").CictPlatformMachine["spec"];
|
|
1353
|
+
}) => Promise<import("./cict.js").CictPlatformMachine & {
|
|
1354
|
+
providerId: string;
|
|
1355
|
+
maxConcurrent: number;
|
|
1356
|
+
}>;
|
|
1357
|
+
};
|
|
1272
1358
|
queues: (namespace: string) => {
|
|
1273
1359
|
mounts: (query?: import("./common.js").Query) => Promise<unknown>;
|
|
1274
1360
|
mount: (input: {
|
|
@@ -1813,7 +1899,7 @@ export declare function createClient(options: ClientOptions): {
|
|
|
1813
1899
|
generate: (input: {
|
|
1814
1900
|
name: string;
|
|
1815
1901
|
entryId?: string;
|
|
1816
|
-
expectedRevision
|
|
1902
|
+
expectedRevision? /** @deprecated Use memos. */: number;
|
|
1817
1903
|
requestId?: string;
|
|
1818
1904
|
}) => Promise<unknown>;
|
|
1819
1905
|
};
|
|
@@ -2613,6 +2699,91 @@ export declare function createClient(options: ClientOptions): {
|
|
|
2613
2699
|
}) => Promise<unknown>;
|
|
2614
2700
|
disconnect: (id: string) => Promise<unknown>;
|
|
2615
2701
|
};
|
|
2702
|
+
/** CI/CT through GitHub Actions: runner pools, GitHub App installations and runs. */
|
|
2703
|
+
cict: (namespace: string) => {
|
|
2704
|
+
runs: {
|
|
2705
|
+
list: (query?: {
|
|
2706
|
+
limit?: number;
|
|
2707
|
+
}) => Promise<import("./cict.js").CictRun[]>;
|
|
2708
|
+
get: (id: string) => Promise<import("./cict.js").CictRun>;
|
|
2709
|
+
result: (id: string, options?: {
|
|
2710
|
+
file?: string;
|
|
2711
|
+
}) => Promise<unknown>;
|
|
2712
|
+
} & {
|
|
2713
|
+
comparison: (id: string) => Promise<import("./cict.js").CictComparison | null>;
|
|
2714
|
+
};
|
|
2715
|
+
jobs: (query?: {
|
|
2716
|
+
limit?: number;
|
|
2717
|
+
}) => Promise<unknown[]>;
|
|
2718
|
+
pools: {
|
|
2719
|
+
list: () => Promise<import("./cict.js").CictPool[]>;
|
|
2720
|
+
create: (input: import("./cict.js").CictPoolInput) => Promise<import("./cict.js").CictPool>;
|
|
2721
|
+
update: (poolId: string, patch: import("./cict.js").CictPoolPatch) => Promise<import("./cict.js").CictPool>;
|
|
2722
|
+
delete: (poolId: string) => Promise<{
|
|
2723
|
+
deleted: true;
|
|
2724
|
+
}>;
|
|
2725
|
+
targets: () => Promise<import("./cict.js").CictRunnerTarget[]>;
|
|
2726
|
+
};
|
|
2727
|
+
platform: {
|
|
2728
|
+
status: () => Promise<import("./cict.js").CictPlatformStatus>;
|
|
2729
|
+
start: (sourceId?: string) => Promise<{
|
|
2730
|
+
access: import("./cict.js").CictPlatformAccess;
|
|
2731
|
+
source: {
|
|
2732
|
+
sourceId: string;
|
|
2733
|
+
name: string;
|
|
2734
|
+
};
|
|
2735
|
+
}>;
|
|
2736
|
+
setSource: (sourceId: string) => Promise<{
|
|
2737
|
+
access: import("./cict.js").CictPlatformAccess;
|
|
2738
|
+
source: {
|
|
2739
|
+
sourceId: string;
|
|
2740
|
+
name: string;
|
|
2741
|
+
};
|
|
2742
|
+
}>;
|
|
2743
|
+
};
|
|
2744
|
+
usage: (month?: string) => Promise<import("./cict.js").CictUsage>;
|
|
2745
|
+
github: {
|
|
2746
|
+
installations: () => Promise<import("./cict.js").CictInstallation[]>;
|
|
2747
|
+
connect: () => Promise<{
|
|
2748
|
+
installUrl: string;
|
|
2749
|
+
}>;
|
|
2750
|
+
};
|
|
2751
|
+
};
|
|
2752
|
+
/** DreamLake GPU administration (admins of the platform namespace). */
|
|
2753
|
+
cictPlatformAdmin: {
|
|
2754
|
+
overview: () => Promise<{
|
|
2755
|
+
machines: Array<import("./cict.js").CictPlatformMachine & {
|
|
2756
|
+
providerId: string;
|
|
2757
|
+
env: Record<string, string>;
|
|
2758
|
+
maxConcurrent: number;
|
|
2759
|
+
}>;
|
|
2760
|
+
namespaces: Array<import("./cict.js").CictPlatformAccess & {
|
|
2761
|
+
namespaceId: string;
|
|
2762
|
+
slug: string | null;
|
|
2763
|
+
running: number;
|
|
2764
|
+
waiting: number;
|
|
2765
|
+
monthGpuSeconds: number;
|
|
2766
|
+
}>;
|
|
2767
|
+
}>;
|
|
2768
|
+
setNamespace: (slug: string, body: {
|
|
2769
|
+
state?: "allowed" | "suspended";
|
|
2770
|
+
maxConcurrent?: number;
|
|
2771
|
+
}) => Promise<import("./cict.js").CictPlatformAccess & {
|
|
2772
|
+
namespaceId: string;
|
|
2773
|
+
}>;
|
|
2774
|
+
putMachine: (id: string, body: {
|
|
2775
|
+
label?: string;
|
|
2776
|
+
title?: string;
|
|
2777
|
+
providerId?: string;
|
|
2778
|
+
enabled?: boolean;
|
|
2779
|
+
maxConcurrent?: number;
|
|
2780
|
+
env?: Record<string, string>;
|
|
2781
|
+
spec?: import("./cict.js").CictPlatformMachine["spec"];
|
|
2782
|
+
}) => Promise<import("./cict.js").CictPlatformMachine & {
|
|
2783
|
+
providerId: string;
|
|
2784
|
+
maxConcurrent: number;
|
|
2785
|
+
}>;
|
|
2786
|
+
};
|
|
2616
2787
|
queues: (namespace: string) => {
|
|
2617
2788
|
mounts: (query?: import("./common.js").Query) => Promise<unknown>;
|
|
2618
2789
|
mount: (input: {
|
package/dist/api/index.js
CHANGED
|
@@ -5,6 +5,8 @@ import { SourceFileAdapter } from "./file-source.js";
|
|
|
5
5
|
export * from "./file-workspace.js";
|
|
6
6
|
export * from "./file-source.js";
|
|
7
7
|
import { appsApi } from './apps.js';
|
|
8
|
+
import { cictApi, cictPlatformAdminApi } from './cict.js';
|
|
9
|
+
export * from './cict.js';
|
|
8
10
|
export * from './apps.js';
|
|
9
11
|
import { lakeshoreConnectionsApi } from './lakeshore-connections.js';
|
|
10
12
|
export * from './lakeshore-connections.js';
|
|
@@ -110,6 +112,10 @@ export function createApi(transport, storageFetch = fetch) {
|
|
|
110
112
|
runs: (namespace) => runsApi(transport, namespace),
|
|
111
113
|
providers: (namespace) => providersApi(transport, namespace),
|
|
112
114
|
lakeshores: (namespace) => lakeshoreConnectionsApi(transport, namespace),
|
|
115
|
+
/** CI/CT through GitHub Actions: runner pools, GitHub App installations and runs. */
|
|
116
|
+
cict: (namespace) => cictApi(transport, namespace),
|
|
117
|
+
/** DreamLake GPU administration (admins of the platform namespace). */
|
|
118
|
+
cictPlatformAdmin: cictPlatformAdminApi(transport),
|
|
113
119
|
queues: (namespace) => queuesApi(transport, namespace),
|
|
114
120
|
ownership: (namespace) => ownershipApi(transport, namespace),
|
|
115
121
|
runConfigs: (namespace) => resourceApi(transport, namespacePath(namespace) + "/run-configs"),
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { ApiError, createTransport } from "../../api/client.js";
|
|
2
|
+
import { cictApi, cictPlatformAdminApi } from "../../api/cict.js";
|
|
3
|
+
import { resolveRemote, resolveToken } from "../config.js";
|
|
4
|
+
// `dl cict` — CI/CT through GitHub Actions (channel A). Thin commands over the
|
|
5
|
+
// `dl.cict(namespace)` TypeScript API; the server owns validation and access.
|
|
6
|
+
const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
|
|
7
|
+
const ID = /^[a-f0-9]{24}$/;
|
|
8
|
+
const RUN = /^\d{1,12}-\d{1,20}$/;
|
|
9
|
+
const ENV = /^([A-Z][A-Z0-9_]{0,63})=(.*)$/s;
|
|
10
|
+
function api(namespace) {
|
|
11
|
+
if (!SEGMENT.test(namespace))
|
|
12
|
+
throw new Error("Use a single namespace name");
|
|
13
|
+
const token = resolveToken();
|
|
14
|
+
if (!token)
|
|
15
|
+
throw new Error("DreamLake authentication is required; run `dl login`");
|
|
16
|
+
return cictApi(createTransport({ remote: resolveRemote(), token }), namespace);
|
|
17
|
+
}
|
|
18
|
+
function adminApi() {
|
|
19
|
+
const token = resolveToken();
|
|
20
|
+
if (!token)
|
|
21
|
+
throw new Error("DreamLake authentication is required; run `dl login`");
|
|
22
|
+
return cictPlatformAdminApi(createTransport({ remote: resolveRemote(), token }));
|
|
23
|
+
}
|
|
24
|
+
function envOf(entries = []) {
|
|
25
|
+
const env = {};
|
|
26
|
+
for (const entry of entries) {
|
|
27
|
+
const m = ENV.exec(entry);
|
|
28
|
+
if (!m)
|
|
29
|
+
throw new Error("--env takes NAME=value with an uppercase NAME");
|
|
30
|
+
env[m[1]] = m[2];
|
|
31
|
+
}
|
|
32
|
+
return env;
|
|
33
|
+
}
|
|
34
|
+
function count(value, flag, max = 1000) {
|
|
35
|
+
const n = Number(value);
|
|
36
|
+
if (!Number.isSafeInteger(n) || n < 0 || n > max)
|
|
37
|
+
throw new Error(`${flag} must be a whole number from 0 to ${max}`);
|
|
38
|
+
return n;
|
|
39
|
+
}
|
|
40
|
+
const gpuMinutes = (seconds) => `${seconds > 0 && seconds < 60 ? "<1" : Math.round(seconds / 60)} GPU min`;
|
|
41
|
+
function selector(value, pattern, shape) {
|
|
42
|
+
const at = value.indexOf("/");
|
|
43
|
+
const [ns, id] = at < 0 ? ["", ""] : [value.slice(0, at), value.slice(at + 1)];
|
|
44
|
+
if (!SEGMENT.test(ns) || !pattern.test(id))
|
|
45
|
+
throw new Error(`Use ${shape}`);
|
|
46
|
+
return [ns, id];
|
|
47
|
+
}
|
|
48
|
+
function limit(value) {
|
|
49
|
+
if (value === undefined)
|
|
50
|
+
return undefined;
|
|
51
|
+
const n = Number(value);
|
|
52
|
+
if (!Number.isSafeInteger(n) || n < 1 || n > 200)
|
|
53
|
+
throw new Error("--limit must be between 1 and 200");
|
|
54
|
+
return n;
|
|
55
|
+
}
|
|
56
|
+
const collect = (value, previous = []) => [...previous, value];
|
|
57
|
+
async function execute(opts, work, text) {
|
|
58
|
+
try {
|
|
59
|
+
const result = await work();
|
|
60
|
+
console.log(opts.json || !text ? JSON.stringify(result, null, opts.json ? undefined : 2) : text(result));
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
const message = error instanceof ApiError ? `CI/CT request failed (HTTP ${error.status}): ${error.code}`
|
|
64
|
+
: error instanceof Error ? error.message : "CI/CT request failed";
|
|
65
|
+
if (opts.json)
|
|
66
|
+
console.log(JSON.stringify({ error: message, ...(error instanceof ApiError ? { code: error.code } : {}) }));
|
|
67
|
+
else
|
|
68
|
+
console.error(message);
|
|
69
|
+
process.exitCode = 1;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const outcome = (run) => run.status === "completed" ? run.conclusion ?? "completed" : run.status;
|
|
73
|
+
function runLine(run) {
|
|
74
|
+
const where = run.routing.kind === "matched" ? run.routing.pool.name
|
|
75
|
+
: run.routing.kind === "platform" ? run.routing.machine.label
|
|
76
|
+
: run.routing.kind === "rejected" || run.routing.kind === "refused" ? `${run.routing.kind}:${run.routing.reason}` : run.routing.kind;
|
|
77
|
+
const state = run.queue ? `queued #${run.queue.position}` : outcome(run);
|
|
78
|
+
return [run.id, state.padEnd(11), run.repository.fullName, run.name ?? "", run.headSha?.slice(0, 7) ?? "", where, run.evidence?.state ?? "-"].join(" ");
|
|
79
|
+
}
|
|
80
|
+
function runText(run) {
|
|
81
|
+
return [
|
|
82
|
+
`${run.repository.fullName} · ${run.name ?? `job ${run.jobId}`} (${run.id})`,
|
|
83
|
+
` status ${outcome(run)}`,
|
|
84
|
+
` commit ${run.headSha ?? "-"}${run.headBranch ? ` on ${run.headBranch}` : ""}`,
|
|
85
|
+
` github ${run.githubUrl}`,
|
|
86
|
+
` routing ${run.routing.kind === "matched" ? `pool ${run.routing.pool.name}` : run.routing.kind === "platform" ? `DreamLake GPU ${run.routing.machine.label}${run.queue ? ` · queued #${run.queue.position}` : ""}` : run.routing.kind === "refused" ? `not started: ${run.routing.reason}` : run.routing.kind}`,
|
|
87
|
+
...(run.sender ? [` by ${run.sender}`] : []),
|
|
88
|
+
...(run.usage ? [` GPU time ${gpuMinutes(run.usage.gpuSeconds)}`] : []),
|
|
89
|
+
...(run.comparison ? [` baseline ${run.comparison.baseline ? `${run.comparison.regressions} regressed, ${run.comparison.fixes} fixed vs ${run.comparison.baseline.branch}` : "none yet"}`] : []),
|
|
90
|
+
` runner ${run.runner ? `${run.runner.name} ${run.runner.state}${run.runner.slurmJobId ? ` (Slurm job ${run.runner.slurmJobId})` : ""}${run.runner.error ? ` error=${run.runner.error}` : ""}` : "-"}`,
|
|
91
|
+
` evidence ${run.evidence ? `${run.evidence.state} → source ${run.evidence.sourceId} ${run.evidence.prefix}` : "-"}`,
|
|
92
|
+
].join("\n");
|
|
93
|
+
}
|
|
94
|
+
export function registerCictCommand(program) {
|
|
95
|
+
const group = program.command("cict").description("CI/CT on DreamLake machines through GitHub Actions: GitHub connection, runner pools and runs");
|
|
96
|
+
const github = group.command("github").description("connect GitHub App installations to a namespace");
|
|
97
|
+
github.command("connect <namespace>").option("--json")
|
|
98
|
+
.description("print the GitHub App install link; open it while signed in to DreamLake as the same user within 15 minutes")
|
|
99
|
+
.action((ns, opts) => execute(opts, () => api(ns).github.connect(), (r) => `Open this link to install the DreamLake CI/CT GitHub App and choose repositories:\n${r.installUrl}\nGitHub returns to DreamLake, which binds the installation to ${ns}. Check with: dl cict github installations ${ns}`));
|
|
100
|
+
github.command("installations <namespace>").option("--json")
|
|
101
|
+
.description("list GitHub App installations bound to the namespace")
|
|
102
|
+
.action((ns, opts) => execute(opts, () => api(ns).github.installations(), (list) => list.length ? list.map((i) => `${i.installationId} ${i.accountLogin} (${i.accountType}) ${i.state}`).join("\n") : "No GitHub installation is bound; run dl cict github connect"));
|
|
103
|
+
const pools = group.command("pools").description("route workflow `runs-on` labels to DreamLake machines");
|
|
104
|
+
pools.command("list <namespace>").option("--json")
|
|
105
|
+
.action((ns, opts) => execute(opts, () => api(ns).pools.list(), (list) => list.length ? list.map((p) => `${p.id} ${p.name} runs-on: ${p.labels.join(",")} ${p.target.kind}:${p.target.providerId ?? p.target.hostId}${p.evidence ? ` evidence: ${p.evidence.sourceId}${p.evidence.prefix ? `/${p.evidence.prefix}` : ""}` : ""}`).join("\n") : "No runner pools"));
|
|
106
|
+
pools.command("create <namespace>")
|
|
107
|
+
.description("create a pool that starts single-use runners on a Slurm provider")
|
|
108
|
+
.requiredOption("--name <name>", "pool name; the default label is dreamlake-<name>")
|
|
109
|
+
.requiredOption("--provider <id>", "Slurm provider ID (your own association decides the partition)")
|
|
110
|
+
.option("--label <label>", "runs-on label to serve instead of dreamlake-<name> (repeatable)", collect)
|
|
111
|
+
.option("--evidence-source <id>", "Source that receives each run's declared evidence")
|
|
112
|
+
.option("--evidence-prefix <path>", "parent folder in that Source (default: ci)")
|
|
113
|
+
.option("--env <NAME=value>", "machine fact every job sees, such as a cache path; not for secrets (repeatable)", collect)
|
|
114
|
+
.option("--json")
|
|
115
|
+
.action((ns, opts) => execute(opts, () => {
|
|
116
|
+
if (!ID.test(opts.provider))
|
|
117
|
+
throw new Error("--provider must be a 24-character provider ID");
|
|
118
|
+
if (opts.evidencePrefix !== undefined && opts.evidenceSource === undefined)
|
|
119
|
+
throw new Error("--evidence-prefix requires --evidence-source");
|
|
120
|
+
if (opts.evidenceSource !== undefined && !ID.test(opts.evidenceSource))
|
|
121
|
+
throw new Error("--evidence-source must be a 24-character Source ID");
|
|
122
|
+
const env = envOf(opts.env);
|
|
123
|
+
const input = {
|
|
124
|
+
name: opts.name,
|
|
125
|
+
target: { kind: "slurm", providerId: opts.provider },
|
|
126
|
+
...(opts.label ? { labels: opts.label } : {}),
|
|
127
|
+
...(opts.evidenceSource ? { evidence: { sourceId: opts.evidenceSource, ...(opts.evidencePrefix ? { prefix: opts.evidencePrefix } : {}) } } : {}),
|
|
128
|
+
...(Object.keys(env).length ? { env } : {}),
|
|
129
|
+
};
|
|
130
|
+
return api(ns).pools.create(input);
|
|
131
|
+
}, (p) => `Created pool ${p.name} (${p.id}). Workflows use: runs-on: ${p.labels[0]}`));
|
|
132
|
+
pools.command("delete <pool>").description("delete a pool; queued jobs for its labels stop getting runners")
|
|
133
|
+
.option("--json")
|
|
134
|
+
.action((value, opts) => execute(opts, () => {
|
|
135
|
+
const [ns, id] = selector(value, ID, "namespace/pool-id");
|
|
136
|
+
return api(ns).pools.delete(id);
|
|
137
|
+
}, () => "Deleted"));
|
|
138
|
+
const runs = group.command("runs").description("CI/CT runs: each GitHub job that asked for a DreamLake runner");
|
|
139
|
+
runs.command("list <namespace>").option("--limit <n>", "most recent runs, 1-200 (default 50)").option("--json")
|
|
140
|
+
.action((ns, opts) => execute(opts, () => api(ns).runs.list({ limit: limit(opts.limit) }), (list) => list.length ? list.map(runLine).join("\n") : "No CI/CT runs yet"));
|
|
141
|
+
runs.command("show <run>").description("show one run: routing, runner, Slurm job and evidence location")
|
|
142
|
+
.option("--result", "also print the run's result.json from its evidence Source")
|
|
143
|
+
.option("--json")
|
|
144
|
+
.action((value, opts) => execute(opts, async () => {
|
|
145
|
+
const [ns, id] = selector(value, RUN, "namespace/<installationId>-<jobId>");
|
|
146
|
+
const client = api(ns);
|
|
147
|
+
const run = await client.runs.get(id);
|
|
148
|
+
return opts.result ? { run, result: await client.runs.result(id) } : run;
|
|
149
|
+
}, (r) => runShowText(r)));
|
|
150
|
+
runs.command("compare <run>").description("compare a run case by case with its baseline: the latest earlier run of the same job on the default branch")
|
|
151
|
+
.option("--all", "list every case, not only the ones that changed")
|
|
152
|
+
.option("--json")
|
|
153
|
+
.action((value, opts) => execute(opts, async () => {
|
|
154
|
+
const [ns, id] = selector(value, RUN, "namespace/<installationId>-<jobId>");
|
|
155
|
+
return api(ns).runs.comparison(id);
|
|
156
|
+
}, (c) => {
|
|
157
|
+
if (!c)
|
|
158
|
+
return "Not compared yet: the run's evidence is not written, or it wrote no result.json.";
|
|
159
|
+
const head = c.baseline
|
|
160
|
+
? `${c.passed}/${c.total} cases passed · ${c.regressions} regressed · ${c.fixes} fixed vs ${c.baseline.branch}@${c.baseline.headSha?.slice(0, 7) ?? "?"} (run ${c.baseline.runId})`
|
|
161
|
+
: `${c.passed}/${c.total} cases passed · no earlier default-branch run to compare with`;
|
|
162
|
+
const rows = (opts.all ? c.cases : c.cases.filter((x) => x.change !== "unchanged")).map((x) => {
|
|
163
|
+
const score = x.score?.delta !== undefined ? ` ${x.score.name} ${x.score.delta >= 0 ? "+" : ""}${Math.round(x.score.delta * 1000) / 1000}` : "";
|
|
164
|
+
return ` ${x.change.padEnd(13)} ${x.id}${score}`;
|
|
165
|
+
});
|
|
166
|
+
return [head, ...rows].join("\n");
|
|
167
|
+
}));
|
|
168
|
+
pools.command("update <pool>").description("edit a pool; running jobs keep what they started with")
|
|
169
|
+
.option("--provider <id>", "Slurm provider ID")
|
|
170
|
+
.option("--label <label>", "replace the runs-on labels (repeatable)", collect)
|
|
171
|
+
.option("--evidence-source <id>", "Source that receives evidence")
|
|
172
|
+
.option("--evidence-prefix <path>", "parent folder in that Source")
|
|
173
|
+
.option("--no-evidence", "stop keeping evidence")
|
|
174
|
+
.option("--env <NAME=value>", "replace every variable (repeatable)", collect)
|
|
175
|
+
.option("--clear-env", "remove every variable")
|
|
176
|
+
.option("--json")
|
|
177
|
+
.action((value, opts) => execute(opts, () => {
|
|
178
|
+
const [ns, id] = selector(value, ID, "namespace/pool-id");
|
|
179
|
+
const patch = {};
|
|
180
|
+
if (opts.provider !== undefined) {
|
|
181
|
+
if (!ID.test(opts.provider))
|
|
182
|
+
throw new Error("--provider must be a 24-character provider ID");
|
|
183
|
+
patch.target = { kind: "slurm", providerId: opts.provider };
|
|
184
|
+
}
|
|
185
|
+
if (opts.label)
|
|
186
|
+
patch.labels = opts.label;
|
|
187
|
+
if (opts.evidence === false)
|
|
188
|
+
patch.evidence = null;
|
|
189
|
+
else if (opts.evidenceSource !== undefined) {
|
|
190
|
+
if (!ID.test(opts.evidenceSource))
|
|
191
|
+
throw new Error("--evidence-source must be a 24-character Source ID");
|
|
192
|
+
patch.evidence = { sourceId: opts.evidenceSource, ...(opts.evidencePrefix ? { prefix: opts.evidencePrefix } : {}) };
|
|
193
|
+
}
|
|
194
|
+
else if (opts.evidencePrefix !== undefined)
|
|
195
|
+
throw new Error("--evidence-prefix requires --evidence-source");
|
|
196
|
+
if (opts.clearEnv)
|
|
197
|
+
patch.env = null;
|
|
198
|
+
else if (opts.env)
|
|
199
|
+
patch.env = envOf(opts.env);
|
|
200
|
+
if (!Object.keys(patch).length)
|
|
201
|
+
throw new Error("Nothing to change: pass --provider, --label, --evidence-source, --no-evidence, --env or --clear-env");
|
|
202
|
+
return api(ns).pools.update(id, patch);
|
|
203
|
+
}, (p) => `Updated ${p.name}; the next job uses it.`));
|
|
204
|
+
pools.command("targets <namespace>").description("Slurm providers a pool can use, and why the others cannot start runners")
|
|
205
|
+
.option("--json")
|
|
206
|
+
.action((ns, opts) => execute(opts, () => api(ns).pools.targets(), (list) => list.length ? list.map((t) => `${t.id} ${t.name} ${t.usable ? "usable" : `not usable: ${t.reason}`}`).join("\n") : "No Slurm provider in this namespace"));
|
|
207
|
+
const platform = group.command("platform").description("DreamLake GPU: machines DreamLake provides, chosen by runs-on label");
|
|
208
|
+
platform.command("show <namespace>").description("machine catalog, whether the namespace can use it, its limit, results Source and this month's GPU time")
|
|
209
|
+
.option("--json")
|
|
210
|
+
.action((ns, opts) => execute(opts, () => api(ns).platform.status(), (st) => {
|
|
211
|
+
if (!st.available)
|
|
212
|
+
return "DreamLake GPU is not offered on this server.";
|
|
213
|
+
const a = st.access;
|
|
214
|
+
const state = !a ? "not enabled for this namespace (ask the DreamLake team)"
|
|
215
|
+
: a.state === "allowed" ? "enabled, not started: run dl cict platform start" : a.state === "suspended" ? "paused" : "in use";
|
|
216
|
+
return [
|
|
217
|
+
`DreamLake GPU: ${state}`,
|
|
218
|
+
...(a?.state === "active" ? [` running ${st.running} of ${a.maxConcurrent} · queued ${st.waiting} · this month ${gpuMinutes(st.monthGpuSeconds)}`,
|
|
219
|
+
` results: source ${a.evidence?.sourceId} / ${a.evidence?.prefix}`] : []),
|
|
220
|
+
"Machines:",
|
|
221
|
+
...st.machines.map((m) => ` ${m.label} ${m.spec.gpus}x ${m.spec.gpu} · ${m.spec.cpus} CPU · ${m.spec.memGiB} GB · ${m.spec.timeLimit}`),
|
|
222
|
+
].join("\n");
|
|
223
|
+
}));
|
|
224
|
+
platform.command("start <namespace>").description("start using DreamLake GPU: create cict-evidence (or use --source) for results; you authorize the writes")
|
|
225
|
+
.option("--source <id>", "an existing Source you can edit")
|
|
226
|
+
.option("--json")
|
|
227
|
+
.action((ns, opts) => execute(opts, () => {
|
|
228
|
+
if (opts.source !== undefined && !ID.test(opts.source))
|
|
229
|
+
throw new Error("--source must be a 24-character Source ID");
|
|
230
|
+
return api(ns).platform.start(opts.source);
|
|
231
|
+
}, (r) => `DreamLake GPU in use; results go to ${r.source.name}.`));
|
|
232
|
+
platform.command("source <namespace>").description("change where DreamLake GPU runs save results")
|
|
233
|
+
.requiredOption("--source <id>", "a Source you can edit")
|
|
234
|
+
.option("--json")
|
|
235
|
+
.action((ns, opts) => execute(opts, () => {
|
|
236
|
+
if (!ID.test(opts.source))
|
|
237
|
+
throw new Error("--source must be a 24-character Source ID");
|
|
238
|
+
return api(ns).platform.setSource(opts.source);
|
|
239
|
+
}, (r) => `Results now go to ${r.source.name}.`));
|
|
240
|
+
group.command("usage <namespace>").description("DreamLake GPU time in a month, by repository and workflow")
|
|
241
|
+
.option("--month <YYYY-MM>", "default: this month")
|
|
242
|
+
.option("--json")
|
|
243
|
+
.action((ns, opts) => execute(opts, async () => api(ns).usage(opts.month), (u) => [
|
|
244
|
+
`${u.month}: ${gpuMinutes(u.gpuSeconds)} over ${u.jobs} job${u.jobs === 1 ? "" : "s"}`,
|
|
245
|
+
...u.byWorkflow.map((r) => ` ${r.repository} ${r.workflowName ?? "-"} ${r.jobs} jobs ${gpuMinutes(r.gpuSeconds)}`),
|
|
246
|
+
].join("\n")));
|
|
247
|
+
const admin = platform.command("admin").description("administer DreamLake GPU (admins of the platform namespace)");
|
|
248
|
+
admin.command("list").description("machine types and namespaces with their limits, occupancy and this month's GPU time")
|
|
249
|
+
.option("--json")
|
|
250
|
+
.action((opts) => execute(opts, () => adminApi().overview(), (o) => [
|
|
251
|
+
"Namespaces:",
|
|
252
|
+
...o.namespaces.map((n) => ` ${n.slug ?? n.namespaceId} ${n.state} limit ${n.maxConcurrent} running ${n.running} queued ${n.waiting} ${gpuMinutes(n.monthGpuSeconds)}`),
|
|
253
|
+
"Machines:",
|
|
254
|
+
...o.machines.map((m) => ` ${m.id} ${m.label} ${m.enabled ? "enabled" : "disabled"} limit ${m.maxConcurrent} provider ${m.providerId}`),
|
|
255
|
+
].join("\n")));
|
|
256
|
+
admin.command("allow <namespace>").description("let a namespace use DreamLake GPU (or resume a paused one)")
|
|
257
|
+
.option("--limit <n>", "jobs at once for this namespace")
|
|
258
|
+
.option("--json")
|
|
259
|
+
.action((slug, opts) => execute(opts, () => adminApi().setNamespace(slug, { state: "allowed", ...(opts.limit !== undefined ? { maxConcurrent: count(opts.limit, "--limit", 100) } : {}) }), (a) => `${slug}: ${a.state}, limit ${a.maxConcurrent}`));
|
|
260
|
+
admin.command("pause <namespace>").description("stop starting new DreamLake GPU jobs for a namespace; running ones finish")
|
|
261
|
+
.option("--json")
|
|
262
|
+
.action((slug, opts) => execute(opts, () => adminApi().setNamespace(slug, { state: "suspended" }), () => `${slug}: paused`));
|
|
263
|
+
admin.command("limit <namespace> <n>").description("set how many DreamLake GPU jobs a namespace runs at once")
|
|
264
|
+
.option("--json")
|
|
265
|
+
.action((slug, n, opts) => execute(opts, () => adminApi().setNamespace(slug, { maxConcurrent: count(n, "limit", 100) }), (a) => `${slug}: limit ${a.maxConcurrent}`));
|
|
266
|
+
admin.command("machine <id>").description("create or change a machine type")
|
|
267
|
+
.option("--label <label>", "runs-on label, dreamlake-…")
|
|
268
|
+
.option("--title <text>")
|
|
269
|
+
.option("--provider <id>", "Slurm provider in the platform namespace")
|
|
270
|
+
.option("--limit <n>", "jobs at once across all namespaces")
|
|
271
|
+
.option("--enable").option("--disable")
|
|
272
|
+
.option("--gpus <n>").option("--gpu <model>").option("--cpus <n>").option("--mem-gib <n>").option("--time-limit <hh:mm:ss>")
|
|
273
|
+
.option("--env <NAME=value>", "replace every variable (repeatable)", collect)
|
|
274
|
+
.option("--json")
|
|
275
|
+
.action((id, opts) => execute(opts, () => {
|
|
276
|
+
if (opts.enable && opts.disable)
|
|
277
|
+
throw new Error("Pass --enable or --disable, not both");
|
|
278
|
+
const specFlags = [opts.gpus, opts.gpu, opts.cpus, opts.memGib, opts.timeLimit];
|
|
279
|
+
if (specFlags.some((v) => v !== undefined) && specFlags.some((v) => v === undefined))
|
|
280
|
+
throw new Error("The catalog spec needs all of --gpus, --gpu, --cpus, --mem-gib and --time-limit");
|
|
281
|
+
if (opts.provider !== undefined && !ID.test(opts.provider))
|
|
282
|
+
throw new Error("--provider must be a 24-character provider ID");
|
|
283
|
+
return adminApi().putMachine(id, {
|
|
284
|
+
...(opts.label ? { label: opts.label } : {}), ...(opts.title ? { title: opts.title } : {}),
|
|
285
|
+
...(opts.provider ? { providerId: opts.provider } : {}),
|
|
286
|
+
...(opts.limit !== undefined ? { maxConcurrent: count(opts.limit, "--limit") } : {}),
|
|
287
|
+
...(opts.enable ? { enabled: true } : opts.disable ? { enabled: false } : {}),
|
|
288
|
+
...(opts.gpus !== undefined ? { spec: { gpus: count(opts.gpus, "--gpus", 16), gpu: opts.gpu, cpus: count(opts.cpus, "--cpus", 512), memGiB: count(opts.memGib, "--mem-gib", 4096), timeLimit: opts.timeLimit } } : {}),
|
|
289
|
+
...(opts.env ? { env: envOf(opts.env) } : {}),
|
|
290
|
+
});
|
|
291
|
+
}, (m) => `${m.id}: ${m.label} ${m.enabled ? "enabled" : "disabled"}, limit ${m.maxConcurrent}`));
|
|
292
|
+
}
|
|
293
|
+
// Text output for `runs show`, with or without the result.
|
|
294
|
+
function runShowText(value) {
|
|
295
|
+
if (!value?.run)
|
|
296
|
+
return runText(value);
|
|
297
|
+
return `${runText(value.run)}\n\nresult.json:\n${value.result === null ? " (none written)" : JSON.stringify(value.result, null, 2)}`;
|
|
298
|
+
}
|
|
@@ -26,6 +26,30 @@ export const helpExamples = {
|
|
|
26
26
|
"runnable bind": "dl runnable bind udf/acme/recipes.process@FULL_SHA256 \\\n --snapshot CODE_REPO_ID --project recipes \\\n --server \"$LAKESHORE_URL\" --cp-namespace acme --dry-run\ndl runnable bind udf/acme/recipes.process@FULL_SHA256 \\\n --snapshot CODE_REPO_ID --project recipes --server \"$LAKESHORE_URL\" --cp-namespace acme",
|
|
27
27
|
"runnable execute": "dl runnable execute exec/acme/recipes.process@EXECUTION_DIGEST \\\n --queue cpu --inputs record.json --timeout-seconds 600 --json\ndl runnable execute exec/acme/recipes.process@EXECUTION_DIGEST \\\n --queue cpu --inputs record.json --timeout-seconds 600 --attempt 2 --json",
|
|
28
28
|
"runnable execution-receipt": "dl runnable execution-receipt exec/acme/recipes.process@EXECUTION_DIGEST \\\n --queue cpu --inputs record.json --timeout-seconds 600 --wait-seconds 30",
|
|
29
|
+
"cict github": "dl cict github connect my-team\ndl cict github installations my-team",
|
|
30
|
+
"cict github connect": "dl cict github connect my-team",
|
|
31
|
+
"cict github installations": "dl cict github installations my-team --json",
|
|
32
|
+
"cict pools": "dl cict pools create my-team --name bos14-gpu \\\n --provider 0123456789abcdef01234567 \\\n --evidence-source 1123456789abcdef01234567\ndl cict pools list my-team\ndl cict pools delete my-team/2123456789abcdef01234567",
|
|
33
|
+
"cict pools create": "dl cict pools create my-team --name bos14-gpu \\\n --provider 0123456789abcdef01234567 \\\n --evidence-source 1123456789abcdef01234567 --evidence-prefix ci \\\n --env DREAMLAKE_ROBOT_ASSETS_CACHE=/work/assets",
|
|
34
|
+
"cict pools list": "dl cict pools list my-team --json",
|
|
35
|
+
"cict pools delete": "dl cict pools delete my-team/2123456789abcdef01234567",
|
|
36
|
+
"cict pools update": "dl cict pools update my-team/2123456789abcdef01234567 --provider 0123456789abcdef01234567\ndl cict pools update my-team/2123456789abcdef01234567 --no-evidence --clear-env",
|
|
37
|
+
"cict pools targets": "dl cict pools targets my-team",
|
|
38
|
+
"cict runs": "dl cict runs list my-team --limit 20\ndl cict runs show my-team/91234567-48812345678 --result",
|
|
39
|
+
"cict runs list": "dl cict runs list my-team --limit 20 --json",
|
|
40
|
+
"cict runs show": "dl cict runs show my-team/91234567-48812345678\ndl cict runs show my-team/91234567-48812345678 --result --json",
|
|
41
|
+
"cict runs compare": "dl cict runs compare my-team/91234567-48812345678\ndl cict runs compare my-team/91234567-48812345678 --all --json",
|
|
42
|
+
"cict platform": "dl cict platform show my-team\ndl cict platform start my-team\ndl cict usage my-team",
|
|
43
|
+
"cict platform show": "dl cict platform show my-team --json",
|
|
44
|
+
"cict platform start": "dl cict platform start my-team\ndl cict platform start my-team --source 1123456789abcdef01234567",
|
|
45
|
+
"cict platform source": "dl cict platform source my-team --source 1123456789abcdef01234567",
|
|
46
|
+
"cict usage": "dl cict usage my-team\ndl cict usage my-team --month 2026-10 --json",
|
|
47
|
+
"cict platform admin": "dl cict platform admin list\ndl cict platform admin allow robotics-lab --limit 2\ndl cict platform admin pause robotics-lab",
|
|
48
|
+
"cict platform admin list": "dl cict platform admin list --json",
|
|
49
|
+
"cict platform admin allow": "dl cict platform admin allow robotics-lab --limit 2",
|
|
50
|
+
"cict platform admin pause": "dl cict platform admin pause robotics-lab",
|
|
51
|
+
"cict platform admin limit": "dl cict platform admin limit robotics-lab 4",
|
|
52
|
+
"cict platform admin machine": "dl cict platform admin machine gpu-rtx5090 --label dreamlake-gpu-rtx5090 \\\n --provider 0123456789abcdef01234567 --limit 6 --enable \\\n --gpus 1 --gpu \"RTX 5090\" --cpus 8 --mem-gib 32 --time-limit 02:00:00\ndl cict platform admin machine gpu-rtx5090 --disable",
|
|
29
53
|
"hosts": "dl hosts\ndl hosts --all\ndl hosts -h\ndl hosts list -h",
|
|
30
54
|
"su": "dl su fortyfive\ndl hosts list\ndl hosts list --namespace geyang\ndl su geyang",
|
|
31
55
|
"logout": "dl logout --env staging",
|
package/dist/cli/index.js
CHANGED
|
@@ -19,6 +19,7 @@ import { registerQueuesCommand } from "./queues/index.js";
|
|
|
19
19
|
import { registerServiceAccountsCommand } from "./service-accounts/index.js";
|
|
20
20
|
import { registerExperimentsCommand } from "./experiments/index.js";
|
|
21
21
|
import { registerProvidersCommand } from "./providers/index.js";
|
|
22
|
+
import { registerCictCommand } from "./cict/index.js";
|
|
22
23
|
import { executeRun, registerRunsCommands } from "./run/index.js";
|
|
23
24
|
import { authFilePath, BUILTIN_ENVS } from "./auth/credentials.js";
|
|
24
25
|
import { migrateLegacyConfig } from "./auth/migrate.js";
|
|
@@ -253,6 +254,7 @@ async function main(argv) {
|
|
|
253
254
|
// because the server itself has one registrar for all three.
|
|
254
255
|
registerProviderCommand(program);
|
|
255
256
|
registerProvidersCommand(program);
|
|
257
|
+
registerCictCommand(program);
|
|
256
258
|
registerServiceAccountsCommand(program);
|
|
257
259
|
registerQueuesCommand(program);
|
|
258
260
|
registerLakeshoreCommand(program);
|
|
@@ -13,7 +13,7 @@ export declare function validateKey(value: string): void;
|
|
|
13
13
|
export declare function prepareTransfers(transfers: Transfer[]): Promise<void>;
|
|
14
14
|
export declare function transferSelected(transfers: Transfer[], call: VaultCall, cancelled?: () => boolean): Promise<void>;
|
|
15
15
|
export declare function transferReport(transfers: Transfer[]): {
|
|
16
|
-
status: "success" | "
|
|
16
|
+
status: "success" | "pending" | "unknown" | "failure";
|
|
17
17
|
detail: string;
|
|
18
18
|
ifMatch?: number | undefined;
|
|
19
19
|
id: string;
|
package/dist/cli/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "0.123.0";
|
package/dist/cli/version.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dreamlake/dreamlake-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.123.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"type": "git",
|
|
6
6
|
"url": "git+https://github.com/dreamlake-ai/dreamlake-cli.git"
|
|
@@ -37,13 +37,13 @@
|
|
|
37
37
|
"node": ">=18"
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
|
-
"@dreamlake/dreamlake-cli-darwin-arm64": "0.
|
|
41
|
-
"@dreamlake/dreamlake-cli-darwin-x64": "0.
|
|
42
|
-
"@dreamlake/dreamlake-cli-linux-x64": "0.
|
|
43
|
-
"@dreamlake/dreamlake-cli-linux-arm64": "0.
|
|
44
|
-
"@dreamlake/dreamlake-cli-linux-x64-musl": "0.
|
|
45
|
-
"@dreamlake/dreamlake-cli-linux-arm64-musl": "0.
|
|
46
|
-
"@dreamlake/dreamlake-cli-win32-x64": "0.
|
|
47
|
-
"@dreamlake/dreamlake-cli-win32-arm64": "0.
|
|
40
|
+
"@dreamlake/dreamlake-cli-darwin-arm64": "0.123.0",
|
|
41
|
+
"@dreamlake/dreamlake-cli-darwin-x64": "0.123.0",
|
|
42
|
+
"@dreamlake/dreamlake-cli-linux-x64": "0.123.0",
|
|
43
|
+
"@dreamlake/dreamlake-cli-linux-arm64": "0.123.0",
|
|
44
|
+
"@dreamlake/dreamlake-cli-linux-x64-musl": "0.123.0",
|
|
45
|
+
"@dreamlake/dreamlake-cli-linux-arm64-musl": "0.123.0",
|
|
46
|
+
"@dreamlake/dreamlake-cli-win32-x64": "0.123.0",
|
|
47
|
+
"@dreamlake/dreamlake-cli-win32-arm64": "0.123.0"
|
|
48
48
|
}
|
|
49
49
|
}
|