@ferronhooi/aihub 0.1.0-next.3 → 0.1.0-next.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/fabric.js +71 -0
- package/dist/commands/robot.js +53 -0
- package/dist/lib/changes.js +14 -0
- package/dist/lib/fabric.js +232 -0
- package/dist/lib/glob.js +28 -0
- package/dist/lib/robot.js +88 -0
- package/dist/router.js +4 -0
- package/dist/tools/lintSkills.js +1 -0
- package/pack/commands/copilot/test.prompt.md +6 -0
- package/pack/commands/opencode/test.md +5 -0
- package/pack/skills/_shared/references/team.schema.yaml +15 -0
- package/pack/skills/test-run/SKILL.md +35 -0
- package/pack/skills/test-run/references/fabric-robot.md +23 -0
- package/package.json +1 -1
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { parseArgs } from "node:util";
|
|
3
|
+
import { changedFiles } from "../lib/changes.js";
|
|
4
|
+
import { AihubError } from "../lib/errors.js";
|
|
5
|
+
import { buildPlan, chooseWorkspace, executionDataFromParams, fabricWorkspaces, FabricClient, formatJob, formatPlan, guardEnv, makeFabricAuth, waitForJob, } from "../lib/fabric.js";
|
|
6
|
+
import { git, repoRoot } from "../lib/git.js";
|
|
7
|
+
import { loadTeam } from "../lib/team.js";
|
|
8
|
+
const USAGE = "usage: aihub fabric plan|run|status [flags]";
|
|
9
|
+
export async function run(argv, ctx) {
|
|
10
|
+
const [sub, ...rest] = argv;
|
|
11
|
+
let v;
|
|
12
|
+
try {
|
|
13
|
+
v = parseArgs({
|
|
14
|
+
args: rest,
|
|
15
|
+
options: {
|
|
16
|
+
branch: { type: "string" },
|
|
17
|
+
target: { type: "string" },
|
|
18
|
+
workspace: { type: "string" },
|
|
19
|
+
item: { type: "string" },
|
|
20
|
+
job: { type: "string" },
|
|
21
|
+
"params-file": { type: "string" },
|
|
22
|
+
"allow-env": { type: "string" },
|
|
23
|
+
wait: { type: "boolean" },
|
|
24
|
+
},
|
|
25
|
+
}).values;
|
|
26
|
+
}
|
|
27
|
+
catch (e) {
|
|
28
|
+
throw new AihubError(e.message);
|
|
29
|
+
}
|
|
30
|
+
if (sub !== "plan" && sub !== "run" && sub !== "status")
|
|
31
|
+
throw new AihubError(USAGE);
|
|
32
|
+
const root = repoRoot(ctx.cwd);
|
|
33
|
+
const team = loadTeam(root);
|
|
34
|
+
const list = fabricWorkspaces(team);
|
|
35
|
+
const given = v.branch;
|
|
36
|
+
if (sub === "plan") {
|
|
37
|
+
const c = changedFiles(root, team, { branch: given, target: v.target });
|
|
38
|
+
ctx.out(formatPlan(buildPlan(c.files, list, { branch: c.branch, workspace: v.workspace }), c.branch, c.target));
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const branch = given ?? git(root, ["rev-parse", "--abbrev-ref", "HEAD"], { allowFail: true }).trim();
|
|
42
|
+
const ws = chooseWorkspace(list, { workspace: v.workspace, branch });
|
|
43
|
+
if (!ws) {
|
|
44
|
+
if (v.workspace)
|
|
45
|
+
throw new AihubError(`workspace '${v.workspace}' is not in team.yaml fabric.workspaces; the CLI only runs workspaces the team listed`);
|
|
46
|
+
throw new AihubError("no dev/feature workspace in team.yaml fabric.workspaces; pass --workspace <name|id>");
|
|
47
|
+
}
|
|
48
|
+
guardEnv(ws, v["allow-env"]);
|
|
49
|
+
if (!v.item)
|
|
50
|
+
throw new AihubError(`aihub fabric ${sub} needs --item <name|id>`);
|
|
51
|
+
if (sub === "status" && !v.job)
|
|
52
|
+
throw new AihubError("aihub fabric status needs --job <id> and --item <name|id>");
|
|
53
|
+
const client = new FabricClient(makeFabricAuth({ env: ctx.env, exec: ctx.exec, cacheFile: ctx.tokenCache && `${ctx.tokenCache}.fabric` }), ctx.fetch);
|
|
54
|
+
const wsId = await client.workspaceId(ws);
|
|
55
|
+
const item = await client.resolveItem(wsId, v.item);
|
|
56
|
+
if (sub === "status") {
|
|
57
|
+
ctx.out(formatJob(await client.jobStatus(wsId, item.id, v.job), item.displayName));
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const exec = v["params-file"] ? executionDataFromParams(readFileSync(v["params-file"], "utf8"), item) : undefined;
|
|
61
|
+
const jobId = await client.startJob(wsId, item, exec);
|
|
62
|
+
ctx.out(`started ${item.type} ${item.displayName} in ${ws.name} (${ws.environment}): job ${jobId}`);
|
|
63
|
+
if (!v.wait) {
|
|
64
|
+
ctx.out(`follow with: aihub fabric status --item ${item.displayName} --job ${jobId}`);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const j = await waitForJob(client, wsId, item.id, jobId, { pollMs: Number(ctx.env.AIHUB_POLL_MS ?? 5000) });
|
|
68
|
+
ctx.out(formatJob(j, item.displayName));
|
|
69
|
+
if (j.status !== "Completed")
|
|
70
|
+
throw new AihubError(`${item.displayName} ended as ${j.status}`);
|
|
71
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { changedFiles } from "../lib/changes.js";
|
|
3
|
+
import { repoClient } from "../lib/context.js";
|
|
4
|
+
import { AihubError } from "../lib/errors.js";
|
|
5
|
+
import { followBuild, formatNeeded, queueRobot, robotConfig, robotNeeded, summarize } from "../lib/robot.js";
|
|
6
|
+
import { loadTeam } from "../lib/team.js";
|
|
7
|
+
export async function run(argv, ctx) {
|
|
8
|
+
const [sub, ...rest] = argv;
|
|
9
|
+
let v;
|
|
10
|
+
try {
|
|
11
|
+
v = parseArgs({
|
|
12
|
+
args: rest,
|
|
13
|
+
options: {
|
|
14
|
+
branch: { type: "string" },
|
|
15
|
+
target: { type: "string" },
|
|
16
|
+
suite: { type: "string" },
|
|
17
|
+
include: { type: "string" },
|
|
18
|
+
pipeline: { type: "string" },
|
|
19
|
+
"no-wait": { type: "boolean" },
|
|
20
|
+
},
|
|
21
|
+
}).values;
|
|
22
|
+
}
|
|
23
|
+
catch (e) {
|
|
24
|
+
throw new AihubError(e.message);
|
|
25
|
+
}
|
|
26
|
+
if (sub !== "needed" && sub !== "run")
|
|
27
|
+
throw new AihubError("usage: aihub robot needed|run [flags]");
|
|
28
|
+
const { root, client } = repoClient(ctx);
|
|
29
|
+
const team = loadTeam(root);
|
|
30
|
+
const cfg = robotConfig(team);
|
|
31
|
+
if (sub === "needed") {
|
|
32
|
+
const c = changedFiles(root, team, { branch: v.branch, target: v.target });
|
|
33
|
+
ctx.out(formatNeeded(robotNeeded(c.files, cfg), cfg, c.branch));
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const pipeline = v.pipeline ?? cfg.pipeline;
|
|
37
|
+
if (!pipeline)
|
|
38
|
+
throw new AihubError("no Robot pipeline: set robot.pipeline in team.yaml or pass --pipeline <name>");
|
|
39
|
+
const branch = cfg.integrationBranch;
|
|
40
|
+
if (!branch)
|
|
41
|
+
throw new AihubError("robot.integrationBranch is not set in team.yaml; the Robot pipeline runs on that branch");
|
|
42
|
+
if (v.suite && cfg.suites.length && !cfg.suites.some((s) => s.name === v.suite)) {
|
|
43
|
+
throw new AihubError(`unknown suite '${v.suite}'; team.yaml suites: ${cfg.suites.map((s) => s.name).join(", ")}`);
|
|
44
|
+
}
|
|
45
|
+
const b = await queueRobot(client, { pipeline, branch, suite: v.suite, include: v.include });
|
|
46
|
+
ctx.out(`queued ${pipeline} build ${b.id} on ${branch}${v.suite ? ` (suite ${v.suite})` : ""}${v.include ? ` (include ${v.include})` : ""}`);
|
|
47
|
+
if (v["no-wait"])
|
|
48
|
+
return;
|
|
49
|
+
const done = await followBuild(client, b.id, Number(ctx.env.AIHUB_POLL_MS ?? 15000));
|
|
50
|
+
ctx.out(await summarize(client, done));
|
|
51
|
+
if (done.result !== "succeeded")
|
|
52
|
+
throw new AihubError(`Robot build ${b.id} ended as ${done.result ?? done.status}`);
|
|
53
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { AihubError } from "./errors.js";
|
|
2
|
+
import { git, hasRef, originDefaultBranch } from "./git.js";
|
|
3
|
+
/** Files changed on `branch` (default HEAD) compared with the target branch (merge-base diff). */
|
|
4
|
+
export function changedFiles(cwd, team, opts = {}) {
|
|
5
|
+
const target = opts.target ?? team.repositories?.[0]?.defaultTarget ?? (originDefaultBranch(cwd) || "main");
|
|
6
|
+
const branch = opts.branch ?? "HEAD";
|
|
7
|
+
const tref = hasRef(cwd, `origin/${target}`) ? `origin/${target}` : target;
|
|
8
|
+
if (!hasRef(cwd, tref))
|
|
9
|
+
throw new AihubError(`target branch '${target}' not found; fetch it or pass --target`);
|
|
10
|
+
const bref = branch !== "HEAD" && !hasRef(cwd, branch) && hasRef(cwd, `origin/${branch}`) ? `origin/${branch}` : branch;
|
|
11
|
+
const out = git(cwd, ["diff", "--name-only", `${tref}...${bref}`]);
|
|
12
|
+
const label = branch === "HEAD" ? git(cwd, ["rev-parse", "--abbrev-ref", "HEAD"], { allowFail: true }).trim() || "HEAD" : branch;
|
|
13
|
+
return { files: out.split("\n").map((s) => s.trim()).filter(Boolean), target, branch: label };
|
|
14
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { defaultExec } from "./auth.js";
|
|
5
|
+
import { AihubError } from "./errors.js";
|
|
6
|
+
import { anyGlob } from "./glob.js";
|
|
7
|
+
export const FABRIC_API = "https://api.fabric.microsoft.com/v1";
|
|
8
|
+
export const FABRIC_RESOURCE = "https://api.fabric.microsoft.com";
|
|
9
|
+
export const ENVS = ["dev", "feature", "test", "acc", "prd"];
|
|
10
|
+
/** Environments the CLI may run in without an explicit --allow-env. */
|
|
11
|
+
const SAFE_ENVS = ["dev", "feature"];
|
|
12
|
+
export function fabricWorkspaces(team) {
|
|
13
|
+
const f = team.fabric;
|
|
14
|
+
return (f?.workspaces ?? [])
|
|
15
|
+
.filter((w) => !!w?.name && ENVS.includes(String(w.environment).toLowerCase()))
|
|
16
|
+
.map((w) => ({ ...w, environment: String(w.environment).toLowerCase() }));
|
|
17
|
+
}
|
|
18
|
+
const ITEM_TYPES = ["Notebook", "DataPipeline", "Lakehouse", "Warehouse", "Environment", "VariableLibrary", "Dataflow", "SemanticModel", "Report"];
|
|
19
|
+
/** Dependencies first, then things that run. */
|
|
20
|
+
const ORDER = ["Environment", "VariableLibrary", "Lakehouse", "Warehouse", "Dataflow", "Notebook", "DataPipeline", "SemanticModel", "Report"];
|
|
21
|
+
export const jobTypeFor = (t) => (t === "Notebook" ? "RunNotebook" : t === "DataPipeline" ? "Pipeline" : undefined);
|
|
22
|
+
/** Map changed paths to Fabric items using the git-integration folder convention `<name>.<Type>/`. */
|
|
23
|
+
export function mapItems(files) {
|
|
24
|
+
const re = new RegExp(`(?:^|/)([^/]+)\\.(${ITEM_TYPES.join("|")})(?:/|$)`);
|
|
25
|
+
const found = new Map();
|
|
26
|
+
for (const f of files) {
|
|
27
|
+
const m = re.exec(f.replace(/\\/g, "/"));
|
|
28
|
+
if (!m)
|
|
29
|
+
continue;
|
|
30
|
+
const type = m[2];
|
|
31
|
+
const key = `${type}:${m[1]}`;
|
|
32
|
+
const cur = found.get(key) ?? { name: m[1], type, jobType: jobTypeFor(type), files: 0 };
|
|
33
|
+
cur.files++;
|
|
34
|
+
found.set(key, cur);
|
|
35
|
+
}
|
|
36
|
+
return [...found.values()].sort((a, b) => ORDER.indexOf(a.type) - ORDER.indexOf(b.type) || a.name.localeCompare(b.name));
|
|
37
|
+
}
|
|
38
|
+
const isGuid = (s) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(s);
|
|
39
|
+
/** Workspace for a request: explicit name/id, else the branch rules, else the first feature, else the first dev workspace. */
|
|
40
|
+
export function chooseWorkspace(list, opts) {
|
|
41
|
+
if (opts.workspace) {
|
|
42
|
+
const w = opts.workspace.toLowerCase();
|
|
43
|
+
return list.find((x) => x.name.toLowerCase() === w || x.id?.toLowerCase() === w);
|
|
44
|
+
}
|
|
45
|
+
const byBranch = opts.branch ? list.find((x) => x.branches && anyGlob(x.branches, opts.branch)) : undefined;
|
|
46
|
+
return byBranch ?? list.find((x) => x.environment === "feature") ?? list.find((x) => x.environment === "dev");
|
|
47
|
+
}
|
|
48
|
+
export function buildPlan(files, list, opts) {
|
|
49
|
+
return { workspace: chooseWorkspace(list, opts), items: mapItems(files) };
|
|
50
|
+
}
|
|
51
|
+
export function formatPlan(p, branch, target) {
|
|
52
|
+
const lines = [`Fabric plan for ${branch} (vs ${target}):`];
|
|
53
|
+
lines.push(p.workspace
|
|
54
|
+
? `workspace: ${p.workspace.name} (${p.workspace.environment})`
|
|
55
|
+
: "workspace: none found in team.yaml fabric.workspaces; pass --workspace <name|id>");
|
|
56
|
+
if (!p.items.length) {
|
|
57
|
+
lines.push("No Fabric items changed: nothing to run in Fabric.");
|
|
58
|
+
return lines.join("\n");
|
|
59
|
+
}
|
|
60
|
+
let n = 0;
|
|
61
|
+
for (const it of p.items) {
|
|
62
|
+
const how = it.jobType ? `run: aihub fabric run --item ${it.name} --wait` : "deploy only (nothing to run)";
|
|
63
|
+
lines.push(`${++n}. ${it.type} ${it.name} (${it.files} file${it.files === 1 ? "" : "s"}): ${how}`);
|
|
64
|
+
}
|
|
65
|
+
return lines.join("\n");
|
|
66
|
+
}
|
|
67
|
+
// ---- guard -----------------------------------------------------------------
|
|
68
|
+
export function guardEnv(w, allowEnv) {
|
|
69
|
+
if (SAFE_ENVS.includes(w.environment))
|
|
70
|
+
return;
|
|
71
|
+
if (allowEnv && allowEnv.toLowerCase() === w.environment)
|
|
72
|
+
return;
|
|
73
|
+
const E = w.environment.toUpperCase();
|
|
74
|
+
throw new AihubError(`workspace '${w.name}' is ${E}: the developer runs ${E}, the CLI only runs dev/feature (--allow-env ${w.environment} only when the developer asked for it)`);
|
|
75
|
+
}
|
|
76
|
+
export function makeFabricAuth(deps) {
|
|
77
|
+
const exec = deps.exec ?? defaultExec;
|
|
78
|
+
const now = deps.now ?? Date.now;
|
|
79
|
+
const cacheFile = deps.cacheFile ?? join(tmpdir(), "aihub-fabric-token.json");
|
|
80
|
+
let cached;
|
|
81
|
+
return async () => {
|
|
82
|
+
if (cached)
|
|
83
|
+
return cached;
|
|
84
|
+
if (deps.env.AIHUB_FABRIC_TOKEN)
|
|
85
|
+
return (cached = `Bearer ${deps.env.AIHUB_FABRIC_TOKEN}`);
|
|
86
|
+
try {
|
|
87
|
+
if (existsSync(cacheFile)) {
|
|
88
|
+
const c = JSON.parse(readFileSync(cacheFile, "utf8"));
|
|
89
|
+
if (c.token && c.expiresAt - 120_000 > now())
|
|
90
|
+
return (cached = `Bearer ${c.token}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
/* ignore a broken cache */
|
|
95
|
+
}
|
|
96
|
+
try {
|
|
97
|
+
const j = JSON.parse(exec("az", ["account", "get-access-token", "--resource", FABRIC_RESOURCE, "--output", "json"]));
|
|
98
|
+
const expiresAt = j.expires_on ? Number(j.expires_on) * 1000 : Date.parse(j.expiresOn ?? "") || now() + 30 * 60_000;
|
|
99
|
+
try {
|
|
100
|
+
writeFileSync(cacheFile, JSON.stringify({ token: j.accessToken, expiresAt }), { mode: 0o600 });
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
/* cache is optional */
|
|
104
|
+
}
|
|
105
|
+
return (cached = `Bearer ${j.accessToken}`);
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
throw new AihubError("not signed in to Fabric: run 'az login' or set AIHUB_FABRIC_TOKEN");
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
export class FabricClient {
|
|
113
|
+
auth;
|
|
114
|
+
fetchFn;
|
|
115
|
+
constructor(auth, fetchFn) {
|
|
116
|
+
this.auth = auth;
|
|
117
|
+
this.fetchFn = fetchFn;
|
|
118
|
+
}
|
|
119
|
+
async request(method, path, body) {
|
|
120
|
+
const res = await this.fetchFn(`${FABRIC_API}${path}`, {
|
|
121
|
+
method,
|
|
122
|
+
headers: {
|
|
123
|
+
Authorization: await this.auth(),
|
|
124
|
+
Accept: "application/json",
|
|
125
|
+
...(body !== undefined ? { "Content-Type": "application/json" } : {}),
|
|
126
|
+
},
|
|
127
|
+
body: body !== undefined ? JSON.stringify(body) : undefined,
|
|
128
|
+
});
|
|
129
|
+
const text = await res.text();
|
|
130
|
+
if (res.status === 401)
|
|
131
|
+
throw new AihubError("not signed in to Fabric: run 'az login' or set AIHUB_FABRIC_TOKEN");
|
|
132
|
+
if (!res.ok) {
|
|
133
|
+
let detail = "";
|
|
134
|
+
try {
|
|
135
|
+
const j = JSON.parse(text);
|
|
136
|
+
detail = j.message ?? j.errorCode ?? "";
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
detail = text.slice(0, 200);
|
|
140
|
+
}
|
|
141
|
+
if (res.status === 403)
|
|
142
|
+
throw new AihubError(`Fabric denied access (403): your account needs access to this workspace${detail ? ` (${detail})` : ""}`);
|
|
143
|
+
throw new AihubError(`Fabric ${method} ${path.split("?")[0]} failed (${res.status})${detail ? `: ${detail}` : ""}`);
|
|
144
|
+
}
|
|
145
|
+
let data = {};
|
|
146
|
+
try {
|
|
147
|
+
data = text ? JSON.parse(text) : {};
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
/* non-JSON body */
|
|
151
|
+
}
|
|
152
|
+
return { status: res.status, data, headers: res.headers };
|
|
153
|
+
}
|
|
154
|
+
async workspaceId(w) {
|
|
155
|
+
if (w.id)
|
|
156
|
+
return w.id;
|
|
157
|
+
if (isGuid(w.name))
|
|
158
|
+
return w.name;
|
|
159
|
+
const r = await this.request("GET", "/workspaces");
|
|
160
|
+
const hit = r.data.value?.find((x) => x.displayName.toLowerCase() === w.name.toLowerCase());
|
|
161
|
+
if (!hit)
|
|
162
|
+
throw new AihubError(`Fabric workspace '${w.name}' not found for your account`);
|
|
163
|
+
return hit.id;
|
|
164
|
+
}
|
|
165
|
+
async resolveItem(wsId, item) {
|
|
166
|
+
const r = await this.request("GET", `/workspaces/${wsId}/items`);
|
|
167
|
+
const items = (r.data.value ?? []);
|
|
168
|
+
const hits = items.filter((i) => i.id.toLowerCase() === item.toLowerCase() || i.displayName.toLowerCase() === item.toLowerCase());
|
|
169
|
+
if (!hits.length)
|
|
170
|
+
throw new AihubError(`item '${item}' not found in this workspace`);
|
|
171
|
+
if (hits.length > 1)
|
|
172
|
+
throw new AihubError(`item name '${item}' is ambiguous (${hits.map((h) => h.type).join(", ")}); pass the item id`);
|
|
173
|
+
return hits[0];
|
|
174
|
+
}
|
|
175
|
+
async startJob(wsId, item, executionData) {
|
|
176
|
+
const jobType = jobTypeFor(item.type);
|
|
177
|
+
if (!jobType)
|
|
178
|
+
throw new AihubError(`${item.type} '${item.displayName}' cannot run on demand (only Notebook and DataPipeline)`);
|
|
179
|
+
const r = await this.request("POST", `/workspaces/${wsId}/items/${item.id}/jobs/instances?jobType=${jobType}`, executionData ? { executionData } : undefined);
|
|
180
|
+
const jobId = /instances\/([^/?]+)/.exec(r.headers.get("location") ?? "")?.[1];
|
|
181
|
+
if (!jobId)
|
|
182
|
+
throw new AihubError("Fabric accepted the job but returned no job id");
|
|
183
|
+
return jobId;
|
|
184
|
+
}
|
|
185
|
+
async jobStatus(wsId, itemId, jobId) {
|
|
186
|
+
const r = await this.request("GET", `/workspaces/${wsId}/items/${itemId}/jobs/instances/${jobId}`);
|
|
187
|
+
return r.data;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
export const isDone = (s) => ["Completed", "Failed", "Cancelled", "Deduped"].includes(s);
|
|
191
|
+
export async function waitForJob(c, wsId, itemId, jobId, opts) {
|
|
192
|
+
const deadline = Date.now() + (opts.timeoutMs ?? 6 * 3600_000);
|
|
193
|
+
for (;;) {
|
|
194
|
+
const j = await c.jobStatus(wsId, itemId, jobId);
|
|
195
|
+
if (isDone(j.status))
|
|
196
|
+
return j;
|
|
197
|
+
if (Date.now() > deadline)
|
|
198
|
+
throw new AihubError(`job ${jobId} still ${j.status} after the timeout; check with: aihub fabric status --job ${jobId} --item ${itemId}`);
|
|
199
|
+
await new Promise((r) => setTimeout(r, opts.pollMs));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
export function formatJob(j, itemName) {
|
|
203
|
+
const lines = [`job ${j.id}: ${j.status} (${itemName})`];
|
|
204
|
+
if (j.startTimeUtc)
|
|
205
|
+
lines.push(`started: ${j.startTimeUtc}${j.endTimeUtc ? `, ended: ${j.endTimeUtc}` : ""}`);
|
|
206
|
+
if (j.status === "Failed") {
|
|
207
|
+
const f = j.failureReason;
|
|
208
|
+
lines.push(`failure: ${f?.errorCode ?? "unknown"}: ${(f?.message ?? "no details from Fabric").replace(/\s+/g, " ").trim()}`);
|
|
209
|
+
}
|
|
210
|
+
return lines.join("\n");
|
|
211
|
+
}
|
|
212
|
+
/** Parameter file: JSON map (`{"limit": 10}`) or `{"parameters": {...}}`, turned into Fabric executionData. */
|
|
213
|
+
export function executionDataFromParams(text, item) {
|
|
214
|
+
let j;
|
|
215
|
+
try {
|
|
216
|
+
j = JSON.parse(text.replace(/^/, ""));
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
throw new AihubError('--params-file must contain JSON, e.g. {"limit": 10}');
|
|
220
|
+
}
|
|
221
|
+
const map = j && typeof j === "object" && j.parameters && typeof j.parameters === "object" ? j.parameters : j;
|
|
222
|
+
if (item.type === "DataPipeline")
|
|
223
|
+
return { parameters: map };
|
|
224
|
+
const typed = {};
|
|
225
|
+
for (const [k, v] of Object.entries(map)) {
|
|
226
|
+
if (v && typeof v === "object" && "value" in v && "type" in v)
|
|
227
|
+
typed[k] = v;
|
|
228
|
+
else
|
|
229
|
+
typed[k] = { value: v, type: typeof v === "boolean" ? "bool" : typeof v === "number" ? (Number.isInteger(v) ? "int" : "float") : "string" };
|
|
230
|
+
}
|
|
231
|
+
return { parameters: typed };
|
|
232
|
+
}
|
package/dist/lib/glob.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** Minimal glob: `**` any depth, `*` within a segment, `?` one character. Case-insensitive, `/` separators. */
|
|
2
|
+
export function globToRegExp(glob) {
|
|
3
|
+
let re = "";
|
|
4
|
+
const g = glob.replace(/\\/g, "/");
|
|
5
|
+
for (let i = 0; i < g.length; i++) {
|
|
6
|
+
const c = g[i];
|
|
7
|
+
if (c === "*") {
|
|
8
|
+
if (g[i + 1] === "*") {
|
|
9
|
+
i++;
|
|
10
|
+
if (g[i + 1] === "/") {
|
|
11
|
+
i++;
|
|
12
|
+
re += "(?:.*/)?";
|
|
13
|
+
}
|
|
14
|
+
else
|
|
15
|
+
re += ".*";
|
|
16
|
+
}
|
|
17
|
+
else
|
|
18
|
+
re += "[^/]*";
|
|
19
|
+
}
|
|
20
|
+
else if (c === "?")
|
|
21
|
+
re += "[^/]";
|
|
22
|
+
else
|
|
23
|
+
re += c.replace(/[.+^${}()|[\]]/g, "\\$&");
|
|
24
|
+
}
|
|
25
|
+
return new RegExp(`^${re}$`, "i");
|
|
26
|
+
}
|
|
27
|
+
export const globMatch = (glob, path) => globToRegExp(glob).test(path.replace(/\\/g, "/"));
|
|
28
|
+
export const anyGlob = (globs, path) => globs.some((g) => globMatch(g, path));
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { AihubError } from "./errors.js";
|
|
2
|
+
import { anyGlob } from "./glob.js";
|
|
3
|
+
export function robotConfig(team) {
|
|
4
|
+
const r = team.robot ?? {};
|
|
5
|
+
const suites = (r.suites ?? []).filter((s) => !!s?.name && Array.isArray(s.paths));
|
|
6
|
+
return { pipeline: r.pipeline, integrationBranch: r.integrationBranch, suites };
|
|
7
|
+
}
|
|
8
|
+
export function robotNeeded(files, cfg) {
|
|
9
|
+
const out = [];
|
|
10
|
+
for (const s of cfg.suites) {
|
|
11
|
+
const flowFiles = files.filter((f) => anyGlob(s.paths, f) && !(s.tests && anyGlob(s.tests, f)));
|
|
12
|
+
const testFiles = s.tests ? files.filter((f) => anyGlob(s.tests, f)) : [];
|
|
13
|
+
if (flowFiles.length || testFiles.length)
|
|
14
|
+
out.push({ suite: s.name, flowFiles, testFiles });
|
|
15
|
+
}
|
|
16
|
+
return out;
|
|
17
|
+
}
|
|
18
|
+
export function formatNeeded(needs, cfg, branch) {
|
|
19
|
+
if (!cfg.suites.length)
|
|
20
|
+
return "robot: no suites in team.yaml robot.suites; nothing to decide";
|
|
21
|
+
if (!needs.length)
|
|
22
|
+
return `robot: not needed for ${branch}: no changed file belongs to a flow with Robot tests`;
|
|
23
|
+
const lines = [`robot: needed for ${branch}: ${needs.length} suite${needs.length === 1 ? "" : "s"}`];
|
|
24
|
+
for (const n of needs) {
|
|
25
|
+
const t = n.testFiles.length
|
|
26
|
+
? `tests changed (${n.testFiles.length})`
|
|
27
|
+
: "no Robot test changed: a new or changed flow needs a new or adjusted test";
|
|
28
|
+
lines.push(`- ${n.suite}: ${n.flowFiles.length} flow file${n.flowFiles.length === 1 ? "" : "s"}; ${t}`);
|
|
29
|
+
}
|
|
30
|
+
lines.push(`run: aihub robot run --suite ${needs[0].suite}`);
|
|
31
|
+
return lines.join("\n");
|
|
32
|
+
}
|
|
33
|
+
export async function queueRobot(c, opts) {
|
|
34
|
+
const defs = await c.get("build/definitions", { name: opts.pipeline });
|
|
35
|
+
const def = (defs.value ?? []).find((d) => d.name.toLowerCase() === opts.pipeline.toLowerCase());
|
|
36
|
+
if (!def)
|
|
37
|
+
throw new AihubError(`pipeline '${opts.pipeline}' not found in Azure DevOps; check robot.pipeline in team.yaml or pass --pipeline`);
|
|
38
|
+
const templateParameters = {};
|
|
39
|
+
if (opts.suite)
|
|
40
|
+
templateParameters.suite = opts.suite;
|
|
41
|
+
if (opts.include)
|
|
42
|
+
templateParameters.include = opts.include;
|
|
43
|
+
return c.post("build/builds", {
|
|
44
|
+
definition: { id: def.id },
|
|
45
|
+
sourceBranch: opts.branch.startsWith("refs/") ? opts.branch : `refs/heads/${opts.branch}`,
|
|
46
|
+
...(Object.keys(templateParameters).length ? { templateParameters } : {}),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
export async function followBuild(c, id, pollMs, timeoutMs = 4 * 3600_000) {
|
|
50
|
+
const deadline = Date.now() + timeoutMs;
|
|
51
|
+
for (;;) {
|
|
52
|
+
const b = await c.get(`build/builds/${id}`);
|
|
53
|
+
if (b.status === "completed")
|
|
54
|
+
return b;
|
|
55
|
+
if (Date.now() > deadline)
|
|
56
|
+
throw new AihubError(`build ${id} still ${b.status} after the timeout; check it in Azure DevOps`);
|
|
57
|
+
await new Promise((r) => setTimeout(r, pollMs));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Results summary: build result, test counts (when published) and failed steps with their first error lines. */
|
|
61
|
+
export async function summarize(c, b) {
|
|
62
|
+
const lines = [`build ${b.buildNumber ?? b.id}: ${b.result ?? b.status}${b._links?.web?.href ? ` ${b._links.web.href}` : ""}`];
|
|
63
|
+
try {
|
|
64
|
+
const runs = await c.get("test/runs", {
|
|
65
|
+
buildUri: `vstfs:///Build/Build/${b.id}`,
|
|
66
|
+
});
|
|
67
|
+
for (const r of runs.value ?? []) {
|
|
68
|
+
const total = r.totalTests ?? 0;
|
|
69
|
+
const passed = r.passedTests ?? 0;
|
|
70
|
+
lines.push(`tests ${r.name ?? "run"}: ${passed}/${total} passed, ${total - passed} failed`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
/* test results are optional */
|
|
75
|
+
}
|
|
76
|
+
try {
|
|
77
|
+
const t = await c.get(`build/builds/${b.id}/timeline`);
|
|
78
|
+
const failed = (t.records ?? []).filter((r) => r.result === "failed" && r.type !== "Stage" && r.type !== "Phase" && r.type !== "Job");
|
|
79
|
+
for (const r of failed.slice(0, 5)) {
|
|
80
|
+
const msg = (r.issues ?? []).find((i) => i.type === "error")?.message ?? "no error message";
|
|
81
|
+
lines.push(`failed: ${r.name}: ${msg.replace(/\s+/g, " ").trim().slice(0, 300)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
/* timeline is optional */
|
|
86
|
+
}
|
|
87
|
+
return lines.join("\n");
|
|
88
|
+
}
|
package/dist/router.js
CHANGED
|
@@ -5,6 +5,8 @@ import * as doctor from "./commands/doctor.js";
|
|
|
5
5
|
import * as init from "./commands/init.js";
|
|
6
6
|
import * as pr from "./commands/pr.js";
|
|
7
7
|
import * as skills from "./commands/skills.js";
|
|
8
|
+
import * as fabric from "./commands/fabric.js";
|
|
9
|
+
import * as robot from "./commands/robot.js";
|
|
8
10
|
/**
|
|
9
11
|
* Command registry. To add a command: create src/commands/<name>.ts exporting
|
|
10
12
|
* `run(argv, ctx)` and add one line here. Commands print through ctx.out and
|
|
@@ -16,6 +18,8 @@ export const commands = {
|
|
|
16
18
|
init: init.run,
|
|
17
19
|
skills: skills.run,
|
|
18
20
|
doctor: doctor.run,
|
|
21
|
+
fabric: fabric.run,
|
|
22
|
+
robot: robot.run,
|
|
19
23
|
};
|
|
20
24
|
export const VERSION = cliVersion();
|
|
21
25
|
/** Returns the process exit code. */
|
package/dist/tools/lintSkills.js
CHANGED
|
@@ -14,6 +14,7 @@ export const COMMANDS = [
|
|
|
14
14
|
{ name: "refine", skill: "story-refine", description: "Refine a story", hint: "story id" },
|
|
15
15
|
{ name: "review", skill: "review-process", description: "Process review comments", hint: "optional PR id" },
|
|
16
16
|
{ name: "watch", skill: "pr-watch", description: "Follow a PR and its pipeline", hint: "optional PR id or link" },
|
|
17
|
+
{ name: "test", skill: "test-run", description: "Run the Fabric or Robot tests for this branch", hint: "fabric or robot, optional item or suite" },
|
|
17
18
|
];
|
|
18
19
|
export function expectedCommand(kind, c) {
|
|
19
20
|
const load = `Load the \`${c.skill}\` skill and follow it exactly.`;
|
|
@@ -39,3 +39,18 @@ pipelines: # pipelines to follow per repository (optiona
|
|
|
39
39
|
- repository: <repository> # repository name this pipeline belongs to
|
|
40
40
|
name: <pipeline> # pipeline name or id
|
|
41
41
|
required: true # true when a red run blocks the PR
|
|
42
|
+
|
|
43
|
+
fabric: # Fabric workspaces for `aihub fabric` (optional)
|
|
44
|
+
workspaces:
|
|
45
|
+
- name: <workspace> # workspace display name, e.g. DEV-test-project (optional)
|
|
46
|
+
id: <guid> # workspace id (optional; looked up by name when absent)
|
|
47
|
+
environment: dev # dev | feature | test | acc | prd (optional); acc and prd are never run by the CLI
|
|
48
|
+
branches: [] # branch patterns tested in this workspace, e.g. ["feature/*"] (optional)
|
|
49
|
+
|
|
50
|
+
robot: # Robot integration tests for `aihub robot` (optional)
|
|
51
|
+
pipeline: <pipeline> # Azure DevOps pipeline that runs the Robot suites (optional; needed for robot run)
|
|
52
|
+
integrationBranch: <branch> # branch the pipeline runs on (optional; needed for robot run)
|
|
53
|
+
suites: # list; a suite belongs to a flow (optional; needed for robot needed)
|
|
54
|
+
- name: <suite> # suite name passed to the pipeline as parameter `suite`
|
|
55
|
+
paths: [] # path patterns of the flow, e.g. ["src/flows/orders/**"]
|
|
56
|
+
tests: [] # path patterns of the suite's Robot files, e.g. ["tests/robot/orders/**"] (optional)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: test-run
|
|
3
|
+
description: "Use when the change must be tested in Fabric or with Robot: 'test dit in Fabric', 'run it in Fabric', 'draai de Robot-tests', 'run the Robot tests', 'moeten de integratietests draaien', before reporting tests in a PR."
|
|
4
|
+
---
|
|
5
|
+
# Run Fabric and Robot tests
|
|
6
|
+
|
|
7
|
+
## When
|
|
8
|
+
The branch changes Fabric items (notebooks, pipelines) or a flow with Robot integration tests, and the PR tests section needs real results.
|
|
9
|
+
|
|
10
|
+
## Steps
|
|
11
|
+
1. Fabric: see what to run, then run it in order and wait:
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
aihub fabric plan
|
|
15
|
+
aihub fabric run --item nb_load_orders --wait
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
2. Robot: check first, then run only when needed:
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
aihub robot needed
|
|
22
|
+
aihub robot run --suite orders
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
3. Parameters: `--params-file params.json` (Fabric), `--include <tag>` (Robot).
|
|
26
|
+
4. A failed run: read the failure line, fix the cause, run again. Never weaken a test.
|
|
27
|
+
5. Report in the PR tests section only what ran, with the result. Not run: write "nog niet uitgevoerd".
|
|
28
|
+
|
|
29
|
+
## What the CLI fills
|
|
30
|
+
Changed items versus the target, the workspace by branch rule, item and job ids, the pipeline and branch from the team file. Do not ask for ids.
|
|
31
|
+
|
|
32
|
+
## If it fails
|
|
33
|
+
Not signed in: `az login` or `aihub doctor`. Item not found: check the plan. Never run ACC or PRD: the developer does that; the CLI refuses them. Details: [Fabric and Robot reference](references/fabric-robot.md).
|
|
34
|
+
|
|
35
|
+
Rules: direct task, 0 questions. A new Robot test is only needed for a new or changed flow.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Fabric and Robot reference
|
|
2
|
+
|
|
3
|
+
## Commands
|
|
4
|
+
|
|
5
|
+
| Command | Does |
|
|
6
|
+
|---|---|
|
|
7
|
+
| `aihub fabric plan [--branch B] [--target T] [--workspace W]` | Maps changed files (`<name>.Notebook/`, `<name>.DataPipeline/`, lakehouse items) to Fabric items, ordered: dependencies, notebooks, pipelines. |
|
|
8
|
+
| `aihub fabric run --item N [--workspace W] [--params-file F] [--wait]` | Starts a notebook or pipeline job; with `--wait` polls and prints status and the failure reason. |
|
|
9
|
+
| `aihub fabric status --item N --job ID [--workspace W]` | Status of one job. |
|
|
10
|
+
| `aihub robot needed [--branch B]` | Which suites cover a changed flow, and whether a test changed with it. |
|
|
11
|
+
| `aihub robot run [--suite S] [--include tag] [--pipeline P] [--no-wait]` | Queues the team Robot pipeline on the integration branch and prints the results summary. |
|
|
12
|
+
|
|
13
|
+
## Environments
|
|
14
|
+
|
|
15
|
+
`dev` and `feature` workspaces run without a flag. `test`, `acc` and `prd` are refused with one line unless `--allow-env <env>` is passed, and that only when the developer asked for it in this session. A workspace that is not in the team file is refused.
|
|
16
|
+
|
|
17
|
+
## Parameters file
|
|
18
|
+
|
|
19
|
+
JSON map, for example `{"limit": 10, "full_load": false}`. Notebooks get typed parameters, pipelines get the map as is.
|
|
20
|
+
|
|
21
|
+
## PR tests section
|
|
22
|
+
|
|
23
|
+
One line per run: what ran, where (workspace or pipeline), result. Anything not run: "nog niet uitgevoerd".
|
package/package.json
CHANGED