@10xagents/connector-multica 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -1
- package/dist/index.d.mts +339 -0
- package/dist/index.mjs +2274 -0
- package/package.json +46 -4
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,2274 @@
|
|
|
1
|
+
import { accessSync, constants, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir, tmpdir } from "node:os";
|
|
3
|
+
import { delimiter, join, relative, sep } from "node:path";
|
|
4
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
5
|
+
import { crc32 } from "node:zlib";
|
|
6
|
+
import { structuredPatch } from "diff";
|
|
7
|
+
// src/multica.ts
|
|
8
|
+
/** Where the macOS Desktop app keeps its copy of the CLI. */
|
|
9
|
+
const DESKTOP_CLI = "/Applications/Multica.app/Contents/Resources/app.asar.unpacked/resources/bin/multica";
|
|
10
|
+
const executable = (path) => {
|
|
11
|
+
try {
|
|
12
|
+
accessSync(path, constants.X_OK);
|
|
13
|
+
return true;
|
|
14
|
+
} catch {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
/** `multica` on `PATH`, as `which` finds it. */
|
|
19
|
+
const onPath = (env) => (env.PATH ?? "").split(delimiter).filter(Boolean).map((dir) => join(dir, "multica")).find(executable);
|
|
20
|
+
/**
|
|
21
|
+
* The CLI to run: `$MULTICA`, then `$MULTICA_CLI_PATH`, then `multica` on `PATH`, then the macOS
|
|
22
|
+
* Desktop app's. Throws when there's none.
|
|
23
|
+
*/
|
|
24
|
+
const findMultica = (env, desktop = DESKTOP_CLI) => {
|
|
25
|
+
const found = env.MULTICA || env.MULTICA_CLI_PATH || onPath(env);
|
|
26
|
+
if (found) return found;
|
|
27
|
+
if (existsSync(desktop)) return desktop;
|
|
28
|
+
throw new Error("multica CLI not found: install it, or set MULTICA to its path");
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* The environment `multica` runs in. Every inherited `MULTICA_*` variable goes, since the CLI
|
|
32
|
+
* prefers them to the profile's own token, server and workspace: an agent run's would otherwise
|
|
33
|
+
* sync another account. Only the workspace this call is for comes back.
|
|
34
|
+
*/
|
|
35
|
+
const childEnv = (env, workspace) => {
|
|
36
|
+
const clean = {};
|
|
37
|
+
for (const [key, value] of Object.entries(env)) if (value !== void 0 && !key.startsWith("MULTICA_")) clean[key] = value;
|
|
38
|
+
if (workspace) clean.MULTICA_WORKSPACE_ID = workspace;
|
|
39
|
+
return clean;
|
|
40
|
+
};
|
|
41
|
+
/** The home directory, which is also where `multica` runs, away from any project's config. */
|
|
42
|
+
const homeOf = (env) => env.HOME || homedir();
|
|
43
|
+
/** The text `multica <args>` prints. Throws with its error output when it fails. */
|
|
44
|
+
const run = (cli, args, workspace, input, label = args) => {
|
|
45
|
+
try {
|
|
46
|
+
return execFileSync(cli.bin, [...cli.profileArgs, ...args], {
|
|
47
|
+
encoding: "utf8",
|
|
48
|
+
maxBuffer: 268435456,
|
|
49
|
+
cwd: homeOf(cli.env),
|
|
50
|
+
env: childEnv(cli.env, workspace),
|
|
51
|
+
stdio: [
|
|
52
|
+
input === void 0 ? "ignore" : "pipe",
|
|
53
|
+
"pipe",
|
|
54
|
+
"pipe"
|
|
55
|
+
],
|
|
56
|
+
...input === void 0 ? {} : { input }
|
|
57
|
+
});
|
|
58
|
+
} catch (error) {
|
|
59
|
+
const output = error && typeof error === "object" && "stderr" in error ? error.stderr : "";
|
|
60
|
+
const message = (typeof output === "string" ? output : Buffer.isBuffer(output) ? output.toString("utf8") : "").trim() || (error instanceof Error ? error.message : String(error));
|
|
61
|
+
throw new Error(`multica ${label.slice(0, 3).join(" ")}: ${message}`, { cause: error });
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
/** `multica <args> --output json`, parsed. */
|
|
65
|
+
const runJson = (cli, args, workspace) => {
|
|
66
|
+
const out = run(cli, [
|
|
67
|
+
...args,
|
|
68
|
+
"--output",
|
|
69
|
+
"json"
|
|
70
|
+
], workspace, void 0, args);
|
|
71
|
+
try {
|
|
72
|
+
return JSON.parse(out);
|
|
73
|
+
} catch (error) {
|
|
74
|
+
throw new Error(`multica ${args.slice(0, 3).join(" ")}: printed no JSON`, { cause: error });
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
// src/profiles.ts
|
|
78
|
+
/**
|
|
79
|
+
* The profiles that hold a token, by name: `""` is the default profile. `multica auth logout`
|
|
80
|
+
* leaves a config without a token, and an unreadable config logs nobody in.
|
|
81
|
+
*/
|
|
82
|
+
const listProfiles = (home) => {
|
|
83
|
+
const root = join(home, ".multica");
|
|
84
|
+
const dir = join(root, "profiles");
|
|
85
|
+
const configs = [["", join(root, "config.json")]];
|
|
86
|
+
if (existsSync(dir)) for (const name of readdirSync(dir).toSorted()) configs.push([name, join(dir, name, "config.json")]);
|
|
87
|
+
return configs.filter(([, config]) => {
|
|
88
|
+
try {
|
|
89
|
+
const parsed = JSON.parse(readFileSync(config, "utf8"));
|
|
90
|
+
return !!parsed && typeof parsed === "object" && "token" in parsed && !!parsed.token;
|
|
91
|
+
} catch {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
}).map(([name]) => name);
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* The profile to use: the one given, the only one there is, or the one the person picks. Throws
|
|
98
|
+
* when there are several and no terminal to ask in. The question holds the numbered list, so the
|
|
99
|
+
* list reaches the person wherever the question does, and never the command's output.
|
|
100
|
+
*/
|
|
101
|
+
const pickProfile = async (home, given, ask) => {
|
|
102
|
+
if (given !== void 0) return given;
|
|
103
|
+
const found = listProfiles(home);
|
|
104
|
+
if (found.length === 0) throw new Error("no logged-in Multica CLI profile under ~/.multica: log in with the Desktop app or `multica login`");
|
|
105
|
+
if (found.length === 1) return found[0] ?? "";
|
|
106
|
+
if (!ask) {
|
|
107
|
+
const names = found.map((name) => name || "\"\" for the default").join(", ");
|
|
108
|
+
throw new Error(`several Multica profiles: pass --profile <name> (${names})`);
|
|
109
|
+
}
|
|
110
|
+
const answer = await ask(`${found.map((name, index) => ` ${index + 1}. ${name || "(default)"}\n`).join("")}Profile (number): `);
|
|
111
|
+
const picked = found[Number(answer) - 1];
|
|
112
|
+
if (picked === void 0) throw new Error(`no profile ${answer}`);
|
|
113
|
+
return picked;
|
|
114
|
+
};
|
|
115
|
+
// src/pull.ts
|
|
116
|
+
const isRow = (value) => !!value && typeof value === "object" && !Array.isArray(value);
|
|
117
|
+
const rows = (value, what) => {
|
|
118
|
+
if (!Array.isArray(value)) throw new Error(`multica ${what}: expected a list`);
|
|
119
|
+
return value.filter(isRow);
|
|
120
|
+
};
|
|
121
|
+
const row = (value, what) => {
|
|
122
|
+
if (!isRow(value)) throw new Error(`multica ${what}: expected an object`);
|
|
123
|
+
return value;
|
|
124
|
+
};
|
|
125
|
+
/** A string or number field as text. Anything else is empty. */
|
|
126
|
+
const text = (value) => typeof value === "string" ? value : typeof value === "number" || typeof value === "boolean" ? String(value) : "";
|
|
127
|
+
/** Reads the workspace `target` syncs to, with the profile and workspace `options` choose. */
|
|
128
|
+
const pull = async (target, options, context) => {
|
|
129
|
+
const log = options.log ?? (() => {});
|
|
130
|
+
const bin = findMultica(context.env, context.desktop);
|
|
131
|
+
const profile = await pickProfile(homeOf(context.env), options.profile, options.ask);
|
|
132
|
+
const cli = {
|
|
133
|
+
bin,
|
|
134
|
+
profileArgs: profile ? ["--profile", profile] : [],
|
|
135
|
+
env: context.env
|
|
136
|
+
};
|
|
137
|
+
log(`Profile: ${profile || "(default)"}`);
|
|
138
|
+
const want = options.workspace ?? target.workspace;
|
|
139
|
+
const found = rows(runJson(cli, ["workspace", "list"]), "workspace list").find((ws) => [
|
|
140
|
+
ws.id,
|
|
141
|
+
ws.slug,
|
|
142
|
+
ws.name
|
|
143
|
+
].includes(want));
|
|
144
|
+
if (!found) throw new Error(`no workspace ${want} for this profile`);
|
|
145
|
+
const workspace = {
|
|
146
|
+
id: text(found.id),
|
|
147
|
+
name: text(found.name),
|
|
148
|
+
slug: text(found.slug)
|
|
149
|
+
};
|
|
150
|
+
log(`Workspace: ${workspace.name}`);
|
|
151
|
+
const id = workspace.id;
|
|
152
|
+
const json = (args, scoped = true) => runJson(cli, args, scoped ? id : void 0);
|
|
153
|
+
const agentContext = text(row(json([
|
|
154
|
+
"workspace",
|
|
155
|
+
"get",
|
|
156
|
+
id
|
|
157
|
+
], false), "workspace get").context);
|
|
158
|
+
const members = rows(json([
|
|
159
|
+
"workspace",
|
|
160
|
+
"member",
|
|
161
|
+
"list",
|
|
162
|
+
id
|
|
163
|
+
], false), "workspace member list").map((member) => ({
|
|
164
|
+
user_id: text(member.user_id),
|
|
165
|
+
name: text(member.name),
|
|
166
|
+
...member.email === void 0 ? {} : { email: text(member.email) }
|
|
167
|
+
}));
|
|
168
|
+
const skills = rows(json(["skill", "list"]), "skill list").map((listed) => {
|
|
169
|
+
const full = row(json([
|
|
170
|
+
"skill",
|
|
171
|
+
"get",
|
|
172
|
+
text(listed.id),
|
|
173
|
+
"--with-content"
|
|
174
|
+
]), "skill get");
|
|
175
|
+
const files = { "SKILL.md": text(full.content) };
|
|
176
|
+
for (const file of rows(full.files ?? [], "skill get")) files[text(file.path)] = text(file.content);
|
|
177
|
+
return {
|
|
178
|
+
id: text(listed.id),
|
|
179
|
+
name: text(listed.name),
|
|
180
|
+
files
|
|
181
|
+
};
|
|
182
|
+
});
|
|
183
|
+
const agents = rows(json(["agent", "list"]), "agent list").map((listed) => {
|
|
184
|
+
const full = row(json([
|
|
185
|
+
"agent",
|
|
186
|
+
"get",
|
|
187
|
+
text(listed.id)
|
|
188
|
+
]), "agent get");
|
|
189
|
+
return {
|
|
190
|
+
id: text(listed.id),
|
|
191
|
+
name: text(listed.name),
|
|
192
|
+
description: text(full.description),
|
|
193
|
+
model: text(full.model),
|
|
194
|
+
thinking_level: text(full.thinking_level),
|
|
195
|
+
max_concurrent_tasks: text(full.max_concurrent_tasks),
|
|
196
|
+
instructions: text(full.instructions),
|
|
197
|
+
skills: rows(full.skills ?? [], "agent get").map((skill) => ({
|
|
198
|
+
name: text(skill.name),
|
|
199
|
+
enabled: skill.enabled !== false
|
|
200
|
+
}))
|
|
201
|
+
};
|
|
202
|
+
});
|
|
203
|
+
let squad = null;
|
|
204
|
+
const squadName = target.squad?.name;
|
|
205
|
+
if (squadName !== void 0) {
|
|
206
|
+
const listed = rows(json(["squad", "list"]), "squad list").find((candidate) => candidate.name === squadName);
|
|
207
|
+
if (listed) {
|
|
208
|
+
const full = row(json([
|
|
209
|
+
"squad",
|
|
210
|
+
"get",
|
|
211
|
+
text(listed.id)
|
|
212
|
+
]), "squad get");
|
|
213
|
+
squad = {
|
|
214
|
+
id: text(listed.id),
|
|
215
|
+
name: squadName,
|
|
216
|
+
instructions: text(full.instructions),
|
|
217
|
+
leader_id: text(full.leader_id),
|
|
218
|
+
members: rows(json([
|
|
219
|
+
"squad",
|
|
220
|
+
"member",
|
|
221
|
+
"list",
|
|
222
|
+
text(listed.id)
|
|
223
|
+
]), "squad member list").map((member) => ({
|
|
224
|
+
member_id: text(member.member_id),
|
|
225
|
+
member_type: text(member.member_type),
|
|
226
|
+
role: text(member.role)
|
|
227
|
+
}))
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
const workflow = pullWorkflow(json);
|
|
232
|
+
const projects = rows(json(["project", "list"]), "project list").map((project) => ({
|
|
233
|
+
id: text(project.id),
|
|
234
|
+
name: text(project.title ?? project.name)
|
|
235
|
+
}));
|
|
236
|
+
return {
|
|
237
|
+
connector: "multica",
|
|
238
|
+
cli,
|
|
239
|
+
profile,
|
|
240
|
+
workspace,
|
|
241
|
+
context: agentContext,
|
|
242
|
+
members,
|
|
243
|
+
skills,
|
|
244
|
+
agents,
|
|
245
|
+
squad,
|
|
246
|
+
workflow,
|
|
247
|
+
projects
|
|
248
|
+
};
|
|
249
|
+
};
|
|
250
|
+
const pullWorkflow = (json) => {
|
|
251
|
+
let listed;
|
|
252
|
+
try {
|
|
253
|
+
listed = rows(json(["workflow", "list"]), "workflow list");
|
|
254
|
+
} catch (error) {
|
|
255
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
256
|
+
if (/unknown command|unknown flag/i.test(message)) return { problem: "this multica CLI has no `workflow` command: update it, or build the workflow by hand" };
|
|
257
|
+
if (/returned 404/.test(message)) return { problem: "this Multica server has no workflows: update it, or build the workflow by hand" };
|
|
258
|
+
throw error;
|
|
259
|
+
}
|
|
260
|
+
const found = listed.find((workflow) => workflow.is_default === true);
|
|
261
|
+
if (!found) return { problem: "Multica listed no Default workflow" };
|
|
262
|
+
const full = row(json([
|
|
263
|
+
"workflow",
|
|
264
|
+
"get",
|
|
265
|
+
text(found.id)
|
|
266
|
+
]), "workflow get");
|
|
267
|
+
return {
|
|
268
|
+
id: text(found.id),
|
|
269
|
+
steps: rows(full.steps ?? [], "workflow get").map(toStep),
|
|
270
|
+
project_ids: Array.isArray(full.project_ids) ? full.project_ids.map(text) : []
|
|
271
|
+
};
|
|
272
|
+
};
|
|
273
|
+
/** A step as Multica returns it, with only the fields the sync compares. */
|
|
274
|
+
const toStep = (step) => {
|
|
275
|
+
const handler = isRow(step.handler) ? step.handler : null;
|
|
276
|
+
return {
|
|
277
|
+
status_key: text(step.status_key),
|
|
278
|
+
...handler ? { handler: {
|
|
279
|
+
type: text(handler.type),
|
|
280
|
+
...handler.id === void 0 || handler.id === null ? {} : { id: text(handler.id) }
|
|
281
|
+
} } : {},
|
|
282
|
+
...step.instructions === void 0 ? {} : { instructions: text(step.instructions) },
|
|
283
|
+
...step.next_status_key ? { next_status_key: text(step.next_status_key) } : {},
|
|
284
|
+
...step.back_status_key ? { back_status_key: text(step.back_status_key) } : {}
|
|
285
|
+
};
|
|
286
|
+
};
|
|
287
|
+
// src/text.ts
|
|
288
|
+
/** Text that ends in exactly one newline, so a stored copy and a file compare alike. */
|
|
289
|
+
const norm = (text) => `${(text ?? "").replace(/\n*$/, "")}\n`;
|
|
290
|
+
/** How `live` differs from `setup`, compared as they are. */
|
|
291
|
+
const textDiff = (live, setup) => {
|
|
292
|
+
if (live === setup) return {
|
|
293
|
+
lines: 0,
|
|
294
|
+
diff: ""
|
|
295
|
+
};
|
|
296
|
+
const patch = structuredPatch("live", "setup", live, setup, "", "", { context: 3 });
|
|
297
|
+
let added = 0;
|
|
298
|
+
let removed = 0;
|
|
299
|
+
const out = [];
|
|
300
|
+
for (const hunk of patch.hunks) {
|
|
301
|
+
out.push(`@@ -${hunk.oldStart},${hunk.oldLines} +${hunk.newStart},${hunk.newLines} @@`);
|
|
302
|
+
for (const line of hunk.lines) {
|
|
303
|
+
if (line.startsWith("+")) added++;
|
|
304
|
+
else if (line.startsWith("-")) removed++;
|
|
305
|
+
out.push(line);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return {
|
|
309
|
+
lines: Math.max(added, removed),
|
|
310
|
+
diff: out.join("\n")
|
|
311
|
+
};
|
|
312
|
+
};
|
|
313
|
+
/** "1 line differs", "3 lines differ". */
|
|
314
|
+
const differ = (lines) => `${lines} ${lines === 1 ? "line differs" : "lines differ"}`;
|
|
315
|
+
/** A value, cut to 70 characters for one line of output. */
|
|
316
|
+
const preview = (text) => text.length > 70 ? `${text.slice(0, 70)}…` : text;
|
|
317
|
+
/** Code points, as Multica counts them. */
|
|
318
|
+
const length = (text) => Array.from(text).length;
|
|
319
|
+
// src/skills.ts
|
|
320
|
+
/** Every file under `dir`, as absolute paths, sorted, without `.DS_Store`. */
|
|
321
|
+
const walk = (dir) => readdirSync(dir).toSorted().flatMap((entry) => {
|
|
322
|
+
const path = join(dir, entry);
|
|
323
|
+
if (entry === ".DS_Store") return [];
|
|
324
|
+
return statSync(path).isDirectory() ? walk(path) : [path];
|
|
325
|
+
});
|
|
326
|
+
const posix = (path) => path.split(sep).join("/");
|
|
327
|
+
/** A skill's files, by path inside its folder, each normalised as Multica stores it. */
|
|
328
|
+
const skillFiles = (skill) => new Map(walk(skill.dir).map((path) => [posix(relative(skill.dir, path)), norm(readFileSync(path, "utf8"))]));
|
|
329
|
+
/** The `name` in a SKILL.md's frontmatter, or `""`. */
|
|
330
|
+
const frontmatterName = (text) => /^---\r?\n([\s\S]*?\r?\n)---/.exec(text)?.[1]?.match(/^name:[ \t]*(.*)$/m)?.[1]?.trim().replace(/^(["'])(.*)\1$/, "$2") ?? "";
|
|
331
|
+
const BINARY = /\.(png|jpe?g|gif|webp|bmp|tiff|ico|heic|ttf|otf|woff2?|eot|zip|gz|tar|bz2|7z|rar|pdf|docx?|xlsx?|pptx?|mp3|mp4|wav|avi|mov|webm|m4a|flac|exe|dll|so|dylib|class|jar|wasm|db|sqlite3?|pyc)$/i;
|
|
332
|
+
/**
|
|
333
|
+
* A file Multica's import leaves out: dotfiles, a second `skill.md`, licences and binaries
|
|
334
|
+
* (`skill_import_archive.go`, `skill/binary.go`). The sync compares every file, so a file like
|
|
335
|
+
* that would show as a change on every run.
|
|
336
|
+
*/
|
|
337
|
+
const leftOut = (path) => path.split("/").some((segment) => segment.startsWith(".")) || path !== "SKILL.md" && /(^|\/)skill\.md$/i.test(path) || /(^|\/)license(\.md|\.txt)?$/i.test(path) || BINARY.test(path);
|
|
338
|
+
/**
|
|
339
|
+
* What Multica's import would change about a skill: a SKILL.md that's missing or names another
|
|
340
|
+
* skill, and files it would leave out.
|
|
341
|
+
*/
|
|
342
|
+
const checkSkill = (skill) => {
|
|
343
|
+
const issues = [];
|
|
344
|
+
const at = (path) => join(skill.dir, path);
|
|
345
|
+
let files;
|
|
346
|
+
try {
|
|
347
|
+
files = walk(skill.dir).map((path) => posix(relative(skill.dir, path)));
|
|
348
|
+
} catch (error) {
|
|
349
|
+
return [{
|
|
350
|
+
file: skill.dir,
|
|
351
|
+
field: "(dir)",
|
|
352
|
+
message: `can't be read: ${error instanceof Error ? error.message : String(error)}`
|
|
353
|
+
}];
|
|
354
|
+
}
|
|
355
|
+
if (!files.includes("SKILL.md")) issues.push({
|
|
356
|
+
file: at("SKILL.md"),
|
|
357
|
+
field: "(file)",
|
|
358
|
+
message: "is missing: every skill needs one"
|
|
359
|
+
});
|
|
360
|
+
else {
|
|
361
|
+
const named = frontmatterName(readFileSync(at("SKILL.md"), "utf8"));
|
|
362
|
+
if (named !== skill.name) issues.push({
|
|
363
|
+
file: at("SKILL.md"),
|
|
364
|
+
field: "name",
|
|
365
|
+
message: named ? `must be ${skill.name}, like its folder (it says ${named}): Multica names a skill after it` : `is missing: write name: ${skill.name}, like its folder`
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
for (const path of files) if (leftOut(path)) issues.push({
|
|
369
|
+
file: at(path),
|
|
370
|
+
field: "(file)",
|
|
371
|
+
message: "Multica leaves this file out of an import: rename or remove it"
|
|
372
|
+
});
|
|
373
|
+
return issues;
|
|
374
|
+
};
|
|
375
|
+
/** A zip of the skill's folder, stored without compression, with the folder at its root. */
|
|
376
|
+
const zipSkill = (skill) => {
|
|
377
|
+
const locals = [];
|
|
378
|
+
const centrals = [];
|
|
379
|
+
let offset = 0;
|
|
380
|
+
let count = 0;
|
|
381
|
+
for (const path of walk(skill.dir)) {
|
|
382
|
+
const name = Buffer.from(`${skill.name}/${posix(relative(skill.dir, path))}`);
|
|
383
|
+
const data = readFileSync(path);
|
|
384
|
+
const sum = crc32(data);
|
|
385
|
+
const local = Buffer.alloc(30);
|
|
386
|
+
local.writeUInt32LE(67324752, 0);
|
|
387
|
+
local.writeUInt16LE(20, 4);
|
|
388
|
+
local.writeUInt16LE(2048, 6);
|
|
389
|
+
local.writeUInt16LE(33, 12);
|
|
390
|
+
local.writeUInt32LE(sum, 14);
|
|
391
|
+
local.writeUInt32LE(data.length, 18);
|
|
392
|
+
local.writeUInt32LE(data.length, 22);
|
|
393
|
+
local.writeUInt16LE(name.length, 26);
|
|
394
|
+
const central = Buffer.alloc(46);
|
|
395
|
+
central.writeUInt32LE(33639248, 0);
|
|
396
|
+
central.writeUInt16LE(20, 4);
|
|
397
|
+
central.writeUInt16LE(20, 6);
|
|
398
|
+
central.writeUInt16LE(2048, 8);
|
|
399
|
+
central.writeUInt16LE(33, 14);
|
|
400
|
+
central.writeUInt32LE(sum, 16);
|
|
401
|
+
central.writeUInt32LE(data.length, 20);
|
|
402
|
+
central.writeUInt32LE(data.length, 24);
|
|
403
|
+
central.writeUInt16LE(name.length, 28);
|
|
404
|
+
central.writeUInt32LE(offset, 42);
|
|
405
|
+
locals.push(local, name, data);
|
|
406
|
+
centrals.push(central, name);
|
|
407
|
+
offset += local.length + name.length + data.length;
|
|
408
|
+
count++;
|
|
409
|
+
}
|
|
410
|
+
const directory = Buffer.concat(centrals);
|
|
411
|
+
const end = Buffer.alloc(22);
|
|
412
|
+
end.writeUInt32LE(101010256, 0);
|
|
413
|
+
end.writeUInt16LE(count, 8);
|
|
414
|
+
end.writeUInt16LE(count, 10);
|
|
415
|
+
end.writeUInt32LE(directory.length, 12);
|
|
416
|
+
end.writeUInt32LE(offset, 16);
|
|
417
|
+
return Buffer.concat([
|
|
418
|
+
...locals,
|
|
419
|
+
directory,
|
|
420
|
+
end
|
|
421
|
+
]);
|
|
422
|
+
};
|
|
423
|
+
// src/workflow.ts
|
|
424
|
+
/** Multica's built-in statuses, by key, with their names. */
|
|
425
|
+
const BUILT_IN = {
|
|
426
|
+
backlog: "Backlog",
|
|
427
|
+
todo: "Todo",
|
|
428
|
+
in_progress: "In Progress",
|
|
429
|
+
in_review: "In Review",
|
|
430
|
+
blocked: "Blocked",
|
|
431
|
+
done: "Done",
|
|
432
|
+
cancelled: "Cancelled"
|
|
433
|
+
};
|
|
434
|
+
const isBuiltIn = (key) => Object.hasOwn(BUILT_IN, key);
|
|
435
|
+
/** Multica keeps these first, in this order. */
|
|
436
|
+
const HEAD = ["backlog", "todo"];
|
|
437
|
+
/** Multica keeps these last, in this order. */
|
|
438
|
+
const TAIL = [
|
|
439
|
+
"blocked",
|
|
440
|
+
"done",
|
|
441
|
+
"cancelled"
|
|
442
|
+
];
|
|
443
|
+
const FIXED = [...HEAD, ...TAIL];
|
|
444
|
+
/** Every workflow has these. */
|
|
445
|
+
const REQUIRED = [
|
|
446
|
+
"todo",
|
|
447
|
+
"done",
|
|
448
|
+
"cancelled"
|
|
449
|
+
];
|
|
450
|
+
/** These take no handoff. */
|
|
451
|
+
const NO_HANDOFF = /* @__PURE__ */ new Set([
|
|
452
|
+
"backlog",
|
|
453
|
+
"done",
|
|
454
|
+
"cancelled"
|
|
455
|
+
]);
|
|
456
|
+
/** The most characters Multica takes in a step's instructions. */
|
|
457
|
+
const MAX_STEP = 8e3;
|
|
458
|
+
/** The most characters Multica takes in an agent's description. */
|
|
459
|
+
const MAX_DESCRIPTION = 255;
|
|
460
|
+
/** `plan_review` → `Plan Review`. */
|
|
461
|
+
const titleCase = (key) => key.split(/[_-]+/).filter(Boolean).map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`).join(" ");
|
|
462
|
+
/**
|
|
463
|
+
* The Multica name of each column: a built-in column's own, the first workflow's custom columns
|
|
464
|
+
* by their `name` (or their key, in title case), and every later workflow's prefixed with the
|
|
465
|
+
* workflow's `prefix`, or its id in title case. The first workflow's `prefix` is ignored, so a
|
|
466
|
+
* workflow can come first in one target and later in another.
|
|
467
|
+
*/
|
|
468
|
+
const columnNames = (workflows) => Object.fromEntries(workflows.map((workflow, index) => [workflow.id, Object.fromEntries(workflow.columns.map((column) => {
|
|
469
|
+
const builtIn = BUILT_IN[column.key];
|
|
470
|
+
if (builtIn !== void 0) return [column.key, builtIn];
|
|
471
|
+
const own = column.name ?? titleCase(column.key);
|
|
472
|
+
if (index === 0) return [column.key, own];
|
|
473
|
+
return [column.key, `${workflow.prefix ?? titleCase(workflow.id)} ${own}`];
|
|
474
|
+
}))]));
|
|
475
|
+
const ref = (workflow, key) => `${workflow}\0${key}`;
|
|
476
|
+
const toColumn = (workflow, column) => ({
|
|
477
|
+
name: column.name,
|
|
478
|
+
...isBuiltIn(column.key) ? { key: column.key } : { category: column.category ?? "started" },
|
|
479
|
+
handoff: column.handoff,
|
|
480
|
+
...column.instructions === void 0 ? {} : { instructions: column.instructions },
|
|
481
|
+
from: [{
|
|
482
|
+
workflow,
|
|
483
|
+
key: column.key
|
|
484
|
+
}]
|
|
485
|
+
});
|
|
486
|
+
/** Merges `rendered`'s workflows, in Multica's order, and reports what can't merge. */
|
|
487
|
+
const mergeWorkflows = (rendered) => {
|
|
488
|
+
const issues = [];
|
|
489
|
+
const workflows = rendered.workflows;
|
|
490
|
+
const place = /* @__PURE__ */ new Map();
|
|
491
|
+
const columns = [];
|
|
492
|
+
const shared = /* @__PURE__ */ new Map();
|
|
493
|
+
const add = (workflow, key, column) => {
|
|
494
|
+
const builtIn = isBuiltIn(key);
|
|
495
|
+
const existing = builtIn ? shared.get(key) : void 0;
|
|
496
|
+
if (existing !== void 0) {
|
|
497
|
+
place.set(ref(workflow, key), existing);
|
|
498
|
+
columns[existing]?.from.push({
|
|
499
|
+
workflow,
|
|
500
|
+
key
|
|
501
|
+
});
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
place.set(ref(workflow, key), columns.length);
|
|
505
|
+
if (builtIn) shared.set(key, columns.length);
|
|
506
|
+
columns.push(column);
|
|
507
|
+
};
|
|
508
|
+
const all = workflows.flatMap((workflow) => workflow.columns.map((column) => ({
|
|
509
|
+
workflow: workflow.id,
|
|
510
|
+
column
|
|
511
|
+
})));
|
|
512
|
+
for (const key of HEAD) for (const { workflow, column } of all.filter((item) => item.column.key === key)) add(workflow, key, toColumn(workflow, column));
|
|
513
|
+
for (const { workflow, column } of all.filter((item) => !FIXED.includes(item.column.key))) add(workflow, column.key, toColumn(workflow, column));
|
|
514
|
+
for (const key of TAIL) for (const { workflow, column } of all.filter((item) => item.column.key === key)) add(workflow, key, toColumn(workflow, column));
|
|
515
|
+
const seen = /* @__PURE__ */ new Set();
|
|
516
|
+
for (const { workflow, column } of all) {
|
|
517
|
+
const at = place.get(ref(workflow, column.key));
|
|
518
|
+
const merged = at === void 0 ? void 0 : columns[at];
|
|
519
|
+
if (at === void 0 || !merged) continue;
|
|
520
|
+
const next = column.next === void 0 ? void 0 : place.get(ref(workflow, column.next));
|
|
521
|
+
const back = column.back === void 0 ? void 0 : place.get(ref(workflow, column.back));
|
|
522
|
+
if (!seen.has(at)) {
|
|
523
|
+
seen.add(at);
|
|
524
|
+
if (next !== void 0) merged.next = next;
|
|
525
|
+
if (back !== void 0) merged.back = back;
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
const first = merged.from[0];
|
|
529
|
+
const conflicts = [
|
|
530
|
+
column.handoff !== merged.handoff ? "handoff" : "",
|
|
531
|
+
(column.instructions ?? "") !== (merged.instructions ?? "") ? "step text" : "",
|
|
532
|
+
next !== merged.next ? "next" : "",
|
|
533
|
+
back !== merged.back ? "back" : ""
|
|
534
|
+
].filter(Boolean);
|
|
535
|
+
if (conflicts.length > 0 && first) issues.push({
|
|
536
|
+
field: `workflows.${workflow}.columns.${column.key}`,
|
|
537
|
+
message: `${merged.name} is shared with workflow ${first.workflow}, which gives it another ${conflicts.join(", ")}: Multica has one ${merged.name}, so give both the same`
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
return {
|
|
541
|
+
columns,
|
|
542
|
+
issues
|
|
543
|
+
};
|
|
544
|
+
};
|
|
545
|
+
/** What Multica would refuse in a target's workflows, or would put back another way. */
|
|
546
|
+
const checkWorkflows = (rendered, merged) => {
|
|
547
|
+
const issues = [...merged.issues];
|
|
548
|
+
for (const workflow of rendered.workflows) {
|
|
549
|
+
const field = (key) => `workflows.${workflow.id}.columns.${key}`;
|
|
550
|
+
const keys = workflow.columns.map((column) => column.key);
|
|
551
|
+
const fixed = keys.filter((key) => FIXED.includes(key));
|
|
552
|
+
const expected = FIXED.filter((key) => keys.includes(key));
|
|
553
|
+
const firstMiddle = keys.findIndex((key) => !FIXED.includes(key));
|
|
554
|
+
const lastMiddle = keys.findLastIndex((key) => !FIXED.includes(key));
|
|
555
|
+
const lastHead = Math.max(-1, ...HEAD.map((key) => keys.indexOf(key)));
|
|
556
|
+
const tailPlaces = TAIL.map((key) => keys.indexOf(key)).filter((index) => index >= 0);
|
|
557
|
+
const firstTail = Math.min(keys.length, ...tailPlaces);
|
|
558
|
+
if (fixed.join() !== expected.join() || firstMiddle >= 0 && (lastHead > firstMiddle || firstTail < lastMiddle)) issues.push({
|
|
559
|
+
field: `workflows.${workflow.id}.columns`,
|
|
560
|
+
message: "list backlog and todo first, and blocked, done and cancelled last, as Multica does"
|
|
561
|
+
});
|
|
562
|
+
for (const column of workflow.columns) {
|
|
563
|
+
if (NO_HANDOFF.has(column.key) && column.handoff !== "none") issues.push({
|
|
564
|
+
field: `${field(column.key)}.handoff`,
|
|
565
|
+
message: `Multica hands nothing off in ${BUILT_IN[column.key]}: make it none`
|
|
566
|
+
});
|
|
567
|
+
const size = length(column.instructions ?? "");
|
|
568
|
+
if (size > 8e3) issues.push({
|
|
569
|
+
field: `workflows.${workflow.id}.steps.${column.key}`,
|
|
570
|
+
message: `is ${size} characters, over Multica's ${MAX_STEP}`
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
const keys = new Set(merged.columns.flatMap((column) => column.key ? [column.key] : []));
|
|
575
|
+
for (const key of REQUIRED) if (!keys.has(key)) issues.push({
|
|
576
|
+
field: "workflows",
|
|
577
|
+
message: `no workflow has a ${key} column, and Multica needs one`
|
|
578
|
+
});
|
|
579
|
+
const names = /* @__PURE__ */ new Map();
|
|
580
|
+
const builtInNames = new Set(Object.values(BUILT_IN).map((name) => name.toLowerCase()));
|
|
581
|
+
for (const column of merged.columns) {
|
|
582
|
+
if (column.key) continue;
|
|
583
|
+
const from = column.from[0];
|
|
584
|
+
const field = from ? `workflows.${from.workflow}.columns.${from.key}.name` : "workflows";
|
|
585
|
+
const lower = column.name.toLowerCase();
|
|
586
|
+
const earlier = names.get(lower);
|
|
587
|
+
if (builtInNames.has(lower)) issues.push({
|
|
588
|
+
field,
|
|
589
|
+
message: `${column.name} is the name of a built-in Multica status`
|
|
590
|
+
});
|
|
591
|
+
else if (earlier) {
|
|
592
|
+
const other = earlier.from[0];
|
|
593
|
+
issues.push({
|
|
594
|
+
field,
|
|
595
|
+
message: `${column.name} is also the name of ${other ? `workflow ${other.workflow}'s ${other.key}` : "another column"}: Multica names a status once`
|
|
596
|
+
});
|
|
597
|
+
} else names.set(lower, column);
|
|
598
|
+
}
|
|
599
|
+
return issues;
|
|
600
|
+
};
|
|
601
|
+
// src/steps.ts
|
|
602
|
+
const builtInKeys = new Set(Object.keys(BUILT_IN));
|
|
603
|
+
/**
|
|
604
|
+
* The custom columns' status keys, by place. Multica keeps a workflow's steps in the order
|
|
605
|
+
* written, and a key says nothing about its status's name, since a renamed status keeps its
|
|
606
|
+
* key. So the n-th custom key is the n-th custom column's, and when the counts differ, none is
|
|
607
|
+
* known.
|
|
608
|
+
*/
|
|
609
|
+
const customKeys = (liveKeys, specs) => {
|
|
610
|
+
const custom = liveKeys.filter((key) => !builtInKeys.has(key));
|
|
611
|
+
const places = specs.flatMap((spec, index) => spec.key ? [] : [index]);
|
|
612
|
+
if (custom.length !== places.length) return /* @__PURE__ */ new Map();
|
|
613
|
+
return new Map(places.map((place, index) => [place, custom[index] ?? ""]));
|
|
614
|
+
};
|
|
615
|
+
/** The status key of the column at `place`, when it's known. */
|
|
616
|
+
const keyAt = (specs, keys, place) => specs[place]?.key ?? keys.get(place);
|
|
617
|
+
/**
|
|
618
|
+
* One step as Multica takes it: by key, or by name for a custom column whose key isn't known
|
|
619
|
+
* (Multica reuses the status with that name, or creates it), with its handoff and the moves
|
|
620
|
+
* whose target has a key.
|
|
621
|
+
*/
|
|
622
|
+
const stepFor = (specs, keys, place) => {
|
|
623
|
+
const spec = specs[place];
|
|
624
|
+
if (!spec) throw new Error(`no column at ${place}`);
|
|
625
|
+
const key = keyAt(specs, keys, place);
|
|
626
|
+
const step = key ? { status_key: key } : { new_status: {
|
|
627
|
+
name: spec.name,
|
|
628
|
+
category: spec.category ?? "started"
|
|
629
|
+
} };
|
|
630
|
+
if (spec.handler) {
|
|
631
|
+
step.handler = spec.handler;
|
|
632
|
+
step.instructions = spec.instructions;
|
|
633
|
+
}
|
|
634
|
+
const next = spec.next === void 0 ? void 0 : keyAt(specs, keys, spec.next);
|
|
635
|
+
const back = spec.back === void 0 ? void 0 : keyAt(specs, keys, spec.back);
|
|
636
|
+
if (next) step.next_status_key = next;
|
|
637
|
+
if (back) step.back_status_key = back;
|
|
638
|
+
return step;
|
|
639
|
+
};
|
|
640
|
+
/** Every step, in order. */
|
|
641
|
+
const stepsFor = (specs, keys) => specs.map((_, place) => stepFor(specs, keys, place));
|
|
642
|
+
// src/apply.ts
|
|
643
|
+
const OPS = /* @__PURE__ */ new Set([
|
|
644
|
+
"skill-import",
|
|
645
|
+
"agent-update",
|
|
646
|
+
"agent-skills-add",
|
|
647
|
+
"squad-instructions",
|
|
648
|
+
"squad-member-add",
|
|
649
|
+
"squad-member-role",
|
|
650
|
+
"squad-leader",
|
|
651
|
+
"workflow",
|
|
652
|
+
"context"
|
|
653
|
+
]);
|
|
654
|
+
const isData = (value) => !!value && typeof value === "object" && "op" in value && typeof value.op === "string" && OPS.has(value.op) && "cli" in value && "workspace" in value;
|
|
655
|
+
/** Makes `changes` in order. One that fails, like a skill another account owns, leaves the rest. */
|
|
656
|
+
const applyChanges = (changes) => {
|
|
657
|
+
const failed = [];
|
|
658
|
+
const dir = mkdtempSync(join(tmpdir(), "10xagents-multica-"));
|
|
659
|
+
let workflowFailed = false;
|
|
660
|
+
try {
|
|
661
|
+
let files = 0;
|
|
662
|
+
const file = (name, content) => {
|
|
663
|
+
const path = join(dir, `${++files}-${name}`);
|
|
664
|
+
writeFileSync(path, content);
|
|
665
|
+
return path;
|
|
666
|
+
};
|
|
667
|
+
for (const change of changes) {
|
|
668
|
+
if (change.manual || change.note) continue;
|
|
669
|
+
try {
|
|
670
|
+
if (!isData(change.data)) throw new Error("not a change the Multica connector planned");
|
|
671
|
+
const data = change.data;
|
|
672
|
+
if (data.op === "workflow") try {
|
|
673
|
+
writeWorkflow(data.cli, data.workspace, data, file);
|
|
674
|
+
} catch (error) {
|
|
675
|
+
workflowFailed = true;
|
|
676
|
+
throw error;
|
|
677
|
+
}
|
|
678
|
+
else if (data.op === "context") {
|
|
679
|
+
if (workflowFailed) throw new Error("held back, since the workflow's write failed");
|
|
680
|
+
run(data.cli, [
|
|
681
|
+
"workspace",
|
|
682
|
+
"update",
|
|
683
|
+
data.workspace,
|
|
684
|
+
"--context-stdin"
|
|
685
|
+
], data.workspace, data.context);
|
|
686
|
+
} else applyOne(data, file);
|
|
687
|
+
} catch (error) {
|
|
688
|
+
failed.push({
|
|
689
|
+
change,
|
|
690
|
+
message: error instanceof Error ? error.message : String(error)
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
} finally {
|
|
695
|
+
rmSync(dir, {
|
|
696
|
+
recursive: true,
|
|
697
|
+
force: true
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
return { failed };
|
|
701
|
+
};
|
|
702
|
+
const applyOne = (data, file) => {
|
|
703
|
+
const { cli, workspace } = data;
|
|
704
|
+
const call = (args) => run(cli, args, workspace);
|
|
705
|
+
switch (data.op) {
|
|
706
|
+
case "skill-import":
|
|
707
|
+
call([
|
|
708
|
+
"skill",
|
|
709
|
+
"import",
|
|
710
|
+
"--file",
|
|
711
|
+
file(`${data.skill.name}.zip`, zipSkill(data.skill)),
|
|
712
|
+
"--on-conflict",
|
|
713
|
+
"overwrite"
|
|
714
|
+
]);
|
|
715
|
+
return;
|
|
716
|
+
case "agent-update":
|
|
717
|
+
call([
|
|
718
|
+
"agent",
|
|
719
|
+
"update",
|
|
720
|
+
data.agent,
|
|
721
|
+
...data.args
|
|
722
|
+
]);
|
|
723
|
+
return;
|
|
724
|
+
case "agent-skills-add": {
|
|
725
|
+
const listed = runJson(cli, ["skill", "list"], workspace);
|
|
726
|
+
const matched = (Array.isArray(listed) ? listed : []).filter((skill) => !!skill && typeof skill === "object" && "name" in skill && "id" in skill).filter((skill) => data.skills.includes(String(skill.name)));
|
|
727
|
+
if (matched.length > 0) {
|
|
728
|
+
const ids = matched.map((skill) => String(skill.id));
|
|
729
|
+
call([
|
|
730
|
+
"agent",
|
|
731
|
+
"skills",
|
|
732
|
+
"add",
|
|
733
|
+
data.agent,
|
|
734
|
+
"--skill-ids",
|
|
735
|
+
ids.join(",")
|
|
736
|
+
]);
|
|
737
|
+
}
|
|
738
|
+
const found = new Set(matched.map((skill) => String(skill.name)));
|
|
739
|
+
const missing = data.skills.filter((name) => !found.has(name));
|
|
740
|
+
if (missing.length > 0) throw new Error(`not in the workspace, so not attached: ${missing.join(", ")}`);
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
case "squad-instructions":
|
|
744
|
+
call([
|
|
745
|
+
"squad",
|
|
746
|
+
"update",
|
|
747
|
+
data.squad,
|
|
748
|
+
"--instructions",
|
|
749
|
+
data.instructions
|
|
750
|
+
]);
|
|
751
|
+
return;
|
|
752
|
+
case "squad-member-add":
|
|
753
|
+
call([
|
|
754
|
+
"squad",
|
|
755
|
+
"member",
|
|
756
|
+
"add",
|
|
757
|
+
data.squad,
|
|
758
|
+
"--member-id",
|
|
759
|
+
data.agent,
|
|
760
|
+
"--type",
|
|
761
|
+
"agent",
|
|
762
|
+
"--role",
|
|
763
|
+
data.role
|
|
764
|
+
]);
|
|
765
|
+
return;
|
|
766
|
+
case "squad-member-role":
|
|
767
|
+
call([
|
|
768
|
+
"squad",
|
|
769
|
+
"member",
|
|
770
|
+
"set-role",
|
|
771
|
+
data.squad,
|
|
772
|
+
"--member-id",
|
|
773
|
+
data.agent,
|
|
774
|
+
"--member-type",
|
|
775
|
+
"agent",
|
|
776
|
+
"--role",
|
|
777
|
+
data.role
|
|
778
|
+
]);
|
|
779
|
+
return;
|
|
780
|
+
case "squad-leader":
|
|
781
|
+
call([
|
|
782
|
+
"squad",
|
|
783
|
+
"update",
|
|
784
|
+
data.squad,
|
|
785
|
+
"--leader",
|
|
786
|
+
data.agent
|
|
787
|
+
]);
|
|
788
|
+
return;
|
|
789
|
+
default: throw new Error(`can't apply ${data.op} here`);
|
|
790
|
+
}
|
|
791
|
+
};
|
|
792
|
+
/**
|
|
793
|
+
* Writes the workflow in two passes. The first names every custom column, so Multica creates or
|
|
794
|
+
* reuses its status, and sets only the moves whose target has a key. The second sets the moves
|
|
795
|
+
* to custom columns, with the keys the first one made.
|
|
796
|
+
*/
|
|
797
|
+
const writeWorkflow = (cli, workspace, data, file) => {
|
|
798
|
+
const steps = (keys) => file("steps.json", JSON.stringify(stepsFor(data.specs, keys), null, 2));
|
|
799
|
+
const first = stepsFor(data.specs, /* @__PURE__ */ new Map());
|
|
800
|
+
const written = runJson(cli, [
|
|
801
|
+
"workflow",
|
|
802
|
+
"update",
|
|
803
|
+
data.workflow,
|
|
804
|
+
"--steps-file",
|
|
805
|
+
steps(/* @__PURE__ */ new Map())
|
|
806
|
+
], workspace);
|
|
807
|
+
const writtenSteps = written && typeof written === "object" && "steps" in written && Array.isArray(written.steps) ? written.steps.filter((step) => !!step && typeof step === "object").map(toStep) : [];
|
|
808
|
+
const made = customKeys(writtenSteps.map((step) => step.status_key), data.specs);
|
|
809
|
+
const custom = data.specs.filter((spec) => !spec.key).length;
|
|
810
|
+
if (made.size !== custom) throw new Error("the written workflow's custom columns don't line up with the setup's");
|
|
811
|
+
const second = stepsFor(data.specs, made);
|
|
812
|
+
if (JSON.stringify(second) !== JSON.stringify(first)) run(cli, [
|
|
813
|
+
"workflow",
|
|
814
|
+
"update",
|
|
815
|
+
data.workflow,
|
|
816
|
+
"--steps-file",
|
|
817
|
+
steps(made)
|
|
818
|
+
], workspace);
|
|
819
|
+
};
|
|
820
|
+
// src/bench/read.ts
|
|
821
|
+
const isObject = (value) => !!value && typeof value === "object" && !Array.isArray(value);
|
|
822
|
+
const isNamed = (value) => isObject(value) && typeof value.id === "string" && typeof value.name === "string";
|
|
823
|
+
const isIssue = (value) => isObject(value) && typeof value.id === "string" && typeof value.created_at === "string";
|
|
824
|
+
const isRun = (value) => isObject(value) && typeof value.id === "string" && typeof value.agent_id === "string" && typeof value.status === "string";
|
|
825
|
+
const isEntry = (value) => isObject(value);
|
|
826
|
+
const isComment = (value) => isObject(value) && typeof value.created_at === "string";
|
|
827
|
+
/** The records in a list that have the fields `guard` checks. */
|
|
828
|
+
const list = (value, guard) => Array.isArray(value) ? value.filter((item) => guard(item)) : [];
|
|
829
|
+
/** The list itself, or the list a field of an object holds. */
|
|
830
|
+
const listIn = (value, field, guard) => Array.isArray(value) ? list(value, guard) : isObject(value) ? list(value[field], guard) : [];
|
|
831
|
+
/** What `multica <args>` printed, and what it wrote to standard error, where it warns of a cut timeline. */
|
|
832
|
+
const spawn = (cli, args, workspace) => {
|
|
833
|
+
const result = spawnSync(cli.bin, [...cli.profileArgs, ...args], {
|
|
834
|
+
encoding: "utf8",
|
|
835
|
+
maxBuffer: 268435456,
|
|
836
|
+
cwd: homeOf(cli.env),
|
|
837
|
+
env: childEnv(cli.env, workspace),
|
|
838
|
+
stdio: [
|
|
839
|
+
"ignore",
|
|
840
|
+
"pipe",
|
|
841
|
+
"pipe"
|
|
842
|
+
]
|
|
843
|
+
});
|
|
844
|
+
if (result.error || result.status !== 0) {
|
|
845
|
+
const reason = (result.stderr || result.error?.message || "").trim();
|
|
846
|
+
throw new Error(`multica ${args.slice(0, 3).join(" ")}: ${reason}`, { cause: result.error });
|
|
847
|
+
}
|
|
848
|
+
return {
|
|
849
|
+
stdout: result.stdout,
|
|
850
|
+
warnings: result.stderr ?? ""
|
|
851
|
+
};
|
|
852
|
+
};
|
|
853
|
+
const spawnJson = (cli, args, workspace) => {
|
|
854
|
+
const { stdout, warnings } = spawn(cli, [
|
|
855
|
+
...args,
|
|
856
|
+
"--output",
|
|
857
|
+
"json"
|
|
858
|
+
], workspace);
|
|
859
|
+
return {
|
|
860
|
+
data: JSON.parse(stdout),
|
|
861
|
+
warnings
|
|
862
|
+
};
|
|
863
|
+
};
|
|
864
|
+
/** Opens the target's workspace for reading, with the profile the query names or the only one. */
|
|
865
|
+
const openSource = async (query, context) => {
|
|
866
|
+
const log = query.log ?? (() => {});
|
|
867
|
+
const bin = findMultica(context.env, context.desktop);
|
|
868
|
+
const profile = await pickProfile(homeOf(context.env), query.profile, query.ask);
|
|
869
|
+
const cli = {
|
|
870
|
+
bin,
|
|
871
|
+
profileArgs: profile ? ["--profile", profile] : [],
|
|
872
|
+
env: context.env
|
|
873
|
+
};
|
|
874
|
+
const want = query.target.workspace;
|
|
875
|
+
const workspaces = list(runJson(cli, ["workspace", "list"]), isNamed);
|
|
876
|
+
if (workspaces.length === 0) throw new Error("this profile reaches no workspace: log in with another one");
|
|
877
|
+
const chosen = workspaces.find((ws) => [
|
|
878
|
+
ws.id,
|
|
879
|
+
ws.slug,
|
|
880
|
+
ws.name
|
|
881
|
+
].includes(want));
|
|
882
|
+
if (!chosen) throw new Error(`no workspace ${want} for this profile: ${workspaces.map((ws) => ws.slug || ws.name).join(", ")}`);
|
|
883
|
+
const workspace = {
|
|
884
|
+
id: chosen.id,
|
|
885
|
+
name: chosen.name,
|
|
886
|
+
slug: chosen.slug ?? ""
|
|
887
|
+
};
|
|
888
|
+
const json = (args) => runJson(cli, args, workspace.id);
|
|
889
|
+
const notes = [];
|
|
890
|
+
const agentNames = new Map(list(json([
|
|
891
|
+
"agent",
|
|
892
|
+
"list",
|
|
893
|
+
"--include-archived"
|
|
894
|
+
]), isNamed).map((a) => [a.id, a.name]));
|
|
895
|
+
const byId = /* @__PURE__ */ new Map();
|
|
896
|
+
const listArgs = [
|
|
897
|
+
"issue",
|
|
898
|
+
"list",
|
|
899
|
+
"--sort",
|
|
900
|
+
"created_at",
|
|
901
|
+
"--direction",
|
|
902
|
+
"asc",
|
|
903
|
+
"--limit",
|
|
904
|
+
"100"
|
|
905
|
+
];
|
|
906
|
+
for (let offset = 0;; offset += 100) {
|
|
907
|
+
const page = json([
|
|
908
|
+
...listArgs,
|
|
909
|
+
"--offset",
|
|
910
|
+
String(offset)
|
|
911
|
+
]);
|
|
912
|
+
const issues = listIn(page, "issues", isIssue);
|
|
913
|
+
for (const issue of issues) byId.set(issue.id, issue);
|
|
914
|
+
if (!(isObject(page) && page.has_more === true) || issues.length === 0) break;
|
|
915
|
+
}
|
|
916
|
+
const runs = /* @__PURE__ */ new Map();
|
|
917
|
+
const timelines = /* @__PURE__ */ new Map();
|
|
918
|
+
const comments = /* @__PURE__ */ new Map();
|
|
919
|
+
let fromServer = false;
|
|
920
|
+
const serverTimeline = async (issueId, statusOnly) => {
|
|
921
|
+
const file = join(homeOf(context.env), ".multica", ...profile ? ["profiles", profile] : [], "config.json");
|
|
922
|
+
let config;
|
|
923
|
+
try {
|
|
924
|
+
config = JSON.parse(readFileSync(file, "utf8"));
|
|
925
|
+
} catch (error) {
|
|
926
|
+
throw new Error(`can't read the server's address and token from ${file}`, { cause: error });
|
|
927
|
+
}
|
|
928
|
+
const fields = isObject(config) ? config : {};
|
|
929
|
+
const token = typeof fields.token === "string" ? fields.token : "";
|
|
930
|
+
if (!token) throw new Error(`${file} holds no token: log in with this profile first`);
|
|
931
|
+
const url = ((typeof fields.server_url === "string" ? fields.server_url : "") || "https://api.multica.ai").replace(/^ws/, "http").replace(/\/+$/, "");
|
|
932
|
+
const response = await fetch(`${url}/api/issues/${encodeURIComponent(issueId)}/timeline`, { headers: {
|
|
933
|
+
authorization: `Bearer ${token}`,
|
|
934
|
+
"x-workspace-id": workspace.id
|
|
935
|
+
} });
|
|
936
|
+
if (!response.ok) throw new Error(`timeline ${issueId}: the server answered HTTP ${response.status}`);
|
|
937
|
+
const body = await response.json();
|
|
938
|
+
const entries = listIn(body, "entries", isEntry);
|
|
939
|
+
const cut = response.headers.get("x-timeline-truncated");
|
|
940
|
+
return {
|
|
941
|
+
data: statusOnly ? entries.filter((e) => e.type === "activity" && e.action === "status_changed") : entries,
|
|
942
|
+
warnings: cut ? `warning: timeline truncated by the server cap (${cut})` : ""
|
|
943
|
+
};
|
|
944
|
+
};
|
|
945
|
+
const readTimeline = async (issueId, statusOnly) => {
|
|
946
|
+
if (!fromServer) try {
|
|
947
|
+
const { data, warnings } = spawnJson(cli, [
|
|
948
|
+
"issue",
|
|
949
|
+
"timeline",
|
|
950
|
+
issueId,
|
|
951
|
+
...statusOnly ? ["--action", "status_changed"] : []
|
|
952
|
+
], workspace.id);
|
|
953
|
+
return {
|
|
954
|
+
data: list(data, isEntry),
|
|
955
|
+
warnings
|
|
956
|
+
};
|
|
957
|
+
} catch (error) {
|
|
958
|
+
if (!(error instanceof Error) || !/unknown (flag: --|command "timeline")/.test(error.message)) throw error;
|
|
959
|
+
fromServer = true;
|
|
960
|
+
notes.push("Your multica CLI has no `issue timeline`, so timelines came from the server, with your profile's token.");
|
|
961
|
+
}
|
|
962
|
+
return serverTimeline(issueId, statusOnly);
|
|
963
|
+
};
|
|
964
|
+
const scratch = mkdtempSync(join(tmpdir(), "10xagents-bench-"));
|
|
965
|
+
return {
|
|
966
|
+
workspace,
|
|
967
|
+
agentNames,
|
|
968
|
+
issues: [...byId.values()],
|
|
969
|
+
notes,
|
|
970
|
+
runs: (issueId) => {
|
|
971
|
+
let read = runs.get(issueId);
|
|
972
|
+
if (!read) {
|
|
973
|
+
read = list(json([
|
|
974
|
+
"issue",
|
|
975
|
+
"runs",
|
|
976
|
+
issueId
|
|
977
|
+
]), isRun);
|
|
978
|
+
runs.set(issueId, read);
|
|
979
|
+
}
|
|
980
|
+
return read;
|
|
981
|
+
},
|
|
982
|
+
timeline: (issueId, statusOnly) => {
|
|
983
|
+
const key = `${issueId}:${statusOnly}`;
|
|
984
|
+
let read = timelines.get(key);
|
|
985
|
+
if (!read) {
|
|
986
|
+
read = readTimeline(issueId, statusOnly);
|
|
987
|
+
timelines.set(key, read);
|
|
988
|
+
}
|
|
989
|
+
return read;
|
|
990
|
+
},
|
|
991
|
+
comments: (issueId) => {
|
|
992
|
+
let read = comments.get(issueId);
|
|
993
|
+
if (!read) {
|
|
994
|
+
read = listIn(json([
|
|
995
|
+
"issue",
|
|
996
|
+
"comment",
|
|
997
|
+
"list",
|
|
998
|
+
issueId,
|
|
999
|
+
"--full"
|
|
1000
|
+
]), "comments", isComment);
|
|
1001
|
+
comments.set(issueId, read);
|
|
1002
|
+
}
|
|
1003
|
+
return read;
|
|
1004
|
+
},
|
|
1005
|
+
attachment: (attachment) => {
|
|
1006
|
+
try {
|
|
1007
|
+
const dir = join(scratch, attachment.id);
|
|
1008
|
+
mkdirSync(dir, { recursive: true });
|
|
1009
|
+
let path = join(dir, attachment.filename ?? "");
|
|
1010
|
+
const printed = spawn(cli, [
|
|
1011
|
+
"attachment",
|
|
1012
|
+
"download",
|
|
1013
|
+
attachment.id,
|
|
1014
|
+
"-o",
|
|
1015
|
+
dir
|
|
1016
|
+
], workspace.id).stdout;
|
|
1017
|
+
try {
|
|
1018
|
+
const parsed = JSON.parse(printed);
|
|
1019
|
+
if (isObject(parsed) && typeof parsed.path === "string") path = parsed.path;
|
|
1020
|
+
} catch {}
|
|
1021
|
+
return readFileSync(path, "utf8");
|
|
1022
|
+
} catch (error) {
|
|
1023
|
+
log(` couldn't read ${attachment.filename ?? attachment.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1024
|
+
return null;
|
|
1025
|
+
}
|
|
1026
|
+
},
|
|
1027
|
+
close: () => rmSync(scratch, {
|
|
1028
|
+
recursive: true,
|
|
1029
|
+
force: true
|
|
1030
|
+
})
|
|
1031
|
+
};
|
|
1032
|
+
};
|
|
1033
|
+
// src/bench/shared.ts
|
|
1034
|
+
/** Milliseconds since the epoch, or `NaN` without a date. */
|
|
1035
|
+
const ms = (value) => value ? Date.parse(value) : NaN;
|
|
1036
|
+
/** The finite values at or above 0, sorted. */
|
|
1037
|
+
const finite = (xs) => xs.filter((x) => typeof x === "number" && Number.isFinite(x) && x >= 0).toSorted((a, b) => a - b);
|
|
1038
|
+
/** Nearest rank: the smallest value with at least `p` of the sorted `xs` at or below it. */
|
|
1039
|
+
const pct = (xs, p) => xs.length ? xs[Math.max(0, Math.ceil(p * xs.length) - 1)] ?? NaN : NaN;
|
|
1040
|
+
const sum = (xs) => xs.reduce((a, b) => a + b, 0);
|
|
1041
|
+
const mean = (xs) => xs.length ? sum(xs) / xs.length : NaN;
|
|
1042
|
+
/** Groups `xs` by `key`, in the order keys first appear. */
|
|
1043
|
+
const by = (xs, key) => {
|
|
1044
|
+
const map = /* @__PURE__ */ new Map();
|
|
1045
|
+
for (const x of xs) map.set(key(x), [...map.get(key(x)) ?? [], x]);
|
|
1046
|
+
return map;
|
|
1047
|
+
};
|
|
1048
|
+
/** Counts each value, most first. */
|
|
1049
|
+
const count = (xs) => {
|
|
1050
|
+
const map = /* @__PURE__ */ new Map();
|
|
1051
|
+
for (const x of xs) map.set(x, (map.get(x) ?? 0) + 1);
|
|
1052
|
+
return [...map].toSorted((a, b) => b[1] - a[1]);
|
|
1053
|
+
};
|
|
1054
|
+
/** Collects metric results for one segment. */
|
|
1055
|
+
const collector = (segment) => {
|
|
1056
|
+
const results = [];
|
|
1057
|
+
const add = (id, x, slice) => {
|
|
1058
|
+
results.push({
|
|
1059
|
+
id,
|
|
1060
|
+
segment,
|
|
1061
|
+
value: x !== null && Number.isFinite(x) ? x : null,
|
|
1062
|
+
...slice ? { by: slice } : {}
|
|
1063
|
+
});
|
|
1064
|
+
};
|
|
1065
|
+
return {
|
|
1066
|
+
results,
|
|
1067
|
+
add
|
|
1068
|
+
};
|
|
1069
|
+
};
|
|
1070
|
+
/**
|
|
1071
|
+
* Dollars per million tokens at Anthropic's list prices, as Multica's own table has them
|
|
1072
|
+
* (`server/internal/metrics/pricing.go`), with five-minute cache writes. A model it doesn't know is
|
|
1073
|
+
* priced as Opus 5.5.
|
|
1074
|
+
*/
|
|
1075
|
+
const PRICES = {
|
|
1076
|
+
"claude-opus-5-5": {
|
|
1077
|
+
input: 4,
|
|
1078
|
+
output: 20,
|
|
1079
|
+
cacheRead: .2,
|
|
1080
|
+
cacheWrite: 5
|
|
1081
|
+
},
|
|
1082
|
+
"claude-sonnet-5-5": {
|
|
1083
|
+
input: 2,
|
|
1084
|
+
output: 10,
|
|
1085
|
+
cacheRead: .1,
|
|
1086
|
+
cacheWrite: 2.5
|
|
1087
|
+
},
|
|
1088
|
+
"claude-haiku-5-5": {
|
|
1089
|
+
input: .1,
|
|
1090
|
+
output: .5,
|
|
1091
|
+
cacheRead: .01,
|
|
1092
|
+
cacheWrite: .125
|
|
1093
|
+
},
|
|
1094
|
+
"claude-fable-5-1": {
|
|
1095
|
+
input: 10,
|
|
1096
|
+
output: 50,
|
|
1097
|
+
cacheRead: .25,
|
|
1098
|
+
cacheWrite: 12.5
|
|
1099
|
+
}
|
|
1100
|
+
};
|
|
1101
|
+
/** Prices usage, and remembers the models it had to price as Opus 5.5. */
|
|
1102
|
+
const pricer = () => {
|
|
1103
|
+
const unpriced = /* @__PURE__ */ new Set();
|
|
1104
|
+
const costOf = (entries) => entries.reduce((total, u) => {
|
|
1105
|
+
const key = Object.keys(PRICES).find((k) => (u.model ?? "").includes(k));
|
|
1106
|
+
if (!key) unpriced.add(u.model || "unnamed");
|
|
1107
|
+
const p = PRICES[key ?? "claude-opus-5-5"] ?? {
|
|
1108
|
+
input: 0,
|
|
1109
|
+
output: 0,
|
|
1110
|
+
cacheRead: 0,
|
|
1111
|
+
cacheWrite: 0
|
|
1112
|
+
};
|
|
1113
|
+
return total + ((u.input_tokens ?? 0) * p.input + (u.output_tokens ?? 0) * p.output + (u.cache_read_tokens ?? 0) * p.cacheRead + (u.cache_write_tokens ?? 0) * p.cacheWrite) / 1e6;
|
|
1114
|
+
}, 0);
|
|
1115
|
+
return {
|
|
1116
|
+
costOf,
|
|
1117
|
+
unpriced
|
|
1118
|
+
};
|
|
1119
|
+
};
|
|
1120
|
+
/** A comment's or summary's first words, without the heading marks and emoji before them. */
|
|
1121
|
+
const heading = (text) => (text ?? "").replace(/^[#\s]*/, "").replace(/^[^\p{L}]+/u, "");
|
|
1122
|
+
/** The workflow's built-in steps, where QA also runs without the key being off. */
|
|
1123
|
+
const BUILT_IN_STEPS = /* @__PURE__ */ new Set([
|
|
1124
|
+
"backlog",
|
|
1125
|
+
"todo",
|
|
1126
|
+
"in_progress",
|
|
1127
|
+
"in_review",
|
|
1128
|
+
"done",
|
|
1129
|
+
"blocked",
|
|
1130
|
+
"cancelled"
|
|
1131
|
+
]);
|
|
1132
|
+
/** A run's cost: unknown when it ran without reporting usage, and nothing when it never started. */
|
|
1133
|
+
const runCost = (usage, started, costOf) => usage?.length ? costOf(usage) : started ? null : 0;
|
|
1134
|
+
// src/bench/requests.ts
|
|
1135
|
+
const REQUEST_METRICS = [
|
|
1136
|
+
"requests-touched",
|
|
1137
|
+
"requests",
|
|
1138
|
+
"requests-done",
|
|
1139
|
+
"requests-stopped",
|
|
1140
|
+
"requests-open",
|
|
1141
|
+
"requests-left-out",
|
|
1142
|
+
"lead-time-p50",
|
|
1143
|
+
"lead-time-p90",
|
|
1144
|
+
"people-actions-p50",
|
|
1145
|
+
"people-actions-p90",
|
|
1146
|
+
"waiting-on-people-p50",
|
|
1147
|
+
"waiting-on-people-p90",
|
|
1148
|
+
"waiting-on-people-share",
|
|
1149
|
+
"review-rounds-mean",
|
|
1150
|
+
"plan-reviews-mean",
|
|
1151
|
+
"runs-per-request-p50",
|
|
1152
|
+
"runs-per-request-p90",
|
|
1153
|
+
"agent-time-per-request-p50",
|
|
1154
|
+
"agent-time-per-request-p90",
|
|
1155
|
+
"cost-per-request-p50",
|
|
1156
|
+
"cost-per-request-p90",
|
|
1157
|
+
"requests-cost-known",
|
|
1158
|
+
"requests-cost",
|
|
1159
|
+
"requests-failed-runs"
|
|
1160
|
+
];
|
|
1161
|
+
const commentKind = (summary) => {
|
|
1162
|
+
const s = heading(summary);
|
|
1163
|
+
if (!s) return "assignment or status";
|
|
1164
|
+
if (/^Wakeup: sub-issues closed/i.test(s)) return "stage barrier";
|
|
1165
|
+
if (/^Wakeup:/i.test(s)) return "wakeup";
|
|
1166
|
+
if (/^(FIX|DELIVERY|RESTACK|SYNTHESIS|STAGE REVIEW)\b/.test(s)) return "review request";
|
|
1167
|
+
if (/^PLAN v\d+.*in review/.test(s)) return "plan";
|
|
1168
|
+
return "comment";
|
|
1169
|
+
};
|
|
1170
|
+
const LIVE = /* @__PURE__ */ new Set([
|
|
1171
|
+
"queued",
|
|
1172
|
+
"dispatched",
|
|
1173
|
+
"claimed",
|
|
1174
|
+
"running"
|
|
1175
|
+
]);
|
|
1176
|
+
const CLOSED = /* @__PURE__ */ new Set(["done", "cancelled"]);
|
|
1177
|
+
const SAME_ACTION = 12e4;
|
|
1178
|
+
const RUN_GRACE = 6e4;
|
|
1179
|
+
const hours = (x) => x / 36e5;
|
|
1180
|
+
const lastTouch = (i) => ms(i.last_activity_at ?? i.updated_at ?? i.created_at);
|
|
1181
|
+
const isReview = (run) => /^QA$/i.test(run.agent);
|
|
1182
|
+
const during = (r, t) => r.started <= t && t <= r.end + RUN_GRACE;
|
|
1183
|
+
const causedBy = (r, t) => ["assignment or status", "stage barrier"].includes(r.kind) && r.created >= t - 1e3 && r.created <= t;
|
|
1184
|
+
const timeWhen = (a, b, moments, holds) => {
|
|
1185
|
+
const cuts = [
|
|
1186
|
+
a,
|
|
1187
|
+
...moments.filter((m) => m > a && m < b).toSorted((x, y) => x - y),
|
|
1188
|
+
b
|
|
1189
|
+
];
|
|
1190
|
+
let total = 0;
|
|
1191
|
+
for (let i = 1; i < cuts.length; i++) {
|
|
1192
|
+
const from = cuts[i - 1] ?? 0;
|
|
1193
|
+
const to = cuts[i] ?? 0;
|
|
1194
|
+
if (to > from && holds((from + to) / 2)) total += to - from;
|
|
1195
|
+
}
|
|
1196
|
+
return total;
|
|
1197
|
+
};
|
|
1198
|
+
const escape = (x) => x.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1199
|
+
const reviews = (run) => isReview(run) && run.attempt <= 1 && !(run.status === "cancelled" && !(run.started > 0));
|
|
1200
|
+
/** Measures the requests whose first run falls in the window. */
|
|
1201
|
+
const measureRequests = async (source, query) => {
|
|
1202
|
+
const { since, until, split } = query;
|
|
1203
|
+
const { costOf, unpriced } = pricer();
|
|
1204
|
+
const notes = [];
|
|
1205
|
+
const log = query.log ?? (() => {});
|
|
1206
|
+
const children = /* @__PURE__ */ new Map();
|
|
1207
|
+
for (const issue of source.issues) {
|
|
1208
|
+
if (!issue.parent_issue_id) continue;
|
|
1209
|
+
children.set(issue.parent_issue_id, [...children.get(issue.parent_issue_id) ?? [], issue]);
|
|
1210
|
+
}
|
|
1211
|
+
const treeOf = (issue) => [issue, ...(children.get(issue.id) ?? []).flatMap(treeOf)];
|
|
1212
|
+
const candidates = source.issues.filter((i) => !i.parent_issue_id && ms(i.created_at) < until).map((i) => ({
|
|
1213
|
+
request: i,
|
|
1214
|
+
tree: treeOf(i)
|
|
1215
|
+
})).filter(({ tree }) => Math.max(...tree.map(lastTouch)) >= since);
|
|
1216
|
+
log(`${candidates.length} requests touched in the window; reading their runs…`);
|
|
1217
|
+
const readIssue = async (issue) => {
|
|
1218
|
+
const { data, warnings } = await source.timeline(issue.id, false);
|
|
1219
|
+
const entries = data.toSorted((a, b) => ms(a.created_at) - ms(b.created_at)).map((e) => ({
|
|
1220
|
+
...e,
|
|
1221
|
+
issue: issue.id
|
|
1222
|
+
}));
|
|
1223
|
+
const changes = entries.filter((e) => e.type === "activity" && e.action === "status_changed");
|
|
1224
|
+
const statusAt = (t) => {
|
|
1225
|
+
let status = changes[0]?.details?.from ?? issue.status;
|
|
1226
|
+
for (const c of changes) {
|
|
1227
|
+
if (ms(c.created_at) > t) break;
|
|
1228
|
+
status = c.details?.to ?? status;
|
|
1229
|
+
}
|
|
1230
|
+
return status;
|
|
1231
|
+
};
|
|
1232
|
+
return {
|
|
1233
|
+
issue,
|
|
1234
|
+
entries,
|
|
1235
|
+
changes,
|
|
1236
|
+
statusAt,
|
|
1237
|
+
runs: source.runs(issue.id).map((r) => {
|
|
1238
|
+
const kind = commentKind(r.trigger_summary);
|
|
1239
|
+
const created = ms(r.created_at);
|
|
1240
|
+
const started = ms(r.started_at);
|
|
1241
|
+
const completed = ms(r.completed_at);
|
|
1242
|
+
return {
|
|
1243
|
+
issue: issue.id,
|
|
1244
|
+
agent: source.agentNames.get(r.agent_id) ?? r.agent_id,
|
|
1245
|
+
status: r.status,
|
|
1246
|
+
attempt: r.attempt ?? 1,
|
|
1247
|
+
kind,
|
|
1248
|
+
step: kind === "assignment or status" ? statusAt(created + 1e3) ?? "" : "",
|
|
1249
|
+
created,
|
|
1250
|
+
started,
|
|
1251
|
+
completed,
|
|
1252
|
+
end: completed > 0 ? completed : LIVE.has(r.status) ? 1e400 : Math.max(created, started || 0),
|
|
1253
|
+
cost: runCost(r.usage, started > 0, costOf)
|
|
1254
|
+
};
|
|
1255
|
+
}),
|
|
1256
|
+
truncated: /truncated/i.test(warnings)
|
|
1257
|
+
};
|
|
1258
|
+
};
|
|
1259
|
+
const startsReview = (run) => reviews(run) && (run.kind === "review request" || query.reviewSteps.includes(run.step));
|
|
1260
|
+
const startsPlanReview = (run) => reviews(run) && (run.kind === "plan" || query.planSteps.includes(run.step));
|
|
1261
|
+
const trees = [];
|
|
1262
|
+
const unreadable = [];
|
|
1263
|
+
for (const { request, tree } of candidates) try {
|
|
1264
|
+
const read = [];
|
|
1265
|
+
for (const issue of tree) read.push(await readIssue(issue));
|
|
1266
|
+
trees.push({
|
|
1267
|
+
request,
|
|
1268
|
+
read
|
|
1269
|
+
});
|
|
1270
|
+
} catch (error) {
|
|
1271
|
+
unreadable.push(request.identifier ?? request.id);
|
|
1272
|
+
log(` skipped ${request.identifier ?? request.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1273
|
+
}
|
|
1274
|
+
if (candidates.length && !trees.length) throw new Error(`couldn't read any of the ${candidates.length} requests`);
|
|
1275
|
+
const filed = trees.filter((t) => t.request.creator_type === "agent").map((t) => ({
|
|
1276
|
+
...t,
|
|
1277
|
+
name: new RegExp(`(?<![\\w-])(${[t.request.identifier, t.request.id].filter((x) => !!x).map(escape).join("|")})(?![\\w-])`)
|
|
1278
|
+
}));
|
|
1279
|
+
const namedIn = (entries) => filed.filter((f) => entries.some((e) => e.type === "comment" && f.name.test(e.content ?? "")));
|
|
1280
|
+
const rows = [];
|
|
1281
|
+
const truncated = [];
|
|
1282
|
+
const unknown = [];
|
|
1283
|
+
let unstarted = 0;
|
|
1284
|
+
let agentMade = 0;
|
|
1285
|
+
let outside = 0;
|
|
1286
|
+
let unplaced = 0;
|
|
1287
|
+
const qaSteps = /* @__PURE__ */ new Map();
|
|
1288
|
+
for (const { request, read } of trees) {
|
|
1289
|
+
const root = read[0];
|
|
1290
|
+
if (!root) continue;
|
|
1291
|
+
const runsAll = read.flatMap((r) => r.runs).toSorted((a, b) => a.created - b.created);
|
|
1292
|
+
const start = runsAll.find((r) => r.started > 0)?.created ?? NaN;
|
|
1293
|
+
if (!Number.isFinite(start)) {
|
|
1294
|
+
unstarted++;
|
|
1295
|
+
continue;
|
|
1296
|
+
}
|
|
1297
|
+
if (!(start >= since && start < until)) {
|
|
1298
|
+
outside++;
|
|
1299
|
+
continue;
|
|
1300
|
+
}
|
|
1301
|
+
if (request.creator_type === "agent") {
|
|
1302
|
+
agentMade++;
|
|
1303
|
+
continue;
|
|
1304
|
+
}
|
|
1305
|
+
const partsAt = (t) => read.slice(1).filter((o) => ms(o.issue.created_at) <= t);
|
|
1306
|
+
const standing = (t) => {
|
|
1307
|
+
const status = root.statusAt(t);
|
|
1308
|
+
const parts = partsAt(t);
|
|
1309
|
+
const outcome = status === "done" && parts.every((o) => CLOSED.has(o.statusAt(t) ?? "")) ? "done" : ["cancelled", "backlog"].includes(status ?? "") ? "stopped" : "open";
|
|
1310
|
+
const lastTo = (r) => r.changes.findLast((c) => ms(c.created_at) <= t && c.details?.to === r.statusAt(t));
|
|
1311
|
+
const closes = [root, ...outcome === "done" ? parts : []].map(lastTo);
|
|
1312
|
+
return {
|
|
1313
|
+
outcome,
|
|
1314
|
+
at: closes.every(Boolean) ? Math.max(...closes.map((c) => ms(c?.created_at))) : NaN
|
|
1315
|
+
};
|
|
1316
|
+
};
|
|
1317
|
+
const placed = split.placeRequest(start, (t) => standing(t).outcome === "open");
|
|
1318
|
+
if (!placed) {
|
|
1319
|
+
unplaced++;
|
|
1320
|
+
continue;
|
|
1321
|
+
}
|
|
1322
|
+
const { segment, horizon } = placed;
|
|
1323
|
+
const { outcome, at } = standing(horizon);
|
|
1324
|
+
if (outcome !== "open" && !Number.isFinite(at)) {
|
|
1325
|
+
unknown.push(request.identifier ?? request.id);
|
|
1326
|
+
continue;
|
|
1327
|
+
}
|
|
1328
|
+
if (outcome !== "open" && at < start) {
|
|
1329
|
+
unstarted++;
|
|
1330
|
+
continue;
|
|
1331
|
+
}
|
|
1332
|
+
const end = outcome === "open" ? NaN : at;
|
|
1333
|
+
const runs = runsAll.filter((r) => r.created < horizon);
|
|
1334
|
+
for (const r of runs) if (isReview(r) && r.step) qaSteps.set(r.step, (qaSteps.get(r.step) ?? 0) + 1);
|
|
1335
|
+
if (read.some((r) => r.truncated)) truncated.push(request.identifier ?? request.id);
|
|
1336
|
+
const entries = read.flatMap((r) => r.entries).filter((e) => ms(e.created_at) < horizon);
|
|
1337
|
+
const linked = namedIn(entries).flatMap((f) => f.read);
|
|
1338
|
+
const blockedAt = (t) => read.some((x) => x.statusAt(t) === "blocked");
|
|
1339
|
+
const counts = (r, t) => !r.linked || blockedAt(t);
|
|
1340
|
+
const squadRuns = [...runs, ...linked.flatMap((r) => r.runs).map((r) => ({
|
|
1341
|
+
...r,
|
|
1342
|
+
linked: true
|
|
1343
|
+
}))];
|
|
1344
|
+
const squadEntries = [...entries, ...linked.flatMap((r) => r.entries).filter((e) => ms(e.created_at) < horizon && blockedAt(ms(e.created_at)))];
|
|
1345
|
+
const byPerson = (o) => squadEntries.some((e) => e.actor_type !== "agent" && ms(e.created_at) >= o.created - 5e3 && ms(e.created_at) <= o.created + 1e3);
|
|
1346
|
+
const idle = squadRuns.filter((r) => r.kind === "wakeup" && !isReview(r) && !squadEntries.some((e) => e.actor_type === "agent" && e.action === "status_changed" && during(r, ms(e.created_at))) && !squadRuns.some((o) => o !== r && during(r, o.created) && !byPerson(o) && !(o.attempt > 1 && o.agent === r.agent && o.issue === r.issue && o.kind === r.kind)));
|
|
1347
|
+
const working = squadRuns.filter((r) => !idle.includes(r));
|
|
1348
|
+
const busy = (t) => working.some((r) => r.created <= t && r.end > t && !causedBy(r, t) && counts(r, t));
|
|
1349
|
+
const qaMerging = (e, t) => runs.some((r) => r.issue === e.issue && isReview(r) && r.started <= t && r.end + RUN_GRACE > t);
|
|
1350
|
+
const parked = (t) => {
|
|
1351
|
+
const status = root.statusAt(t);
|
|
1352
|
+
const closedAt = ms(root.changes.findLast((c) => ms(c.created_at) <= t)?.created_at);
|
|
1353
|
+
const takenUp = status !== "backlog" && working.some((r) => r.issue === root.issue.id && r.kind === "comment" && r.created > closedAt && r.created <= t);
|
|
1354
|
+
return (status === "backlog" || CLOSED.has(status ?? "") && !takenUp && partsAt(t).every((o) => CLOSED.has(o.statusAt(t) ?? ""))) && !working.some((r) => r.created <= t && r.end > t && counts(r, t));
|
|
1355
|
+
};
|
|
1356
|
+
const moments = [...read.flatMap((r) => [ms(r.issue.created_at), ...r.changes.map((c) => ms(c.created_at))]), ...working.flatMap((r) => [r.created, r.end])].filter(Number.isFinite);
|
|
1357
|
+
const parkedIn = (a, b) => b > a ? timeWhen(a, b, moments, parked) : 0;
|
|
1358
|
+
const last = outcome === "open" ? horizon : end;
|
|
1359
|
+
const byPeople = entries.filter((e) => {
|
|
1360
|
+
const t = ms(e.created_at);
|
|
1361
|
+
if (!(t > start + SAME_ACTION && t <= last)) return false;
|
|
1362
|
+
if (e.actor_type === "member") return e.type === "comment" || ["status_changed", "assignee_changed"].includes(e.action ?? "");
|
|
1363
|
+
return e.type === "activity" && e.action === "status_changed" && e.details?.to === "done" && e.actor_type !== "agent" && !qaMerging(e, t);
|
|
1364
|
+
}).map((e) => ms(e.created_at)).toSorted((a, b) => a - b);
|
|
1365
|
+
const actions = [];
|
|
1366
|
+
for (const t of byPeople) if (!(t - (actions.at(-1) ?? -1e400) <= SAME_ACTION)) actions.push(t);
|
|
1367
|
+
const idleEntry = (e) => idle.some((r) => r.issue === e.issue && during(r, ms(e.created_at))) && e.action !== "status_changed";
|
|
1368
|
+
const agentEntries = squadEntries.filter((e) => e.actor_type === "agent" && !idleEntry(e)).map((e) => ms(e.created_at));
|
|
1369
|
+
let waited = 0;
|
|
1370
|
+
let previous = start;
|
|
1371
|
+
for (const t of actions) {
|
|
1372
|
+
const lastSquad = Math.max(start, ...agentEntries.filter((m) => m < t), ...working.filter((r) => !causedBy(r, t)).flatMap((r) => [
|
|
1373
|
+
r.created,
|
|
1374
|
+
r.started,
|
|
1375
|
+
r.completed
|
|
1376
|
+
].filter((m) => counts(r, m))).filter((m) => m < t));
|
|
1377
|
+
const from = Math.max(lastSquad, previous);
|
|
1378
|
+
if (!busy(t)) waited += t - from - parkedIn(from, t);
|
|
1379
|
+
previous = t;
|
|
1380
|
+
}
|
|
1381
|
+
const costs = runs.map((r) => r.cost);
|
|
1382
|
+
rows.push({
|
|
1383
|
+
request: request.identifier ?? request.id,
|
|
1384
|
+
segment,
|
|
1385
|
+
created: request.created_at,
|
|
1386
|
+
firstRun: new Date(start).toISOString(),
|
|
1387
|
+
outcome,
|
|
1388
|
+
leadHours: hours(end - start - parkedIn(start, end)),
|
|
1389
|
+
issues: read.length,
|
|
1390
|
+
linkedIssues: namedIn(entries).map(({ request: f }) => f.identifier ?? f.id),
|
|
1391
|
+
runs: runs.length,
|
|
1392
|
+
failedRuns: runs.filter((r) => r.status === "failed").length,
|
|
1393
|
+
agentHours: hours(runs.filter((r) => r.started > 0 && r.completed > 0).reduce((total, r) => total + Math.max(0, r.completed - r.started), 0)),
|
|
1394
|
+
cost: costs.includes(null) ? null : sum(costs.map((c) => c ?? 0)),
|
|
1395
|
+
costKnown: sum(costs.map((c) => c ?? 0)),
|
|
1396
|
+
reviewRounds: runs.filter(startsReview).length,
|
|
1397
|
+
planReviews: runs.filter(startsPlanReview).length,
|
|
1398
|
+
peopleActions: actions.length,
|
|
1399
|
+
waitedHours: hours(waited)
|
|
1400
|
+
});
|
|
1401
|
+
}
|
|
1402
|
+
const metrics = [];
|
|
1403
|
+
for (const segment of split.segments) {
|
|
1404
|
+
const { results, add } = collector(segment);
|
|
1405
|
+
const all = rows.filter((r) => r.segment === segment);
|
|
1406
|
+
const done = all.filter((r) => r.outcome === "done");
|
|
1407
|
+
const spread = (id, xs) => {
|
|
1408
|
+
const values = finite(xs);
|
|
1409
|
+
add(`${id}-p50`, pct(values, .5));
|
|
1410
|
+
add(`${id}-p90`, pct(values, .9));
|
|
1411
|
+
};
|
|
1412
|
+
add("requests", all.length);
|
|
1413
|
+
for (const outcome of [
|
|
1414
|
+
"done",
|
|
1415
|
+
"stopped",
|
|
1416
|
+
"open"
|
|
1417
|
+
]) add(`requests-${outcome}`, all.filter((r) => r.outcome === outcome).length);
|
|
1418
|
+
spread("lead-time", done.map((r) => r.leadHours));
|
|
1419
|
+
spread("runs-per-request", done.map((r) => r.runs));
|
|
1420
|
+
spread("agent-time-per-request", done.map((r) => r.agentHours));
|
|
1421
|
+
spread("cost-per-request", done.map((r) => r.cost));
|
|
1422
|
+
add("requests-cost-known", done.filter((r) => r.cost !== null).length);
|
|
1423
|
+
add("review-rounds-mean", mean(done.map((r) => r.reviewRounds)));
|
|
1424
|
+
add("plan-reviews-mean", mean(done.map((r) => r.planReviews)));
|
|
1425
|
+
spread("people-actions", done.map((r) => r.peopleActions));
|
|
1426
|
+
spread("waiting-on-people", done.map((r) => r.waitedHours));
|
|
1427
|
+
const lead = sum(finite(done.map((r) => r.leadHours)));
|
|
1428
|
+
add("waiting-on-people-share", lead ? sum(done.map((r) => r.waitedHours)) / lead : NaN);
|
|
1429
|
+
add("requests-cost", sum(all.map((r) => r.costKnown)));
|
|
1430
|
+
add("requests-failed-runs", sum(all.map((r) => r.failedRuns)));
|
|
1431
|
+
metrics.push(...results);
|
|
1432
|
+
}
|
|
1433
|
+
metrics.push({
|
|
1434
|
+
id: "requests-touched",
|
|
1435
|
+
value: candidates.length
|
|
1436
|
+
});
|
|
1437
|
+
const reasons = [
|
|
1438
|
+
["first run before --since or after --until", outside],
|
|
1439
|
+
["no run worked on it yet", unstarted],
|
|
1440
|
+
["an agent created it", agentMade],
|
|
1441
|
+
["the timeline lost the move that closed it", unknown.length],
|
|
1442
|
+
["the CLI couldn't read it", unreadable.length],
|
|
1443
|
+
["started while neither compared setup was live", unplaced]
|
|
1444
|
+
];
|
|
1445
|
+
for (const [reason, n] of reasons) metrics.push({
|
|
1446
|
+
id: "requests-left-out",
|
|
1447
|
+
by: { reason },
|
|
1448
|
+
value: n
|
|
1449
|
+
});
|
|
1450
|
+
if (unknown.length) notes.push(`The timeline lost the move that closed: ${unknown.join(", ")}.`);
|
|
1451
|
+
if (unreadable.length) notes.push(`The CLI couldn't read: ${unreadable.join(", ")}.`);
|
|
1452
|
+
if (qaSteps.size) {
|
|
1453
|
+
const stepsLine = [...qaSteps].map(([step, n]) => `${step} ${n}`).join(", ");
|
|
1454
|
+
notes.push(`QA's runs from a move or an assignment were at: ${stepsLine}.`);
|
|
1455
|
+
}
|
|
1456
|
+
const strange = [...qaSteps.keys()].filter((step) => !BUILT_IN_STEPS.has(step) && !query.reviewSteps.includes(step) && !query.planSteps.includes(step));
|
|
1457
|
+
if (strange.length) notes.push(`QA ran at ${strange.join(", ")}, which is neither a --review-step (${query.reviewSteps.join(", ")}) nor a --plan-step (${query.planSteps.join(", ")}): if one is your review or plan step, pass its key.`);
|
|
1458
|
+
if (truncated.length) notes.push(`The server cut short a timeline in these requests, so their counts miss the oldest entries: ${truncated.join(", ")}.`);
|
|
1459
|
+
if (unpriced.size) notes.push(`Priced as Opus 5.5: ${[...unpriced].join(", ")}.`);
|
|
1460
|
+
return {
|
|
1461
|
+
metrics,
|
|
1462
|
+
rows,
|
|
1463
|
+
notes
|
|
1464
|
+
};
|
|
1465
|
+
};
|
|
1466
|
+
// src/bench/timing.ts
|
|
1467
|
+
const TIMING_METRICS = [
|
|
1468
|
+
"runs",
|
|
1469
|
+
"failed-runs",
|
|
1470
|
+
"queue-wait-p50",
|
|
1471
|
+
"queue-wait-p90",
|
|
1472
|
+
"run-time-p50",
|
|
1473
|
+
"run-time-p90",
|
|
1474
|
+
"queue-hours",
|
|
1475
|
+
"run-hours",
|
|
1476
|
+
"run-cost-p50",
|
|
1477
|
+
"run-cost",
|
|
1478
|
+
"failure-causes",
|
|
1479
|
+
"peak-concurrent-runs",
|
|
1480
|
+
"review-cycles",
|
|
1481
|
+
"review-cycles-with-wakeup",
|
|
1482
|
+
"review-cycle-p50",
|
|
1483
|
+
"review-cycle-p90",
|
|
1484
|
+
"review-cycle-share"
|
|
1485
|
+
];
|
|
1486
|
+
const KINDS = [
|
|
1487
|
+
[/^FIX\s+[—-]\s+mechanical/, "FIX — mechanical"],
|
|
1488
|
+
[/^FIX\s+[—-]\s+conflict/, "FIX — conflict"],
|
|
1489
|
+
[/^FIX\s+[—-]\s+owner/, "FIX — owner"],
|
|
1490
|
+
[/^FIX\s+[—-]\s+PLAN/, "FIX — PLAN"],
|
|
1491
|
+
[/^FIX\b/, "FIX"],
|
|
1492
|
+
[/^DELIVERY\b/, "DELIVERY"],
|
|
1493
|
+
[/^RESTACK\b/, "RESTACK"],
|
|
1494
|
+
[/^VERDICT:\s*FAIL/, "VERDICT FAIL"],
|
|
1495
|
+
[/^VERDICT:\s*CONFLICT/, "VERDICT CONFLICT"],
|
|
1496
|
+
[/^VERDICT:\s*PLAN WRONG/, "PLAN WRONG"],
|
|
1497
|
+
[/^VERDICT/, "VERDICT (other)"],
|
|
1498
|
+
[/^PLAN v\d+.*in review/, "PLAN in review"],
|
|
1499
|
+
[/^PLAN v\d+.*approved/, "PLAN approved"],
|
|
1500
|
+
[/^PLAN REVIEW/, "PLAN REVIEW"],
|
|
1501
|
+
[/^PLAN INFEASIBLE/, "PLAN INFEASIBLE"],
|
|
1502
|
+
[/^REPLAN/, "REPLAN"],
|
|
1503
|
+
[/^SYNTHESIS\b/, "SYNTHESIS"],
|
|
1504
|
+
[/^STAGE REVIEW\b/, "STAGE REVIEW"],
|
|
1505
|
+
[/^FINISHED/, "FINISHED"],
|
|
1506
|
+
[/^QUESTIONS/, "QUESTIONS"],
|
|
1507
|
+
[/^BLOCKED/, "BLOCKED"],
|
|
1508
|
+
[/^ESCALATE/, "ESCALATE"],
|
|
1509
|
+
[/^Delegated task/i, "failure notice"]
|
|
1510
|
+
];
|
|
1511
|
+
const kindOf = (summary) => {
|
|
1512
|
+
const s = heading(summary);
|
|
1513
|
+
if (!s) return "assignment or status";
|
|
1514
|
+
if (/^Wakeup: sub-issues closed/i.test(s)) return "stage barrier";
|
|
1515
|
+
if (/^Wakeup:/i.test(s)) return "wakeup";
|
|
1516
|
+
return KINDS.find(([re]) => re.test(s))?.[1] ?? "comment";
|
|
1517
|
+
};
|
|
1518
|
+
const HANDOFFS = /* @__PURE__ */ new Set([
|
|
1519
|
+
"DELIVERY",
|
|
1520
|
+
"FIX",
|
|
1521
|
+
"FIX — mechanical",
|
|
1522
|
+
"FIX — conflict",
|
|
1523
|
+
"FIX — owner",
|
|
1524
|
+
"FIX — PLAN",
|
|
1525
|
+
"RESTACK",
|
|
1526
|
+
"SYNTHESIS",
|
|
1527
|
+
"STAGE REVIEW"
|
|
1528
|
+
]);
|
|
1529
|
+
const statusAt = (issue, changes, t) => {
|
|
1530
|
+
let status = changes[0]?.details?.from ?? issue.status;
|
|
1531
|
+
for (const c of changes) {
|
|
1532
|
+
if (ms(c.created_at) > t) break;
|
|
1533
|
+
status = c.details?.to ?? status;
|
|
1534
|
+
}
|
|
1535
|
+
return status;
|
|
1536
|
+
};
|
|
1537
|
+
/** Measures the runs created in the window, and the review cycles that start in it. */
|
|
1538
|
+
const measureTiming = async (source, query) => {
|
|
1539
|
+
const { since, until, split } = query;
|
|
1540
|
+
const { costOf, unpriced } = pricer();
|
|
1541
|
+
const log = query.log ?? (() => {});
|
|
1542
|
+
const notes = [];
|
|
1543
|
+
const inWindow = (t) => t >= since && t < until;
|
|
1544
|
+
const recent = source.issues.filter((i) => ms(i.last_activity_at ?? i.updated_at ?? i.created_at) >= since && ms(i.created_at) < until);
|
|
1545
|
+
log(`${source.issues.length} issues, ${recent.length} touched in the window; reading their runs…`);
|
|
1546
|
+
const runs = [];
|
|
1547
|
+
const histories = [];
|
|
1548
|
+
let unread = 0;
|
|
1549
|
+
const truncated = [];
|
|
1550
|
+
for (const issue of recent) {
|
|
1551
|
+
let list;
|
|
1552
|
+
try {
|
|
1553
|
+
list = source.runs(issue.id);
|
|
1554
|
+
} catch (error) {
|
|
1555
|
+
log(` skipped ${issue.identifier ?? issue.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1556
|
+
continue;
|
|
1557
|
+
}
|
|
1558
|
+
let changes = null;
|
|
1559
|
+
if (list.some((r) => kindOf(r.trigger_summary) === "assignment or status")) try {
|
|
1560
|
+
const { data, warnings } = await source.timeline(issue.id, true);
|
|
1561
|
+
const cut = /truncated by the server cap \(([^)]*)\)/i.exec(warnings);
|
|
1562
|
+
if (cut ? /activit/i.test(cut[1] ?? "") : /truncated/i.test(warnings)) truncated.push(issue.identifier ?? issue.id);
|
|
1563
|
+
changes = data.toSorted((a, b) => ms(a.created_at) - ms(b.created_at));
|
|
1564
|
+
} catch (error) {
|
|
1565
|
+
unread++;
|
|
1566
|
+
log(` no steps for ${issue.identifier ?? issue.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1567
|
+
}
|
|
1568
|
+
const history = [];
|
|
1569
|
+
for (const r of list) {
|
|
1570
|
+
const kind = kindOf(r.trigger_summary);
|
|
1571
|
+
const usage = (r.usage ?? []).reduce((total, u) => ({
|
|
1572
|
+
input: total.input + (u.input_tokens ?? 0),
|
|
1573
|
+
output: total.output + (u.output_tokens ?? 0),
|
|
1574
|
+
cacheRead: total.cacheRead + (u.cache_read_tokens ?? 0),
|
|
1575
|
+
cacheWrite: total.cacheWrite + (u.cache_write_tokens ?? 0)
|
|
1576
|
+
}), {
|
|
1577
|
+
input: 0,
|
|
1578
|
+
output: 0,
|
|
1579
|
+
cacheRead: 0,
|
|
1580
|
+
cacheWrite: 0
|
|
1581
|
+
});
|
|
1582
|
+
const created = ms(r.created_at);
|
|
1583
|
+
const record = {
|
|
1584
|
+
run: r.id,
|
|
1585
|
+
segment: inWindow(created) ? split.segmentOf(created) ?? null : null,
|
|
1586
|
+
issue: issue.identifier ?? issue.id,
|
|
1587
|
+
issueId: issue.id,
|
|
1588
|
+
parent: issue.parent_issue_id ?? null,
|
|
1589
|
+
agent: source.agentNames.get(r.agent_id) ?? r.agent_id,
|
|
1590
|
+
leader: Boolean(r.is_leader_task),
|
|
1591
|
+
priority: r.priority ?? null,
|
|
1592
|
+
status: r.status,
|
|
1593
|
+
failureReason: r.status === "failed" ? r.failure_reason || "unknown" : null,
|
|
1594
|
+
kind: kind === "assignment or status" && changes ? `at ${statusAt(issue, changes, created + 1e3) ?? ""}` : kind,
|
|
1595
|
+
attempt: r.attempt,
|
|
1596
|
+
created: r.created_at,
|
|
1597
|
+
started: r.started_at,
|
|
1598
|
+
completed: r.completed_at,
|
|
1599
|
+
queueMin: (ms(r.started_at) - created) / 6e4,
|
|
1600
|
+
runMin: (ms(r.completed_at) - ms(r.started_at)) / 6e4,
|
|
1601
|
+
usage,
|
|
1602
|
+
models: [...new Set((r.usage ?? []).map((u) => u.model).filter((m) => !!m))],
|
|
1603
|
+
cost: runCost(r.usage, ms(r.started_at) > 0, costOf)
|
|
1604
|
+
};
|
|
1605
|
+
history.push(record);
|
|
1606
|
+
if (inWindow(created)) runs.push(record);
|
|
1607
|
+
}
|
|
1608
|
+
histories.push({
|
|
1609
|
+
issue,
|
|
1610
|
+
history
|
|
1611
|
+
});
|
|
1612
|
+
}
|
|
1613
|
+
const reviewer = runs.find((r) => /^QA$/i.test(r.agent))?.agent ?? "QA";
|
|
1614
|
+
const reviewKinds = new Set(query.reviewSteps.map((step) => `at ${step}`));
|
|
1615
|
+
const cycles = [];
|
|
1616
|
+
for (const { issue, history } of histories) {
|
|
1617
|
+
const ordered = history.filter((r) => r.created).toSorted((a, b) => ms(a.created) - ms(b.created));
|
|
1618
|
+
let current = null;
|
|
1619
|
+
for (const r of ordered) {
|
|
1620
|
+
if (r.agent !== reviewer) {
|
|
1621
|
+
if (r.kind.startsWith("at ")) current = null;
|
|
1622
|
+
continue;
|
|
1623
|
+
}
|
|
1624
|
+
if (current && (r.kind === "wakeup" || (r.attempt ?? 0) > 1)) current.runs.push(r);
|
|
1625
|
+
else if (HANDOFFS.has(r.kind) || reviewKinds.has(r.kind)) {
|
|
1626
|
+
current = {
|
|
1627
|
+
issue: issue.identifier ?? issue.id,
|
|
1628
|
+
kind: r.kind,
|
|
1629
|
+
runs: [r],
|
|
1630
|
+
start: ms(r.created)
|
|
1631
|
+
};
|
|
1632
|
+
if (inWindow(current.start)) cycles.push(current);
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
if (current && cycles.at(-1) === current && query.reviewSteps.includes(issue.status ?? "")) current.pending = true;
|
|
1636
|
+
}
|
|
1637
|
+
const cycleRows = [];
|
|
1638
|
+
for (const c of cycles) {
|
|
1639
|
+
const first = c.runs[0];
|
|
1640
|
+
const last = c.runs.at(-1);
|
|
1641
|
+
const segment = split.segmentOf(c.start);
|
|
1642
|
+
if (!first || !last || segment === void 0 || c.pending) continue;
|
|
1643
|
+
if (last.status !== "completed" || !c.runs.every((r) => r.completed)) continue;
|
|
1644
|
+
const total = (ms(last.completed) - ms(first.created)) / 6e4;
|
|
1645
|
+
const queue = sum(c.runs.map((r) => Math.max(0, r.queueMin || 0)));
|
|
1646
|
+
const work = sum(c.runs.map((r) => Math.max(0, r.runMin || 0)));
|
|
1647
|
+
const costs = c.runs.map((r) => r.cost);
|
|
1648
|
+
cycleRows.push({
|
|
1649
|
+
issue: c.issue,
|
|
1650
|
+
segment,
|
|
1651
|
+
kind: c.kind,
|
|
1652
|
+
wakeups: c.runs.filter((r) => r.kind === "wakeup").length,
|
|
1653
|
+
total,
|
|
1654
|
+
queue,
|
|
1655
|
+
work,
|
|
1656
|
+
ciGap: Math.max(0, total - queue - work),
|
|
1657
|
+
cost: costs.includes(null) ? null : sum(costs.map((x) => x ?? 0))
|
|
1658
|
+
});
|
|
1659
|
+
}
|
|
1660
|
+
const metrics = [];
|
|
1661
|
+
for (const segment of split.segments) {
|
|
1662
|
+
const { results, add } = collector(segment);
|
|
1663
|
+
const inSegment = runs.filter((r) => r.segment === segment);
|
|
1664
|
+
const summarise = (group, slice) => {
|
|
1665
|
+
const queue = finite(group.map((r) => r.queueMin));
|
|
1666
|
+
const run = finite(group.map((r) => r.runMin));
|
|
1667
|
+
const cost = finite(group.filter((r) => ms(r.started) > 0 || r.cost !== 0).map((r) => r.cost));
|
|
1668
|
+
add("runs", group.length, slice);
|
|
1669
|
+
add("failed-runs", group.filter((r) => r.status === "failed").length, slice);
|
|
1670
|
+
add("queue-wait-p50", pct(queue, .5), slice);
|
|
1671
|
+
add("queue-wait-p90", pct(queue, .9), slice);
|
|
1672
|
+
add("run-time-p50", pct(run, .5), slice);
|
|
1673
|
+
add("run-time-p90", pct(run, .9), slice);
|
|
1674
|
+
add("queue-hours", sum(queue) / 60, slice);
|
|
1675
|
+
add("run-hours", sum(run) / 60, slice);
|
|
1676
|
+
add("run-cost-p50", pct(cost, .5), slice);
|
|
1677
|
+
add("run-cost", cost.length ? sum(cost) : NaN, slice);
|
|
1678
|
+
};
|
|
1679
|
+
add("runs", inSegment.length);
|
|
1680
|
+
for (const [seat, group] of by(inSegment, (r) => r.agent)) summarise(group, { seat });
|
|
1681
|
+
for (const [, group] of by(inSegment, (r) => `${r.agent} · ${r.kind}`)) {
|
|
1682
|
+
const first = group[0];
|
|
1683
|
+
if (first) summarise(group, {
|
|
1684
|
+
seat: first.agent,
|
|
1685
|
+
kind: first.kind
|
|
1686
|
+
});
|
|
1687
|
+
}
|
|
1688
|
+
const failures = inSegment.filter((r) => r.failureReason);
|
|
1689
|
+
for (const [, group] of by(failures, (r) => `${r.agent}\u0000${r.failureReason ?? ""}`)) {
|
|
1690
|
+
const first = group[0];
|
|
1691
|
+
if (first) add("failure-causes", group.length, {
|
|
1692
|
+
seat: first.agent,
|
|
1693
|
+
reason: first.failureReason ?? ""
|
|
1694
|
+
});
|
|
1695
|
+
}
|
|
1696
|
+
for (const [seat, group] of by(inSegment, (r) => r.agent)) {
|
|
1697
|
+
const events = group.filter((r) => r.started && r.completed).flatMap((r) => [[ms(r.started), 1], [ms(r.completed), -1]]).toSorted((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
1698
|
+
let now = 0;
|
|
1699
|
+
let peak = 0;
|
|
1700
|
+
for (const [, d] of events) peak = Math.max(peak, now += d);
|
|
1701
|
+
add("peak-concurrent-runs", peak, { seat });
|
|
1702
|
+
}
|
|
1703
|
+
const segmentCycles = cycleRows.filter((c) => c.segment === segment);
|
|
1704
|
+
add("review-cycles", segmentCycles.length);
|
|
1705
|
+
add("review-cycles-with-wakeup", segmentCycles.filter((c) => c.wakeups > 0).length);
|
|
1706
|
+
const totalTime = Math.max(1, sum(segmentCycles.map((c) => c.total)));
|
|
1707
|
+
for (const [part, key] of [
|
|
1708
|
+
["total", "total"],
|
|
1709
|
+
["queue", "queue"],
|
|
1710
|
+
["work", "work"],
|
|
1711
|
+
["ci-gap", "ciGap"]
|
|
1712
|
+
]) {
|
|
1713
|
+
const xs = finite(segmentCycles.map((c) => c[key]));
|
|
1714
|
+
add("review-cycle-p50", pct(xs, .5), { part });
|
|
1715
|
+
add("review-cycle-p90", pct(xs, .9), { part });
|
|
1716
|
+
add("review-cycle-share", segmentCycles.length ? sum(xs) / totalTime : NaN, { part });
|
|
1717
|
+
}
|
|
1718
|
+
metrics.push(...results);
|
|
1719
|
+
}
|
|
1720
|
+
const strange = [...new Set(runs.filter((r) => r.agent === reviewer && r.kind.startsWith("at ")).map((r) => r.kind.slice(3)).filter((step) => !BUILT_IN_STEPS.has(step) && !query.reviewSteps.includes(step) && !query.planSteps.includes(step)))];
|
|
1721
|
+
if (strange.length) notes.push(`${reviewer} ran at ${strange.join(", ")}, which is neither a --review-step (${query.reviewSteps.join(", ")}) nor a --plan-step (${query.planSteps.join(", ")}): if one is your review or plan step, pass its key.`);
|
|
1722
|
+
if (unread) notes.push(`${unread} issues had an unreadable timeline.`);
|
|
1723
|
+
if (truncated.length) notes.push(`The server cut these timelines short, so early steps may be off: ${truncated.join(", ")}.`);
|
|
1724
|
+
if (unpriced.size) notes.push(`Priced as Opus 5.5: ${[...unpriced].join(", ")}.`);
|
|
1725
|
+
return {
|
|
1726
|
+
metrics,
|
|
1727
|
+
runs,
|
|
1728
|
+
cycles: cycleRows,
|
|
1729
|
+
notes
|
|
1730
|
+
};
|
|
1731
|
+
};
|
|
1732
|
+
// src/bench/verdicts.ts
|
|
1733
|
+
const VERDICT_METRICS = [
|
|
1734
|
+
"first-verdicts",
|
|
1735
|
+
"first-fail-blockers",
|
|
1736
|
+
"blockers-per-first-fail",
|
|
1737
|
+
"first-fails-only-mechanical",
|
|
1738
|
+
"blocker-outcomes",
|
|
1739
|
+
"fix-answers",
|
|
1740
|
+
"issues-by-pass-round"
|
|
1741
|
+
];
|
|
1742
|
+
const section = (text, name) => {
|
|
1743
|
+
const lines = (text ?? "").split("\n");
|
|
1744
|
+
const start = lines.findIndex((l) => new RegExp(`^##\\s+${name}\\b`, "i").test(l));
|
|
1745
|
+
if (start < 0) return [];
|
|
1746
|
+
const end = lines.findIndex((l, i) => i > start && /^##\s/.test(l));
|
|
1747
|
+
return lines.slice(start + 1, end < 0 ? void 0 : end).filter((l) => l.trim());
|
|
1748
|
+
};
|
|
1749
|
+
const ledger = (line) => {
|
|
1750
|
+
const m = /^\s*(?:\d+\.|-)\s*(B\d+)\s*[·—-]?\s*(.*)$/.exec(line);
|
|
1751
|
+
if (!m) return null;
|
|
1752
|
+
const rest = m[2] ?? "";
|
|
1753
|
+
const raised = /raised r(\d+) as `?([a-z]+)`?/.exec(rest);
|
|
1754
|
+
const state = /\b(open|fixed r\d+|withdrawn r\d+|nit r\d+)\b/.exec(rest);
|
|
1755
|
+
return {
|
|
1756
|
+
id: m[1] ?? "",
|
|
1757
|
+
raisedRound: raised ? Number(raised[1]) : null,
|
|
1758
|
+
origin: raised?.[2] ?? null,
|
|
1759
|
+
state: state?.[1]?.replace(/ r\d+$/, "") ?? null,
|
|
1760
|
+
text: rest
|
|
1761
|
+
};
|
|
1762
|
+
};
|
|
1763
|
+
const answer = (line) => {
|
|
1764
|
+
const m = /^\s*\d+\.\s*(B\d+)\s*(?:→|->)\s*(fixed|rebutted|reading|not mechanical|not fixed)\b[:\s]*(.*)$/i.exec(line);
|
|
1765
|
+
return m ? {
|
|
1766
|
+
id: m[1] ?? "",
|
|
1767
|
+
answer: (m[2] ?? "").toLowerCase()
|
|
1768
|
+
} : null;
|
|
1769
|
+
};
|
|
1770
|
+
const finding = ({ id, origin, state, raisedRound }) => ({
|
|
1771
|
+
id,
|
|
1772
|
+
origin,
|
|
1773
|
+
state,
|
|
1774
|
+
raisedRound
|
|
1775
|
+
});
|
|
1776
|
+
const isVerdict = (e) => e.type === "verdict";
|
|
1777
|
+
/** Measures the work issues with a verdict in the window. */
|
|
1778
|
+
const measureVerdicts = (source, query) => {
|
|
1779
|
+
const { since, until, split } = query;
|
|
1780
|
+
const log = query.log ?? (() => {});
|
|
1781
|
+
const inWindow = (at) => ms(at) >= since && ms(at) < until;
|
|
1782
|
+
const recent = source.issues.filter((i) => ms(i.last_activity_at ?? i.updated_at ?? i.created_at) >= since && ms(i.created_at) < until);
|
|
1783
|
+
log(`${recent.length} issues touched in the window; reading their verdicts…`);
|
|
1784
|
+
const results = [];
|
|
1785
|
+
for (const issue of recent) {
|
|
1786
|
+
let comments;
|
|
1787
|
+
try {
|
|
1788
|
+
comments = source.comments(issue.id);
|
|
1789
|
+
} catch (error) {
|
|
1790
|
+
log(` skipped ${issue.identifier ?? issue.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1791
|
+
continue;
|
|
1792
|
+
}
|
|
1793
|
+
const events = [];
|
|
1794
|
+
for (const c of comments.toSorted((a, b) => ms(a.created_at) - ms(b.created_at))) {
|
|
1795
|
+
const head = heading((c.content ?? "").split("\n")[0]);
|
|
1796
|
+
const author = source.agentNames.get(c.author_id ?? "") ?? c.author_type ?? "";
|
|
1797
|
+
if (head.startsWith("VERDICT:")) {
|
|
1798
|
+
const result = /^VERDICT:\s*(PASS|FAIL|PLAN WRONG|CONFLICT|ESCALATE|PENDING CI)\b/.exec(head)?.[1]?.toUpperCase() ?? "?";
|
|
1799
|
+
const report = (c.attachments ?? []).find((a) => /^(verdict-r\d+|integration-review-.+|escalate)\.md$/.test(a.filename ?? ""));
|
|
1800
|
+
const text = report ? source.attachment(report) : null;
|
|
1801
|
+
const reasons = new Set(section(c.content, "Blocking").map(ledger).filter((f) => f !== null).map((f) => f.id));
|
|
1802
|
+
const blocking = section(text, "Blocking").map(ledger).filter((f) => f !== null).map(finding);
|
|
1803
|
+
for (const id of reasons) if (!blocking.some((f) => f.id === id)) blocking.push({
|
|
1804
|
+
id,
|
|
1805
|
+
origin: null,
|
|
1806
|
+
state: null,
|
|
1807
|
+
raisedRound: null
|
|
1808
|
+
});
|
|
1809
|
+
events.push({
|
|
1810
|
+
type: "verdict",
|
|
1811
|
+
at: c.created_at,
|
|
1812
|
+
author,
|
|
1813
|
+
result,
|
|
1814
|
+
round: Number(/\(round (\d+)\)/.exec(head)?.[1]) || null,
|
|
1815
|
+
report: report?.filename ?? null,
|
|
1816
|
+
blocking,
|
|
1817
|
+
settled: section(text, "Settled").map(ledger).filter((f) => f !== null).map(finding),
|
|
1818
|
+
nits: section(text, "Nits").length,
|
|
1819
|
+
needsDecision: section(c.content, "Needs a decision").length
|
|
1820
|
+
});
|
|
1821
|
+
} else if (/^(FIX|DELIVERY|RESTACK)\b/.test(head)) events.push({
|
|
1822
|
+
type: (head.split(/\s/)[0] ?? "").toUpperCase(),
|
|
1823
|
+
variant: /^FIX\s+[—-]\s+(mechanical|owner|conflict|PLAN)\b/i.exec(head)?.[1]?.toLowerCase() ?? null,
|
|
1824
|
+
at: c.created_at,
|
|
1825
|
+
author,
|
|
1826
|
+
answers: (c.content ?? "").split("\n").map(answer).filter((a) => a !== null)
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
const before = events.filter((e) => ms(e.at) < until);
|
|
1830
|
+
const firstInWindow = before.find((e) => e.type === "verdict" && ms(e.at) >= since);
|
|
1831
|
+
if (!firstInWindow) continue;
|
|
1832
|
+
results.push({
|
|
1833
|
+
issue: issue.identifier ?? issue.id,
|
|
1834
|
+
parent: issue.parent_issue_id ?? null,
|
|
1835
|
+
repository: /github\.com\/([^/]+\/[^/]+)\//.exec(issue.metadata?.pr_url ?? "")?.[1] ?? null,
|
|
1836
|
+
segment: split.segmentOf(ms(firstInWindow.at)) ?? null,
|
|
1837
|
+
events: before
|
|
1838
|
+
});
|
|
1839
|
+
}
|
|
1840
|
+
const firsts = results.map((r) => ({
|
|
1841
|
+
r,
|
|
1842
|
+
v: r.events.filter(isVerdict).find((e) => !["PENDING CI", "?"].includes(e.result))
|
|
1843
|
+
})).filter((x) => !!x.v && inWindow(x.v.at));
|
|
1844
|
+
const endOf = (r, id) => {
|
|
1845
|
+
for (const e of r.events.toReversed()) {
|
|
1846
|
+
if (!isVerdict(e)) continue;
|
|
1847
|
+
const f = [...e.blocking, ...e.settled].find((x) => x.id === id);
|
|
1848
|
+
if (f) return f.state ?? (e.blocking.includes(f) ? "open" : "settled");
|
|
1849
|
+
}
|
|
1850
|
+
return "unknown";
|
|
1851
|
+
};
|
|
1852
|
+
const metrics = [];
|
|
1853
|
+
for (const segment of split.segments) {
|
|
1854
|
+
const issues = results.filter((r) => r.segment === segment);
|
|
1855
|
+
const segmentFirsts = firsts.filter(({ v }) => split.segmentOf(ms(v.at)) === segment);
|
|
1856
|
+
const answers = results.flatMap((r) => r.events.filter((e) => ms(e.at) >= since && split.segmentOf(ms(e.at)) === segment)).flatMap((e) => "answers" in e ? e.answers : []);
|
|
1857
|
+
if (issues.length === 0 && segmentFirsts.length === 0) continue;
|
|
1858
|
+
const { results: out, add } = collector(segment);
|
|
1859
|
+
add("first-verdicts", segmentFirsts.length);
|
|
1860
|
+
for (const [result, n] of count(segmentFirsts.map(({ v }) => v.result))) add("first-verdicts", n, { result });
|
|
1861
|
+
const failed = segmentFirsts.filter(({ v }) => v.result === "FAIL");
|
|
1862
|
+
const blockers = failed.flatMap(({ r, v }) => v.blocking.map((f) => ({
|
|
1863
|
+
r,
|
|
1864
|
+
f
|
|
1865
|
+
})));
|
|
1866
|
+
add("first-fail-blockers", blockers.length);
|
|
1867
|
+
for (const [origin, n] of count(blockers.map(({ f }) => f.origin ?? "not in a report"))) add("first-fail-blockers", n, { origin });
|
|
1868
|
+
add("blockers-per-first-fail", blockers.length / Math.max(1, failed.length));
|
|
1869
|
+
add("first-fails-only-mechanical", failed.filter(({ v }) => v.blocking.length && v.blocking.every((f) => f.origin === "mechanical")).length);
|
|
1870
|
+
for (const [state, n] of count(blockers.map(({ r, f }) => endOf(r, f.id)))) add("blocker-outcomes", n, { state });
|
|
1871
|
+
add("fix-answers", answers.length);
|
|
1872
|
+
for (const [kind, n] of count(answers.map((a) => a.answer))) add("fix-answers", n, { answer: kind });
|
|
1873
|
+
const rounds = issues.map((r) => {
|
|
1874
|
+
const pass = r.events.filter(isVerdict).find((e) => e.result === "PASS");
|
|
1875
|
+
return pass ? `PASS at round ${pass.round ?? "?"}` : "no PASS yet";
|
|
1876
|
+
});
|
|
1877
|
+
for (const [round, n] of count(rounds)) add("issues-by-pass-round", n, { round });
|
|
1878
|
+
metrics.push(...out);
|
|
1879
|
+
}
|
|
1880
|
+
return {
|
|
1881
|
+
metrics,
|
|
1882
|
+
rows: results,
|
|
1883
|
+
notes: []
|
|
1884
|
+
};
|
|
1885
|
+
};
|
|
1886
|
+
// src/bench/index.ts
|
|
1887
|
+
/** Every metric the Multica connector reports. */
|
|
1888
|
+
const MULTICA_METRICS = [
|
|
1889
|
+
...REQUEST_METRICS,
|
|
1890
|
+
...TIMING_METRICS,
|
|
1891
|
+
...VERDICT_METRICS
|
|
1892
|
+
];
|
|
1893
|
+
/** Measures everything from `source`. Tests hand it a source of fixtures. */
|
|
1894
|
+
const benchFrom = async (source, query) => {
|
|
1895
|
+
(query.log ?? (() => {}))(`Workspace ${source.workspace.name}`);
|
|
1896
|
+
const requests = await measureRequests(source, query);
|
|
1897
|
+
const timing = await measureTiming(source, query);
|
|
1898
|
+
const verdicts = measureVerdicts(source, query);
|
|
1899
|
+
return {
|
|
1900
|
+
metrics: [
|
|
1901
|
+
...requests.metrics,
|
|
1902
|
+
...timing.metrics,
|
|
1903
|
+
...verdicts.metrics
|
|
1904
|
+
],
|
|
1905
|
+
rows: {
|
|
1906
|
+
requests: requests.rows,
|
|
1907
|
+
runs: timing.runs,
|
|
1908
|
+
cycles: timing.cycles,
|
|
1909
|
+
verdicts: verdicts.rows
|
|
1910
|
+
},
|
|
1911
|
+
notes: [
|
|
1912
|
+
...source.notes,
|
|
1913
|
+
...requests.notes,
|
|
1914
|
+
...timing.notes,
|
|
1915
|
+
...verdicts.notes
|
|
1916
|
+
]
|
|
1917
|
+
};
|
|
1918
|
+
};
|
|
1919
|
+
/** Measures the target's workspace on Multica. */
|
|
1920
|
+
const bench = async (query, context) => {
|
|
1921
|
+
const source = await openSource(query, context);
|
|
1922
|
+
try {
|
|
1923
|
+
return await benchFrom(source, query);
|
|
1924
|
+
} finally {
|
|
1925
|
+
source.close();
|
|
1926
|
+
}
|
|
1927
|
+
};
|
|
1928
|
+
// src/plan.ts
|
|
1929
|
+
/** The agent fields `agent update` sets, with their flags. */
|
|
1930
|
+
const SETTINGS = [
|
|
1931
|
+
["description", "description"],
|
|
1932
|
+
["model", "model"],
|
|
1933
|
+
["thinking_level", "thinking-level"],
|
|
1934
|
+
["max_concurrent_tasks", "max-concurrent-tasks"]
|
|
1935
|
+
];
|
|
1936
|
+
const memberName = (member) => member.name.toLowerCase();
|
|
1937
|
+
/** Every change that makes `live` match `rendered`, in the order `apply` makes them. */
|
|
1938
|
+
const planChanges = (rendered, live) => {
|
|
1939
|
+
const changes = [];
|
|
1940
|
+
const data = (operation) => ({
|
|
1941
|
+
...operation,
|
|
1942
|
+
cli: live.cli,
|
|
1943
|
+
workspace: live.workspace.id
|
|
1944
|
+
});
|
|
1945
|
+
const change = (kind, id, summary, operation, diff = "") => changes.push({
|
|
1946
|
+
kind,
|
|
1947
|
+
id,
|
|
1948
|
+
summary,
|
|
1949
|
+
...diff ? { diff } : {},
|
|
1950
|
+
data: data(operation)
|
|
1951
|
+
});
|
|
1952
|
+
const manual = (kind, id, summary) => changes.push({
|
|
1953
|
+
kind,
|
|
1954
|
+
id,
|
|
1955
|
+
summary: `! ${summary}`,
|
|
1956
|
+
manual: true
|
|
1957
|
+
});
|
|
1958
|
+
const note = (kind, id, summary) => changes.push({
|
|
1959
|
+
kind,
|
|
1960
|
+
id,
|
|
1961
|
+
summary: `· ${summary}`,
|
|
1962
|
+
note: true
|
|
1963
|
+
});
|
|
1964
|
+
for (const skill of rendered.skills) {
|
|
1965
|
+
const files = skillFiles(skill);
|
|
1966
|
+
const found = live.skills.find((candidate) => candidate.name === skill.name);
|
|
1967
|
+
if (!found) {
|
|
1968
|
+
change("skill", skill.name, `+ ${skill.name} (new)`, {
|
|
1969
|
+
op: "skill-import",
|
|
1970
|
+
skill
|
|
1971
|
+
});
|
|
1972
|
+
continue;
|
|
1973
|
+
}
|
|
1974
|
+
const notes = [];
|
|
1975
|
+
let diffs = "";
|
|
1976
|
+
for (const [path, text] of files) {
|
|
1977
|
+
const have = found.files[path];
|
|
1978
|
+
if (have === void 0) {
|
|
1979
|
+
notes.push(`+ ${path}`);
|
|
1980
|
+
continue;
|
|
1981
|
+
}
|
|
1982
|
+
const { lines, diff } = textDiff(norm(have), text);
|
|
1983
|
+
if (lines) {
|
|
1984
|
+
notes.push(`~ ${path} (${differ(lines)})`);
|
|
1985
|
+
diffs += `--- ${path}\n${diff}\n`;
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
for (const path of Object.keys(found.files)) if (!files.has(path)) notes.push(`- ${path}`);
|
|
1989
|
+
if (notes.length > 0) change("skill", skill.name, `~ ${skill.name}: ${notes.join(", ")}`, {
|
|
1990
|
+
op: "skill-import",
|
|
1991
|
+
skill
|
|
1992
|
+
}, diffs);
|
|
1993
|
+
}
|
|
1994
|
+
const skillNames = new Set(rendered.skills.map((skill) => skill.name));
|
|
1995
|
+
const liveById = /* @__PURE__ */ new Map();
|
|
1996
|
+
for (const agent of rendered.agents) {
|
|
1997
|
+
const found = live.agents.find((candidate) => candidate.name === agent.name);
|
|
1998
|
+
if (!found) {
|
|
1999
|
+
manual("agent", agent.id, `${agent.name}: no agent with this name; create it first`);
|
|
2000
|
+
continue;
|
|
2001
|
+
}
|
|
2002
|
+
liveById.set(agent.id, found);
|
|
2003
|
+
const args = [];
|
|
2004
|
+
const notes = [];
|
|
2005
|
+
const { lines, diff } = textDiff(norm(found.instructions), norm(agent.instructions));
|
|
2006
|
+
if (lines) {
|
|
2007
|
+
args.push("--instructions", agent.instructions.replace(/\n+$/, ""));
|
|
2008
|
+
notes.push(`instructions (${differ(lines)})`);
|
|
2009
|
+
}
|
|
2010
|
+
const want = {
|
|
2011
|
+
description: agent.description,
|
|
2012
|
+
model: agent.model.id,
|
|
2013
|
+
thinking_level: agent.model.thinking,
|
|
2014
|
+
max_concurrent_tasks: String(agent.concurrency)
|
|
2015
|
+
};
|
|
2016
|
+
for (const [key, flag] of SETTINGS) {
|
|
2017
|
+
const have = found[key];
|
|
2018
|
+
if (want[key] && have !== want[key]) {
|
|
2019
|
+
args.push(`--${flag}`, want[key]);
|
|
2020
|
+
notes.push(`${flag}: ${preview(have)} → ${preview(want[key])}`);
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
if (args.length > 0) change("agent", agent.id, `~ ${agent.name} ${notes.join(", ")}`, {
|
|
2024
|
+
op: "agent-update",
|
|
2025
|
+
agent: found.id,
|
|
2026
|
+
args
|
|
2027
|
+
}, diff);
|
|
2028
|
+
const wanted = [...new Set(agent.skills)].filter((name) => skillNames.has(name));
|
|
2029
|
+
const missing = wanted.filter((name) => !found.skills.some((skill) => skill.name === name));
|
|
2030
|
+
for (const name of wanted) if (found.skills.some((skill) => skill.name === name && !skill.enabled)) manual("agent", agent.id, `${agent.name}: ${name} is attached but disabled; switch it on in the agent's Skills tab`);
|
|
2031
|
+
if (missing.length > 0) change("agent", agent.id, `+ ${agent.name} skills: ${missing.join(", ")}`, {
|
|
2032
|
+
op: "agent-skills-add",
|
|
2033
|
+
agent: found.id,
|
|
2034
|
+
skills: missing
|
|
2035
|
+
});
|
|
2036
|
+
}
|
|
2037
|
+
if (rendered.squad) {
|
|
2038
|
+
const squad = rendered.squad;
|
|
2039
|
+
const found = live.squad;
|
|
2040
|
+
if (!found) manual("squad", squad.name, `no squad named ${squad.name}; create it first`);
|
|
2041
|
+
else {
|
|
2042
|
+
const { lines, diff } = textDiff(norm(found.instructions), norm(squad.instructions));
|
|
2043
|
+
if (lines) change("squad", squad.name, `~ ${squad.name} instructions (${differ(lines)})`, {
|
|
2044
|
+
op: "squad-instructions",
|
|
2045
|
+
squad: found.id,
|
|
2046
|
+
instructions: squad.instructions.replace(/\n+$/, "")
|
|
2047
|
+
}, diff);
|
|
2048
|
+
for (const agent of rendered.agents) {
|
|
2049
|
+
const liveAgent = liveById.get(agent.id);
|
|
2050
|
+
if (!liveAgent) continue;
|
|
2051
|
+
const role = squad.members[agent.id] ?? agent.role;
|
|
2052
|
+
const member = found.members.find((candidate) => candidate.member_id === liveAgent.id && candidate.member_type === "agent");
|
|
2053
|
+
if (!member) change("squad", squad.name, `+ ${agent.name} joins ${squad.name}`, {
|
|
2054
|
+
op: "squad-member-add",
|
|
2055
|
+
squad: found.id,
|
|
2056
|
+
agent: liveAgent.id,
|
|
2057
|
+
role
|
|
2058
|
+
});
|
|
2059
|
+
else if (role && member.role !== role) change("squad", squad.name, `~ ${agent.name} role: ${preview(member.role)} → ${preview(role)}`, {
|
|
2060
|
+
op: "squad-member-role",
|
|
2061
|
+
squad: found.id,
|
|
2062
|
+
agent: liveAgent.id,
|
|
2063
|
+
role
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
const leader = rendered.agents.find((agent) => agent.id === squad.leader);
|
|
2067
|
+
const liveLeader = liveById.get(squad.leader);
|
|
2068
|
+
if (leader && liveLeader && found.leader_id !== liveLeader.id) change("squad", squad.name, `~ leader → ${leader.name}`, {
|
|
2069
|
+
op: "squad-leader",
|
|
2070
|
+
squad: found.id,
|
|
2071
|
+
agent: liveLeader.id
|
|
2072
|
+
});
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
const held = planWorkflow(rendered, live, {
|
|
2076
|
+
change,
|
|
2077
|
+
manual,
|
|
2078
|
+
note,
|
|
2079
|
+
liveById
|
|
2080
|
+
});
|
|
2081
|
+
if (!rendered.sharedContext) note("context", "context", "the setup gives this workspace no Agent context: left alone");
|
|
2082
|
+
else {
|
|
2083
|
+
const { lines, diff } = textDiff(norm(live.context), norm(rendered.sharedContext));
|
|
2084
|
+
if (lines && held) note("context", "context", `Agent context (${differ(lines)}): waits for the workflow`);
|
|
2085
|
+
else if (lines) change("context", "context", `~ Agent context (${differ(lines)})`, {
|
|
2086
|
+
op: "context",
|
|
2087
|
+
context: rendered.sharedContext
|
|
2088
|
+
}, diff);
|
|
2089
|
+
}
|
|
2090
|
+
return changes;
|
|
2091
|
+
};
|
|
2092
|
+
/** A move's status key as `plan` compares it: `""` for none, `?` while it isn't known. */
|
|
2093
|
+
const moveKey = (specs, keys, to) => to === void 0 ? "" : keyAt(specs, keys, to) ?? "?";
|
|
2094
|
+
/** Plans the Default workflow. Returns true when something only a person can do holds it back. */
|
|
2095
|
+
const planWorkflow = (rendered, live, { change, manual, note, liveById }) => {
|
|
2096
|
+
if (rendered.workflows.length === 0) return false;
|
|
2097
|
+
if ("problem" in live.workflow) {
|
|
2098
|
+
manual("workflow", "default", live.workflow.problem);
|
|
2099
|
+
return true;
|
|
2100
|
+
}
|
|
2101
|
+
const workflow = live.workflow;
|
|
2102
|
+
const merged = mergeWorkflows(rendered);
|
|
2103
|
+
let unresolved = false;
|
|
2104
|
+
const reported = /* @__PURE__ */ new Set();
|
|
2105
|
+
const once = (key, summary) => {
|
|
2106
|
+
unresolved = true;
|
|
2107
|
+
if (!reported.has(key)) manual("workflow", "default", summary);
|
|
2108
|
+
reported.add(key);
|
|
2109
|
+
};
|
|
2110
|
+
const owner = rendered.target.vars.owner;
|
|
2111
|
+
const handlerOf = (handoff) => {
|
|
2112
|
+
if (handoff.startsWith("agent:")) {
|
|
2113
|
+
const id = handoff.slice(6);
|
|
2114
|
+
const agent = rendered.agents.find((candidate) => candidate.id === id);
|
|
2115
|
+
const found = liveById.get(id) ?? live.agents.find((candidate) => candidate.name === agent?.name);
|
|
2116
|
+
if (found) return {
|
|
2117
|
+
type: "agent",
|
|
2118
|
+
id: found.id
|
|
2119
|
+
};
|
|
2120
|
+
once(`agent:${id}`, `Agent: ${agent?.name ?? id}: no agent with this name; create it first`);
|
|
2121
|
+
return;
|
|
2122
|
+
}
|
|
2123
|
+
if (handoff === "owner") {
|
|
2124
|
+
if (owner === void 0 || owner === "") {
|
|
2125
|
+
once("owner", "owner: the target sets no vars.owner, so no member takes the owner's steps");
|
|
2126
|
+
return;
|
|
2127
|
+
}
|
|
2128
|
+
const want = String(owner).toLowerCase();
|
|
2129
|
+
let found = live.members.filter((member) => memberName(member) === want);
|
|
2130
|
+
if (found.length === 0) found = live.members.filter((member) => memberName(member).split(/\s+/)[0] === want);
|
|
2131
|
+
const [only] = found;
|
|
2132
|
+
if (found.length === 1 && only) {
|
|
2133
|
+
if (!reported.has("owner")) note("workflow", "default", `owner: ${String(owner)} is ${only.name}${only.email ? ` <${only.email}>` : ""}`);
|
|
2134
|
+
reported.add("owner");
|
|
2135
|
+
return {
|
|
2136
|
+
type: "member",
|
|
2137
|
+
id: only.user_id
|
|
2138
|
+
};
|
|
2139
|
+
}
|
|
2140
|
+
once("owner", found.length > 0 ? `owner: ${String(owner)} matches ${found.map((member) => member.name).join(" and ")}: write the full name in vars.owner` : `owner: ${String(owner)}: no workspace member has this name`);
|
|
2141
|
+
return;
|
|
2142
|
+
}
|
|
2143
|
+
};
|
|
2144
|
+
const specs = merged.columns.map((column) => {
|
|
2145
|
+
const handler = handlerOf(column.handoff);
|
|
2146
|
+
return {
|
|
2147
|
+
name: column.name,
|
|
2148
|
+
...column.key ? { key: column.key } : { category: column.category ?? "started" },
|
|
2149
|
+
...handler ? { handler } : {},
|
|
2150
|
+
instructions: column.instructions ?? "",
|
|
2151
|
+
...column.next === void 0 ? {} : { next: column.next },
|
|
2152
|
+
...column.back === void 0 ? {} : { back: column.back }
|
|
2153
|
+
};
|
|
2154
|
+
});
|
|
2155
|
+
if (unresolved) return true;
|
|
2156
|
+
const liveSteps = workflow.steps;
|
|
2157
|
+
const liveKeys = liveSteps.map((step) => step.status_key);
|
|
2158
|
+
const keys = customKeys(liveKeys, specs);
|
|
2159
|
+
const want = stepsFor(specs, keys);
|
|
2160
|
+
const wanted = specs.map((_, place) => keyAt(specs, keys, place));
|
|
2161
|
+
const notes = [];
|
|
2162
|
+
let diffs = "";
|
|
2163
|
+
specs.forEach((spec, place) => {
|
|
2164
|
+
const key = wanted[place];
|
|
2165
|
+
if (!key || !liveKeys.includes(key)) notes.push(`+ ${spec.name}`);
|
|
2166
|
+
});
|
|
2167
|
+
for (const dropped of liveKeys.filter((key) => !wanted.includes(key))) notes.push(`- ${dropped}`);
|
|
2168
|
+
const order = liveKeys.filter((key) => wanted.includes(key));
|
|
2169
|
+
if (order.join() !== wanted.filter((key) => key && order.includes(key)).join()) notes.push("column order");
|
|
2170
|
+
specs.forEach((spec, place) => {
|
|
2171
|
+
const step = liveSteps.find((candidate) => candidate.status_key === wanted[place]);
|
|
2172
|
+
const target = want[place];
|
|
2173
|
+
if (!step || !target) return;
|
|
2174
|
+
const have = step.handler ?? { type: "none" };
|
|
2175
|
+
const wantHandler = target.handler ?? { type: "none" };
|
|
2176
|
+
const haveId = "id" in have ? have.id ?? "" : "";
|
|
2177
|
+
const wantId = "id" in wantHandler ? wantHandler.id : "";
|
|
2178
|
+
if (have.type !== wantHandler.type || haveId !== wantId) notes.push(`${spec.name} handoff`);
|
|
2179
|
+
else if (target.handler) {
|
|
2180
|
+
const { lines, diff } = textDiff(norm(step.instructions), norm(target.instructions));
|
|
2181
|
+
if (lines) {
|
|
2182
|
+
notes.push(`${spec.name} instructions (${differ(lines)})`);
|
|
2183
|
+
diffs += `--- ${spec.name}\n${diff}\n`;
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
const wantMoves = `${moveKey(specs, keys, spec.next)}|${moveKey(specs, keys, spec.back)}`;
|
|
2187
|
+
if (`${step.next_status_key ?? ""}|${step.back_status_key ?? ""}` !== wantMoves) notes.push(`${spec.name} moves`);
|
|
2188
|
+
});
|
|
2189
|
+
if (notes.length > 0) change("workflow", "default", `~ Default workflow: ${notes.join(", ")}`, {
|
|
2190
|
+
op: "workflow",
|
|
2191
|
+
workflow: workflow.id,
|
|
2192
|
+
specs
|
|
2193
|
+
}, diffs);
|
|
2194
|
+
const on = new Set(workflow.project_ids);
|
|
2195
|
+
const command = [
|
|
2196
|
+
"multica",
|
|
2197
|
+
...live.cli.profileArgs,
|
|
2198
|
+
"--workspace-id",
|
|
2199
|
+
live.workspace.id
|
|
2200
|
+
].join(" ");
|
|
2201
|
+
for (const project of live.projects.filter((candidate) => !on.has(candidate.id))) note("project", project.id, `${project.name}: on another workflow; switch it back with \`${command} project workflow set ${project.id} default\``);
|
|
2202
|
+
return false;
|
|
2203
|
+
};
|
|
2204
|
+
// src/index.ts
|
|
2205
|
+
/** The platform this connector syncs to. */
|
|
2206
|
+
const platform = "multica";
|
|
2207
|
+
const isLive = (live) => live.connector === platform;
|
|
2208
|
+
/** What Multica would refuse in `rendered`, checked before anything is read. */
|
|
2209
|
+
const validate = (rendered) => {
|
|
2210
|
+
const issues = [];
|
|
2211
|
+
for (const agent of rendered.agents) {
|
|
2212
|
+
const size = length(agent.description);
|
|
2213
|
+
if (size > 255) issues.push({
|
|
2214
|
+
field: `agents.${agent.id}.description`,
|
|
2215
|
+
message: `is ${size} characters, over Multica's 255`
|
|
2216
|
+
});
|
|
2217
|
+
}
|
|
2218
|
+
for (const skill of rendered.skills) issues.push(...checkSkill(skill));
|
|
2219
|
+
if (rendered.squad && !rendered.squad.instructions.trim()) issues.push({
|
|
2220
|
+
field: "squad.instructions",
|
|
2221
|
+
message: "renders empty: add squad.md to the setup or the target, or point squad.instructions at a file"
|
|
2222
|
+
});
|
|
2223
|
+
issues.push(...checkWorkflows(rendered, mergeWorkflows(rendered)));
|
|
2224
|
+
return issues;
|
|
2225
|
+
};
|
|
2226
|
+
/** A Multica connector that reads its environment from `options`. */
|
|
2227
|
+
const createConnector = (options = {}) => ({
|
|
2228
|
+
id: platform,
|
|
2229
|
+
capabilities: {
|
|
2230
|
+
sharedContext: true,
|
|
2231
|
+
workflowPerIssue: false,
|
|
2232
|
+
squads: true,
|
|
2233
|
+
skills: true,
|
|
2234
|
+
metrics: MULTICA_METRICS
|
|
2235
|
+
},
|
|
2236
|
+
runFolders: /multica.workspaces/i,
|
|
2237
|
+
columnNames,
|
|
2238
|
+
validate,
|
|
2239
|
+
place: (live) => {
|
|
2240
|
+
if (!isLive(live)) throw new Error("not a Multica connector's live state");
|
|
2241
|
+
return {
|
|
2242
|
+
name: live.workspace.name,
|
|
2243
|
+
pull: {
|
|
2244
|
+
profile: live.profile,
|
|
2245
|
+
workspace: live.workspace.id
|
|
2246
|
+
}
|
|
2247
|
+
};
|
|
2248
|
+
},
|
|
2249
|
+
pull: (target, pullOptions = {}) => pull(target, pullOptions, {
|
|
2250
|
+
env: options.env ?? process.env,
|
|
2251
|
+
...options.desktop === void 0 ? {} : { desktop: options.desktop }
|
|
2252
|
+
}),
|
|
2253
|
+
plan: (rendered, live) => {
|
|
2254
|
+
if (!isLive(live)) return Promise.reject(/* @__PURE__ */ new Error("not a Multica connector's live state"));
|
|
2255
|
+
try {
|
|
2256
|
+
return Promise.resolve(planChanges(rendered, live));
|
|
2257
|
+
} catch (error) {
|
|
2258
|
+
return Promise.reject(error instanceof Error ? error : new Error(String(error)));
|
|
2259
|
+
}
|
|
2260
|
+
},
|
|
2261
|
+
bench: (query) => bench(query, {
|
|
2262
|
+
env: options.env ?? process.env,
|
|
2263
|
+
...options.desktop === void 0 ? {} : { desktop: options.desktop }
|
|
2264
|
+
}),
|
|
2265
|
+
apply: (changes) => {
|
|
2266
|
+
try {
|
|
2267
|
+
return Promise.resolve(applyChanges(changes));
|
|
2268
|
+
} catch (error) {
|
|
2269
|
+
return Promise.reject(error instanceof Error ? error : new Error(String(error)));
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
});
|
|
2273
|
+
const connector = createConnector();
|
|
2274
|
+
export { BUILT_IN, DESKTOP_CLI, MAX_DESCRIPTION, MAX_STEP, MULTICA_METRICS, benchFrom, childEnv, columnNames, connector, connector as default, createConnector, findMultica, listProfiles, mergeWorkflows, platform, validate };
|