@deksden-com/dd-flow-cli 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -4
- package/dist/build-info.json +6 -6
- package/dist/cli/help.js +14 -3
- package/dist/cli/run-cli.js +32 -16
- package/dist/domain/flow-contract.js +81 -2
- package/dist/domain/validation.js +56 -28
- package/dist/protocol/local-files.js +1 -16
- package/dist/schemas/code-stage-report.schema.json +2 -2
- package/dist/schemas/flow-contract.schema.json +150 -94
- package/dist/schemas/flow-run.schema.json +129 -23
- package/dist/schemas/mb-upgrade-review-data.schema.json +2 -2
- package/dist/schemas/memorybank-permissions-preflight.schema.json +13 -73
- package/dist/schemas/merge-stage-report.schema.json +2 -2
- package/dist/schemas/plan-stage-report.schema.json +38 -335
- package/dist/schemas/project-flow-pack-manifest.schema.json +4 -4
- package/dist/schemas/protocol-plan.schema.json +197 -0
- package/dist/schemas/release-impact.schema.json +9 -5
- package/dist/schemas/session-usage.schema.json +16 -0
- package/dist/schemas/stage-finish-input.schema.json +20 -0
- package/dist/schemas/stage-prompt.schema.json +31 -0
- package/dist/schemas/stage-report.schema.json +20 -0
- package/dist/schemas/stage-start-response.schema.json +29 -0
- package/dist/schemas/timeline-event.schema.json +29 -0
- package/dist/schemas/worktrunk-workspace.schema.json +19 -0
- package/dist/services/branch-context.js +9 -4
- package/dist/services/dashboard.js +51 -26
- package/dist/services/engines.js +84 -18
- package/dist/services/hooks.js +80 -246
- package/dist/services/memory-permissions.js +77 -69
- package/dist/services/plan-runtime.js +124 -0
- package/dist/services/plans.js +22 -84
- package/dist/services/projects.js +2 -1
- package/dist/services/prompts.js +26 -21
- package/dist/services/protocols.js +29 -25
- package/dist/services/run-projection.js +77 -11
- package/dist/services/runs.js +95 -61
- package/dist/services/schema-validation.js +168 -7
- package/dist/services/sessions.js +132 -68
- package/dist/services/stage-lifecycle.js +572 -0
- package/dist/services/tooling.js +285 -0
- package/dist/services/usage.js +183 -30
- package/dist/services/version-status.js +1 -1
- package/dist/services/worktrees.js +88 -39
- package/dist/storage/database.js +72 -30
- package/dist/storage/paths.js +0 -9
- package/package.json +14 -13
- package/tools/worktrunk-manifest.json +34 -0
- package/dist/schemas/flow-run-index-v3.schema.json +0 -203
- package/dist/schemas/flow-run-index.schema.json +0 -175
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import { AppError } from "../shared/errors.js";
|
|
7
|
+
import { ensureDir } from "../storage/paths.js";
|
|
8
|
+
const manifestRelativePath = path.join("tools", "worktrunk-manifest.json");
|
|
9
|
+
const managedToolRootName = "wt";
|
|
10
|
+
const manifestSchemaId = "dd-flow/managed-tool-manifest@1";
|
|
11
|
+
export function worktrunkManifest() {
|
|
12
|
+
const packageRoot = findPackageRoot(path.dirname(fileURLToPath(import.meta.url)));
|
|
13
|
+
if (!packageRoot)
|
|
14
|
+
throw new AppError("tool_manifest_missing", "Cannot locate dd-flow package root", 1);
|
|
15
|
+
const file = path.join(packageRoot, manifestRelativePath);
|
|
16
|
+
try {
|
|
17
|
+
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
18
|
+
validateManifest(parsed, file);
|
|
19
|
+
return parsed;
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
if (error instanceof AppError)
|
|
23
|
+
throw error;
|
|
24
|
+
throw new AppError("tool_manifest_missing", `Managed Worktrunk manifest is unreadable: ${file}`, 1, {
|
|
25
|
+
manifest: file,
|
|
26
|
+
cause: String(error)
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function worktrunkPlatform() {
|
|
31
|
+
if (process.platform === "darwin" && process.arch === "arm64")
|
|
32
|
+
return "darwin-arm64";
|
|
33
|
+
if (process.platform === "darwin" && process.arch === "x64")
|
|
34
|
+
return "darwin-x64";
|
|
35
|
+
if (process.platform === "linux" && process.arch === "arm64")
|
|
36
|
+
return "linux-arm64";
|
|
37
|
+
if (process.platform === "linux" && process.arch === "x64")
|
|
38
|
+
return "linux-x64";
|
|
39
|
+
if (process.platform === "win32" && process.arch === "x64")
|
|
40
|
+
return "windows-x64";
|
|
41
|
+
throw new AppError("tool_platform_unsupported", "No pinned Worktrunk artifact supports this platform", 1, {
|
|
42
|
+
platform: process.platform,
|
|
43
|
+
architecture: process.arch
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
export function worktrunkCacheRoot(context, platform = worktrunkPlatform()) {
|
|
47
|
+
return path.join(context.ddFlowHome, "tools", managedToolRootName, worktrunkManifest().version, platform);
|
|
48
|
+
}
|
|
49
|
+
export function worktrunkCacheBinary(context, platform = worktrunkPlatform()) {
|
|
50
|
+
return path.join(worktrunkCacheRoot(context, platform), process.platform === "win32" ? "wt.exe" : "wt");
|
|
51
|
+
}
|
|
52
|
+
export function inspectManagedWorktrunk(context) {
|
|
53
|
+
try {
|
|
54
|
+
const manifest = worktrunkManifest();
|
|
55
|
+
const platform = worktrunkPlatform();
|
|
56
|
+
const root = worktrunkCacheRoot(context, platform);
|
|
57
|
+
if (!fs.existsSync(root)) {
|
|
58
|
+
return { available: false, bin: null, version: manifest.version, platform, diagnostics: ["managed_tool_not_installed"] };
|
|
59
|
+
}
|
|
60
|
+
const verified = verifyCachedWorktrunk(context, manifest, platform);
|
|
61
|
+
return { available: true, bin: verified.bin, version: manifest.version, platform, diagnostics: [] };
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
const details = error instanceof AppError ? error.details : undefined;
|
|
65
|
+
return {
|
|
66
|
+
available: false,
|
|
67
|
+
bin: null,
|
|
68
|
+
version: "unknown",
|
|
69
|
+
platform: `${process.platform}-${process.arch}`,
|
|
70
|
+
diagnostics: [error instanceof Error ? error.message : String(error), ...(details ? [JSON.stringify(details)] : [])]
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export function ensureManagedWorktrunk(context) {
|
|
75
|
+
const manifest = worktrunkManifest();
|
|
76
|
+
const platform = worktrunkPlatform();
|
|
77
|
+
const cachedRoot = worktrunkCacheRoot(context, platform);
|
|
78
|
+
if (fs.existsSync(cachedRoot))
|
|
79
|
+
return verifyCachedWorktrunk(context, manifest, platform);
|
|
80
|
+
const lockDir = `${cachedRoot}.lock`;
|
|
81
|
+
acquireLock(lockDir);
|
|
82
|
+
try {
|
|
83
|
+
if (fs.existsSync(cachedRoot))
|
|
84
|
+
return verifyCachedWorktrunk(context, manifest, platform);
|
|
85
|
+
if (context.env.DD_FLOW_TOOL_OFFLINE === "1") {
|
|
86
|
+
throw new AppError("tool_unavailable", "Pinned Worktrunk is not cached and tool acquisition is offline", 1, {
|
|
87
|
+
tool: manifest.tool,
|
|
88
|
+
version: manifest.version,
|
|
89
|
+
platform,
|
|
90
|
+
cache_root: cachedRoot,
|
|
91
|
+
artifact_url: artifactFor(manifest, platform).url,
|
|
92
|
+
next_action: "Place the verified official artifact in the managed tool cache and retry."
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
installWorktrunkArtifact(manifest, platform, cachedRoot);
|
|
96
|
+
return verifyCachedWorktrunk(context, manifest, platform);
|
|
97
|
+
}
|
|
98
|
+
finally {
|
|
99
|
+
fs.rmSync(lockDir, { recursive: true, force: true });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
function installWorktrunkArtifact(manifest, platform, targetRoot) {
|
|
103
|
+
const artifact = artifactFor(manifest, platform);
|
|
104
|
+
const parent = path.dirname(targetRoot);
|
|
105
|
+
ensureDir(parent);
|
|
106
|
+
const tempRoot = fs.mkdtempSync(path.join(parent, `.wt-${process.pid}-`));
|
|
107
|
+
const archivePath = path.join(tempRoot, `worktrunk.${artifact.archive}`);
|
|
108
|
+
const extractedRoot = path.join(tempRoot, "extracted");
|
|
109
|
+
try {
|
|
110
|
+
const download = spawnSync("curl", ["--fail", "--location", "--silent", "--show-error", "--max-time", "120", "-o", archivePath, artifact.url], { encoding: "utf8" });
|
|
111
|
+
if (download.status !== 0) {
|
|
112
|
+
throw new AppError("tool_unavailable", "Unable to download the pinned Worktrunk artifact", 1, {
|
|
113
|
+
tool: manifest.tool,
|
|
114
|
+
version: manifest.version,
|
|
115
|
+
platform,
|
|
116
|
+
artifact_url: artifact.url,
|
|
117
|
+
stdout: sanitize(download.stdout),
|
|
118
|
+
stderr: sanitize(download.stderr)
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
const archiveChecksum = sha256File(archivePath);
|
|
122
|
+
if (archiveChecksum !== artifact.sha256) {
|
|
123
|
+
throw new AppError("tool_integrity_failed", "Pinned Worktrunk artifact checksum mismatch", 1, {
|
|
124
|
+
tool: manifest.tool,
|
|
125
|
+
version: manifest.version,
|
|
126
|
+
platform,
|
|
127
|
+
expected_sha256: artifact.sha256,
|
|
128
|
+
actual_sha256: archiveChecksum
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
ensureDir(extractedRoot);
|
|
132
|
+
extractArchive(archivePath, extractedRoot, artifact.archive);
|
|
133
|
+
const extractedBinary = findBinary(extractedRoot);
|
|
134
|
+
if (!extractedBinary)
|
|
135
|
+
throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive does not contain wt", 1, { platform });
|
|
136
|
+
fs.chmodSync(extractedBinary, 0o755);
|
|
137
|
+
verifyWorktrunkVersion(extractedBinary, manifest.version);
|
|
138
|
+
const binaryChecksum = sha256File(extractedBinary);
|
|
139
|
+
const stagedRoot = path.join(tempRoot, "managed");
|
|
140
|
+
ensureDir(stagedRoot);
|
|
141
|
+
const stagedBinary = path.join(stagedRoot, process.platform === "win32" ? "wt.exe" : "wt");
|
|
142
|
+
fs.copyFileSync(extractedBinary, stagedBinary);
|
|
143
|
+
fs.chmodSync(stagedBinary, 0o755);
|
|
144
|
+
fs.writeFileSync(path.join(stagedRoot, "metadata.json"), `${JSON.stringify({
|
|
145
|
+
schema_id: manifestSchemaId,
|
|
146
|
+
tool: manifest.tool,
|
|
147
|
+
version: manifest.version,
|
|
148
|
+
platform,
|
|
149
|
+
archive_sha256: artifact.sha256,
|
|
150
|
+
binary_sha256: binaryChecksum
|
|
151
|
+
}, null, 2)}\n`);
|
|
152
|
+
if (fs.existsSync(targetRoot))
|
|
153
|
+
throw new AppError("tool_integrity_failed", "Managed Worktrunk cache appeared during installation", 1, { cache_root: targetRoot });
|
|
154
|
+
fs.renameSync(stagedRoot, targetRoot);
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
fs.rmSync(tempRoot, { recursive: true, force: true });
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
function verifyCachedWorktrunk(context, manifest, platform) {
|
|
161
|
+
const root = worktrunkCacheRoot(context, platform);
|
|
162
|
+
const bin = path.join(root, process.platform === "win32" ? "wt.exe" : "wt");
|
|
163
|
+
const metadataFile = path.join(root, "metadata.json");
|
|
164
|
+
if (!fs.existsSync(bin) || !fs.existsSync(metadataFile)) {
|
|
165
|
+
throw new AppError("tool_integrity_failed", "Managed Worktrunk cache is incomplete", 1, { cache_root: root });
|
|
166
|
+
}
|
|
167
|
+
let metadata;
|
|
168
|
+
try {
|
|
169
|
+
metadata = JSON.parse(fs.readFileSync(metadataFile, "utf8"));
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
throw new AppError("tool_integrity_failed", "Managed Worktrunk cache metadata is invalid", 1, { cache_root: root, cause: String(error) });
|
|
173
|
+
}
|
|
174
|
+
const artifact = artifactFor(manifest, platform);
|
|
175
|
+
const actualBinaryChecksum = sha256File(bin);
|
|
176
|
+
if (metadata.schema_id !== manifestSchemaId ||
|
|
177
|
+
metadata.tool !== manifest.tool ||
|
|
178
|
+
metadata.version !== manifest.version ||
|
|
179
|
+
metadata.platform !== platform ||
|
|
180
|
+
metadata.archive_sha256 !== artifact.sha256 ||
|
|
181
|
+
metadata.binary_sha256 !== actualBinaryChecksum) {
|
|
182
|
+
throw new AppError("tool_integrity_failed", "Managed Worktrunk cache verification failed", 1, {
|
|
183
|
+
cache_root: root,
|
|
184
|
+
expected_version: manifest.version,
|
|
185
|
+
expected_archive_sha256: artifact.sha256,
|
|
186
|
+
actual_binary_sha256: actualBinaryChecksum
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
verifyWorktrunkVersion(bin, manifest.version);
|
|
190
|
+
return {
|
|
191
|
+
bin,
|
|
192
|
+
version: manifest.version,
|
|
193
|
+
platform,
|
|
194
|
+
archive_sha256: artifact.sha256,
|
|
195
|
+
binary_sha256: actualBinaryChecksum
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
function extractArchive(archive, destination, format) {
|
|
199
|
+
if (format === "tar.xz") {
|
|
200
|
+
const listing = spawnSync("tar", ["-tJf", archive], { encoding: "utf8" });
|
|
201
|
+
if (listing.status !== 0 || listing.stdout.split("\n").some(unsafeArchivePath)) {
|
|
202
|
+
throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive has an unsafe or unreadable file list", 1);
|
|
203
|
+
}
|
|
204
|
+
const extracted = spawnSync("tar", ["-xJf", archive, "-C", destination], { encoding: "utf8" });
|
|
205
|
+
if (extracted.status !== 0)
|
|
206
|
+
throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive extraction failed", 1, { stderr: sanitize(extracted.stderr) });
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
const listing = spawnSync("unzip", ["-Z1", archive], { encoding: "utf8" });
|
|
210
|
+
if (listing.status !== 0 || listing.stdout.split("\n").some(unsafeArchivePath)) {
|
|
211
|
+
throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive has an unsafe or unreadable file list", 1);
|
|
212
|
+
}
|
|
213
|
+
const extracted = spawnSync("unzip", ["-q", archive, "-d", destination], { encoding: "utf8" });
|
|
214
|
+
if (extracted.status !== 0)
|
|
215
|
+
throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive extraction failed", 1, { stderr: sanitize(extracted.stderr) });
|
|
216
|
+
}
|
|
217
|
+
function findBinary(root) {
|
|
218
|
+
const expected = process.platform === "win32" ? "wt.exe" : "wt";
|
|
219
|
+
const entries = fs.readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
|
|
220
|
+
for (const entry of entries) {
|
|
221
|
+
const candidate = path.join(root, entry.name);
|
|
222
|
+
if (entry.isFile() && entry.name === expected)
|
|
223
|
+
return candidate;
|
|
224
|
+
if (entry.isDirectory()) {
|
|
225
|
+
const nested = findBinary(candidate);
|
|
226
|
+
if (nested)
|
|
227
|
+
return nested;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
|
+
function verifyWorktrunkVersion(bin, expected) {
|
|
233
|
+
const result = spawnSync(bin, ["--version"], { encoding: "utf8" });
|
|
234
|
+
const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`;
|
|
235
|
+
if (result.status !== 0 || !output.includes(expected)) {
|
|
236
|
+
throw new AppError("tool_integrity_failed", "Managed Worktrunk version verification failed", 1, {
|
|
237
|
+
expected_version: expected,
|
|
238
|
+
status: result.status,
|
|
239
|
+
output: sanitize(output)
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
function artifactFor(manifest, platform) {
|
|
244
|
+
const artifact = manifest.artifacts[platform];
|
|
245
|
+
if (!artifact)
|
|
246
|
+
throw new AppError("tool_platform_unsupported", "Pinned Worktrunk manifest has no artifact for this platform", 1, { platform });
|
|
247
|
+
return artifact;
|
|
248
|
+
}
|
|
249
|
+
function validateManifest(value, file) {
|
|
250
|
+
if (!value || value.schema_id !== manifestSchemaId || value.tool !== "wt" || !value.version || !value.artifacts) {
|
|
251
|
+
throw new AppError("tool_manifest_invalid", `Managed Worktrunk manifest is invalid: ${file}`, 1, { manifest: file });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
function acquireLock(lockDir) {
|
|
255
|
+
ensureDir(path.dirname(lockDir));
|
|
256
|
+
try {
|
|
257
|
+
fs.mkdirSync(lockDir);
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
throw new AppError("tool_install_locked", "Another Worktrunk installation is in progress", 1, { lock_dir: lockDir });
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
function sha256File(file) {
|
|
264
|
+
return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
|
265
|
+
}
|
|
266
|
+
function unsafeArchivePath(value) {
|
|
267
|
+
const normalized = value.trim();
|
|
268
|
+
if (!normalized)
|
|
269
|
+
return false;
|
|
270
|
+
return path.isAbsolute(normalized) || normalized.split(/[\\/]/u).includes("..");
|
|
271
|
+
}
|
|
272
|
+
function sanitize(value) {
|
|
273
|
+
return value.replace(/(token|secret|password)=\S+/gi, "$1=<redacted>").slice(0, 2000);
|
|
274
|
+
}
|
|
275
|
+
function findPackageRoot(start) {
|
|
276
|
+
let current = path.resolve(start);
|
|
277
|
+
while (true) {
|
|
278
|
+
if (fs.existsSync(path.join(current, "package.json")))
|
|
279
|
+
return current;
|
|
280
|
+
const parent = path.dirname(current);
|
|
281
|
+
if (parent === current)
|
|
282
|
+
return null;
|
|
283
|
+
current = parent;
|
|
284
|
+
}
|
|
285
|
+
}
|
package/dist/services/usage.js
CHANGED
|
@@ -12,7 +12,10 @@ export function checkpointSessionUsage(context, session, input) {
|
|
|
12
12
|
observed_at: observedAt,
|
|
13
13
|
total_tokens: result.counter?.usage.total_tokens ?? null,
|
|
14
14
|
input_tokens: result.counter?.usage.input_tokens ?? null,
|
|
15
|
-
cached_input_tokens: result.counter?.usage.
|
|
15
|
+
cached_input_tokens: result.counter?.usage.cache_read_input_tokens ?? null,
|
|
16
|
+
cache_read_input_tokens: result.counter?.usage.cache_read_input_tokens ?? null,
|
|
17
|
+
cache_write_input_tokens: result.counter?.usage.cache_write_input_tokens ?? null,
|
|
18
|
+
uncached_input_tokens: result.counter?.usage.uncached_input_tokens ?? null,
|
|
16
19
|
output_tokens: result.counter?.usage.output_tokens ?? null,
|
|
17
20
|
reasoning_output_tokens: result.counter?.usage.reasoning_output_tokens ?? null,
|
|
18
21
|
source_kind: "codex_transcript_v1",
|
|
@@ -33,12 +36,14 @@ export function checkpointSessionUsage(context, session, input) {
|
|
|
33
36
|
}
|
|
34
37
|
context.db.run(`INSERT INTO flow_run_usage_snapshots
|
|
35
38
|
(id, project_id, run_id, session_id, checkpoint, stage, stage_attempt, observed_at,
|
|
36
|
-
total_tokens, input_tokens, cached_input_tokens,
|
|
39
|
+
total_tokens, input_tokens, cached_input_tokens, cache_read_input_tokens, cache_write_input_tokens, uncached_input_tokens,
|
|
40
|
+
output_tokens, reasoning_output_tokens,
|
|
37
41
|
source_kind, token_event_at, turn_id, turn_attribution, parser_version, extraction_status, diagnostic_code, created_at)
|
|
38
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
42
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
39
43
|
snapshot.id, snapshot.project_id, snapshot.run_id, snapshot.session_id, snapshot.checkpoint, snapshot.stage,
|
|
40
44
|
snapshot.stage_attempt, snapshot.observed_at, snapshot.total_tokens, snapshot.input_tokens,
|
|
41
|
-
snapshot.cached_input_tokens, snapshot.
|
|
45
|
+
snapshot.cached_input_tokens, snapshot.cache_read_input_tokens, snapshot.cache_write_input_tokens,
|
|
46
|
+
snapshot.uncached_input_tokens, snapshot.output_tokens, snapshot.reasoning_output_tokens, snapshot.source_kind,
|
|
42
47
|
snapshot.token_event_at, snapshot.turn_id, snapshot.turn_attribution, snapshot.parser_version,
|
|
43
48
|
snapshot.extraction_status, snapshot.diagnostic_code, observedAt
|
|
44
49
|
]);
|
|
@@ -50,20 +55,26 @@ export function checkpointRunUsage(context, input) {
|
|
|
50
55
|
return sessions.map((session) => checkpointSessionUsage(context, session, input));
|
|
51
56
|
}
|
|
52
57
|
export function usageForRun(context, input) {
|
|
53
|
-
const sessions = context.db.all(`SELECT session_id, project_id, run_id, transcript_path, parent_session_id, role, aspect_id, plan_item_id, session_kind
|
|
54
|
-
FROM flow_sessions WHERE project_id = ? AND run_id =
|
|
58
|
+
const sessions = context.db.all(`SELECT DISTINCT session_id, project_id, ? AS run_id, transcript_path, parent_session_id, role, aspect_id, plan_item_id, session_kind
|
|
59
|
+
FROM flow_sessions WHERE project_id = ? AND run_id = ?
|
|
60
|
+
UNION
|
|
61
|
+
SELECT DISTINCT f.session_id, f.project_id, ? AS run_id, f.transcript_path, f.parent_session_id, f.role, f.aspect_id, f.plan_item_id, f.session_kind
|
|
62
|
+
FROM flow_session_segments s JOIN flow_sessions f ON f.project_id = s.project_id AND f.session_id = s.session_id
|
|
63
|
+
WHERE s.project_id = ? AND s.run_id = ?`, [input.runId, input.projectId, input.runId, input.runId, input.projectId, input.runId]);
|
|
55
64
|
const refreshed = sessions.map((session) => checkpointSessionUsage(context, session, { checkpoint: "manual_sync" }));
|
|
56
65
|
const rows = context.db.all(`SELECT s.*, f.parent_session_id, f.role, f.aspect_id, f.plan_item_id, f.session_kind
|
|
57
66
|
FROM flow_run_usage_snapshots s
|
|
58
67
|
LEFT JOIN flow_sessions f ON f.project_id = s.project_id AND f.session_id = s.session_id
|
|
59
68
|
WHERE s.project_id = ? AND s.run_id = ? ORDER BY s.session_id, s.observed_at, s.id`, [input.projectId, input.runId]);
|
|
60
69
|
const deltas = usageDeltas(rows);
|
|
70
|
+
const segments = context.db.all(`SELECT id, session_id, run_id, protocol_id, started_at, ended_at, cwd, tool_name
|
|
71
|
+
FROM flow_session_segments WHERE project_id = ? AND run_id = ? ORDER BY started_at, id`, [input.projectId, input.runId]);
|
|
61
72
|
const supported = new Set(["session", "role", "aspect", "plan-item", "stage", "protocol"]);
|
|
62
73
|
const groupBy = supported.has(input.groupBy) ? input.groupBy : "session";
|
|
63
74
|
const groups = new Map();
|
|
64
75
|
for (const delta of deltas) {
|
|
65
76
|
const key = groupKey(delta, groupBy);
|
|
66
|
-
const group = groups.get(key) ?? { tokens:
|
|
77
|
+
const group = groups.get(key) ?? { tokens: emptyUsage(), snapshots: 0, statuses: {}, transition_buckets: 0 };
|
|
67
78
|
group.snapshots += 1;
|
|
68
79
|
group.statuses[delta.extraction_status] = (group.statuses[delta.extraction_status] ?? 0) + 1;
|
|
69
80
|
if (delta.usage)
|
|
@@ -81,11 +92,26 @@ export function usageForRun(context, input) {
|
|
|
81
92
|
session_id: session.session_id, parent_session_id: session.parent_session_id ?? null, role: session.role ?? null,
|
|
82
93
|
aspect_id: session.aspect_id ?? null, plan_item_id: session.plan_item_id ?? null, session_kind: session.session_kind ?? null
|
|
83
94
|
})),
|
|
84
|
-
groups: [...groups.entries()].map(([key, group]) => ({ key, ...group })),
|
|
95
|
+
groups: [...groups.entries()].map(([key, group]) => ({ key, ...group, tokens: outputUsageTotals(group.tokens) })),
|
|
85
96
|
deltas,
|
|
97
|
+
segments: segments.map((segment) => ({
|
|
98
|
+
segment_id: segment.id,
|
|
99
|
+
session_id: segment.session_id,
|
|
100
|
+
run_id: segment.run_id,
|
|
101
|
+
protocol_id: segment.protocol_id,
|
|
102
|
+
started_at: segment.started_at,
|
|
103
|
+
ended_at: segment.ended_at,
|
|
104
|
+
usage: segmentUsage(context, sessions.find((session) => session.session_id === segment.session_id), segment)
|
|
105
|
+
})),
|
|
86
106
|
coverage: coverageForRows(rows)
|
|
87
107
|
};
|
|
88
108
|
}
|
|
109
|
+
function segmentUsage(context, session, segment) {
|
|
110
|
+
if (!session)
|
|
111
|
+
return { status: "unavailable", diagnostic: "session_record_missing" };
|
|
112
|
+
const result = readCodexTranscriptWindow(session.transcript_path, session.session_id, segment.started_at, segment.ended_at ?? context.now());
|
|
113
|
+
return result.counter ? { status: result.status, tokens: result.counter.usage, token_event_at: result.counter.token_event_at } : { status: result.status, diagnostic: result.diagnostic ?? null };
|
|
114
|
+
}
|
|
89
115
|
function readCodexTranscript(transcriptPath, expectedSessionId) {
|
|
90
116
|
if (!transcriptPath)
|
|
91
117
|
return { status: "ephemeral_session", diagnostic: "transcript_path_missing" };
|
|
@@ -114,8 +140,8 @@ function readCodexTranscript(transcriptPath, expectedSessionId) {
|
|
|
114
140
|
if (event.type !== "event_msg" || payload?.type !== "token_count")
|
|
115
141
|
continue;
|
|
116
142
|
const info = object(payload.info);
|
|
117
|
-
const total = object(info?.total_token_usage);
|
|
118
|
-
const parsed =
|
|
143
|
+
const total = object(info?.total_token_usage) ?? object(payload?.usage) ?? object(event.usage);
|
|
144
|
+
const parsed = parseProviderUsage(total);
|
|
119
145
|
if (parsed)
|
|
120
146
|
latest = { usage: parsed, token_event_at: stringValue(event.timestamp) ?? null, turn_id: nearestTurn, turn_attribution: nearestTurn ? "inferred_nearest_task_started" : "unavailable" };
|
|
121
147
|
}
|
|
@@ -129,6 +155,78 @@ function readCodexTranscript(transcriptPath, expectedSessionId) {
|
|
|
129
155
|
return { status: "malformed_source", diagnostic: "transcript_read_failed" };
|
|
130
156
|
}
|
|
131
157
|
}
|
|
158
|
+
function readCodexTranscriptWindow(transcriptPath, expectedSessionId, startedAt, endedAt) {
|
|
159
|
+
const parsed = readTranscriptEvents(transcriptPath, expectedSessionId);
|
|
160
|
+
if (parsed.status !== "measured")
|
|
161
|
+
return parsed;
|
|
162
|
+
const from = Date.parse(startedAt);
|
|
163
|
+
const to = Date.parse(endedAt);
|
|
164
|
+
const events = parsed.events ?? [];
|
|
165
|
+
const inWindow = events.filter((event) => {
|
|
166
|
+
const at = Date.parse(event.token_event_at ?? "");
|
|
167
|
+
return !Number.isFinite(at) || (!Number.isFinite(from) || at >= from) && (!Number.isFinite(to) || at <= to);
|
|
168
|
+
});
|
|
169
|
+
const last = inWindow[inWindow.length - 1];
|
|
170
|
+
if (!last)
|
|
171
|
+
return { status: "not_yet_emitted", diagnostic: "token_count_not_in_segment" };
|
|
172
|
+
const before = events.filter((event) => Number.isFinite(from) && Number.isFinite(Date.parse(event.token_event_at ?? "")) && Date.parse(event.token_event_at ?? "") < from).at(-1);
|
|
173
|
+
const usage = before ? subtractParsedUsage(last.usage, before.usage) : last.usage;
|
|
174
|
+
return { status: "measured", counter: { ...last, usage } };
|
|
175
|
+
}
|
|
176
|
+
function readTranscriptEvents(transcriptPath, expectedSessionId) {
|
|
177
|
+
if (!transcriptPath)
|
|
178
|
+
return { status: "ephemeral_session", diagnostic: "transcript_path_missing" };
|
|
179
|
+
if (!fs.existsSync(transcriptPath))
|
|
180
|
+
return { status: "transcript_missing", diagnostic: "transcript_path_missing" };
|
|
181
|
+
try {
|
|
182
|
+
const lines = fs.readFileSync(transcriptPath, "utf8").split(/\r?\n/);
|
|
183
|
+
let sessionId;
|
|
184
|
+
let nearestTurn = null;
|
|
185
|
+
const events = [];
|
|
186
|
+
for (const line of lines) {
|
|
187
|
+
if (!line.trim())
|
|
188
|
+
continue;
|
|
189
|
+
let event;
|
|
190
|
+
try {
|
|
191
|
+
event = JSON.parse(line);
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const payload = object(event.payload);
|
|
197
|
+
if (event.type === "session_meta")
|
|
198
|
+
sessionId = stringValue(payload?.id) ?? sessionId;
|
|
199
|
+
if (event.type === "event_msg" && payload?.type === "task_started")
|
|
200
|
+
nearestTurn = stringValue(payload.turn_id) ?? nearestTurn;
|
|
201
|
+
if (event.type !== "event_msg" || payload?.type !== "token_count")
|
|
202
|
+
continue;
|
|
203
|
+
const info = object(payload.info);
|
|
204
|
+
const total = object(info?.total_token_usage) ?? object(payload?.usage) ?? object(event.usage);
|
|
205
|
+
const usage = parseProviderUsage(total);
|
|
206
|
+
if (usage)
|
|
207
|
+
events.push({ usage, token_event_at: stringValue(event.timestamp) ?? null, turn_id: nearestTurn, turn_attribution: nearestTurn ? "inferred_nearest_task_started" : "unavailable" });
|
|
208
|
+
}
|
|
209
|
+
if (sessionId && sessionId !== expectedSessionId)
|
|
210
|
+
return { status: "session_mismatch", diagnostic: "transcript_session_id_mismatch" };
|
|
211
|
+
if (events.length === 0)
|
|
212
|
+
return { status: "not_yet_emitted", diagnostic: "token_count_not_found" };
|
|
213
|
+
return { status: "measured", events };
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
return { status: "malformed_source", diagnostic: "transcript_read_failed" };
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
function subtractParsedUsage(current, previous) {
|
|
220
|
+
return {
|
|
221
|
+
total_tokens: Math.max(0, current.total_tokens - previous.total_tokens),
|
|
222
|
+
input_tokens: current.input_tokens === null || previous.input_tokens === null ? null : Math.max(0, current.input_tokens - previous.input_tokens),
|
|
223
|
+
cache_read_input_tokens: current.cache_read_input_tokens === null || previous.cache_read_input_tokens === null ? null : Math.max(0, current.cache_read_input_tokens - previous.cache_read_input_tokens),
|
|
224
|
+
cache_write_input_tokens: current.cache_write_input_tokens === null || previous.cache_write_input_tokens === null ? null : Math.max(0, current.cache_write_input_tokens - previous.cache_write_input_tokens),
|
|
225
|
+
uncached_input_tokens: current.uncached_input_tokens === null || previous.uncached_input_tokens === null ? null : Math.max(0, current.uncached_input_tokens - previous.uncached_input_tokens),
|
|
226
|
+
output_tokens: current.output_tokens === null || previous.output_tokens === null ? null : Math.max(0, current.output_tokens - previous.output_tokens),
|
|
227
|
+
reasoning_output_tokens: current.reasoning_output_tokens === null || previous.reasoning_output_tokens === null ? null : Math.max(0, current.reasoning_output_tokens - previous.reasoning_output_tokens)
|
|
228
|
+
};
|
|
229
|
+
}
|
|
132
230
|
function usageDeltas(rows) {
|
|
133
231
|
const previous = new Map();
|
|
134
232
|
return rows.map((row) => {
|
|
@@ -154,7 +252,9 @@ function subtractUsage(current, previous) {
|
|
|
154
252
|
return {
|
|
155
253
|
total_tokens: (current.total_tokens ?? 0) - (previous.total_tokens ?? 0),
|
|
156
254
|
input_tokens: current.input_tokens === null || previous.input_tokens === null ? null : current.input_tokens - previous.input_tokens,
|
|
157
|
-
|
|
255
|
+
cache_read_input_tokens: current.cache_read_input_tokens === null || previous.cache_read_input_tokens === null ? null : current.cache_read_input_tokens - previous.cache_read_input_tokens,
|
|
256
|
+
cache_write_input_tokens: current.cache_write_input_tokens === null || previous.cache_write_input_tokens === null ? null : current.cache_write_input_tokens - previous.cache_write_input_tokens,
|
|
257
|
+
uncached_input_tokens: current.uncached_input_tokens === null || previous.uncached_input_tokens === null ? null : current.uncached_input_tokens - previous.uncached_input_tokens,
|
|
158
258
|
output_tokens: current.output_tokens === null || previous.output_tokens === null ? null : current.output_tokens - previous.output_tokens,
|
|
159
259
|
reasoning_output_tokens: current.reasoning_output_tokens === null || previous.reasoning_output_tokens === null ? null : current.reasoning_output_tokens - previous.reasoning_output_tokens
|
|
160
260
|
};
|
|
@@ -163,7 +263,9 @@ function counterReset(current, previous) {
|
|
|
163
263
|
return [
|
|
164
264
|
[current.total_tokens, previous.total_tokens],
|
|
165
265
|
[current.input_tokens, previous.input_tokens],
|
|
166
|
-
[current.
|
|
266
|
+
[current.cache_read_input_tokens, previous.cache_read_input_tokens],
|
|
267
|
+
[current.cache_write_input_tokens, previous.cache_write_input_tokens],
|
|
268
|
+
[current.uncached_input_tokens, previous.uncached_input_tokens],
|
|
167
269
|
[current.output_tokens, previous.output_tokens],
|
|
168
270
|
[current.reasoning_output_tokens, previous.reasoning_output_tokens]
|
|
169
271
|
].some(([now, before]) => typeof now === "number" && typeof before === "number" && now < before);
|
|
@@ -188,6 +290,7 @@ function coverageForRows(rows) {
|
|
|
188
290
|
function snapshotForOutput(snapshot) {
|
|
189
291
|
const safe = { ...snapshot };
|
|
190
292
|
delete safe.transcript_path;
|
|
293
|
+
delete safe.cached_input_tokens;
|
|
191
294
|
return { schema_id: "dd-flow/session-usage-snapshot@1", ...safe };
|
|
192
295
|
}
|
|
193
296
|
function sameUsageSnapshot(previous, current) {
|
|
@@ -196,38 +299,88 @@ function sameUsageSnapshot(previous, current) {
|
|
|
196
299
|
&& previous.extraction_status === current.extraction_status
|
|
197
300
|
&& previous.total_tokens === current.total_tokens
|
|
198
301
|
&& previous.input_tokens === current.input_tokens
|
|
199
|
-
&& previous.
|
|
302
|
+
&& previous.cache_read_input_tokens === current.cache_read_input_tokens
|
|
303
|
+
&& previous.cache_write_input_tokens === current.cache_write_input_tokens
|
|
304
|
+
&& previous.uncached_input_tokens === current.uncached_input_tokens
|
|
200
305
|
&& previous.output_tokens === current.output_tokens
|
|
201
306
|
&& previous.reasoning_output_tokens === current.reasoning_output_tokens
|
|
202
307
|
&& previous.token_event_at === current.token_event_at
|
|
203
308
|
&& previous.turn_id === current.turn_id;
|
|
204
309
|
}
|
|
205
|
-
function
|
|
310
|
+
export function parseProviderUsage(value) {
|
|
206
311
|
if (!value)
|
|
207
312
|
return undefined;
|
|
208
|
-
const total =
|
|
313
|
+
const total = tokenField(value, ["total_tokens"]);
|
|
314
|
+
if (total === null || total === undefined)
|
|
315
|
+
return undefined;
|
|
316
|
+
const input = tokenField(value, ["input_tokens"]);
|
|
317
|
+
const cacheRead = tokenField(value, ["cache_read_input_tokens", "cached_input_tokens"]);
|
|
318
|
+
const cacheWrite = tokenField(value, ["cache_write_input_tokens", "cache_creation_input_tokens"]);
|
|
319
|
+
const explicitUncached = tokenField(value, ["uncached_input_tokens"]);
|
|
320
|
+
const derivedUncached = explicitUncached !== undefined
|
|
321
|
+
? explicitUncached
|
|
322
|
+
: input !== null && input !== undefined && cacheRead !== null && cacheRead !== undefined && input >= cacheRead
|
|
323
|
+
? input - cacheRead
|
|
324
|
+
: null;
|
|
209
325
|
if (total === undefined)
|
|
210
326
|
return undefined;
|
|
211
327
|
return {
|
|
212
328
|
total_tokens: total,
|
|
213
|
-
input_tokens:
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
329
|
+
input_tokens: input ?? null,
|
|
330
|
+
cache_read_input_tokens: cacheRead ?? null,
|
|
331
|
+
cache_write_input_tokens: cacheWrite ?? null,
|
|
332
|
+
uncached_input_tokens: derivedUncached,
|
|
333
|
+
output_tokens: tokenField(value, ["output_tokens"]) ?? null,
|
|
334
|
+
reasoning_output_tokens: tokenField(value, ["reasoning_output_tokens"]) ?? null
|
|
217
335
|
};
|
|
218
336
|
}
|
|
219
337
|
function addUsage(target, source) {
|
|
220
|
-
target.total_tokens
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
338
|
+
target.total_tokens = addNullable(target.total_tokens, source.total_tokens);
|
|
339
|
+
target.input_tokens = addNullable(target.input_tokens, source.input_tokens);
|
|
340
|
+
target.cache_read_input_tokens = addNullable(target.cache_read_input_tokens, source.cache_read_input_tokens);
|
|
341
|
+
target.cache_write_input_tokens = addNullable(target.cache_write_input_tokens, source.cache_write_input_tokens);
|
|
342
|
+
target.uncached_input_tokens = addNullable(target.uncached_input_tokens, source.uncached_input_tokens);
|
|
343
|
+
target.output_tokens = addNullable(target.output_tokens, source.output_tokens);
|
|
344
|
+
target.reasoning_output_tokens = addNullable(target.reasoning_output_tokens, source.reasoning_output_tokens);
|
|
345
|
+
}
|
|
346
|
+
function emptyUsage() {
|
|
347
|
+
return {
|
|
348
|
+
total_tokens: undefined,
|
|
349
|
+
input_tokens: undefined,
|
|
350
|
+
cache_read_input_tokens: undefined,
|
|
351
|
+
cache_write_input_tokens: undefined,
|
|
352
|
+
uncached_input_tokens: undefined,
|
|
353
|
+
output_tokens: undefined,
|
|
354
|
+
reasoning_output_tokens: undefined
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
function outputUsageTotals(value) {
|
|
358
|
+
return {
|
|
359
|
+
total_tokens: value.total_tokens ?? null,
|
|
360
|
+
input_tokens: value.input_tokens ?? null,
|
|
361
|
+
cache_read_input_tokens: value.cache_read_input_tokens ?? null,
|
|
362
|
+
cache_write_input_tokens: value.cache_write_input_tokens ?? null,
|
|
363
|
+
uncached_input_tokens: value.uncached_input_tokens ?? null,
|
|
364
|
+
output_tokens: value.output_tokens ?? null,
|
|
365
|
+
reasoning_output_tokens: value.reasoning_output_tokens ?? null
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
function addNullable(current, next) {
|
|
369
|
+
if (current === undefined)
|
|
370
|
+
return next;
|
|
371
|
+
if (current === null || next === null)
|
|
372
|
+
return null;
|
|
373
|
+
return current + next;
|
|
374
|
+
}
|
|
375
|
+
function tokenField(value, keys) {
|
|
376
|
+
for (const key of keys) {
|
|
377
|
+
if (!Object.prototype.hasOwnProperty.call(value, key))
|
|
378
|
+
continue;
|
|
379
|
+
const token = numberValue(value[key]);
|
|
380
|
+
return token === undefined || token < 0 ? null : token;
|
|
381
|
+
}
|
|
382
|
+
return undefined;
|
|
383
|
+
}
|
|
231
384
|
function object(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : undefined; }
|
|
232
385
|
function stringValue(value) { return typeof value === "string" && value ? value : undefined; }
|
|
233
386
|
function numberValue(value) { return typeof value === "number" && Number.isFinite(value) ? value : undefined; }
|
|
@@ -2,7 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
4
|
import { parse as parseYaml } from "yaml";
|
|
5
|
-
const canonicalOnlyFlowNames = ["mb-init.md", "mb-upgrade.md", "mb-
|
|
5
|
+
const canonicalOnlyFlowNames = ["mb-init.md", "mb-upgrade.md", "mb-distill.md"];
|
|
6
6
|
export function resolveStatusProjectRoot(input) {
|
|
7
7
|
if (input.requestedRoot) {
|
|
8
8
|
const absolute = path.resolve(input.requestedRoot);
|