@deksden-com/dd-flow-cli 0.9.0-beta.14 → 0.9.0-beta.140

package/dist/services/tooling.js

@@ -2,7 +2,8 @@ import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
-import { spawnSync } from "node:child_process";
+import { commandResult, commandText } from '../harness-runtime/lib/process-json.mjs';
+import { publishSetup, SETUP_STARTUP_MS } from '../harness-runtime/lib/setup-progress.mjs';
 import { AppError } from "../shared/errors.js";
 import { ensureDir } from "../storage/paths.js";
 const manifestRelativePath = path.join("tools", "worktrunk-manifest.json");
@@ -71,7 +72,7 @@ export function inspectManagedWorktrunk(context) {
         };
     }
 }
-export function ensureManagedWorktrunk(context) {
+export async function ensureManagedWorktrunk(context) {
     const manifest = worktrunkManifest();
     const platform = worktrunkPlatform();
     const cachedRoot = worktrunkCacheRoot(context, platform);
@@ -92,14 +93,14 @@ export function ensureManagedWorktrunk(context) {
                 next_action: "Place the verified official artifact in the managed tool cache and retry."
             });
         }
-        installWorktrunkArtifact(manifest, platform, cachedRoot);
+        await installWorktrunkArtifact(manifest, platform, cachedRoot);
         return verifyCachedWorktrunk(context, manifest, platform);
     }
     finally {
         fs.rmSync(lockDir, { recursive: true, force: true });
     }
 }
-function installWorktrunkArtifact(manifest, platform, targetRoot) {
+async function installWorktrunkArtifact(manifest, platform, targetRoot) {
     const artifact = artifactFor(manifest, platform);
     const parent = path.dirname(targetRoot);
     ensureDir(parent);
@@ -107,8 +108,17 @@ function installWorktrunkArtifact(manifest, platform, targetRoot) {
     const archivePath = path.join(tempRoot, `worktrunk.${artifact.archive}`);
     const extractedRoot = path.join(tempRoot, "extracted");
     try {
-        const download = spawnSync("curl", ["--fail", "--location", "--silent", "--show-error", "--max-time", "120", "-o", archivePath, artifact.url], { encoding: "utf8" });
-        if (download.status !== 0) {
+        let observedBytes = 0;
+        publishSetup('probe_started', 'worktrunk-download');
+        const download = await commandResult("curl", ["--fail", "--location", "--silent", "--show-error", "-o", archivePath, artifact.url], { startupMs: SETUP_STARTUP_MS, progress: () => {
+                const bytes = fs.existsSync(archivePath) ? fs.statSync(archivePath).size : 0;
+                if (bytes > observedBytes) {
+                    observedBytes = bytes;
+                    publishSetup('transfer_progress', `worktrunk-transfer:${bytes}`);
+                }
+                return observedBytes;
+            } });
+        if (download.exit_code !== 0) {
             throw new AppError("tool_unavailable", "Unable to download the pinned Worktrunk artifact", 1, {
                 tool: manifest.tool,
                 version: manifest.version,
@@ -118,6 +128,7 @@ function installWorktrunkArtifact(manifest, platform, targetRoot) {
                 stderr: sanitize(download.stderr)
             });
         }
+        publishSetup('probe_completed', 'worktrunk-download');
         const archiveChecksum = sha256File(archivePath);
         if (archiveChecksum !== artifact.sha256) {
             throw new AppError("tool_integrity_failed", "Pinned Worktrunk artifact checksum mismatch", 1, {
@@ -129,12 +140,12 @@ function installWorktrunkArtifact(manifest, platform, targetRoot) {
             });
         }
         ensureDir(extractedRoot);
-        extractArchive(archivePath, extractedRoot, artifact.archive);
+        await extractArchive(archivePath, extractedRoot, artifact.archive);
         const extractedBinary = findBinary(extractedRoot);
         if (!extractedBinary)
             throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive does not contain wt", 1, { platform });
         fs.chmodSync(extractedBinary, 0o755);
-        verifyWorktrunkVersion(extractedBinary, manifest.version);
+        await verifyWorktrunkVersion(extractedBinary, manifest.version);
         const binaryChecksum = sha256File(extractedBinary);
         const stagedRoot = path.join(tempRoot, "managed");
         ensureDir(stagedRoot);
@@ -186,7 +197,7 @@ function verifyCachedWorktrunk(context, manifest, platform) {
             actual_binary_sha256: actualBinaryChecksum
         });
     }
-    verifyWorktrunkVersion(bin, manifest.version);
+    // The admitted binary checksum binds the version probe performed at install.
     return {
         bin,
         version: manifest.version,
@@ -195,23 +206,23 @@ function verifyCachedWorktrunk(context, manifest, platform) {
         binary_sha256: actualBinaryChecksum
     };
 }
-function extractArchive(archive, destination, format) {
+async function extractArchive(archive, destination, format) {
     if (format === "tar.xz") {
-        const listing = spawnSync("tar", ["-tJf", archive], { encoding: "utf8" });
-        if (listing.status !== 0 || listing.stdout.split("\n").some(unsafeArchivePath)) {
+        const listing = await commandResult("tar", ["-tJf", archive], { startupMs: SETUP_STARTUP_MS });
+        if (listing.exit_code !== 0 || listing.stdout.split("\n").some(unsafeArchivePath)) {
             throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive has an unsafe or unreadable file list", 1);
         }
-        const extracted = spawnSync("tar", ["-xJf", archive, "-C", destination], { encoding: "utf8" });
-        if (extracted.status !== 0)
+        const extracted = await commandResult("tar", ["-xJvf", archive, "-C", destination], { startupMs: SETUP_STARTUP_MS });
+        if (extracted.exit_code !== 0)
             throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive extraction failed", 1, { stderr: sanitize(extracted.stderr) });
         return;
     }
-    const listing = spawnSync("unzip", ["-Z1", archive], { encoding: "utf8" });
-    if (listing.status !== 0 || listing.stdout.split("\n").some(unsafeArchivePath)) {
+    const listing = await commandResult("unzip", ["-Z1", archive], { startupMs: SETUP_STARTUP_MS });
+    if (listing.exit_code !== 0 || listing.stdout.split("\n").some(unsafeArchivePath)) {
         throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive has an unsafe or unreadable file list", 1);
     }
-    const extracted = spawnSync("unzip", ["-q", archive, "-d", destination], { encoding: "utf8" });
-    if (extracted.status !== 0)
+    const extracted = await commandResult("unzip", [archive, "-d", destination], { startupMs: SETUP_STARTUP_MS });
+    if (extracted.exit_code !== 0)
         throw new AppError("tool_integrity_failed", "Pinned Worktrunk archive extraction failed", 1, { stderr: sanitize(extracted.stderr) });
 }
 function findBinary(root) {
@@ -229,13 +240,11 @@ function findBinary(root) {
     }
     return null;
 }
-function verifyWorktrunkVersion(bin, expected) {
-    const result = spawnSync(bin, ["--version"], { encoding: "utf8" });
-    const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`;
-    if (result.status !== 0 || !output.includes(expected)) {
+async function verifyWorktrunkVersion(bin, expected) {
+    const output = await commandText(bin, ['--version'], { startupMs: SETUP_STARTUP_MS, phase: 'native-version' });
+    if (!output.includes(expected)) {
         throw new AppError("tool_integrity_failed", "Managed Worktrunk version verification failed", 1, {
             expected_version: expected,
-            status: result.status,
             output: sanitize(output)
         });
     }

package/dist/services/usage.js

@@ -4,6 +4,8 @@ import { parseJsonObject } from "../shared/json.js";
 import { resolveProjectRoot } from "../storage/paths.js";
 import { registerProject, requireProjectByRoot } from "./projects.js";
 import { publicSessionIdentity } from "./session-identity.js";
+import { runObservations } from "./run-observations.js";
+import { ToolObservations, codexToolObservation } from "../harness-runtime/lib/tool-observations.mjs";
 export function ingestZcodeUsage(context, input) {
     return ingestHarnessUsage(context, input, { harness: "zcode-acp", label: "ZCode", idPrefix: "ZUSG", sourceKind: "zcode_session_usage_v1" });
 }
@@ -19,6 +21,14 @@ export function ingestOpenCodeUsage(context, input) {
 export function ingestAgyUsage(context, input) {
     return ingestHarnessUsage(context, input, { harness: "antigravity-cli", label: "Antigravity", idPrefix: "AUSG", sourceKind: "antigravity_cli_session_usage_v1" });
 }
+/** Droid counters must describe one physical Session; inclusive roots double-count children. */
+export function ingestDroidUsage(context, input) {
+    const payload = parseJsonObject(input.stdin || "{}", "Droid usage snapshot");
+    if (payload.usage_scope !== "physical_session" || !["complete", "partial"].includes(String(payload.completeness))) {
+        throw new Error("Droid usage requires physical_session scope and complete or partial completeness");
+    }
+    return ingestHarnessUsage(context, input, { harness: "droid-cli", label: "Droid", idPrefix: "DUSG", sourceKind: "droid_session_usage_v1" });
+}
 function ingestHarnessUsage(context, input, harness) {
     const payload = parseJsonObject(input.stdin || "{}", `${harness.label} usage snapshot`);
     const usage = object(payload.usage);
@@ -157,8 +167,7 @@ export function recalculateRunUsage(context, input) {
     const usageFinal = final && unsettledSessions.length === 0;
     const persistRunProjection = !input.sessionId && !input.stage;
     if (persistRunProjection) {
-        context.db.exec("BEGIN IMMEDIATE");
-        try {
+        context.db.writeTransaction(() => {
             context.db.run("DELETE FROM usage WHERE project_id = ? AND run_id = ?", [input.projectId, input.runId]);
             for (const row of rows) {
                 context.db.run(`INSERT INTO usage (id, project_id, run_id, session_id, checkpoint, stage, stage_attempt, observed_at, total_tokens, input_tokens, cached_input_tokens, cache_read_input_tokens, cache_write_input_tokens, uncached_input_tokens, output_tokens, reasoning_output_tokens, source_kind, source_locator, source_sha256, source_size_bytes, source_mtime_ms, token_event_at, turn_id, turn_attribution, parser_version, extraction_status, diagnostic_code, created_at)
@@ -177,13 +186,10 @@ export function recalculateRunUsage(context, input) {
             AND status = 'idle'
             AND NOT EXISTS (SELECT 1 FROM work_sessions WHERE session_id = ? AND status = 'running')`, [observedAt, session.completed_at, input.projectId, input.runId, session.session_id, session.session_id]);
             }
-            context.db.exec("COMMIT");
-        }
-        catch (error) {
-            context.db.exec("ROLLBACK");
-            throw error;
-        }
+        });
     }
+    const observations = persistRunProjection ? runObservations(context, input.projectId, input.runId) : null;
+    const legacyTools = toolCallsForLinks(context, links, accountingAt);
     return {
         ok: true,
         schema_id: "dd-flow/run-usage@2",
@@ -198,7 +204,8 @@ export function recalculateRunUsage(context, input) {
             stopped: publicSessionIdentities(context, input.projectId, settledSessions.map((session) => session.session_id)),
             unsettled: publicSessionIdentities(context, input.projectId, unsettledSessions)
         },
-        tool_calls: toolCallsForLinks(context, links, accountingAt),
+        tool_calls: observations?.tools.observed_sessions ? observations.tools : legacyTools,
+        ...(observations ? { observations, legacy_tool_calls: legacyTools } : {}),
         coverage: rows.reduce((coverage, row) => ({ ...coverage, [row.status]: (coverage[row.status] ?? 0) + 1 }), {})
     };
 }
@@ -241,6 +248,7 @@ function publicSessionIdentities(context, projectId, storageIds) {
 }
 function toolCallsForLinks(context, links, observedAt) {
     const result = { status: "unavailable", total: 0, failures: 0, by_tool: {}, observed_sessions: 0, unavailable_sessions: [] };
+    let incomplete = false;
     const sessions = new Map();
     for (const link of links)
         sessions.set(link.session_id, [...(sessions.get(link.session_id) ?? []), link]);
@@ -253,18 +261,20 @@ function toolCallsForLinks(context, links, observedAt) {
             result.unavailable_sessions.push(sessionId);
             continue;
         }
+        if ("completeness" in parsed && parsed.completeness !== "complete")
+            incomplete = true;
         result.observed_sessions += 1;
         result.total += parsed.total;
         result.failures += parsed.failures;
         for (const [name, count] of Object.entries(parsed.by_tool))
             result.by_tool[name] = (result.by_tool[name] ?? 0) + count;
     }
-    result.status = result.observed_sessions === 0 ? "unavailable" : result.unavailable_sessions.length ? "partial" : "measured";
+    result.status = result.observed_sessions === 0 ? "unavailable" : result.unavailable_sessions.length || incomplete ? "partial" : "measured";
     return result;
 }
-function isExternalHarness(harness) { return harness === "zcode-acp" || harness === "grok-acp" || harness === "opencode-server" || harness === "antigravity-cli"; }
+function isExternalHarness(harness) { return harness === "droid-cli" || harness === "zcode-acp" || harness === "grok-acp" || harness === "opencode-server" || harness === "antigravity-cli"; }
 function harnessSourceKind(harness) {
-    return harness === "grok-acp" ? "grok_session_usage_v1" : harness === "zcode-acp" ? "zcode_session_usage_v1" : harness === "antigravity-cli" ? "antigravity_cli_session_usage_v1" : "opencode_session_usage_v1";
+    return harness === "droid-cli" ? "droid_session_usage_v1" : harness === "grok-acp" ? "grok_session_usage_v1" : harness === "zcode-acp" ? "zcode_session_usage_v1" : harness === "antigravity-cli" ? "antigravity_cli_session_usage_v1" : "opencode_session_usage_v1";
 }
 function readHarnessToolCallsWindow(context, harness, projectId, providerSessionId, startedAt, endedAt, observedAt) {
     if (!providerSessionId)
@@ -302,9 +312,7 @@ function readToolCallsForWindows(transcriptPath, expectedSessionId, windows, obs
         return null;
     try {
         let sessionId;
-        let total = 0;
-        let failures = 0;
-        const byTool = {};
+        const tools = new ToolObservations();
         for (const line of fs.readFileSync(transcriptPath, "utf8").split(/\r?\n/)) {
             if (!line.trim())
                 continue;
@@ -313,6 +321,7 @@ function readToolCallsForWindows(transcriptPath, expectedSessionId, windows, obs
                 event = JSON.parse(line);
             }
             catch {
+                tools.gaps.add("journal_invalid_record");
                 continue;
             }
             const payload = object(event.payload);
@@ -321,20 +330,13 @@ function readToolCallsForWindows(transcriptPath, expectedSessionId, windows, obs
             if (event.type !== "response_item" || !payload)
                 continue;
             const eventAt = stringValue(event.timestamp);
-            if (eventAt && !windows.some((window) => eventInsideWindow(eventAt, window, observedAt)))
+            if (eventAt && windows.length && !windows.some((window) => eventInsideWindow(eventAt, window, observedAt)))
                 continue;
-            const type = stringValue(payload.type);
-            if (type === "function_call" || type === "custom_tool_call") {
-                const name = stringValue(payload.name) ?? "unknown";
-                byTool[name] = (byTool[name] ?? 0) + 1;
-                total += 1;
-            }
-            if ((type === "function_call_output" || type === "custom_tool_call_output") && outputIndicatesFailure(payload))
-                failures += 1;
+            tools.add(codexToolObservation(event, expectedSessionId));
         }
         if (sessionId && sessionId !== expectedSessionId)
             return null;
-        return { total, failures, by_tool: Object.fromEntries(Object.entries(byTool).sort(([left], [right]) => left.localeCompare(right))) };
+        return tools.summary();
     }
     catch {
         return null;
@@ -346,37 +348,6 @@ function eventInsideWindow(eventAt, window, observedAt) {
     const end = Date.parse(window.completed_at ?? observedAt);
     return Number.isFinite(at) && (!Number.isFinite(start) || at >= start) && (!Number.isFinite(end) || at <= end);
 }
-function outputIndicatesFailure(payload) {
-    if (payload.error === true || typeof payload.error === "string")
-        return true;
-    return containsFailure(payload.output);
-}
-function containsFailure(value) {
-    if (Array.isArray(value))
-        return value.some(containsFailure);
-    if (typeof value === "string") {
-        if (/(?:^|\n)exit=(?:[1-9]\d*)\b|Process exited with code [1-9]\d*/.test(value))
-            return true;
-        const trimmed = value.trim();
-        if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
-            try {
-                return containsFailure(JSON.parse(trimmed));
-            }
-            catch {
-                return false;
-            }
-        }
-        return false;
-    }
-    const item = object(value);
-    if (!item)
-        return false;
-    if (item.ok === false || item.is_error === true || item.isError === true || item.error === true || typeof item.error === "string")
-        return true;
-    if (typeof item.exit_code === "number" && item.exit_code !== 0)
-        return true;
-    return Object.values(item).some(containsFailure);
-}
 /** RUN accounting belongs to the physical provider Session, not to every Work link. */
 function usageRowsForPhysicalSessions(context, links, observedAt, stage) {
     const sessions = new Map();
@@ -398,7 +369,7 @@ function usageRowsForPhysicalSessions(context, links, observedAt, stage) {
             : completedTurn?.completed_at ?? sessionLinks.map((item) => item.completed_at ?? observedAt).sort().at(-1);
         const endedAt = stage?.completedAt ?? (Date.parse(naturalEnd) > Date.parse(observedAt) ? observedAt : naturalEnd);
         const harnessUsage = isExternalHarness(link.harness)
-            ? readHarnessUsageWindow(context, link.harness, link.project_id, link.provider_session_id, startedAt, endedAt, observedAt)
+            ? readHarnessUsageWindow(context, link.harness, link.project_id, link.provider_session_id, startedAt, endedAt, observedAt, Boolean(stage))
             : null;
         const parsed = harnessUsage ?? readCodexTranscriptWindow(link.transcript_path, nativeSessionId(link), startedAt, endedAt);
         return {
@@ -422,18 +393,31 @@ function hasInclusiveUsageSnapshot(context, projectId, harness, providerSessionI
         return false;
     return Boolean(context.db.get("SELECT 1 FROM harness_usage_snapshots WHERE project_id = ? AND harness = ? AND provider_session_id = ? AND usage_scope = 'execution_tree_inclusive' AND observed_at <= ? LIMIT 1", [projectId, harness, providerSessionId, observedAt]));
 }
-function readHarnessUsageWindow(context, harness, projectId, providerSessionId, startedAt, endedAt, observedAt) {
+function readHarnessUsageWindow(context, harness, projectId, providerSessionId, startedAt, endedAt, observedAt, stageWindow) {
     if (!providerSessionId)
         return { status: "not_observable", diagnostic: "provider_session_id_missing" };
-    const snapshots = context.db.all(`SELECT observed_at, total_tokens, input_tokens, cache_read_input_tokens, cache_write_input_tokens, output_tokens, reasoning_output_tokens
+    const snapshots = context.db.all(`SELECT completeness, observed_at, total_tokens, input_tokens, cache_read_input_tokens, cache_write_input_tokens, output_tokens, reasoning_output_tokens
        FROM harness_usage_snapshots
       WHERE project_id = ? AND harness = ? AND provider_session_id = ? AND observed_at <= ?
       ORDER BY observed_at, id`, [projectId, harness, providerSessionId, observedAt]);
-    const last = snapshots.find((snapshot) => Date.parse(snapshot.observed_at) > Date.parse(endedAt))
+    const last = (!stageWindow ? snapshots.find((snapshot) => Date.parse(snapshot.observed_at) > Date.parse(endedAt)) : undefined)
         ?? snapshots.filter((snapshot) => Date.parse(snapshot.observed_at) <= Date.parse(endedAt)).at(-1);
     const before = snapshots.filter((snapshot) => Date.parse(snapshot.observed_at) <= Date.parse(startedAt)).at(-1);
     if (!last || !before || last === before)
         return { status: "not_yet_emitted", diagnostic: `${harness}_usage_window_incomplete` };
+    // Cumulative counters outside an attempt's boundaries cannot establish its
+    // spend: adjacent attempts would otherwise charge overlapping intervals.
+    if (stageWindow && (Date.parse(before.observed_at) !== Date.parse(startedAt)
+        || Date.parse(last.observed_at) !== Date.parse(endedAt)
+        || before.completeness !== "complete" || last.completeness !== "complete")) {
+        return { status: "not_observable", diagnostic: `${harness}_stage_window_unconfirmed` };
+    }
+    if (stageWindow && ["total_tokens", "input_tokens", "cache_read_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_output_tokens"]
+        .some(key => last[key] !== null && before[key] !== null && last[key] < before[key])) {
+        return { status: "not_observable", diagnostic: `${harness}_stage_counter_reset` };
+    }
+    if (harness === "droid-cli" && (last.completeness !== "complete" || before.completeness !== "complete"))
+        return { status: "not_observable", diagnostic: "droid-cli_usage_window_partial" };
     const delta = (value, baseline) => value === null || baseline === null ? null : Math.max(0, value - baseline);
     const input = delta(last.input_tokens, before.input_tokens);
     const cacheRead = delta(last.cache_read_input_tokens, before.cache_read_input_tokens);
@@ -575,62 +559,26 @@ function usageReport(context, input, refresh) {
 }
 function toolCallsForSessions(sessions) {
     const result = { status: "unavailable", total: 0, failures: 0, by_tool: {}, observed_sessions: 0, unavailable_sessions: [] };
+    let incomplete = false;
     for (const session of sessions) {
         const parsed = readToolCalls(session.transcript_path, nativeSessionId(session));
         if (!parsed) {
             result.unavailable_sessions.push(session.session_id);
             continue;
         }
+        if (parsed.completeness !== "complete")
+            incomplete = true;
         result.observed_sessions += 1;
         result.total += parsed.total;
         result.failures += parsed.failures;
         for (const [name, count] of Object.entries(parsed.by_tool))
             result.by_tool[name] = (result.by_tool[name] ?? 0) + count;
     }
-    result.status = result.observed_sessions === 0 ? "unavailable" : result.unavailable_sessions.length === 0 ? "measured" : "partial";
+    result.status = result.observed_sessions === 0 ? "unavailable" : result.unavailable_sessions.length || incomplete ? "partial" : "measured";
     return result;
 }
 function readToolCalls(transcriptPath, expectedSessionId) {
-    if (!transcriptPath || !fs.existsSync(transcriptPath))
-        return null;
-    try {
-        let sessionId;
-        const byTool = {};
-        let total = 0;
-        let failures = 0;
-        for (const line of fs.readFileSync(transcriptPath, "utf8").split(/\r?\n/)) {
-            if (!line.trim())
-                continue;
-            let event;
-            try {
-                event = JSON.parse(line);
-            }
-            catch {
-                continue;
-            }
-            const payload = object(event.payload);
-            if (!payload)
-                continue;
-            if (event.type === "session_meta")
-                sessionId = stringValue(payload.id) ?? sessionId;
-            if (event.type !== "response_item")
-                continue;
-            const type = stringValue(payload.type);
-            if (type === "function_call" || type === "custom_tool_call") {
-                const name = stringValue(payload.name) ?? "unknown";
-                byTool[name] = (byTool[name] ?? 0) + 1;
-                total += 1;
-            }
-            if ((type === "function_call_output" || type === "custom_tool_call_output") && outputIndicatesFailure(payload))
-                failures += 1;
-        }
-        if (sessionId && sessionId !== expectedSessionId)
-            return null;
-        return { total, failures, by_tool: Object.fromEntries(Object.entries(byTool).sort(([left], [right]) => left.localeCompare(right))) };
-    }
-    catch {
-        return null;
-    }
+    return readToolCallsForWindows(transcriptPath, expectedSessionId, [], new Date().toISOString());
 }
 function segmentUsage(context, session, segment) {
     if (!session)