@andorra-labs/code 0.0.0-stage → 0.1.4

package/dist/src/telemetry/capture-core.js

@@ -0,0 +1,374 @@
+import { bucketEndpoint, bucketModel, bucketProviderType, bucketToolName } from './schema.js';
+import { isDocumentationPath, isTestPath } from './classify.js';
+import { usageCostMicroUsd } from '../model-tiers.js';
+export function createEfficiencyTally() {
+    return {
+        prepared: { steps: 0, preparationChars: 0, deterministic: 0, jevSelected: 0, executed: 0, rejected: 0, invalidated: 0 },
+        postflight: { skipped: 0, awaited: 0, awaitedMs: 0 },
+        toolResults: { filtered: 0, originalChars: 0, keptChars: 0, expansions: 0, expandedChars: 0, expansionsRejected: 0 },
+        context: { retrievalSkipped: 0, candidates: 0, selected: 0, omitted: 0, duplicates: 0, selectedChars: 0, omittedChars: 0 },
+        catalog: { selections: 0, omittedSchemas: 0, schemaCharsBefore: 0, schemaCharsAfter: 0, restorations: 0, skillCatalogDeduplicated: 0, skillCatalogChars: 0 },
+        routing: { decisions: 0, routed: 0, baseline: 0, escalations: 0 },
+    };
+}
+const count = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0;
+/**
+ * Fold a runtime `efficiency.*` event into the tally.
+ *
+ * Each event names its feature and outcome in its type (`efficiency.<feature>.<outcome>`); sizes
+ * ride as integer counters. An event the tally does not know is ignored rather than guessed at.
+ */
+export function recordEfficiencyEvent(telemetry, type, data) {
+    if (!type.startsWith('efficiency.'))
+        return;
+    const tally = telemetry.efficiency ??= createEfficiencyTally();
+    switch (type) {
+        case 'efficiency.prepared.declared':
+            tally.prepared.steps += count(data.steps);
+            tally.prepared.preparationChars += count(data.chars);
+            return;
+        case 'efficiency.prepared.selected':
+            if (data.reason === 'deterministic_continuation')
+                tally.prepared.deterministic += 1;
+            else
+                tally.prepared.jevSelected += 1;
+            return;
+        case 'efficiency.prepared.executed':
+            tally.prepared.executed += 1;
+            return;
+        case 'efficiency.prepared.rejected':
+            tally.prepared.rejected += 1;
+            return;
+        case 'efficiency.prepared.invalidated':
+            tally.prepared.invalidated += count(data.steps);
+            return;
+        case 'efficiency.postflight.skipped':
+            tally.postflight.skipped += 1;
+            return;
+        case 'efficiency.postflight.awaited':
+            tally.postflight.awaited += 1;
+            tally.postflight.awaitedMs += count(data.waitMs);
+            return;
+        case 'efficiency.tool_result.filtered':
+            tally.toolResults.filtered += 1;
+            tally.toolResults.originalChars += count(data.originalChars);
+            tally.toolResults.keptChars += count(data.keptChars);
+            return;
+        case 'efficiency.tool_result.expanded':
+            tally.toolResults.expansions += 1;
+            tally.toolResults.expandedChars += count(data.chars);
+            return;
+        case 'efficiency.tool_result.expansion_rejected':
+            tally.toolResults.expansionsRejected += 1;
+            return;
+        case 'efficiency.context.retrieval_skipped':
+            tally.context.retrievalSkipped += 1;
+            return;
+        case 'efficiency.context.selected':
+            tally.context.candidates += count(data.candidates);
+            tally.context.selected += count(data.selected);
+            tally.context.omitted += count(data.omitted);
+            tally.context.selectedChars += count(data.selectedChars);
+            tally.context.omittedChars += count(data.omittedChars);
+            return;
+        case 'efficiency.context.duplicate':
+            tally.context.duplicates += 1;
+            tally.context.omittedChars += count(data.chars);
+            return;
+        case 'efficiency.catalog.selected':
+            tally.catalog.selections += 1;
+            tally.catalog.omittedSchemas += count(data.omitted);
+            tally.catalog.schemaCharsBefore += count(data.schemaCharsBefore);
+            tally.catalog.schemaCharsAfter += count(data.schemaCharsAfter);
+            return;
+        case 'efficiency.catalog.restored':
+            tally.catalog.restorations += count(data.count) || 1;
+            return;
+        case 'efficiency.skills.catalog':
+            tally.catalog.skillCatalogChars += count(data.chars);
+            if (data.reason === 'duplicate')
+                tally.catalog.skillCatalogDeduplicated += 1;
+            return;
+        case 'efficiency.routing.decided':
+            tally.routing.decisions += 1;
+            if (data.applied === true)
+                tally.routing.routed += 1;
+            else
+                tally.routing.baseline += 1;
+            if (data.reason === 'escalated')
+                tally.routing.escalations += 1;
+            return;
+        default: return;
+    }
+}
+function mergeEfficiency(into, from) {
+    for (const group of Object.keys(from)) {
+        const target = into[group], source = from[group];
+        for (const [key, value] of Object.entries(source))
+            target[key] = (target[key] ?? 0) + value;
+    }
+}
+/** Every counter a settled shadow preflight may still add after a merge. */
+const JEV_FIELDS = ['requested', 'completed', 'applied', 'fallback', 'unavailable'];
+/** The failure reasons the ADR counts as JEV being unavailable. */
+const UNAVAILABLE_REASONS = new Set([
+    'provider_unavailable', 'provider_error', 'active_unsupported', 'timeout',
+    'authentication_required', 'decision_service_unavailable', 'decision_input_too_large',
+]);
+export function createRunTelemetry() {
+    return {
+        attempts: 0, retries: 0, toolCalls: 0, toolFailures: 0, tools: {}, models: [],
+        jev: { requested: 0, completed: 0, applied: 0, fallback: 0, unavailable: 0 },
+        pendingQuestions: 0,
+        pendingByInvocation: {},
+        pendingDecisions: [],
+        task: { documentation: 0, test: 0, otherEdits: 0, created: 0, testCommand: false, activity: false },
+    };
+}
+/** Whether a run carried telemetry at all; a result without it reports nothing. */
+export function hasRunTelemetry(value) {
+    return !!value && typeof value === 'object' && Array.isArray(value.models) && !!value.task;
+}
+/**
+ * Record one provider response.
+ *
+ * The tuple key includes the role because a graph can use the same model for an orchestrator and a
+ * subagent; reporting them together would bill one role's work to the other.
+ */
+export function recordModelUse(telemetry, input) {
+    const providerType = bucketProviderType(input.providerType);
+    const endpoint = bucketEndpoint(input.baseUrl);
+    const model = bucketModel(input.model);
+    const key = `${providerType}\0${endpoint}\0${model}\0${input.role}`;
+    let tuple = telemetry.models.find(candidate => `${candidate.providerType}\0${candidate.endpoint}\0${candidate.model}\0${candidate.role}` === key);
+    if (!tuple) {
+        tuple = {
+            providerType, endpoint, model, role: input.role, requests: 0,
+            reported: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, costVersions: new Set() },
+        };
+        telemetry.models.push(tuple);
+    }
+    tuple.requests += input.usage.requests ?? 1;
+    const usage = input.usage;
+    if (typeof usage.inputTokens === 'number') {
+        tuple.reported.input += 1;
+        tuple.inputTokens = (tuple.inputTokens ?? 0) + usage.inputTokens;
+    }
+    if (typeof usage.outputTokens === 'number') {
+        tuple.reported.output += 1;
+        tuple.outputTokens = (tuple.outputTokens ?? 0) + usage.outputTokens;
+    }
+    if (typeof usage.cacheRead === 'number') {
+        tuple.reported.cacheRead += 1;
+        tuple.cacheRead = (tuple.cacheRead ?? 0) + usage.cacheRead;
+    }
+    if (typeof usage.cacheWrite === 'number') {
+        tuple.reported.cacheWrite += 1;
+        tuple.cacheWrite = (tuple.cacheWrite ?? 0) + usage.cacheWrite;
+    }
+    // A price makes the usage a cost, folded exactly as `aggregateModels` folds it: the tuple's cost
+    // stays absent unless every request in it priced its usage, so a partially priced tuple reports
+    // nothing rather than a number the reader would take as complete. No pricing version exists yet
+    // (prices come from the user's configuration), so the version set holds `undefined` — one entry,
+    // which is what "one provenance" means to the aggregation.
+    const cost = usageCostMicroUsd(usage, input.pricing);
+    if (cost !== undefined) {
+        tuple.reported.cost += 1;
+        tuple.costMicroUsd = (tuple.costMicroUsd ?? 0) + cost;
+        tuple.reported.costVersions.add(undefined);
+    }
+    // Deliberately not `task.activity`: the contract omits the task category when no tool ran, and a
+    // model that answers without calling one is exactly the tool-free run it describes.
+}
+/**
+ * Record an executed tool call.
+ *
+ * `failed` is a property of the call, not of the batch: a call interrupted by a stop is not a tool
+ * failure. Runtime bookkeeping calls are excluded by `bucketToolName`, exactly as `RunResult.toolCalls`
+ * excludes them.
+ */
+export function recordToolUse(telemetry, name, failed) {
+    const bucket = bucketToolName(name);
+    if (!bucket)
+        return;
+    const entry = telemetry.tools[bucket] ?? (telemetry.tools[bucket] = { calls: 0, failures: 0 });
+    entry.calls += 1;
+    if (failed)
+        entry.failures += 1;
+    telemetry.toolCalls += 1;
+    if (failed)
+        telemetry.toolFailures += 1;
+    telemetry.task.activity = true;
+}
+/** One node execution started; every attempt after the first is a re-execution. */
+export function recordAttempt(telemetry, retry) {
+    telemetry.attempts += 1;
+    if (retry)
+        telemetry.retries += 1;
+}
+/**
+ * Record a file mutation the run made.
+ *
+ * Only the classification is retained: whether the path is documentation, a test, an ordinary
+ * source file, and whether the file existed when the run started. The path itself is compared here
+ * and discarded, so a persisted run record carries no user-chosen name from telemetry.
+ */
+export function recordEdit(telemetry, filePath, created) {
+    telemetry.task.activity = true;
+    if (created)
+        telemetry.task.created += 1;
+    if (isDocumentationPath(filePath))
+        telemetry.task.documentation += 1;
+    else if (isTestPath(filePath))
+        telemetry.task.test += 1;
+    else
+        telemetry.task.otherEdits += 1;
+}
+/** Note that an executed command was a test command, using the fixed schema-v1 runner list. */
+export function recordTestCommand(telemetry) {
+    telemetry.task.activity = true;
+    telemetry.task.testCommand = true;
+}
+/**
+ * Fold a runtime `decision.*` event into the JEV tally.
+ *
+ * Every counter is a count of decision questions, never of batches or invocations.
+ * `decision.provider_error` is not a counting source: the error path emits it together with a
+ * fallback for the same request, so counting both would report one outage twice.
+ */
+export function recordDecisionEvent(telemetry, type, data) {
+    // Events carry the invocation that asked; a request is attributed to it rather than to whichever
+    // request happened to be counted last.
+    const invocation = typeof data.invocationId === 'string' ? data.invocationId : '';
+    const settle = (count) => {
+        if (invocation === '')
+            telemetry.pendingQuestions = Math.max(0, telemetry.pendingQuestions - count);
+        else {
+            delete telemetry.pendingByInvocation[invocation];
+            telemetry.pendingQuestions = Object.values(telemetry.pendingByInvocation).reduce((sum, value) => sum + value, 0);
+        }
+    };
+    if (type === 'decision.tools_restored') {
+        // The runtime's restoration event, folded into the catalog tally: recovery after an omission
+        // is efficiency overhead the local accounting must reflect. The `efficiency.catalog.restored`
+        // name has no emitter; this is the event the runtime actually sends.
+        (telemetry.efficiency ??= createEfficiencyTally()).catalog.restorations += count(data.count) || 1;
+        return;
+    }
+    if (type === 'decision.requested') {
+        const questions = Array.isArray(data.questions) ? data.questions.length : 0;
+        telemetry.jev.requested += questions;
+        if (invocation === '')
+            telemetry.pendingQuestions = questions;
+        else {
+            telemetry.pendingByInvocation[invocation] = questions;
+            telemetry.pendingQuestions = Object.values(telemetry.pendingByInvocation).reduce((sum, value) => sum + value, 0);
+        }
+        return;
+    }
+    if (type === 'decision.completed') {
+        const decisions = data.decisions;
+        telemetry.jev.completed += decisions && typeof decisions === 'object' ? Object.keys(decisions).length : 0;
+        settle(telemetry.pendingByInvocation[invocation] ?? telemetry.pendingQuestions);
+        return;
+    }
+    if (type === 'decision.applied') {
+        telemetry.jev.applied += 1;
+        return;
+    }
+    if (type === 'decision.fallback') {
+        const reason = typeof data.reason === 'string' ? data.reason : '';
+        // A fallback that names a question counts one; a request-level fallback is attributed to the
+        // questions of the request that emitted it.
+        const pending = telemetry.pendingByInvocation[invocation] ?? telemetry.pendingQuestions;
+        const count = typeof data.question === 'string' ? 1 : pending;
+        telemetry.jev.fallback += count;
+        if (UNAVAILABLE_REASONS.has(reason))
+            telemetry.jev.unavailable += count;
+        settle(pending);
+    }
+}
+/** Count a failure for a call already counted, without counting the call twice. */
+export function recordToolFailure(telemetry, name) {
+    const bucket = bucketToolName(name);
+    if (!bucket)
+        return;
+    const entry = telemetry.tools[bucket] ?? (telemetry.tools[bucket] = { calls: 0, failures: 0 });
+    entry.failures += 1;
+    telemetry.toolFailures += 1;
+}
+/**
+ * Merge one attempt's capture into another.
+ *
+ * A node that retries discards the failed attempt's result object, but its token spend and tool
+ * calls happened. Merging keeps the run's totals truthful and preserves the per-tuple coverage
+ * counts, so a measurement that was partial in one attempt stays partial in the merged figure.
+ */
+export function mergeRunTelemetry(into, from) {
+    if (from.pendingDecisions?.length) {
+        // A shadow preflight settles after this merge: its terminal callback counts into the capture
+        // it was created with, so those later counts have to be carried over here as well. The transfer
+        // waits for every pending decision and takes the cumulative delta once — a per-promise delta
+        // would see the same baseline and count an earlier promise's counters again for each later one,
+        // which can exceed `requested` and make the schema drop the whole finish.
+        const before = { ...from.jev };
+        (into.pendingDecisions ??= []).push(Promise.allSettled(from.pendingDecisions).then(() => {
+            for (const field of JEV_FIELDS) {
+                const delta = from.jev[field] - before[field];
+                if (delta !== 0)
+                    into.jev[field] += delta;
+            }
+        }));
+    }
+    into.attempts += from.attempts;
+    into.retries += from.retries;
+    into.toolCalls += from.toolCalls;
+    into.toolFailures += from.toolFailures;
+    for (const [name, counts] of Object.entries(from.tools)) {
+        const entry = into.tools[name] ?? (into.tools[name] = { calls: 0, failures: 0 });
+        entry.calls += counts.calls;
+        entry.failures += counts.failures;
+    }
+    for (const source of from.models) {
+        const key = `${source.providerType}\0${source.endpoint}\0${source.model}\0${source.role}`;
+        let target = into.models.find(candidate => `${candidate.providerType}\0${candidate.endpoint}\0${candidate.model}\0${candidate.role}` === key);
+        if (!target) {
+            target = { ...source, reported: { ...source.reported, costVersions: new Set(source.reported.costVersions) } };
+            into.models.push(target);
+            continue;
+        }
+        target.requests += source.requests;
+        for (const field of ['input', 'output', 'cacheRead', 'cacheWrite', 'cost'])
+            target.reported[field] += source.reported[field];
+        for (const version of source.reported.costVersions)
+            target.reported.costVersions.add(version);
+        if (source.inputTokens !== undefined)
+            target.inputTokens = (target.inputTokens ?? 0) + source.inputTokens;
+        if (source.outputTokens !== undefined)
+            target.outputTokens = (target.outputTokens ?? 0) + source.outputTokens;
+        if (source.cacheRead !== undefined)
+            target.cacheRead = (target.cacheRead ?? 0) + source.cacheRead;
+        if (source.cacheWrite !== undefined)
+            target.cacheWrite = (target.cacheWrite ?? 0) + source.cacheWrite;
+        if (source.costMicroUsd !== undefined)
+            target.costMicroUsd = (target.costMicroUsd ?? 0) + source.costMicroUsd;
+    }
+    for (const field of JEV_FIELDS)
+        into.jev[field] += from.jev[field];
+    if (from.efficiency)
+        mergeEfficiency(into.efficiency ??= createEfficiencyTally(), from.efficiency);
+    into.task.documentation += from.task.documentation;
+    into.task.test += from.task.test;
+    into.task.otherEdits += from.task.otherEdits;
+    into.task.created += from.task.created;
+    into.task.testCommand ||= from.task.testCommand;
+    into.task.activity ||= from.task.activity;
+    into.providerOverflow ||= from.providerOverflow;
+    // Attribution follows the request, so a request still in flight keeps its own invocation entry;
+    // a fallback that arrives without one is attributed to the merged total as before.
+    for (const [invocation, questions] of Object.entries(from.pendingByInvocation))
+        into.pendingByInvocation[invocation] = questions;
+    const keyedPending = Object.values(into.pendingByInvocation).reduce((sum, questions) => sum + questions, 0);
+    into.pendingQuestions = keyedPending === 0 ? from.pendingQuestions : keyedPending;
+    return into;
+}

package/dist/src/telemetry/capture.d.ts

@@ -0,0 +1,5 @@
+/** Model/tool capture is unchanged; decision rounds require per-request accounting. */
+export * from './capture-core.js';
+import { type RunTelemetry } from './capture-core.js';
+export declare function recordDecisionEvent(telemetry: RunTelemetry, type: string, data: Record<string, unknown>): void;
+export declare function recordEfficiencyEvent(telemetry: RunTelemetry, type: string, data: Record<string, unknown>): void;

package/dist/src/telemetry/capture.js

@@ -0,0 +1,16 @@
+/** Model/tool capture is unchanged; decision rounds require per-request accounting. */
+export * from './capture-core.js';
+import { recordDecisionEvent as legacyDecisionEvent, recordEfficiencyEvent as coreEfficiencyEvent } from './capture-core.js';
+import { captureDecision } from './decision-capture.js';
+export function recordDecisionEvent(telemetry, type, data) {
+    if (!captureDecision(telemetry, type, data))
+        legacyDecisionEvent(telemetry, type, data);
+}
+export function recordEfficiencyEvent(telemetry, type, data) {
+    // Facts are awaited post-tool work too. Fold elapsed batch waits, not the sum of parallel
+    // network latencies, into the existing local critical-path accounting.
+    if (type === 'efficiency.facts.answered')
+        coreEfficiencyEvent(telemetry, 'efficiency.postflight.awaited', data);
+    else
+        coreEfficiencyEvent(telemetry, type, data);
+}

package/dist/src/telemetry/classify.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Deterministic local classification.
+ *
+ * The ADR replaces free text with closed, locally derived labels: a failure class from the
+ * runtime's own reason, a task category from the tool calls and local path comparisons the run
+ * already made, and a language/framework mix from file extensions and manifest files. Nothing here
+ * reads a prompt, a response, or a file's contents beyond a package manifest, and nothing here
+ * sends a path — the paths are compared and discarded.
+ */
+import { type TelemetryFailureClass, type TelemetryFramework, type TelemetryLanguage, type TelemetryOutcome, type TelemetryTaskCategory, type TelemetryWorkspaceBucket } from './schema.js';
+/** The statuses a finalizer may hold: the run-record names plus the runtime's own result statuses. */
+export type ClassifiableStatus = 'succeeded' | 'exhausted' | 'cancelled' | 'infrastructure_error' | 'needs_clarification' | 'interrupted';
+export interface FailureEvidence {
+    reason?: string;
+    /** Set by the runtime when a `ProviderRequestError` ended the run. */
+    providerRequestFailed?: boolean;
+    /** Set when the ended request was the classified context-overflow condition. */
+    providerOverflow?: boolean;
+}
+/**
+ * Map a terminal status onto the ADR's total outcome/failure mapping.
+ *
+ * The last `exhausted` row is the important one: the runtime's reason set is not closed, and one
+ * value embeds a user-chosen graph node id. An unrecognized reason degrades to `limit` rather than
+ * being sent as text or rejecting the record.
+ */
+export declare function classifyOutcome(status: ClassifiableStatus, evidence?: FailureEvidence): {
+    outcome: TelemetryOutcome;
+    failureClass?: TelemetryFailureClass;
+};
+/** The language a path names, if the table knows its extension. */
+export declare function languageForPath(filePath: string): TelemetryLanguage | undefined;
+/** A documentation path, by extension or by a basename a project uses for the same purpose. */
+export declare function isDocumentationPath(filePath: string): boolean;
+/** A test path: a conventional directory, or a conventional file suffix. */
+export declare function isTestPath(filePath: string): boolean;
+/**
+ * Whether an executed command is a test command.
+ *
+ * Closed for schema version 1: the executable and first argument naming `test`, or one of the
+ * fixed runner names. Changing this list is a schema change.
+ */
+export declare function isTestCommand(argv: readonly string[]): boolean;
+/**
+ * Tokenize a shell command line just far enough to recognize a test runner.
+ *
+ * The command is never sent: this runs locally to answer one closed question.
+ */
+export declare function isTestCommandLine(command: string): boolean;
+export interface TaskEvidence {
+    /** Paths written or edited by the run, in the order the run touched them. */
+    editedPaths: string[];
+    /** The subset of `editedPaths` that did not exist when the run started. */
+    createdPaths: string[];
+    ranTestCommand: boolean;
+    /** Whether the run made any tool call at all. A run with none reports no task category. */
+    hadToolActivity: boolean;
+}
+/**
+ * The task category, from the fixed ordered rules.
+ *
+ * `refactor` and `review` are absent because neither is observable without reading the task.
+ */
+export declare function classifyTaskCategory(evidence: TaskEvidence): TelemetryTaskCategory | undefined;
+export interface WorkspaceShape {
+    languages: TelemetryLanguage[];
+    frameworks: TelemetryFramework[];
+    workspaceFiles: TelemetryWorkspaceBucket;
+}
+export interface WorkspaceObservation {
+    /** Paths found by the bounded scan, relative to the workspace root. */
+    filePaths: string[];
+    /** Total files seen by the scan, which may exceed the retained path list. */
+    totalFiles: number;
+    /** Manifest basenames mapped to their bounded contents. */
+    manifests: Record<string, string>;
+    /** A `.csproj` file seen by the scan. */
+    sawCsproj?: boolean;
+}
+/**
+ * The language and framework mix, ranked by matching file count with ties in table order, capped at
+ * five distinct values. Unknown values are omitted rather than bucketed, because a workspace whose
+ * mix is unknown reports fewer values rather than a misleading `other`.
+ */
+export declare function classifyWorkspace(observation: WorkspaceObservation): WorkspaceShape;
+/** The retained form of `TaskEvidence`: counts instead of paths, so nothing user-chosen persists. */
+export interface TaskTallyInput {
+    documentation: number;
+    test: number;
+    otherEdits: number;
+    created: number;
+    testCommand: boolean;
+    activity: boolean;
+}
+/**
+ * The same ordered rules over retained counts.
+ *
+ * This is the form the runtime capture uses: the pictures the rules need are "every edit was
+ * documentation", "every edit was a test", and "an edit created a file", and counts answer all
+ * three without keeping a path.
+ */
+export declare function classifyTaskCounts(evidence: TaskTallyInput): TelemetryTaskCategory | undefined;