@aid-framework/cli 0.1.59 → 0.1.60

package/dist/trace.js

@@ -0,0 +1,212 @@
+/**
+ * Reading the local trace sink back into numbers.
+ *
+ * The format is the one `runtimes/{python,typescript}/src/.../local_trace` writes: one
+ * JSON object per line, one line per span. This module is the third consumer of that
+ * contract and the only one written in TypeScript that does not live in the runtime, so
+ * it repeats the field names in one place rather than importing a runtime that drags a
+ * provider SDK into the CLI.
+ *
+ * Two rules keep the report honest:
+ *
+ * - A span that cannot be parsed is **counted and skipped**, never guessed at. A crash
+ *   truncated the last line of a file is normal; a line that is not a span at all is not,
+ *   and the caller reports the count either way.
+ * - A span with tokens but no `aid.cost.usd` is **unpriced**, not free. Summing it as
+ *   zero would make a missing cost look like a cheap run, which is the same failure the
+ *   IR's cost gate refuses.
+ */
+/** The attribute keys the report reads, from the runtime's own conventions table. */
+const ATTRIBUTES = {
+    alias: 'aid.model.alias',
+    costUsd: 'aid.cost.usd',
+    inputTokens: 'gen_ai.usage.input_tokens',
+    outputTokens: 'gen_ai.usage.output_tokens',
+    responseModel: 'gen_ai.response.model',
+    requestModel: 'gen_ai.request.model',
+};
+const UNKNOWN_MODEL = '(unknown)';
+/** The model a span is attributed to: the pinned alias, else the model the provider served. */
+function modelOf(span) {
+    const attributes = span.attributes ?? {};
+    for (const key of [ATTRIBUTES.alias, ATTRIBUTES.responseModel, ATTRIBUTES.requestModel]) {
+        const value = attributes[key];
+        if (typeof value === 'string' && value !== '')
+            return value;
+    }
+    return UNKNOWN_MODEL;
+}
+/** A non-negative finite number, accepting the string form an attribute may carry. */
+function numeric(value) {
+    if (typeof value === 'number')
+        return Number.isFinite(value) && value >= 0 ? value : undefined;
+    if (typeof value === 'string' && value.trim() !== '') {
+        const parsed = Number(value);
+        return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
+    }
+    return undefined;
+}
+function isSpan(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const candidate = value;
+    return (typeof candidate.trace_id === 'string' &&
+        typeof candidate.span_id === 'string' &&
+        typeof candidate.name === 'string' &&
+        typeof candidate.started_at === 'string');
+}
+/**
+ * Parse one JSONL file. A blank line is not a failure -- the sink never writes one, but
+ * a file being appended to while it is read can end in a partial line, and that is the
+ * one case a reader has to tolerate.
+ */
+export function parseTraceLines(text) {
+    const spans = [];
+    let skipped = 0;
+    for (const line of text.split(/\r?\n/)) {
+        if (line.trim() === '')
+            continue;
+        try {
+            const parsed = JSON.parse(line);
+            if (isSpan(parsed))
+                spans.push(parsed);
+            else
+                skipped += 1;
+        }
+        catch {
+            skipped += 1;
+        }
+    }
+    return { spans, skipped };
+}
+/**
+ * Normalise `--since` to the ISO instant the sink writes, or `undefined` when the value
+ * is not a date. A plain `YYYY-MM-DD` is UTC midnight; anything else must be ISO 8601
+ * with an explicit offset, because a bare local time would mean a different instant on
+ * every machine reading the same repository.
+ */
+export function parseSince(value) {
+    const trimmed = value.trim();
+    const day = /^(\d{4})-(\d{2})-(\d{2})$/;
+    if (day.test(trimmed)) {
+        const date = new Date(`${trimmed}T00:00:00.000Z`);
+        // A round trip rejects 2026-02-31, which `Date` would otherwise roll into March.
+        if (Number.isNaN(date.getTime()) || date.toISOString().slice(0, 10) !== trimmed) {
+            return undefined;
+        }
+        return date.toISOString();
+    }
+    if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/.test(trimmed)) {
+        return undefined;
+    }
+    const date = new Date(trimmed);
+    return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
+}
+/** Nearest-rank percentile: the smallest value at or above the given fraction. */
+function percentile(sorted, fraction) {
+    if (sorted.length === 0)
+        return 0;
+    const rank = Math.ceil(fraction * sorted.length);
+    const index = Math.min(Math.max(rank - 1, 0), sorted.length - 1);
+    return sorted[index] ?? 0;
+}
+/**
+ * Aggregate spans into one report.
+ *
+ * `since` filters on `started_at` by string comparison, which is exact because both
+ * sides are the UTC ISO form the sink writes.
+ */
+export function summariseSpans(spans, options = {}) {
+    const counted = options.since === undefined
+        ? spans
+        : spans.filter((span) => span.started_at >= options.since);
+    const traceIds = new Set();
+    const byModel = new Map();
+    const durations = [];
+    let errors = 0;
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let costUsd = 0;
+    let unpricedSpans = 0;
+    for (const span of counted) {
+        traceIds.add(span.trace_id);
+        if (span.status?.code === 'ERROR')
+            errors += 1;
+        const attributes = span.attributes ?? {};
+        const cost = numeric(attributes[ATTRIBUTES.costUsd]);
+        const input = numeric(attributes[ATTRIBUTES.inputTokens]);
+        const output = numeric(attributes[ATTRIBUTES.outputTokens]);
+        const hasUsage = input !== undefined || output !== undefined;
+        if (cost !== undefined)
+            costUsd += cost;
+        if (input !== undefined)
+            inputTokens += input;
+        if (output !== undefined)
+            outputTokens += output;
+        if (hasUsage && cost === undefined)
+            unpricedSpans += 1;
+        if (hasUsage) {
+            const duration = span.duration_ms;
+            if (typeof duration === 'number' && Number.isFinite(duration) && duration >= 0) {
+                durations.push(duration);
+            }
+        }
+        if (!hasUsage && cost === undefined)
+            continue;
+        const model = modelOf(span);
+        const existing = byModel.get(model);
+        const addition = {
+            input: input ?? 0,
+            output: output ?? 0,
+            cost: cost ?? 0,
+            unpriced: hasUsage && cost === undefined ? 1 : 0,
+        };
+        if (existing === undefined) {
+            byModel.set(model, {
+                model,
+                calls: 1,
+                inputTokens: addition.input,
+                outputTokens: addition.output,
+                costUsd: addition.cost,
+                unpricedCalls: addition.unpriced,
+            });
+        }
+        else {
+            byModel.set(model, {
+                model,
+                calls: existing.calls + 1,
+                inputTokens: existing.inputTokens + addition.input,
+                outputTokens: existing.outputTokens + addition.output,
+                costUsd: existing.costUsd + addition.cost,
+                unpricedCalls: existing.unpricedCalls + addition.unpriced,
+            });
+        }
+    }
+    // Priciest first, then busiest, then by name, so two runs over the same spans print
+    // the same order without depending on the order the files happened to be read in.
+    const models = [...byModel.values()].sort((left, right) => right.costUsd - left.costUsd ||
+        right.calls - left.calls ||
+        (left.model < right.model ? -1 : left.model > right.model ? 1 : 0));
+    durations.sort((left, right) => left - right);
+    const summary = {
+        spans: counted.length,
+        traces: traceIds.size,
+        errors,
+        inputTokens,
+        outputTokens,
+        costUsd,
+        unpricedSpans,
+        models,
+        ...(durations.length === 0
+            ? {}
+            : {
+                latency: {
+                    p50: Math.round(percentile(durations, 0.5)),
+                    p95: Math.round(percentile(durations, 0.95)),
+                    samples: durations.length,
+                },
+            }),
+    };
+    return summary;
+}
+//# sourceMappingURL=trace.js.map

package/dist/trace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace.js","sourceRoot":"","sources":["../src/trace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAqBH,qFAAqF;AACrF,MAAM,UAAU,GAAG;IACjB,KAAK,EAAE,iBAAiB;IACxB,OAAO,EAAE,cAAc;IACvB,WAAW,EAAE,2BAA2B;IACxC,YAAY,EAAE,4BAA4B;IAC1C,aAAa,EAAE,uBAAuB;IACtC,YAAY,EAAE,sBAAsB;CAC5B,CAAC;AAEX,MAAM,aAAa,GAAG,WAAW,CAAC;AAElC,+FAA+F;AAC/F,SAAS,OAAO,CAAC,IAAe;IAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACxF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,EAAE;YAAE,OAAO,KAAK,CAAC;IAC9D,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,sFAAsF;AACtF,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/F,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,SAAS,GAAG,KAAgC,CAAC;IACnD,OAAO,CACL,OAAO,SAAS,CAAC,QAAQ,KAAK,QAAQ;QACtC,OAAO,SAAS,CAAC,OAAO,KAAK,QAAQ;QACrC,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ;QAClC,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ,CACzC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QACjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;gBAClC,OAAO,IAAI,CAAC,CAAC;QACpB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,2BAA2B,CAAC;IACxC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,OAAO,gBAAgB,CAAC,CAAC;QAClD,iFAAiF;QACjF,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC;YAChF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,CAAC,qEAAqE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACzF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;AACvE,CAAC;AAkCD,kFAAkF;AAClF,SAAS,UAAU,CAAC,MAAyB,EAAE,QAAgB;IAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjE,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,KAA2B,EAC3B,OAAO,GAAgC,EAAE;IAEzC,MAAM,OAAO,GACX,OAAO,CAAC,KAAK,KAAK,SAAS;QACzB,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,IAAK,OAAO,CAAC,KAAgB,CAAC,CAAC;IAE3E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,KAAK,OAAO;YAAE,MAAM,IAAI,CAAC,CAAC;QAE/C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,CAAC;QAE7D,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,IAAI,CAAC;QACxC,IAAI,KAAK,KAAK,SAAS;YAAE,WAAW,IAAI,KAAK,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS;YAAE,YAAY,IAAI,MAAM,CAAC;QACjD,IAAI,QAAQ,IAAI,IAAI,KAAK,SAAS;YAAE,aAAa,IAAI,CAAC,CAAC;QAEvD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAC/E,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,SAAS;YAAE,SAAS;QAE9C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG;YACf,KAAK,EAAE,KAAK,IAAI,CAAC;YACjB,MAAM,EAAE,MAAM,IAAI,CAAC;YACnB,IAAI,EAAE,IAAI,IAAI,CAAC;YACf,QAAQ,EAAE,QAAQ,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACjD,CAAC;QAEF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE;gBACjB,KAAK;gBACL,KAAK,EAAE,CAAC;gBACR,WAAW,EAAE,QAAQ,CAAC,KAAK;gBAC3B,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,OAAO,EAAE,QAAQ,CAAC,IAAI;gBACtB,aAAa,EAAE,QAAQ,CAAC,QAAQ;aACjC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE;gBACjB,KAAK;gBACL,KAAK,EAAE,QAAQ,CAAC,KAAK,GAAG,CAAC;gBACzB,WAAW,EAAE,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,KAAK;gBAClD,YAAY,EAAE,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,MAAM;gBACrD,OAAO,EAAE,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,IAAI;gBACzC,aAAa,EAAE,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC,QAAQ;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,kFAAkF;IAClF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CACvC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO;QAC5B,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;QACxB,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACrE,CAAC;IAEF,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAgB;QAC3B,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM,EAAE,QAAQ,CAAC,IAAI;QACrB,MAAM;QACN,WAAW;QACX,YAAY;QACZ,OAAO;QACP,aAAa;QACb,MAAM;QACN,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YACxB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC;gBACE,OAAO,EAAE;oBACP,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;oBAC3C,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;oBAC5C,OAAO,EAAE,SAAS,CAAC,MAAM;iBAC1B;aACF,CAAC;KACP,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aid-framework/cli",
-  "version": "0.1.59",
+  "version": "0.1.60",
   "description": "The `aid` command: scaffold a project, validate its spec, generate its code, and run its eval gate.",
   "license": "Apache-2.0",
   "repository": {
@@ -33,11 +33,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@aid-framework/generator-py-fastapi": "0.1.59",
-    "@aid-framework/generator-sdk": "0.1.59",
-    "@aid-framework/generator-ts-vercel": "0.1.59",
-    "@aid-framework/ir": "0.1.59",
-    "@aid-framework/spec": "0.1.59"
+    "@aid-framework/generator-py-fastapi": "0.1.60",
+    "@aid-framework/generator-sdk": "0.1.60",
+    "@aid-framework/generator-ts-vercel": "0.1.60",
+    "@aid-framework/ir": "0.1.60",
+    "@aid-framework/spec": "0.1.60"
   },
   "devDependencies": {
     "@types/node": "^26.6.5"