@bububuger/spanory 0.1.16 → 0.1.18

package/dist/runtime/shared/normalize.js

@@ -1,5 +1,6 @@
 // @ts-nocheck
 import { createHash } from 'node:crypto';
+import { calibratedEstimate, calibrate, CONTEXT_SOURCE_KINDS, estimateTokens, pollutionScoreV1, } from '@bububuger/core';
 function toNumber(value) {
     const n = Number(value);
     return Number.isFinite(n) ? n : undefined;
@@ -48,24 +49,15 @@ function usageAttributes(usage) {
         attrs['gen_ai.usage.total_tokens'] = usage.total_tokens;
     }
     if (usage.cache_read_input_tokens !== undefined) {
-        attrs['gen_ai.usage.details.cache_read_input_tokens'] = usage.cache_read_input_tokens;
+        attrs['gen_ai.usage.cache_read.input_tokens'] = usage.cache_read_input_tokens;
     }
     if (usage.cache_creation_input_tokens !== undefined) {
-        attrs['gen_ai.usage.details.cache_creation_input_tokens'] = usage.cache_creation_input_tokens;
+        attrs['gen_ai.usage.cache_creation.input_tokens'] = usage.cache_creation_input_tokens;
     }
     const cacheRead = usage.cache_read_input_tokens ?? 0;
     const denominator = (usage.input_tokens ?? 0) + cacheRead;
     const cacheHitRate = denominator > 0 ? cacheRead / denominator : 0;
     attrs['gen_ai.usage.details.cache_hit_rate'] = Number(cacheHitRate.toFixed(6));
-    attrs['langfuse.observation.usage_details'] = JSON.stringify({
-        ...(usage.input_tokens !== undefined ? { input: usage.input_tokens } : {}),
-        ...(usage.output_tokens !== undefined ? { output: usage.output_tokens } : {}),
-        ...(usage.total_tokens !== undefined ? { total: usage.total_tokens } : {}),
-        ...(usage.cache_read_input_tokens !== undefined ? { input_cache_read: usage.cache_read_input_tokens } : {}),
-        ...(usage.cache_creation_input_tokens !== undefined
-            ? { input_cache_creation: usage.cache_creation_input_tokens }
-            : {}),
-    });
     return attrs;
 }
 function modelAttributes(model) {
@@ -217,7 +209,6 @@ function parseBashCommandAttributes(commandLine) {
             'agentic.command.name': '',
             'agentic.command.args': '',
             'agentic.command.pipe_count': 0,
-            'agentic.command.raw': '',
         };
     }
     const segments = raw.split(/\|(?!\|)/);
@@ -229,7 +220,6 @@ function parseBashCommandAttributes(commandLine) {
         'agentic.command.name': name,
         'agentic.command.args': args,
         'agentic.command.pipe_count': Math.max(segments.length - 1, 0),
-        'agentic.command.raw': raw,
     };
 }
 function isMcpToolName(name) {
@@ -268,6 +258,368 @@ function similarityScore(a, b) {
         return 1;
     return Number((intersection / union).toFixed(6));
 }
+const DEFAULT_CONTEXT_WINDOW_TOKENS = 200000;
+const CONTEXT_ENABLED_RUNTIMES = new Set(['claude-code', 'codex', 'openclaw', 'opencode']);
+const CONTEXT_PARSING_ENABLED = process.env.SPANORY_CONTEXT_ENABLED !== '0';
+function contextWindowTokens() {
+    const raw = Number(process.env.SPANORY_CONTEXT_WINDOW_TOKENS);
+    if (Number.isFinite(raw) && raw > 0)
+        return Math.floor(raw);
+    return DEFAULT_CONTEXT_WINDOW_TOKENS;
+}
+function clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+}
+function round6(value) {
+    return Number(value.toFixed(6));
+}
+function detectCompactInferred(currentTokens, previousTokens) {
+    if (!(previousTokens > 0))
+        return { detected: false, compactionRatio: 0 };
+    const dropRatio = (previousTokens - currentTokens) / previousTokens;
+    if (dropRatio > 0.4)
+        return { detected: true, compactionRatio: round6(dropRatio) };
+    return { detected: false, compactionRatio: 0 };
+}
+function createSourceDeltaMap() {
+    const out = {};
+    for (const kind of CONTEXT_SOURCE_KINDS)
+        out[kind] = 0;
+    return out;
+}
+function addSourceDelta(map, kind, delta) {
+    if (!kind || !Object.prototype.hasOwnProperty.call(map, kind))
+        return;
+    const n = Number(delta);
+    if (!Number.isFinite(n) || n <= 0)
+        return;
+    map[kind] += n;
+}
+function moveSourceDelta(map, fromKind, toKind, delta) {
+    const n = Number(delta);
+    if (!Number.isFinite(n) || n <= 0)
+        return;
+    if (!Object.prototype.hasOwnProperty.call(map, fromKind))
+        return;
+    if (!Object.prototype.hasOwnProperty.call(map, toKind))
+        return;
+    const moved = Math.min(map[fromKind], n);
+    if (moved <= 0)
+        return;
+    map[fromKind] -= moved;
+    map[toKind] += moved;
+}
+function parseJsonObject(raw) {
+    const text = String(raw ?? '').trim();
+    if (!text || !text.startsWith('{'))
+        return null;
+    try {
+        const parsed = JSON.parse(text);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+            return parsed;
+    }
+    catch {
+        // ignore parse errors
+    }
+    return null;
+}
+function extractMentionFileSignals(text) {
+    const input = String(text ?? '');
+    if (!input)
+        return [];
+    const matches = input.match(/(?:\/|\b)[^\s"'`]+?\.[a-zA-Z0-9]{1,8}\b/g) ?? [];
+    return [...new Set(matches.map((m) => m.trim()).filter(Boolean))];
+}
+function extractSourceName(event) {
+    const attrs = event?.attributes ?? {};
+    return String(attrs['gen_ai.tool.name'] ?? attrs['agentic.command.name'] ?? event?.name ?? '').trim();
+}
+function classifyContextSignals(turnEvents) {
+    const sourceDelta = createSourceDeltaMap();
+    const sourceNames = new Map();
+    let compactRequested = false;
+    let restoreRequested = false;
+    function markSourceName(kind, name) {
+        const text = String(name ?? '').trim();
+        if (!text)
+            return;
+        if (!sourceNames.has(kind))
+            sourceNames.set(kind, new Set());
+        sourceNames.get(kind).add(text);
+    }
+    function absorbFileSignals(text, fromKind) {
+        const mentions = extractMentionFileSignals(text);
+        if (!mentions.length)
+            return;
+        const mentionTokens = estimateTokens(mentions.join('\n'));
+        addSourceDelta(sourceDelta, 'mention_file', mentionTokens);
+        moveSourceDelta(sourceDelta, fromKind, 'mention_file', mentionTokens);
+        markSourceName('mention_file', mentions[0]);
+        const hasClaudeMd = mentions.some((item) => /(?:^|\/)(?:claude|agents)\.md$/i.test(item));
+        if (hasClaudeMd) {
+            addSourceDelta(sourceDelta, 'claude_md', mentionTokens);
+            moveSourceDelta(sourceDelta, 'mention_file', 'claude_md', mentionTokens);
+            markSourceName('claude_md', mentions.find((item) => /(?:^|\/)(?:claude|agents)\.md$/i.test(item)));
+        }
+    }
+    for (const event of turnEvents) {
+        const category = String(event?.category ?? '');
+        const attrs = event?.attributes ?? {};
+        const input = String(event?.input ?? '');
+        const output = String(event?.output ?? '');
+        const inputTokens = estimateTokens(input);
+        const outputTokens = estimateTokens(output);
+        const sourceName = extractSourceName(event);
+        if (category === 'turn') {
+            addSourceDelta(sourceDelta, 'turn', inputTokens + outputTokens);
+            absorbFileSignals(input, 'turn');
+            const sender = String(attrs['agentic.input.sender'] ?? '').toLowerCase();
+            if (sender === 'system') {
+                addSourceDelta(sourceDelta, 'system_prompt', inputTokens);
+                moveSourceDelta(sourceDelta, 'turn', 'system_prompt', inputTokens);
+                markSourceName('system_prompt', 'system_input');
+            }
+            if (/\b(delegate|handoff|coordinate|sync)\b/i.test(input)) {
+                const coordinationTokens = Math.min(estimateTokens(input), Math.max(1, Math.floor(inputTokens / 2)));
+                addSourceDelta(sourceDelta, 'team_coordination', coordinationTokens);
+                moveSourceDelta(sourceDelta, 'turn', 'team_coordination', coordinationTokens);
+                markSourceName('team_coordination', 'turn_coordination');
+            }
+            continue;
+        }
+        if (category === 'agent_command') {
+            const commandName = String(attrs['agentic.command.name'] ?? '').trim().toLowerCase();
+            addSourceDelta(sourceDelta, 'skill', inputTokens + outputTokens);
+            markSourceName('skill', sourceName ? `/${sourceName}` : 'slash_command');
+            if (commandName === 'compact')
+                compactRequested = true;
+            if (commandName === 'restore' || commandName === 'resume')
+                restoreRequested = true;
+            continue;
+        }
+        if (category === 'agent_task') {
+            addSourceDelta(sourceDelta, 'subagent', inputTokens + outputTokens);
+            addSourceDelta(sourceDelta, 'team_coordination', inputTokens);
+            markSourceName('subagent', sourceName || 'Task');
+            markSourceName('team_coordination', sourceName || 'Task');
+            continue;
+        }
+        if (category === 'tool' || category === 'mcp' || category === 'shell_command') {
+            addSourceDelta(sourceDelta, 'tool_input', inputTokens);
+            addSourceDelta(sourceDelta, 'tool_output', outputTokens);
+            if (sourceName) {
+                markSourceName('tool_input', sourceName);
+                markSourceName('tool_output', sourceName);
+            }
+            absorbFileSignals(input, 'tool_input');
+            const toolName = String(attrs['gen_ai.tool.name'] ?? '').toLowerCase();
+            if (toolName.includes('memory') || /\bmemory\b/i.test(input)) {
+                const memoryTokens = Math.max(1, Math.floor(inputTokens * 0.6));
+                addSourceDelta(sourceDelta, 'memory', memoryTokens);
+                moveSourceDelta(sourceDelta, 'tool_input', 'memory', memoryTokens);
+                markSourceName('memory', sourceName || 'memory');
+            }
+            continue;
+        }
+    }
+    const knownTotal = CONTEXT_SOURCE_KINDS
+        .filter((kind) => kind !== 'unknown')
+        .reduce((acc, kind) => acc + Number(sourceDelta[kind] ?? 0), 0);
+    if (knownTotal <= 0) {
+        sourceDelta.unknown = 1;
+        markSourceName('unknown', 'unclassified');
+    }
+    return { sourceDelta, sourceNames, compactRequested, restoreRequested };
+}
+function composeContextEvents({ runtime, projectId, sessionId, turnEvent, turnEvents, previousEstimatedTotal, recentSourceKindsWindow, calibrationState, }) {
+    const attrs = turnEvent?.attributes ?? {};
+    const usageInput = Number(attrs['gen_ai.usage.input_tokens'] ?? 0);
+    const usageCacheRead = Number(attrs['gen_ai.usage.cache_read.input_tokens'] ?? 0);
+    const usageEstimated = Math.max(0, usageInput + usageCacheRead);
+    const fallbackEstimated = estimateTokens(`${turnEvent?.input ?? ''}\n${turnEvent?.output ?? ''}`);
+    let estimationMethod = 'heuristic';
+    let estimationConfidence = 0.4;
+    let estimatedTotalTokens = Math.max(0, fallbackEstimated);
+    let nextCalibrationState = calibrationState;
+    if (usageEstimated > 0) {
+        estimationMethod = 'usage';
+        estimationConfidence = 1;
+        estimatedTotalTokens = usageEstimated;
+        nextCalibrationState = calibrate(calibrationState, usageEstimated, Math.max(fallbackEstimated, 1));
+    }
+    else if (Number(calibrationState?.sampleCount ?? 0) >= 2) {
+        estimationMethod = 'calibrated';
+        estimationConfidence = round6(0.7 + 0.03 * Math.min(Number(calibrationState.sampleCount ?? 0), 10));
+        estimatedTotalTokens = Math.max(0, calibratedEstimate(fallbackEstimated, calibrationState));
+    }
+    const { sourceDelta, sourceNames, compactRequested, restoreRequested } = classifyContextSignals(turnEvents);
+    const deltaTokens = previousEstimatedTotal > 0 ? (estimatedTotalTokens - previousEstimatedTotal) : 0;
+    const windowLimitTokens = contextWindowTokens();
+    const fillRatio = round6(clamp(estimatedTotalTokens / windowLimitTokens, 0, 1));
+    const compositionEntries = Object.entries(sourceDelta)
+        .filter(([, value]) => Number(value) > 0)
+        .sort((a, b) => Number(b[1]) - Number(a[1]));
+    const activeKinds = compositionEntries.map(([kind]) => kind);
+    const nextRecentSourceKindsWindow = [...recentSourceKindsWindow, activeKinds].slice(-5);
+    const repeatCountByKind = {};
+    for (const kind of CONTEXT_SOURCE_KINDS) {
+        repeatCountByKind[kind] = nextRecentSourceKindsWindow.reduce((count, row) => count + (row.includes(kind) ? 1 : 0), 0);
+    }
+    const totalDelta = compositionEntries.reduce((sum, [, value]) => sum + Number(value), 0);
+    const composition = Object.fromEntries(compositionEntries);
+    const topSources = compositionEntries.slice(0, 3).map(([kind]) => kind);
+    const passthroughAttrs = {};
+    const passthroughKeys = [
+        'gen_ai.agent.id',
+        'agentic.parent.session_id',
+        'agentic.parent.turn_id',
+        'agentic.parent.tool_call_id',
+        'agentic.parent.link.confidence',
+        'agentic.runtime.version',
+    ];
+    for (const key of passthroughKeys) {
+        if (attrs[key] !== undefined)
+            passthroughAttrs[key] = attrs[key];
+    }
+    const baseAttrs = {
+        'agentic.event.category': 'context',
+        'agentic.context.event_type': 'context_snapshot',
+        'agentic.context.estimated_total_tokens': estimatedTotalTokens,
+        'agentic.context.fill_ratio': fillRatio,
+        'agentic.context.delta_tokens': deltaTokens,
+        'agentic.context.composition': JSON.stringify(composition),
+        'agentic.context.top_sources': JSON.stringify(topSources),
+        'agentic.context.estimation_method': estimationMethod,
+        'agentic.context.estimation_confidence': estimationConfidence,
+        ...passthroughAttrs,
+    };
+    const out = [
+        {
+            runtime,
+            projectId,
+            sessionId,
+            turnId: turnEvent.turnId,
+            category: 'context',
+            name: 'Context Snapshot',
+            startedAt: turnEvent.startedAt,
+            endedAt: turnEvent.endedAt,
+            input: '',
+            output: '',
+            attributes: baseAttrs,
+        },
+    ];
+    if (compactRequested && previousEstimatedTotal > 0) {
+        out.push({
+            runtime,
+            projectId,
+            sessionId,
+            turnId: turnEvent.turnId,
+            category: 'context',
+            name: 'Context Boundary',
+            startedAt: turnEvent.startedAt,
+            endedAt: turnEvent.startedAt,
+            input: '',
+            output: '',
+            attributes: {
+                'agentic.event.category': 'context',
+                'agentic.context.event_type': 'context_boundary',
+                'agentic.context.boundary_kind': 'compact_before',
+                'agentic.context.compaction_ratio': 0,
+                'agentic.context.detection_method': 'hook',
+                ...passthroughAttrs,
+            },
+        });
+    }
+    const inferred = detectCompactInferred(estimatedTotalTokens, previousEstimatedTotal);
+    if (inferred.detected) {
+        out.push({
+            runtime,
+            projectId,
+            sessionId,
+            turnId: turnEvent.turnId,
+            category: 'context',
+            name: 'Context Boundary',
+            startedAt: turnEvent.startedAt,
+            endedAt: turnEvent.endedAt,
+            input: '',
+            output: '',
+            attributes: {
+                'agentic.event.category': 'context',
+                'agentic.context.event_type': 'context_boundary',
+                'agentic.context.boundary_kind': 'compact_after',
+                'agentic.context.compaction_ratio': inferred.compactionRatio,
+                'agentic.context.detection_method': 'inferred',
+                ...passthroughAttrs,
+            },
+        });
+    }
+    if (restoreRequested) {
+        out.push({
+            runtime,
+            projectId,
+            sessionId,
+            turnId: turnEvent.turnId,
+            category: 'context',
+            name: 'Context Boundary',
+            startedAt: turnEvent.startedAt,
+            endedAt: turnEvent.startedAt,
+            input: '',
+            output: '',
+            attributes: {
+                'agentic.event.category': 'context',
+                'agentic.context.event_type': 'context_boundary',
+                'agentic.context.boundary_kind': 'restore',
+                'agentic.context.compaction_ratio': 0,
+                'agentic.context.detection_method': 'hook',
+                ...passthroughAttrs,
+            },
+        });
+    }
+    for (const [kind, value] of compositionEntries) {
+        const tokenDelta = Number(value);
+        if (tokenDelta <= 0)
+            continue;
+        const sourceShare = totalDelta > 0 ? round6(tokenDelta / totalDelta) : 0;
+        const repeatCountRecent = Number(repeatCountByKind[kind] ?? 0);
+        const names = [...(sourceNames.get(kind) ?? [])];
+        out.push({
+            runtime,
+            projectId,
+            sessionId,
+            turnId: turnEvent.turnId,
+            category: 'context',
+            name: 'Context Source Attribution',
+            startedAt: turnEvent.startedAt,
+            endedAt: turnEvent.endedAt,
+            input: '',
+            output: '',
+            attributes: {
+                'agentic.event.category': 'context',
+                'agentic.context.event_type': 'context_source_attribution',
+                'agentic.context.source_kind': kind,
+                'agentic.context.source_name': names[0] ?? kind,
+                'agentic.context.token_delta': tokenDelta,
+                'agentic.context.source_share': sourceShare,
+                'agentic.context.repeat_count_recent': repeatCountRecent,
+                'agentic.context.pollution_score': pollutionScoreV1({
+                    tokenDelta,
+                    windowLimitTokens,
+                    sourceShare,
+                    repeatCountRecent,
+                    sourceKind: kind,
+                }),
+                'agentic.context.score_version': 'pollution_score_v1',
+                ...passthroughAttrs,
+            },
+        });
+    }
+    return {
+        events: out,
+        estimatedTotalTokens,
+        recentSourceKindsWindow: nextRecentSourceKindsWindow,
+        calibrationState: nextCalibrationState,
+    };
+}
 function actorHeuristic(messages) {
     const hasSidechainSignal = messages.some((m) => m?.isSidechain === true || (typeof m?.agentId === 'string' && m.agentId.trim().length > 0));
     if (hasSidechainSignal)
@@ -304,8 +656,9 @@ function inferParentLinkAttributes(messages) {
         ?? m?.sessionMeta?.parentToolCallId));
     const explicitConfidence = firstNonEmptyString(messages.map((m) => m?.parentLinkConfidence ?? m?.parent_link_confidence));
     const attrs = {};
-    if (agentId)
-        attrs['agentic.agent_id'] = agentId;
+    if (agentId) {
+        attrs['gen_ai.agent.id'] = agentId;
+    }
     if (parentSessionId)
         attrs['agentic.parent.session_id'] = parentSessionId;
     if (parentTurnId)
@@ -602,6 +955,9 @@ export function normalizeTranscriptMessages({ runtime, projectId, sessionId, mes
     let previousInput = '';
     let previousHash = '';
     let hasPreviousTurn = false;
+    let previousEstimatedTotal = 0;
+    let recentSourceKindsWindow = [];
+    let calibrationState = { ema: 1, sampleCount: 0 };
     for (let i = 0; i < turns.length; i += 1) {
         const turnEvents = createTurn(turns[i], `turn-${i + 1}`, projectId, sessionId, runtime);
         if (!turnEvents.length)
@@ -629,6 +985,22 @@ export function normalizeTranscriptMessages({ runtime, projectId, sessionId, mes
         previousInput = input;
         previousHash = inputHash;
         events.push(...turnEvents);
+        if (CONTEXT_PARSING_ENABLED && CONTEXT_ENABLED_RUNTIMES.has(runtime)) {
+            const contextResult = composeContextEvents({
+                runtime,
+                projectId,
+                sessionId,
+                turnEvent,
+                turnEvents,
+                previousEstimatedTotal,
+                recentSourceKindsWindow,
+                calibrationState,
+            });
+            events.push(...contextResult.events);
+            previousEstimatedTotal = contextResult.estimatedTotalTokens;
+            recentSourceKindsWindow = contextResult.recentSourceKindsWindow;
+            calibrationState = contextResult.calibrationState;
+        }
     }
     return events;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bububuger/spanory",
-  "version": "0.1.16",
+  "version": "0.1.18",
   "description": "Spanory CLI for cross-runtime agent observability",
   "license": "MIT",
   "type": "module",
@@ -10,7 +10,8 @@
     "directory": "packages/cli"
   },
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
   },
   "files": [
     "dist"
@@ -19,18 +20,19 @@
     "spanory": "dist/index.js"
   },
   "scripts": {
-    "check": "tsc -p tsconfig.runtime.json --noEmit",
-    "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.runtime.json",
-    "test": "vitest run test/unit",
+    "check": "npm run --workspace @bububuger/core build && tsc -p tsconfig.runtime.json --noEmit",
+    "build": "npm run --workspace @bububuger/core build && node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.runtime.json && chmod +x dist/index.js",
+    "test": "npm run build && vitest run test/unit",
     "test:bdd": "npm run build && vitest run test/bdd",
     "test:golden:update": "npm run build && node test/fixtures/golden/otlp/update-golden.mjs && node test/fixtures/golden/codex/update-golden.mjs",
-    "build:bundle": "npm run --workspace @bububuger/backend-langfuse build && npm run --workspace @bububuger/otlp-core build && npm run build && mkdir -p build && esbuild dist/index.js --bundle --platform=node --format=cjs --target=node18 --outfile=build/spanory.cjs",
+    "build:bundle": "npm run --workspace @bububuger/backend-langfuse build && npm run --workspace @bububuger/otlp-core build && npm run build && mkdir -p build && esbuild dist/index.js --bundle --platform=node --format=cjs --target=node18 --define:process.env.SPANORY_VERSION=\\\"$npm_package_version\\\" --outfile=build/spanory.cjs",
     "build:bin:macos-arm64": "npm run build:bundle && pkg build/spanory.cjs --targets node18-macos-arm64 --output ../../dist/spanory-macos-arm64",
     "build:bin:macos-x64": "npm run build:bundle && pkg build/spanory.cjs --targets node18-macos-x64 --output ../../dist/spanory-macos-x64",
     "build:bin:linux-x64": "npm run build:bundle && pkg build/spanory.cjs --targets node18-linux-x64 --output ../../dist/spanory-linux-x64",
     "build:bin:win-x64": "npm run build:bundle && pkg build/spanory.cjs --targets node18-win-x64 --output ../../dist/spanory-win-x64.exe"
   },
   "dependencies": {
+    "@bububuger/core": "^0.1.1",
     "commander": "^14.0.1"
   },
   "devDependencies": {