@pepys/source-telemetry 0.0.0-stage → 0.0.1

package/README.md

@@ -1,3 +1,34 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/source-telemetry</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/source-telemetry"><img src="https://img.shields.io/npm/v/@pepys/source-telemetry" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a> •
+    <a href="../../docs/design/v1-plug-ins.md">The plug-ins of v1</a>
+  </p>
+</div>
+
+The pepys telemetry plug-in, for Bun.
+
+The plug-in receives the OTLP log exports of coding agents as facts, and builds one row per model
+request and one per reported turn cost with its two consumers.
+
+## Install
+
+```sh
+bun add @pepys/source-telemetry
+```
+
+## Features
+
+- `telemetryPlugin`: the plug-in, to add to a deployment.
+- `telemetryModelRequestConsumer` and `telemetryTurnCostConsumer`: the consumers of the log record
+  facts, with their SQL schemas.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of the plug-ins.

package/package.json

@@ -1,6 +1,32 @@
 {
   "name": "@pepys/source-telemetry",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.0.1",
+  "description": "The pepys telemetry plug-in: the OTLP log records of coding agents, with their model requests and turn costs",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/source-telemetry"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pepys/kit": "0.0.1",
+    "@zgeoff/std": "0.4.3",
+    "tiny-invariant": "1.3.3",
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@pepys/cli": "0.0.1",
+    "@pepys/ingest": "0.0.1",
+    "@pepys/testing": "0.0.1"
+  }
+}

package/src/consumers/products/find-claude-code-model-request.ts

@@ -0,0 +1,41 @@
+import { findRecordInteger, findRecordNumber, findRecordText } from './find-record-number.ts';
+import type { ModelRequestFields, StoredRecord } from './types.ts';
+
+// One `api_request` record is one model request, with the cost Claude Code reports for it. A record
+// with no model or no token count makes no row: no figure is invented.
+export function findClaudeCodeModelRequest(
+  event: string,
+  record: StoredRecord,
+): ModelRequestFields | undefined {
+  if (event !== 'api_request') {
+    return undefined;
+  }
+
+  const model = findRecordText(record, 'model');
+  const inputTokens = findRecordInteger(record, 'input_tokens');
+  const cacheReadTokens = findRecordInteger(record, 'cache_read_tokens');
+  const cacheCreationTokens = findRecordInteger(record, 'cache_creation_tokens');
+  const outputTokens = findRecordInteger(record, 'output_tokens');
+
+  if (
+    model === undefined ||
+    inputTokens === undefined ||
+    cacheReadTokens === undefined ||
+    cacheCreationTokens === undefined ||
+    outputTokens === undefined
+  ) {
+    return undefined;
+  }
+
+  return {
+    agent_session: findRecordText(record, 'session.id') ?? null,
+    model,
+    input_tokens: inputTokens,
+    cache_read_tokens: cacheReadTokens,
+    cache_creation_tokens: cacheCreationTokens,
+    output_tokens: outputTokens,
+    reported_cost_usd: findRecordNumber(record, 'cost_usd') ?? null,
+    duration_ms: findRecordInteger(record, 'duration_ms') ?? null,
+    agent: null,
+  };
+}

package/src/consumers/products/find-codex-model-request.ts

@@ -0,0 +1,34 @@
+import { findRecordInteger, findRecordText } from './find-record-number.ts';
+import type { ModelRequestFields, StoredRecord } from './types.ts';
+
+// One `sse_event` record of the kind `response.completed` is one model request. Codex reports no
+// cost for one request, and counts the cached tokens inside its input count.
+export function findCodexModelRequest(
+  event: string,
+  record: StoredRecord,
+): ModelRequestFields | undefined {
+  if (event !== 'sse_event' || findRecordText(record, 'event.kind') !== 'response.completed') {
+    return undefined;
+  }
+
+  const model = findRecordText(record, 'model');
+  const inputTokens = findRecordInteger(record, 'input_token_count');
+  const outputTokens = findRecordInteger(record, 'output_token_count');
+  const cachedTokens = findRecordInteger(record, 'cached_token_count') ?? 0;
+
+  if (model === undefined || inputTokens === undefined || outputTokens === undefined) {
+    return undefined;
+  }
+
+  return {
+    agent_session: findRecordText(record, 'conversation.id') ?? null,
+    model,
+    input_tokens: Math.max(inputTokens - cachedTokens, 0),
+    cache_read_tokens: cachedTokens,
+    cache_creation_tokens: 0,
+    output_tokens: outputTokens,
+    reported_cost_usd: null,
+    duration_ms: findRecordInteger(record, 'duration_ms') ?? null,
+    agent: null,
+  };
+}

package/src/consumers/products/find-codex-turn-cost.ts

@@ -0,0 +1,21 @@
+import { findRecordNumber, findRecordText } from './find-record-number.ts';
+import type { StoredRecord, TurnCostFields } from './types.ts';
+
+// One `turn_cost` record is Codex's own estimate for one turn. A record with no number makes no row.
+export function findCodexTurnCost(event: string, record: StoredRecord): TurnCostFields | undefined {
+  if (event !== 'turn_cost') {
+    return undefined;
+  }
+
+  const cost = findRecordNumber(record, 'usage.estimated_usd');
+
+  if (cost === undefined) {
+    return undefined;
+  }
+
+  return {
+    agent_session: findRecordText(record, 'conversation.id') ?? null,
+    model: findRecordText(record, 'model') ?? null,
+    cost_usd: cost,
+  };
+}

package/src/consumers/products/find-record-number.ts

@@ -0,0 +1,101 @@
+import type { KeyValue, StoredRecord } from './types.ts';
+
+// Whether an exporter writes a count as a number or as decimal text is not checked, so both count.
+export function findRecordNumber(record: StoredRecord, name: string): number | undefined {
+  const value = findRecordAttribute(record, name);
+
+  if (typeof value === 'number') {
+    return Number.isFinite(value) ? value : undefined;
+  }
+
+  if (typeof value !== 'string' || !isDecimalText(value)) {
+    return undefined;
+  }
+
+  const number = Number(value);
+
+  return Number.isFinite(number) ? number : undefined;
+}
+
+export function findRecordInteger(record: StoredRecord, name: string): number | undefined {
+  const number = findRecordNumber(record, name);
+
+  return number !== undefined && Number.isSafeInteger(number) ? number : undefined;
+}
+
+// non-empty text; a number becomes its decimal text
+export function findRecordText(record: StoredRecord, name: string): string | undefined {
+  const value = findRecordAttribute(record, name);
+
+  if (typeof value === 'number') {
+    return String(value);
+  }
+
+  return typeof value === 'string' && value !== '' ? value : undefined;
+}
+
+type Scalar = string | number | boolean;
+
+// The record's own attribute first, then the resource's. An OTLP value is a one-key object, such as
+// `{ "intValue": "12" }`, and a list or a map value counts as absent.
+function findRecordAttribute(record: StoredRecord, name: string): Scalar | undefined {
+  const own = findValue(record.logRecord.attributes ?? [], name);
+
+  return own ?? findValue(record.resource.attributes, name);
+}
+
+function findValue(attributes: readonly KeyValue[], name: string): Scalar | undefined {
+  for (const attribute of attributes) {
+    if (attribute.key === name) {
+      return toScalar(attribute.value);
+    }
+  }
+
+  return undefined;
+}
+
+function toScalar(value: unknown): Scalar | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const text = 'stringValue' in value ? value.stringValue : undefined;
+  const integer = 'intValue' in value ? value.intValue : undefined;
+  const double = 'doubleValue' in value ? value.doubleValue : undefined;
+  const flag = 'boolValue' in value ? value.boolValue : undefined;
+
+  if (typeof text === 'string') {
+    return text;
+  }
+
+  if (typeof integer === 'number' || typeof integer === 'string') {
+    return integer;
+  }
+
+  if (typeof double === 'number' || typeof double === 'string') {
+    return double;
+  }
+
+  return typeof flag === 'boolean' ? flag : undefined;
+}
+
+// Digits with one optional leading minus and one optional point, such as `-12` or `0.4`. `Number`
+// alone would also take an empty text, hex and `Infinity`.
+function isDecimalText(text: string): boolean {
+  let digits = 0;
+  let points = 0;
+
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text.charAt(index);
+
+    if (character >= '0' && character <= '9') {
+      digits += 1;
+    } else if (character === '.') {
+      points += 1;
+    } else if (character !== '-' || index !== 0) {
+      return false;
+    }
+  }
+
+  return digits > 0 && points <= 1;
+}

package/src/consumers/products/parse-fact-type.ts

@@ -0,0 +1,21 @@
+import type { ProductName } from './types.ts';
+
+const productNames: readonly ProductName[] = ['claude_code', 'codex'];
+
+// `telemetry.<product>.<event>`, with `unlisted` as the event of a record the product does not list
+export function parseFactType(type: string): { product: ProductName; event: string } | undefined {
+  const parts = type.split('.');
+  const [source, productPart, event] = parts;
+  const product = productNames.find((name) => name === productPart);
+
+  if (
+    parts.length !== 3 ||
+    source !== 'telemetry' ||
+    product === undefined ||
+    event === undefined
+  ) {
+    return undefined;
+  }
+
+  return { product, event };
+}

package/src/consumers/products/types.ts

@@ -0,0 +1,34 @@
+// The consumers import no ingest code, so they keep their own list of product names. A test holds
+// it equal to the product list of the source.
+export type ProductName = 'claude_code' | 'codex';
+
+export interface KeyValue {
+  key: string;
+  value: unknown;
+}
+
+// The part of a stored log record that a product's reader looks at.
+export interface StoredRecord {
+  resource: { attributes: readonly KeyValue[] };
+  logRecord: { attributes?: readonly KeyValue[] | undefined };
+}
+
+export interface ModelRequestFields {
+  agent_session: string | null;
+  model: string;
+  input_tokens: number;
+  cache_read_tokens: number;
+  cache_creation_tokens: number;
+  output_tokens: number;
+
+  // null when the product reports no cost for one request
+  reported_cost_usd: number | null;
+  duration_ms: number | null;
+  agent: string | null;
+}
+
+export interface TurnCostFields {
+  agent_session: string | null;
+  model: string | null;
+  cost_usd: number;
+}

package/src/consumers/telemetry-model-request.ts

@@ -0,0 +1,91 @@
+import {
+  initTelemetryModelRequestModel,
+  parseTelemetryModelRequestFact,
+  parseTelemetrySessionID,
+  toTelemetryModelRequestID,
+  updateTelemetryModelRequestModel,
+} from '../generated/telemetry-model-request-boundary.ts';
+import type {
+  TelemetryModelRequestMessage,
+  TelemetryModelRequestModel,
+  TelemetryModelRequestOperation,
+} from '../generated/telemetry-model-request-boundary.ts';
+import { findClaudeCodeModelRequest } from './products/find-claude-code-model-request.ts';
+import { findCodexModelRequest } from './products/find-codex-model-request.ts';
+import { parseFactType } from './products/parse-fact-type.ts';
+import type { ModelRequestFields, ProductName, StoredRecord } from './products/types.ts';
+
+export type {
+  TelemetryModelRequestMessage,
+  TelemetryModelRequestModel,
+} from '../generated/telemetry-model-request-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): TelemetryModelRequestModel {
+  return initTelemetryModelRequestModel();
+}
+
+export function update(
+  model: TelemetryModelRequestModel,
+  message: TelemetryModelRequestMessage,
+): TelemetryModelRequestModel {
+  return updateTelemetryModelRequestModel(model, message);
+}
+
+type ModelRequestReader = (event: string, record: StoredRecord) => ModelRequestFields | undefined;
+
+// One reader per product: a product with no entry here fails the type check.
+const readers: Readonly<Record<ProductName, ModelRequestReader>> = {
+  claude_code: findClaudeCodeModelRequest,
+  codex: findCodexModelRequest,
+};
+
+const factIDPrefix = 'event:telemetry:';
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+// The host calls it once per fact ID, so a copy never gets here, and the row's key comes from the
+// fact's ID, so the order of the facts changes nothing.
+function mapFact(factJSON: string): TelemetryModelRequestOperation[] {
+  const fact = parseTelemetryModelRequestFact(factJSON);
+  const type = fact === undefined ? undefined : parseFactType(fact.type);
+
+  if (fact === undefined || type === undefined || !fact.id.startsWith(factIDPrefix)) {
+    return [];
+  }
+
+  // ingest chose the session, so the row copies the fact's entity and reads no attribute for it
+  const session = parseTelemetrySessionID(fact.entity);
+  const fields = readers[type.product](type.event, fact.payload);
+
+  if (session === undefined || fields === undefined) {
+    return [];
+  }
+
+  return [
+    {
+      upsert: {
+        telemetry_model_request: {
+          id: toTelemetryModelRequestID(fact.id.slice(factIDPrefix.length)),
+          session_id: session,
+          agent_session: fields.agent_session,
+          product: type.product,
+          model: fields.model,
+          input_tokens: fields.input_tokens,
+          cache_read_tokens: fields.cache_read_tokens,
+          cache_creation_tokens: fields.cache_creation_tokens,
+          output_tokens: fields.output_tokens,
+          cost_usd: fields.reported_cost_usd,
+          cost_basis: fields.reported_cost_usd === null ? 'none' : 'reported',
+          duration_ms: fields.duration_ms,
+          agent: fields.agent,
+          origin_issuer: fact.payload.origin.issuer,
+          origin_subject: fact.payload.origin.subject,
+          at: fact.at,
+        },
+      },
+    },
+  ];
+}

package/src/consumers/telemetry-turn-cost.ts

@@ -0,0 +1,85 @@
+import {
+  initTelemetryTurnCostModel,
+  parseTelemetrySessionID,
+  parseTelemetryTurnCostFact,
+  toTelemetryTurnCostID,
+  updateTelemetryTurnCostModel,
+} from '../generated/telemetry-turn-cost-boundary.ts';
+import type {
+  TelemetryTurnCostMessage,
+  TelemetryTurnCostModel,
+  TelemetryTurnCostOperation,
+} from '../generated/telemetry-turn-cost-boundary.ts';
+import { findCodexTurnCost } from './products/find-codex-turn-cost.ts';
+import { parseFactType } from './products/parse-fact-type.ts';
+import type { ProductName, StoredRecord, TurnCostFields } from './products/types.ts';
+
+export type {
+  TelemetryTurnCostMessage,
+  TelemetryTurnCostModel,
+} from '../generated/telemetry-turn-cost-boundary.ts';
+
+// scriptc takes the model and message types through a re-export, but not the two functions
+export function init(): TelemetryTurnCostModel {
+  return initTelemetryTurnCostModel();
+}
+
+export function update(
+  model: TelemetryTurnCostModel,
+  message: TelemetryTurnCostMessage,
+): TelemetryTurnCostModel {
+  return updateTelemetryTurnCostModel(model, message);
+}
+
+type TurnCostReader = (event: string, record: StoredRecord) => TurnCostFields | undefined;
+
+// One reader per product: a product with no entry here fails the type check. Claude Code reports
+// its cost per request, so it has no turn cost, and a sum over both tables counts nothing twice.
+const readers: Readonly<Record<ProductName, TurnCostReader>> = {
+  claude_code: findNoTurnCost,
+  codex: findCodexTurnCost,
+};
+
+const factIDPrefix = 'event:telemetry:';
+
+export function applyFact(factJSON: string): string {
+  return JSON.stringify(mapFact(factJSON));
+}
+
+function mapFact(factJSON: string): TelemetryTurnCostOperation[] {
+  const fact = parseTelemetryTurnCostFact(factJSON);
+  const type = fact === undefined ? undefined : parseFactType(fact.type);
+
+  if (fact === undefined || type === undefined || !fact.id.startsWith(factIDPrefix)) {
+    return [];
+  }
+
+  const session = parseTelemetrySessionID(fact.entity);
+  const fields = readers[type.product](type.event, fact.payload);
+
+  if (session === undefined || fields === undefined) {
+    return [];
+  }
+
+  return [
+    {
+      upsert: {
+        telemetry_turn_cost: {
+          id: toTelemetryTurnCostID(fact.id.slice(factIDPrefix.length)),
+          session_id: session,
+          agent_session: fields.agent_session,
+          product: type.product,
+          model: fields.model,
+          cost_usd: fields.cost_usd,
+          origin_issuer: fact.payload.origin.issuer,
+          origin_subject: fact.payload.origin.subject,
+          at: fact.at,
+        },
+      },
+    },
+  ];
+}
+
+function findNoTurnCost(): TurnCostFields | undefined {
+  return undefined;
+}