@pepys/source-telemetry 0.0.0-stage → 0.0.1

package/src/generated/telemetry-turn-cost-boundary.ts

@@ -0,0 +1,361 @@
+// Generated by `pepys build`. Do not edit.
+
+declare const idTag: unique symbol;
+declare const kindTag: unique symbol;
+
+export type EntityID = string & { readonly [idTag]: 'EntityID' };
+
+type KindedID<Kind extends string> = EntityID & { readonly [kindTag]: Kind };
+
+export type TelemetrySessionID = KindedID<'telemetry:session'>;
+
+export function toTelemetrySessionID(id: string): TelemetrySessionID {
+  const text = 'telemetry:session:' + id;
+
+  if (!isTelemetrySessionID(text)) {
+    throw new Error('an ID of the kind telemetry:session is never empty');
+  }
+
+  return text;
+}
+
+export function parseTelemetrySessionID(text: string): TelemetrySessionID | undefined {
+  return isTelemetrySessionID(text) ? text : undefined;
+}
+
+function isTelemetrySessionID(text: string): text is TelemetrySessionID {
+  return text.length > 'telemetry:session:'.length && text.startsWith('telemetry:session:');
+}
+
+export type TelemetryTurnCostID = KindedID<'telemetry:turn_cost'>;
+
+export function toTelemetryTurnCostID(id: string): TelemetryTurnCostID {
+  const text = 'telemetry:turn_cost:' + id;
+
+  if (!isTelemetryTurnCostID(text)) {
+    throw new Error('an ID of the kind telemetry:turn_cost is never empty');
+  }
+
+  return text;
+}
+
+export function parseTelemetryTurnCostID(text: string): TelemetryTurnCostID | undefined {
+  return isTelemetryTurnCostID(text) ? text : undefined;
+}
+
+function isTelemetryTurnCostID(text: string): text is TelemetryTurnCostID {
+  return text.length > 'telemetry:turn_cost:'.length && text.startsWith('telemetry:turn_cost:');
+}
+
+export interface TelemetryTurnCostModel {
+  input: string;
+}
+
+export type TelemetryTurnCostMessage = { kind: 'apply'; input: string } | { kind: 'reset' };
+
+export function initTelemetryTurnCostModel(): TelemetryTurnCostModel {
+  return { input: '' };
+}
+
+export function updateTelemetryTurnCostModel(model: TelemetryTurnCostModel, message: TelemetryTurnCostMessage): TelemetryTurnCostModel {
+  return message.kind === 'reset' ? initTelemetryTurnCostModel() : { ...model, input: message.input };
+}
+
+export interface TelemetryPayload {
+  resource: TelemetryPayloadResource;
+  scope: TelemetryPayloadScope;
+  logRecord: TelemetryPayloadLogRecord;
+  origin: TelemetryPayloadOrigin;
+}
+
+export interface TelemetryPayloadResource {
+  attributes: TelemetryPayloadResourceAttributesItem[];
+}
+
+export interface TelemetryPayloadResourceAttributesItem {
+  key: string;
+  value: unknown;
+}
+
+export interface TelemetryPayloadScope {
+  name: string;
+  version?: string;
+}
+
+export interface TelemetryPayloadLogRecord {
+  attributes?: TelemetryPayloadLogRecordAttributesItem[];
+}
+
+export interface TelemetryPayloadLogRecordAttributesItem {
+  key: string;
+  value: unknown;
+}
+
+export interface TelemetryPayloadOrigin {
+  issuer: string;
+  subject: string;
+}
+
+export type TelemetryTurnCostFact =
+  | { id: string; type: 'telemetry.claude_code.api_request' | 'telemetry.claude_code.api_error' | 'telemetry.claude_code.api_refusal' | 'telemetry.claude_code.user_prompt' | 'telemetry.claude_code.assistant_response' | 'telemetry.claude_code.tool_result' | 'telemetry.claude_code.tool_decision' | 'telemetry.claude_code.unlisted' | 'telemetry.codex.sse_event' | 'telemetry.codex.api_request' | 'telemetry.codex.turn_cost' | 'telemetry.codex.conversation_starts' | 'telemetry.codex.user_prompt' | 'telemetry.codex.tool_decision' | 'telemetry.codex.tool_result' | 'telemetry.codex.unlisted'; entity: string; at: string; payload: TelemetryPayload };
+
+export function parseTelemetryTurnCostFact(factJSON: string): TelemetryTurnCostFact | undefined {
+  const parsed: unknown = JSON.parse(factJSON);
+
+  return decodeTelemetryTurnCostFact(parsed);
+}
+
+function decodeTelemetryTurnCostFact(value: unknown): TelemetryTurnCostFact | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const id = 'id' in value ? value.id : undefined;
+  const type = 'type' in value ? value.type : undefined;
+  const entity = 'entity' in value ? value.entity : undefined;
+  const at = 'at' in value ? value.at : undefined;
+  const payload = 'payload' in value ? value.payload : undefined;
+
+  if (typeof id !== 'string' || typeof entity !== 'string' || typeof at !== 'string') {
+    return undefined;
+  }
+
+  if (type === 'telemetry.claude_code.api_request' || type === 'telemetry.claude_code.api_error' || type === 'telemetry.claude_code.api_refusal' || type === 'telemetry.claude_code.user_prompt' || type === 'telemetry.claude_code.assistant_response' || type === 'telemetry.claude_code.tool_result' || type === 'telemetry.claude_code.tool_decision' || type === 'telemetry.claude_code.unlisted' || type === 'telemetry.codex.sse_event' || type === 'telemetry.codex.api_request' || type === 'telemetry.codex.turn_cost' || type === 'telemetry.codex.conversation_starts' || type === 'telemetry.codex.user_prompt' || type === 'telemetry.codex.tool_decision' || type === 'telemetry.codex.tool_result' || type === 'telemetry.codex.unlisted') {
+    const decoded = decodeTelemetryPayload(payload);
+
+    if (decoded === undefined) {
+      return undefined;
+    }
+
+    return { id, type, entity, at, payload: decoded };
+  }
+
+  return undefined;
+}
+
+function decodeTelemetryPayload(value: unknown): TelemetryPayload | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'resource' in value ? value.resource : undefined;
+  const field0 = decodeTelemetryPayloadResource(raw0);
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  const raw1 = 'scope' in value ? value.scope : undefined;
+  const field1 = decodeTelemetryPayloadScope(raw1);
+
+  if (field1 === undefined) {
+    return undefined;
+  }
+
+  const raw2 = 'logRecord' in value ? value.logRecord : undefined;
+  const field2 = decodeTelemetryPayloadLogRecord(raw2);
+
+  if (field2 === undefined) {
+    return undefined;
+  }
+
+  const raw3 = 'origin' in value ? value.origin : undefined;
+  const field3 = decodeTelemetryPayloadOrigin(raw3);
+
+  if (field3 === undefined) {
+    return undefined;
+  }
+
+  return {
+    resource: field0,
+    scope: field1,
+    logRecord: field2,
+    origin: field3,
+  };
+}
+
+function decodeTelemetryPayloadResource(value: unknown): TelemetryPayloadResource | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'attributes' in value ? value.attributes : undefined;
+  const field0 = decodeTelemetryPayloadResourceAttributes(raw0);
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  return {
+    attributes: field0,
+  };
+}
+
+function decodeTelemetryPayloadResourceAttributes(value: unknown): TelemetryPayloadResourceAttributesItem[] | undefined {
+  if (!Array.isArray(value)) {
+    return undefined;
+  }
+
+  const items: TelemetryPayloadResourceAttributesItem[] = [];
+
+  for (let index = 0; index < value.length; index += 1) {
+    const element: unknown = value[index];
+    const item = decodeTelemetryPayloadResourceAttributesItem(element);
+
+    if (item === undefined) {
+      return undefined;
+    }
+
+    items.push(item);
+  }
+
+  return items;
+}
+
+function decodeTelemetryPayloadResourceAttributesItem(value: unknown): TelemetryPayloadResourceAttributesItem | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'key' in value ? value.key : undefined;
+  const field0 = typeof raw0 === 'string' ? raw0 : undefined;
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  const field1 = 'value' in value ? value.value : undefined;
+
+  return {
+    key: field0,
+    value: field1,
+  };
+}
+
+function decodeTelemetryPayloadScope(value: unknown): TelemetryPayloadScope | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'name' in value ? value.name : undefined;
+  const field0 = typeof raw0 === 'string' ? raw0 : undefined;
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  const raw1 = 'version' in value ? value.version : undefined;
+  const field1 = raw1 === undefined ? undefined : typeof raw1 === 'string' ? raw1 : undefined;
+
+  if (raw1 !== undefined && field1 === undefined) {
+    return undefined;
+  }
+
+  return {
+    name: field0,
+    ...(field1 === undefined ? {} : { version: field1 }),
+  };
+}
+
+function decodeTelemetryPayloadLogRecord(value: unknown): TelemetryPayloadLogRecord | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'attributes' in value ? value.attributes : undefined;
+  const field0 = raw0 === undefined ? undefined : decodeTelemetryPayloadLogRecordAttributes(raw0);
+
+  if (raw0 !== undefined && field0 === undefined) {
+    return undefined;
+  }
+
+  return {
+    ...(field0 === undefined ? {} : { attributes: field0 }),
+  };
+}
+
+function decodeTelemetryPayloadLogRecordAttributes(value: unknown): TelemetryPayloadLogRecordAttributesItem[] | undefined {
+  if (!Array.isArray(value)) {
+    return undefined;
+  }
+
+  const items: TelemetryPayloadLogRecordAttributesItem[] = [];
+
+  for (let index = 0; index < value.length; index += 1) {
+    const element: unknown = value[index];
+    const item = decodeTelemetryPayloadLogRecordAttributesItem(element);
+
+    if (item === undefined) {
+      return undefined;
+    }
+
+    items.push(item);
+  }
+
+  return items;
+}
+
+function decodeTelemetryPayloadLogRecordAttributesItem(value: unknown): TelemetryPayloadLogRecordAttributesItem | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'key' in value ? value.key : undefined;
+  const field0 = typeof raw0 === 'string' ? raw0 : undefined;
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  const field1 = 'value' in value ? value.value : undefined;
+
+  return {
+    key: field0,
+    value: field1,
+  };
+}
+
+function decodeTelemetryPayloadOrigin(value: unknown): TelemetryPayloadOrigin | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const raw0 = 'issuer' in value ? value.issuer : undefined;
+  const field0 = typeof raw0 === 'string' ? raw0 : undefined;
+
+  if (field0 === undefined) {
+    return undefined;
+  }
+
+  const raw1 = 'subject' in value ? value.subject : undefined;
+  const field1 = typeof raw1 === 'string' ? raw1 : undefined;
+
+  if (field1 === undefined) {
+    return undefined;
+  }
+
+  return {
+    issuer: field0,
+    subject: field1,
+  };
+}
+
+export interface TelemetryTurnCostRow {
+  id: TelemetryTurnCostID;
+  session_id: EntityID;
+  agent_session: string | null;
+  product: string;
+  model: string | null;
+  cost_usd: number;
+  origin_issuer: string;
+  origin_subject: string;
+  at: string;
+}
+
+export interface TelemetryTurnCostKey {
+  id: TelemetryTurnCostID;
+}
+
+export type TelemetryTurnCostOperation =
+  | { upsert: { telemetry_turn_cost: TelemetryTurnCostRow } }
+  | { delete: { telemetry_turn_cost: TelemetryTurnCostKey } };

package/src/generated/telemetry-turn-cost.sql

@@ -0,0 +1,26 @@
+-- Generated by `pepys build`. Do not edit.
+
+CREATE TABLE "telemetry_turn_cost" (
+  "id" TEXT NOT NULL,
+  "session_id" TEXT NOT NULL,
+  "agent_session" TEXT,
+  "product" TEXT NOT NULL,
+  "model" TEXT,
+  "cost_usd" REAL NOT NULL,
+  "origin_issuer" TEXT NOT NULL,
+  "origin_subject" TEXT NOT NULL,
+  "at" TEXT NOT NULL,
+  PRIMARY KEY ("id")
+);
+
+CREATE INDEX "telemetry_turn_cost_session_id_idx" ON "telemetry_turn_cost" ("session_id");
+
+CREATE VIEW "entity" AS
+SELECT
+  "id" AS "id",
+  'telemetry:turn_cost' AS "kind",
+  "product" AS "title",
+  NULL AS "url",
+  NULL AS "actor",
+  "at" AS "updated_at"
+FROM "telemetry_turn_cost";

package/src/index.ts

@@ -0,0 +1,25 @@
+export {
+  buildLogRecordTypes,
+  logRecordFamily,
+  logRecordPayloadSchema,
+} from './log-record-family.ts';
+
+export type { LogRecordPayload } from './log-record-family.ts';
+export { readOtlpFacts } from './otlp/read-otlp-facts.ts';
+
+export type {
+  DroppedRecord,
+  OtlpFacts,
+  OtlpReadOptions,
+  TelemetryFact,
+} from './otlp/read-otlp-facts.ts';
+
+export { receiveOtlpLogs } from './otlp/receive-otlp-logs.ts';
+export type * from './otlp/types.ts';
+export { defineTelemetryProduct } from './products/define-telemetry-product.ts';
+export { products } from './products/products.ts';
+export type { ProductName } from './products/products.ts';
+export type { TelemetryProduct } from './products/types.ts';
+export { telemetryModelRequestConsumer } from './telemetry-model-request-consumer.ts';
+export { telemetryPlugin, telemetrySource } from './telemetry-plugin.ts';
+export { telemetryTurnCostConsumer } from './telemetry-turn-cost-consumer.ts';

package/src/log-record-family.ts

@@ -0,0 +1,41 @@
+import { defineFactFamily } from '@pepys/kit';
+import { z } from 'zod';
+import { products } from './products/products.ts';
+import type { TelemetryProduct } from './products/types.ts';
+
+const keyValueSchema = z.object({ key: z.string(), value: z.unknown() });
+
+// What the exporter wrote, whole, and the proven sender. The glue types the attribute lists and the
+// origin; an attribute's value is an OTLP union, which a consumer narrows itself.
+export const logRecordPayloadSchema = z.object({
+  resource: z.object({ attributes: z.array(keyValueSchema) }),
+  scope: z.object({ name: z.string(), version: z.string().optional() }),
+  logRecord: z.looseObject({ attributes: z.array(keyValueSchema).optional() }),
+
+  // The only part of the payload that the sender did not write: ingest copies it from the proof.
+  // It is not in the fact ID, which depends on the record alone.
+  origin: z.object({ issuer: z.string(), subject: z.string() }),
+});
+
+export type LogRecordPayload = z.infer<typeof logRecordPayloadSchema>;
+
+// `telemetry.<product>.<event>` for each listed event, and `telemetry.<product>.unlisted`.
+export function buildLogRecordTypes(
+  list: readonly TelemetryProduct[],
+): Readonly<Record<string, typeof logRecordPayloadSchema>> {
+  const entries = list.flatMap((product) =>
+    [...Object.keys(product.events), 'unlisted'].map(
+      (event) => [`telemetry.${product.name}.${event}`, logRecordPayloadSchema] as const,
+    ),
+  );
+
+  return Object.fromEntries(entries);
+}
+
+// One family and one stream: one log record of a known product is one fact, stored whole.
+export const logRecordFamily = defineFactFamily({
+  stream: 'facts.telemetry.log_record',
+  api: 'otlp-logs/1',
+  entityKind: 'telemetry:session',
+  types: buildLogRecordTypes(products),
+});

package/src/otlp/format-canonical-json.ts

@@ -0,0 +1,24 @@
+import { isRecord } from '@zgeoff/std';
+
+// JSON with the keys of every object sorted and no white space, so one value has one text whatever
+// the key order the exporter wrote.
+export function formatCanonicalJSON(value: unknown): string {
+  return JSON.stringify(toSortedValue(value));
+}
+
+function toSortedValue(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map((item: unknown) => toSortedValue(item));
+  }
+
+  if (!isRecord(value)) {
+    return value;
+  }
+
+  // `fromEntries` defines each key as an own property, so a `__proto__` key stays a key
+  return Object.fromEntries(
+    Object.keys(value)
+      .toSorted()
+      .map((key) => [key, toSortedValue(value[key])] as const),
+  );
+}

package/src/otlp/parse-logs-export.ts

@@ -0,0 +1,71 @@
+import { isRecord } from '@zgeoff/std';
+import type { LogsExport, OtlpKeyValue, OtlpLogRecord, ResourceLogs, ScopeLogs } from './types.ts';
+
+// The export in a JSON body, or undefined for a value that is no logs export at all. Inside an
+// export, a part of the wrong shape is skipped, so one odd record never refuses the others: an
+// exporter drops an export that gets a 4xx answer.
+export function parseLogsExport(value: unknown): LogsExport | undefined {
+  if (!isRecord(value)) {
+    return undefined;
+  }
+
+  const resourceLogs = value['resourceLogs'];
+
+  if (resourceLogs === undefined) {
+    return { resourceLogs: [] };
+  }
+
+  if (!Array.isArray(resourceLogs)) {
+    return undefined;
+  }
+
+  return { resourceLogs: collectObjects(resourceLogs).map((item) => parseResourceLogs(item)) };
+}
+
+type JSONObject = Readonly<Record<string, unknown>>;
+
+function collectObjects(value: unknown): JSONObject[] {
+  const items: readonly unknown[] = Array.isArray(value) ? value : [];
+
+  return items.filter((item): item is JSONObject => isRecord(item));
+}
+
+function parseResourceLogs(value: JSONObject): ResourceLogs {
+  const resource = value['resource'];
+  const attributes = isRecord(resource) ? resource['attributes'] : undefined;
+
+  return {
+    resource: { attributes: parseAttributes(attributes) },
+    scopeLogs: collectObjects(value['scopeLogs']).map((item) => parseScopeLogs(item)),
+  };
+}
+
+function parseAttributes(value: unknown): OtlpKeyValue[] {
+  return collectObjects(value).flatMap((entry) => {
+    const key = entry['key'];
+
+    return typeof key === 'string' ? [{ key, value: entry['value'] }] : [];
+  });
+}
+
+function parseScopeLogs(value: JSONObject): ScopeLogs {
+  const scope = isRecord(value['scope']) ? value['scope'] : {};
+  const name = scope['name'];
+  const version = scope['version'];
+
+  return {
+    scope: {
+      name: typeof name === 'string' ? name : '',
+      ...(typeof version === 'string' ? { version } : {}),
+    },
+    logRecords: collectObjects(value['logRecords']).map((item) => parseLogRecord(item)),
+  };
+}
+
+function parseLogRecord(value: JSONObject): OtlpLogRecord {
+  if (value['attributes'] === undefined) {
+    return { ...value, attributes: undefined };
+  }
+
+  return { ...value, attributes: parseAttributes(value['attributes']) };
+}