@contractspec/lib.ai-agent 1.61.0 → 2.0.0

package/dist/telemetry/adapter.d.ts

@@ -65,4 +65,3 @@ export declare function createInMemoryTelemetryCollector(): InMemoryTelemetryCol
  * No-op telemetry collector that discards all metrics.
  */
 export declare const noopTelemetryCollector: TelemetryCollector;
-//# sourceMappingURL=adapter.d.ts.map

package/dist/telemetry/index.d.ts

@@ -1,2 +1,3 @@
 export * from './adapter';
-//# sourceMappingURL=index.d.ts.map
+export * from './posthog-types';
+export * from './posthog';

package/dist/telemetry/index.js

@@ -76,11 +76,127 @@ var init_adapter = __esm(() => {
   };
 });
 
+// src/telemetry/posthog.ts
+var exports_posthog = {};
+__export(exports_posthog, {
+  createPostHogTracedModel: () => createPostHogTracedModel,
+  createPostHogTelemetryCollector: () => createPostHogTelemetryCollector,
+  createCompositeTelemetryCollector: () => createCompositeTelemetryCollector,
+  PostHogTelemetryCollector: () => PostHogTelemetryCollector,
+  CompositeTelemetryCollector: () => CompositeTelemetryCollector
+});
+async function createPostHogTracedModel(model, config, overrides) {
+  const { withTracing } = await importPostHogAI();
+  const phClient = await resolvePostHogClient(config);
+  const tracingOptions = {
+    ...config.defaults,
+    ...overrides
+  };
+  return withTracing(model, phClient, tracingOptions);
+}
+
+class PostHogTelemetryCollector {
+  phClient;
+  config;
+  initPromise;
+  constructor(config) {
+    this.config = config;
+    this.phClient = config.client;
+  }
+  async collect(sample) {
+    const client = await this.getClient();
+    const distinctId = this.config.defaults?.posthogDistinctId ?? sample.metadata?.["actorId"] ?? "system";
+    client.capture({
+      distinctId,
+      event: "$ai_generation",
+      properties: {
+        $ai_model: sample.operation.name,
+        $ai_provider: "contractspec",
+        $ai_latency: sample.durationMs / 1000,
+        $ai_is_error: !sample.success,
+        $ai_trace_id: this.config.defaults?.posthogTraceId,
+        ...sample.metadata?.["tokenUsage"] ? mapTokenUsage(sample.metadata["tokenUsage"]) : {},
+        ...this.config.defaults?.posthogProperties,
+        contractspec_operation: sample.operation.name,
+        contractspec_version: sample.operation.version,
+        contractspec_agent_id: sample.metadata?.["agentId"],
+        contractspec_finish_reason: sample.metadata?.["finishReason"],
+        contractspec_tool_count: sample.metadata?.["toolCallCount"]
+      },
+      groups: this.config.defaults?.posthogGroups
+    });
+  }
+  async shutdown() {
+    if (this.phClient?.shutdown) {
+      await this.phClient.shutdown();
+    }
+  }
+  async getClient() {
+    if (this.phClient)
+      return this.phClient;
+    if (!this.initPromise) {
+      this.initPromise = resolvePostHogClient(this.config).then((client) => {
+        this.phClient = client;
+        return client;
+      });
+    }
+    return this.initPromise;
+  }
+}
+function createPostHogTelemetryCollector(config) {
+  return new PostHogTelemetryCollector(config);
+}
+
+class CompositeTelemetryCollector {
+  collectors;
+  constructor(collectors) {
+    this.collectors = collectors;
+  }
+  async collect(sample) {
+    await Promise.all(this.collectors.map((c) => c.collect(sample)));
+  }
+}
+function createCompositeTelemetryCollector(collectors) {
+  return new CompositeTelemetryCollector(collectors);
+}
+async function importPostHogAI() {
+  try {
+    return await import("@posthog/ai");
+  } catch {
+    throw new Error("PostHog LLM Analytics requires @posthog/ai to be installed. Run: npm install @posthog/ai posthog-node");
+  }
+}
+async function resolvePostHogClient(config) {
+  if (config.client)
+    return config.client;
+  if (!config.apiKey) {
+    throw new Error("PostHog LLM Analytics requires either a client instance or an apiKey.");
+  }
+  try {
+    const { PostHog } = await import("posthog-node");
+    return new PostHog(config.apiKey, {
+      host: config.host ?? "https://us.i.posthog.com"
+    });
+  } catch {
+    throw new Error("PostHog LLM Analytics requires posthog-node to be installed. Run: npm install posthog-node");
+  }
+}
+function mapTokenUsage(usage) {
+  return {
+    $ai_input_tokens: usage["promptTokens"],
+    $ai_output_tokens: usage["completionTokens"]
+  };
+}
 // src/telemetry/index.ts
 init_adapter();
 export {
   trackAgentStep,
   noopTelemetryCollector,
+  createPostHogTracedModel,
+  createPostHogTelemetryCollector,
   createInMemoryTelemetryCollector,
-  InMemoryTelemetryCollector
+  createCompositeTelemetryCollector,
+  PostHogTelemetryCollector,
+  InMemoryTelemetryCollector,
+  CompositeTelemetryCollector
 };

package/dist/telemetry/posthog-types.d.ts

@@ -0,0 +1,59 @@
+/**
+ * PostHog LLM Analytics type definitions.
+ *
+ * Shared types for PostHog integration. Separated from implementation
+ * to allow importing types without pulling in dynamic imports.
+ */
+/**
+ * PostHog tracing options passed to @posthog/ai withTracing wrapper.
+ * These enrich captured $ai_generation events with contextual data.
+ *
+ * @see https://posthog.com/docs/llm-analytics/installation/vercel-ai
+ */
+export interface PostHogTracingOptions {
+    /** Distinct user ID for attribution */
+    posthogDistinctId?: string;
+    /** Trace ID for grouping related generations */
+    posthogTraceId?: string;
+    /** Custom properties attached to every generation event */
+    posthogProperties?: Record<string, unknown>;
+    /** When true, input/output content is redacted */
+    posthogPrivacyMode?: boolean;
+    /** Group analytics (e.g., { company: "companyId" }) */
+    posthogGroups?: Record<string, string>;
+}
+/**
+ * Minimal interface for posthog-node PostHog client.
+ * Avoids hard dependency on posthog-node types.
+ */
+export interface PostHogClient {
+    capture(params: {
+        distinctId: string;
+        event: string;
+        properties?: Record<string, unknown>;
+        groups?: Record<string, string>;
+    }): void;
+    shutdown?(): Promise<void>;
+}
+/**
+ * Configuration for PostHog LLM Analytics integration.
+ *
+ * Provide either an existing PostHog client instance or
+ * apiKey + host to create one internally.
+ *
+ * PostHog Evaluations (Relevance, Helpfulness, Hallucination, Toxicity,
+ * Jailbreak, and custom prompts) run server-side on captured $ai_generation
+ * events. Configure them in PostHog UI under LLM Analytics > Evaluations.
+ *
+ * @see https://posthog.com/docs/llm-analytics/evaluations
+ */
+export interface PostHogLLMConfig {
+    /** Existing posthog-node client instance */
+    client?: PostHogClient;
+    /** PostHog project API key (used when client is not provided) */
+    apiKey?: string;
+    /** PostHog ingestion host (defaults to https://us.i.posthog.com) */
+    host?: string;
+    /** Default tracing options applied to all wrapped models */
+    defaults?: PostHogTracingOptions;
+}

package/dist/telemetry/posthog-types.js

@@ -0,0 +1 @@
+// @bun

package/dist/telemetry/posthog.d.ts

@@ -0,0 +1,80 @@
+import type { LanguageModel } from 'ai';
+import type { TelemetryCollector, OperationMetricSample } from './adapter';
+import type { PostHogLLMConfig, PostHogTracingOptions } from './posthog-types';
+export type { PostHogClient, PostHogLLMConfig, PostHogTracingOptions, } from './posthog-types';
+/**
+ * Wrap a Vercel AI SDK LanguageModel with PostHog LLM tracing.
+ *
+ * Requires `@posthog/ai` and `posthog-node` to be installed as peer deps.
+ * Automatically captures `$ai_generation` events for every LLM call,
+ * including token usage, latency, cost, input/output, and tool metadata.
+ *
+ * These events power PostHog's LLM Analytics dashboard (Traces, Generations)
+ * and are the basis for server-side Evaluations.
+ *
+ * @param model - AI SDK LanguageModel to wrap
+ * @param config - PostHog configuration
+ * @param overrides - Per-call tracing option overrides
+ * @returns Wrapped LanguageModel that auto-reports to PostHog
+ *
+ * @example
+ * ```typescript
+ * import { createPostHogTracedModel } from '@contractspec/lib.ai-agent/telemetry/posthog';
+ *
+ * const model = await createPostHogTracedModel(
+ *   anthropic('claude-sonnet-4-20250514'),
+ *   { apiKey: process.env.POSTHOG_API_KEY },
+ *   { posthogDistinctId: 'user_123' },
+ * );
+ * ```
+ */
+export declare function createPostHogTracedModel(model: LanguageModel, config: PostHogLLMConfig, overrides?: PostHogTracingOptions): Promise<LanguageModel>;
+/**
+ * PostHog-backed telemetry collector.
+ *
+ * Implements the existing TelemetryCollector interface so it integrates
+ * seamlessly with trackAgentStep(). Each OperationMetricSample is captured
+ * as a PostHog $ai_generation event, bridging ContractSpec's evolution
+ * engine with PostHog's LLM Analytics dashboard.
+ *
+ * @example
+ * ```typescript
+ * const collector = createPostHogTelemetryCollector({
+ *   apiKey: process.env.POSTHOG_API_KEY,
+ *   defaults: { posthogDistinctId: 'system' },
+ * });
+ * ```
+ */
+export declare class PostHogTelemetryCollector implements TelemetryCollector {
+    private phClient;
+    private readonly config;
+    private initPromise;
+    constructor(config: PostHogLLMConfig);
+    collect(sample: OperationMetricSample): Promise<void>;
+    /** Shut down the PostHog client (flushes pending events). */
+    shutdown(): Promise<void>;
+    private getClient;
+}
+/** Create a PostHog-backed telemetry collector. */
+export declare function createPostHogTelemetryCollector(config: PostHogLLMConfig): PostHogTelemetryCollector;
+/**
+ * Composite telemetry collector that forwards samples to multiple collectors.
+ *
+ * Useful when you want both PostHog LLM Analytics and the existing
+ * evolution engine to receive the same telemetry data.
+ *
+ * @example
+ * ```typescript
+ * const composite = createCompositeTelemetryCollector([
+ *   evolutionCollector,
+ *   createPostHogTelemetryCollector({ apiKey: '...' }),
+ * ]);
+ * ```
+ */
+export declare class CompositeTelemetryCollector implements TelemetryCollector {
+    private readonly collectors;
+    constructor(collectors: TelemetryCollector[]);
+    collect(sample: OperationMetricSample): Promise<void>;
+}
+/** Create a composite telemetry collector. */
+export declare function createCompositeTelemetryCollector(collectors: TelemetryCollector[]): CompositeTelemetryCollector;

package/dist/telemetry/posthog.js

@@ -0,0 +1,132 @@
+// @bun
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: (newValue) => all[name] = () => newValue
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __require = import.meta.require;
+
+// src/telemetry/posthog.ts
+var exports_posthog = {};
+__export(exports_posthog, {
+  createPostHogTracedModel: () => createPostHogTracedModel,
+  createPostHogTelemetryCollector: () => createPostHogTelemetryCollector,
+  createCompositeTelemetryCollector: () => createCompositeTelemetryCollector,
+  PostHogTelemetryCollector: () => PostHogTelemetryCollector,
+  CompositeTelemetryCollector: () => CompositeTelemetryCollector
+});
+async function createPostHogTracedModel(model, config, overrides) {
+  const { withTracing } = await importPostHogAI();
+  const phClient = await resolvePostHogClient(config);
+  const tracingOptions = {
+    ...config.defaults,
+    ...overrides
+  };
+  return withTracing(model, phClient, tracingOptions);
+}
+
+class PostHogTelemetryCollector {
+  phClient;
+  config;
+  initPromise;
+  constructor(config) {
+    this.config = config;
+    this.phClient = config.client;
+  }
+  async collect(sample) {
+    const client = await this.getClient();
+    const distinctId = this.config.defaults?.posthogDistinctId ?? sample.metadata?.["actorId"] ?? "system";
+    client.capture({
+      distinctId,
+      event: "$ai_generation",
+      properties: {
+        $ai_model: sample.operation.name,
+        $ai_provider: "contractspec",
+        $ai_latency: sample.durationMs / 1000,
+        $ai_is_error: !sample.success,
+        $ai_trace_id: this.config.defaults?.posthogTraceId,
+        ...sample.metadata?.["tokenUsage"] ? mapTokenUsage(sample.metadata["tokenUsage"]) : {},
+        ...this.config.defaults?.posthogProperties,
+        contractspec_operation: sample.operation.name,
+        contractspec_version: sample.operation.version,
+        contractspec_agent_id: sample.metadata?.["agentId"],
+        contractspec_finish_reason: sample.metadata?.["finishReason"],
+        contractspec_tool_count: sample.metadata?.["toolCallCount"]
+      },
+      groups: this.config.defaults?.posthogGroups
+    });
+  }
+  async shutdown() {
+    if (this.phClient?.shutdown) {
+      await this.phClient.shutdown();
+    }
+  }
+  async getClient() {
+    if (this.phClient)
+      return this.phClient;
+    if (!this.initPromise) {
+      this.initPromise = resolvePostHogClient(this.config).then((client) => {
+        this.phClient = client;
+        return client;
+      });
+    }
+    return this.initPromise;
+  }
+}
+function createPostHogTelemetryCollector(config) {
+  return new PostHogTelemetryCollector(config);
+}
+
+class CompositeTelemetryCollector {
+  collectors;
+  constructor(collectors) {
+    this.collectors = collectors;
+  }
+  async collect(sample) {
+    await Promise.all(this.collectors.map((c) => c.collect(sample)));
+  }
+}
+function createCompositeTelemetryCollector(collectors) {
+  return new CompositeTelemetryCollector(collectors);
+}
+async function importPostHogAI() {
+  try {
+    return await import("@posthog/ai");
+  } catch {
+    throw new Error("PostHog LLM Analytics requires @posthog/ai to be installed. Run: npm install @posthog/ai posthog-node");
+  }
+}
+async function resolvePostHogClient(config) {
+  if (config.client)
+    return config.client;
+  if (!config.apiKey) {
+    throw new Error("PostHog LLM Analytics requires either a client instance or an apiKey.");
+  }
+  try {
+    const { PostHog } = await import("posthog-node");
+    return new PostHog(config.apiKey, {
+      host: config.host ?? "https://us.i.posthog.com"
+    });
+  } catch {
+    throw new Error("PostHog LLM Analytics requires posthog-node to be installed. Run: npm install posthog-node");
+  }
+}
+function mapTokenUsage(usage) {
+  return {
+    $ai_input_tokens: usage["promptTokens"],
+    $ai_output_tokens: usage["completionTokens"]
+  };
+}
+export {
+  createPostHogTracedModel,
+  createPostHogTelemetryCollector,
+  createCompositeTelemetryCollector,
+  PostHogTelemetryCollector,
+  CompositeTelemetryCollector
+};

package/dist/telemetry/posthog.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tools/index.d.ts

@@ -2,4 +2,3 @@ export * from './tool-adapter';
 export * from './knowledge-tool';
 export * from './mcp-client';
 export * from './mcp-server';
-//# sourceMappingURL=index.d.ts.map