@contractspec/lib.contracts 1.61.0 → 1.62.0

package/dist/node/integrations/index.js

@@ -1891,13 +1891,13 @@ function registerPowensIntegration(registry2) {
 var posthogIntegrationSpec = defineIntegration({
   meta: {
     key: "analytics.posthog",
-    version: "1.0.0",
+    version: "1.1.0",
     category: "analytics",
     title: "PostHog",
-    description: "PostHog integration for product analytics, feature flags, and HogQL queries.",
+    description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
     domain: "analytics",
     owners: ["@platform.integrations"],
-    tags: ["analytics", "posthog"],
+    tags: ["analytics", "posthog", "llm", "ai"],
     stability: StabilityEnum.Beta
   },
   supportedModes: ["managed", "byok"],
@@ -1910,7 +1910,9 @@ var posthogIntegrationSpec = defineIntegration({
       { key: "analytics.persons", version: "1.0.0" },
       { key: "analytics.insights", version: "1.0.0" },
       { key: "analytics.cohorts", version: "1.0.0" },
-      { key: "analytics.annotations", version: "1.0.0" }
+      { key: "analytics.annotations", version: "1.0.0" },
+      { key: "analytics.llm-tracing", version: "1.0.0" },
+      { key: "analytics.llm-evaluations", version: "1.0.0" }
     ]
   },
   configSchema: {
@@ -1969,6 +1971,174 @@ function registerPosthogIntegration(registry2) {
   return registry2.register(posthogIntegrationSpec);
 }
 
+// src/integrations/providers/posthog-llm-telemetry.ts
+var posthogLLMTelemetrySpec = {
+  meta: {
+    key: "analytics.posthog.llm",
+    version: "1.0.0",
+    title: "PostHog LLM Analytics Telemetry",
+    description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
+    domain: "analytics",
+    owners: ["@platform.integrations"],
+    tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
+    stability: StabilityEnum.Beta
+  },
+  events: [
+    {
+      key: "posthog.llm.generation",
+      version: "1.0.0",
+      semantics: {
+        who: "AI agent or LLM consumer",
+        what: "Captures an LLM generation call with full observability metadata",
+        why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
+      },
+      properties: {
+        $ai_model: {
+          type: "string",
+          required: true,
+          description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
+        },
+        $ai_provider: {
+          type: "string",
+          required: true,
+          description: "The LLM provider (e.g., openai, anthropic, contractspec)"
+        },
+        $ai_latency: {
+          type: "number",
+          required: false,
+          description: "LLM call latency in seconds"
+        },
+        $ai_input_tokens: {
+          type: "number",
+          required: false,
+          description: "Number of input/prompt tokens consumed"
+        },
+        $ai_output_tokens: {
+          type: "number",
+          required: false,
+          description: "Number of output/completion tokens generated"
+        },
+        $ai_total_cost_usd: {
+          type: "number",
+          required: false,
+          description: "Total cost in USD (input + output)"
+        },
+        $ai_is_error: {
+          type: "boolean",
+          required: false,
+          description: "Whether the generation resulted in an error"
+        },
+        $ai_trace_id: {
+          type: "string",
+          required: false,
+          description: "Trace ID for grouping related generations"
+        },
+        $ai_stream: {
+          type: "boolean",
+          required: false,
+          description: "Whether the response was streamed"
+        },
+        $ai_time_to_first_token: {
+          type: "number",
+          required: false,
+          description: "Time to first token in seconds (streaming only)"
+        },
+        $ai_tools: {
+          type: "json",
+          required: false,
+          description: "Tools/functions available to the LLM"
+        },
+        $ai_input: {
+          type: "json",
+          required: false,
+          pii: true,
+          redact: true,
+          description: "Messages sent to the LLM (may contain PII)"
+        },
+        $ai_output_choices: {
+          type: "json",
+          required: false,
+          pii: true,
+          redact: true,
+          description: "Response choices from the LLM (may contain PII)"
+        },
+        contractspec_operation: {
+          type: "string",
+          required: false,
+          description: "ContractSpec operation name"
+        },
+        contractspec_version: {
+          type: "string",
+          required: false,
+          description: "ContractSpec operation version"
+        },
+        contractspec_agent_id: {
+          type: "string",
+          required: false,
+          description: "ContractSpec agent identifier"
+        },
+        contractspec_finish_reason: {
+          type: "string",
+          required: false,
+          description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
+        },
+        contractspec_tool_count: {
+          type: "number",
+          required: false,
+          description: "Number of tool calls made in this step"
+        }
+      },
+      privacy: "internal",
+      tags: ["llm", "generation", "posthog"]
+    }
+  ],
+  config: {
+    defaultRetentionDays: 90,
+    defaultSamplingRate: 1,
+    providers: [
+      {
+        type: "posthog",
+        config: {
+          eventName: "$ai_generation",
+          enableEvaluations: true,
+          evaluationTemplates: [
+            "relevance",
+            "helpfulness",
+            "jailbreak",
+            "hallucination",
+            "toxicity"
+          ]
+        }
+      }
+    ]
+  }
+};
+var POSTHOG_LLM_PII_FIELDS = [
+  "$ai_input",
+  "$ai_output_choices"
+];
+var POSTHOG_LLM_TELEMETRY_EVENTS = {
+  generation: "$ai_generation",
+  span: "$ai_span",
+  evaluation: "$ai_evaluation"
+};
+var POSTHOG_EVALUATION_TEMPLATES = {
+  relevance: "relevance",
+  helpfulness: "helpfulness",
+  jailbreak: "jailbreak",
+  hallucination: "hallucination",
+  toxicity: "toxicity"
+};
+function redactPostHogLLMTelemetryPayload(payload) {
+  const redacted = { ...payload };
+  for (const field of POSTHOG_LLM_PII_FIELDS) {
+    if (field in redacted) {
+      redacted[field] = "[REDACTED]";
+    }
+  }
+  return redacted;
+}
+
 // src/integrations/providers/linear.ts
 var linearIntegrationSpec = defineIntegration({
   meta: {
@@ -3755,11 +3925,13 @@ export {
   registerFathomIntegration,
   registerFalIntegration,
   registerElevenLabsIntegration,
+  redactPostHogLLMTelemetryPayload,
   redactOpenBankingTelemetryPayload,
   redactMeetingRecorderTelemetryPayload,
   qdrantIntegrationSpec,
   powensIntegrationSpec,
   postmarkIntegrationSpec,
+  posthogLLMTelemetrySpec,
   posthogIntegrationSpec,
   notionIntegrationSpec,
   mistralIntegrationSpec,
@@ -3779,6 +3951,9 @@ export {
   defineIntegration,
   createDefaultIntegrationSpecRegistry,
   assertPrimaryOpenBankingReady,
+  POSTHOG_LLM_TELEMETRY_EVENTS,
+  POSTHOG_LLM_PII_FIELDS,
+  POSTHOG_EVALUATION_TEMPLATES,
   OpenBankingSyncTransactions,
   OpenBankingSyncAccounts,
   OpenBankingRefreshBalances,

package/dist/node/integrations/providers/index.js

@@ -1370,13 +1370,13 @@ function registerPowensIntegration(registry) {
 var posthogIntegrationSpec = defineIntegration({
   meta: {
     key: "analytics.posthog",
-    version: "1.0.0",
+    version: "1.1.0",
     category: "analytics",
     title: "PostHog",
-    description: "PostHog integration for product analytics, feature flags, and HogQL queries.",
+    description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
     domain: "analytics",
     owners: ["@platform.integrations"],
-    tags: ["analytics", "posthog"],
+    tags: ["analytics", "posthog", "llm", "ai"],
     stability: StabilityEnum.Beta
   },
   supportedModes: ["managed", "byok"],
@@ -1389,7 +1389,9 @@ var posthogIntegrationSpec = defineIntegration({
       { key: "analytics.persons", version: "1.0.0" },
       { key: "analytics.insights", version: "1.0.0" },
       { key: "analytics.cohorts", version: "1.0.0" },
-      { key: "analytics.annotations", version: "1.0.0" }
+      { key: "analytics.annotations", version: "1.0.0" },
+      { key: "analytics.llm-tracing", version: "1.0.0" },
+      { key: "analytics.llm-evaluations", version: "1.0.0" }
     ]
   },
   configSchema: {
@@ -1448,6 +1450,174 @@ function registerPosthogIntegration(registry) {
   return registry.register(posthogIntegrationSpec);
 }
 
+// src/integrations/providers/posthog-llm-telemetry.ts
+var posthogLLMTelemetrySpec = {
+  meta: {
+    key: "analytics.posthog.llm",
+    version: "1.0.0",
+    title: "PostHog LLM Analytics Telemetry",
+    description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
+    domain: "analytics",
+    owners: ["@platform.integrations"],
+    tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
+    stability: StabilityEnum.Beta
+  },
+  events: [
+    {
+      key: "posthog.llm.generation",
+      version: "1.0.0",
+      semantics: {
+        who: "AI agent or LLM consumer",
+        what: "Captures an LLM generation call with full observability metadata",
+        why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
+      },
+      properties: {
+        $ai_model: {
+          type: "string",
+          required: true,
+          description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
+        },
+        $ai_provider: {
+          type: "string",
+          required: true,
+          description: "The LLM provider (e.g., openai, anthropic, contractspec)"
+        },
+        $ai_latency: {
+          type: "number",
+          required: false,
+          description: "LLM call latency in seconds"
+        },
+        $ai_input_tokens: {
+          type: "number",
+          required: false,
+          description: "Number of input/prompt tokens consumed"
+        },
+        $ai_output_tokens: {
+          type: "number",
+          required: false,
+          description: "Number of output/completion tokens generated"
+        },
+        $ai_total_cost_usd: {
+          type: "number",
+          required: false,
+          description: "Total cost in USD (input + output)"
+        },
+        $ai_is_error: {
+          type: "boolean",
+          required: false,
+          description: "Whether the generation resulted in an error"
+        },
+        $ai_trace_id: {
+          type: "string",
+          required: false,
+          description: "Trace ID for grouping related generations"
+        },
+        $ai_stream: {
+          type: "boolean",
+          required: false,
+          description: "Whether the response was streamed"
+        },
+        $ai_time_to_first_token: {
+          type: "number",
+          required: false,
+          description: "Time to first token in seconds (streaming only)"
+        },
+        $ai_tools: {
+          type: "json",
+          required: false,
+          description: "Tools/functions available to the LLM"
+        },
+        $ai_input: {
+          type: "json",
+          required: false,
+          pii: true,
+          redact: true,
+          description: "Messages sent to the LLM (may contain PII)"
+        },
+        $ai_output_choices: {
+          type: "json",
+          required: false,
+          pii: true,
+          redact: true,
+          description: "Response choices from the LLM (may contain PII)"
+        },
+        contractspec_operation: {
+          type: "string",
+          required: false,
+          description: "ContractSpec operation name"
+        },
+        contractspec_version: {
+          type: "string",
+          required: false,
+          description: "ContractSpec operation version"
+        },
+        contractspec_agent_id: {
+          type: "string",
+          required: false,
+          description: "ContractSpec agent identifier"
+        },
+        contractspec_finish_reason: {
+          type: "string",
+          required: false,
+          description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
+        },
+        contractspec_tool_count: {
+          type: "number",
+          required: false,
+          description: "Number of tool calls made in this step"
+        }
+      },
+      privacy: "internal",
+      tags: ["llm", "generation", "posthog"]
+    }
+  ],
+  config: {
+    defaultRetentionDays: 90,
+    defaultSamplingRate: 1,
+    providers: [
+      {
+        type: "posthog",
+        config: {
+          eventName: "$ai_generation",
+          enableEvaluations: true,
+          evaluationTemplates: [
+            "relevance",
+            "helpfulness",
+            "jailbreak",
+            "hallucination",
+            "toxicity"
+          ]
+        }
+      }
+    ]
+  }
+};
+var POSTHOG_LLM_PII_FIELDS = [
+  "$ai_input",
+  "$ai_output_choices"
+];
+var POSTHOG_LLM_TELEMETRY_EVENTS = {
+  generation: "$ai_generation",
+  span: "$ai_span",
+  evaluation: "$ai_evaluation"
+};
+var POSTHOG_EVALUATION_TEMPLATES = {
+  relevance: "relevance",
+  helpfulness: "helpfulness",
+  jailbreak: "jailbreak",
+  hallucination: "hallucination",
+  toxicity: "toxicity"
+};
+function redactPostHogLLMTelemetryPayload(payload) {
+  const redacted = { ...payload };
+  for (const field of POSTHOG_LLM_PII_FIELDS) {
+    if (field in redacted) {
+      redacted[field] = "[REDACTED]";
+    }
+  }
+  return redacted;
+}
+
 // src/integrations/providers/linear.ts
 var linearIntegrationSpec = defineIntegration({
   meta: {
@@ -2121,9 +2291,11 @@ export {
   registerFathomIntegration,
   registerFalIntegration,
   registerElevenLabsIntegration,
+  redactPostHogLLMTelemetryPayload,
   qdrantIntegrationSpec,
   powensIntegrationSpec,
   postmarkIntegrationSpec,
+  posthogLLMTelemetrySpec,
   posthogIntegrationSpec,
   notionIntegrationSpec,
   mistralIntegrationSpec,
@@ -2138,5 +2310,8 @@ export {
   fathomIntegrationSpec,
   falIntegrationSpec,
   elevenLabsIntegrationSpec,
-  createDefaultIntegrationSpecRegistry
+  createDefaultIntegrationSpecRegistry,
+  POSTHOG_LLM_TELEMETRY_EVENTS,
+  POSTHOG_LLM_PII_FIELDS,
+  POSTHOG_EVALUATION_TEMPLATES
 };

package/dist/node/integrations/providers/posthog-llm-telemetry.js

@@ -0,0 +1,241 @@
+import { createRequire } from "node:module";
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __moduleCache = /* @__PURE__ */ new WeakMap;
+var __toCommonJS = (from) => {
+  var entry = __moduleCache.get(from), desc;
+  if (entry)
+    return entry;
+  entry = __defProp({}, "__esModule", { value: true });
+  if (from && typeof from === "object" || typeof from === "function")
+    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    }));
+  __moduleCache.set(from, entry);
+  return entry;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: (newValue) => all[name] = () => newValue
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
+// src/ownership.ts
+var StabilityEnum = {
+  Idea: "idea",
+  InCreation: "in_creation",
+  Experimental: "experimental",
+  Beta: "beta",
+  Stable: "stable",
+  Deprecated: "deprecated"
+};
+var OwnersEnum = {
+  PlatformCore: "platform.core",
+  PlatformSigil: "platform.sigil",
+  PlatformMarketplace: "platform.marketplace",
+  PlatformMessaging: "platform.messaging",
+  PlatformContent: "platform.content",
+  PlatformFeatureFlags: "platform.featureflags",
+  PlatformFinance: "platform.finance"
+};
+var Owners = OwnersEnum;
+var TagsEnum = {
+  Spots: "spots",
+  Collectivity: "collectivity",
+  Marketplace: "marketplace",
+  Sellers: "sellers",
+  Auth: "auth",
+  Login: "login",
+  Signup: "signup",
+  Guide: "guide",
+  Docs: "docs",
+  I18n: "i18n",
+  Incident: "incident",
+  Automation: "automation",
+  Hygiene: "hygiene"
+};
+var Tags = TagsEnum;
+
+// src/integrations/providers/posthog-llm-telemetry.ts
+var posthogLLMTelemetrySpec = {
+  meta: {
+    key: "analytics.posthog.llm",
+    version: "1.0.0",
+    title: "PostHog LLM Analytics Telemetry",
+    description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
+    domain: "analytics",
+    owners: ["@platform.integrations"],
+    tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
+    stability: StabilityEnum.Beta
+  },
+  events: [
+    {
+      key: "posthog.llm.generation",
+      version: "1.0.0",
+      semantics: {
+        who: "AI agent or LLM consumer",
+        what: "Captures an LLM generation call with full observability metadata",
+        why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
+      },
+      properties: {
+        $ai_model: {
+          type: "string",
+          required: true,
+          description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
+        },
+        $ai_provider: {
+          type: "string",
+          required: true,
+          description: "The LLM provider (e.g., openai, anthropic, contractspec)"
+        },
+        $ai_latency: {
+          type: "number",
+          required: false,
+          description: "LLM call latency in seconds"
+        },
+        $ai_input_tokens: {
+          type: "number",
+          required: false,
+          description: "Number of input/prompt tokens consumed"
+        },
+        $ai_output_tokens: {
+          type: "number",
+          required: false,
+          description: "Number of output/completion tokens generated"
+        },
+        $ai_total_cost_usd: {
+          type: "number",
+          required: false,
+          description: "Total cost in USD (input + output)"
+        },
+        $ai_is_error: {
+          type: "boolean",
+          required: false,
+          description: "Whether the generation resulted in an error"
+        },
+        $ai_trace_id: {
+          type: "string",
+          required: false,
+          description: "Trace ID for grouping related generations"
+        },
+        $ai_stream: {
+          type: "boolean",
+          required: false,
+          description: "Whether the response was streamed"
+        },
+        $ai_time_to_first_token: {
+          type: "number",
+          required: false,
+          description: "Time to first token in seconds (streaming only)"
+        },
+        $ai_tools: {
+          type: "json",
+          required: false,
+          description: "Tools/functions available to the LLM"
+        },
+        $ai_input: {
+          type: "json",
+          required: false,
+          pii: true,
+          redact: true,
+          description: "Messages sent to the LLM (may contain PII)"
+        },
+        $ai_output_choices: {
+          type: "json",
+          required: false,
+          pii: true,
+          redact: true,
+          description: "Response choices from the LLM (may contain PII)"
+        },
+        contractspec_operation: {
+          type: "string",
+          required: false,
+          description: "ContractSpec operation name"
+        },
+        contractspec_version: {
+          type: "string",
+          required: false,
+          description: "ContractSpec operation version"
+        },
+        contractspec_agent_id: {
+          type: "string",
+          required: false,
+          description: "ContractSpec agent identifier"
+        },
+        contractspec_finish_reason: {
+          type: "string",
+          required: false,
+          description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
+        },
+        contractspec_tool_count: {
+          type: "number",
+          required: false,
+          description: "Number of tool calls made in this step"
+        }
+      },
+      privacy: "internal",
+      tags: ["llm", "generation", "posthog"]
+    }
+  ],
+  config: {
+    defaultRetentionDays: 90,
+    defaultSamplingRate: 1,
+    providers: [
+      {
+        type: "posthog",
+        config: {
+          eventName: "$ai_generation",
+          enableEvaluations: true,
+          evaluationTemplates: [
+            "relevance",
+            "helpfulness",
+            "jailbreak",
+            "hallucination",
+            "toxicity"
+          ]
+        }
+      }
+    ]
+  }
+};
+var POSTHOG_LLM_PII_FIELDS = [
+  "$ai_input",
+  "$ai_output_choices"
+];
+var POSTHOG_LLM_TELEMETRY_EVENTS = {
+  generation: "$ai_generation",
+  span: "$ai_span",
+  evaluation: "$ai_evaluation"
+};
+var POSTHOG_EVALUATION_TEMPLATES = {
+  relevance: "relevance",
+  helpfulness: "helpfulness",
+  jailbreak: "jailbreak",
+  hallucination: "hallucination",
+  toxicity: "toxicity"
+};
+function redactPostHogLLMTelemetryPayload(payload) {
+  const redacted = { ...payload };
+  for (const field of POSTHOG_LLM_PII_FIELDS) {
+    if (field in redacted) {
+      redacted[field] = "[REDACTED]";
+    }
+  }
+  return redacted;
+}
+export {
+  redactPostHogLLMTelemetryPayload,
+  posthogLLMTelemetrySpec,
+  POSTHOG_LLM_TELEMETRY_EVENTS,
+  POSTHOG_LLM_PII_FIELDS,
+  POSTHOG_EVALUATION_TEMPLATES
+};