@contractspec/lib.contracts 1.61.0 → 1.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +179 -4
- package/dist/integrations/index.js +179 -4
- package/dist/integrations/providers/index.d.ts +1 -0
- package/dist/integrations/providers/index.d.ts.map +1 -1
- package/dist/integrations/providers/index.js +180 -5
- package/dist/integrations/providers/posthog-llm-telemetry.d.ts +52 -0
- package/dist/integrations/providers/posthog-llm-telemetry.d.ts.map +1 -0
- package/dist/integrations/providers/posthog-llm-telemetry.js +241 -0
- package/dist/integrations/providers/posthog.d.ts.map +1 -1
- package/dist/integrations/providers/posthog.js +6 -4
- package/dist/integrations/providers/registry.js +6 -4
- package/dist/node/index.js +179 -4
- package/dist/node/integrations/index.js +179 -4
- package/dist/node/integrations/providers/index.js +180 -5
- package/dist/node/integrations/providers/posthog-llm-telemetry.js +241 -0
- package/dist/node/integrations/providers/posthog.js +6 -4
- package/dist/node/integrations/providers/registry.js +6 -4
- package/package.json +22 -10
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { TelemetrySpec } from '../../telemetry/spec';
|
|
2
|
+
/**
|
|
3
|
+
* PostHog LLM Analytics telemetry spec.
|
|
4
|
+
*
|
|
5
|
+
* Defines the telemetry events captured by the @posthog/ai integration
|
|
6
|
+
* for LLM observability. Events are captured via withTracing() model wrapping
|
|
7
|
+
* or the PostHogTelemetryCollector bridge.
|
|
8
|
+
*
|
|
9
|
+
* PostHog Evaluations (Relevance, Helpfulness, Hallucination, Toxicity,
|
|
10
|
+
* Jailbreak, and custom prompts) run server-side on these events.
|
|
11
|
+
* No client-side code is needed beyond capturing $ai_generation events.
|
|
12
|
+
*/
|
|
13
|
+
export declare const posthogLLMTelemetrySpec: TelemetrySpec;
|
|
14
|
+
/**
|
|
15
|
+
* PostHog LLM Analytics PII fields that may appear in generation events.
|
|
16
|
+
* These should be redacted when privacyMode is enabled.
|
|
17
|
+
*/
|
|
18
|
+
export declare const POSTHOG_LLM_PII_FIELDS: readonly ["$ai_input", "$ai_output_choices"];
|
|
19
|
+
/**
|
|
20
|
+
* PostHog LLM Analytics telemetry event name constants.
|
|
21
|
+
*/
|
|
22
|
+
export declare const POSTHOG_LLM_TELEMETRY_EVENTS: {
|
|
23
|
+
/** Captured for every LLM generation call */
|
|
24
|
+
readonly generation: "$ai_generation";
|
|
25
|
+
/** Captured for LLM trace spans */
|
|
26
|
+
readonly span: "$ai_span";
|
|
27
|
+
/** Captured for LLM evaluation results (server-side) */
|
|
28
|
+
readonly evaluation: "$ai_evaluation";
|
|
29
|
+
};
|
|
30
|
+
export type PostHogLLMTelemetryEvent = (typeof POSTHOG_LLM_TELEMETRY_EVENTS)[keyof typeof POSTHOG_LLM_TELEMETRY_EVENTS];
|
|
31
|
+
/**
|
|
32
|
+
* Built-in PostHog evaluation templates.
|
|
33
|
+
* These are configured in PostHog UI under LLM Analytics > Evaluations.
|
|
34
|
+
*/
|
|
35
|
+
export declare const POSTHOG_EVALUATION_TEMPLATES: {
|
|
36
|
+
/** Whether the output addresses the user's input */
|
|
37
|
+
readonly relevance: "relevance";
|
|
38
|
+
/** Whether the response is useful and actionable */
|
|
39
|
+
readonly helpfulness: "helpfulness";
|
|
40
|
+
/** Attempts to bypass safety guardrails */
|
|
41
|
+
readonly jailbreak: "jailbreak";
|
|
42
|
+
/** Made-up facts or unsupported claims */
|
|
43
|
+
readonly hallucination: "hallucination";
|
|
44
|
+
/** Harmful, offensive, or inappropriate content */
|
|
45
|
+
readonly toxicity: "toxicity";
|
|
46
|
+
};
|
|
47
|
+
export type PostHogEvaluationTemplate = (typeof POSTHOG_EVALUATION_TEMPLATES)[keyof typeof POSTHOG_EVALUATION_TEMPLATES];
|
|
48
|
+
/**
|
|
49
|
+
* Redact PII fields from PostHog LLM telemetry payload.
|
|
50
|
+
*/
|
|
51
|
+
export declare function redactPostHogLLMTelemetryPayload<T extends Record<string, unknown>>(payload: T): T;
|
|
52
|
+
//# sourceMappingURL=posthog-llm-telemetry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"posthog-llm-telemetry.d.ts","sourceRoot":"","sources":["../../../src/integrations/providers/posthog-llm-telemetry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D;;;;;;;;;;GAUG;AACH,eAAO,MAAM,uBAAuB,EAAE,aA+IrC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,sBAAsB,8CAGzB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,4BAA4B;IACvC,6CAA6C;;IAE7C,mCAAmC;;IAEnC,wDAAwD;;CAEhD,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAClC,CAAC,OAAO,4BAA4B,CAAC,CAAC,MAAM,OAAO,4BAA4B,CAAC,CAAC;AAEnF;;;GAGG;AACH,eAAO,MAAM,4BAA4B;IACvC,oDAAoD;;IAEpD,oDAAoD;;IAEpD,2CAA2C;;IAE3C,0CAA0C;;IAE1C,mDAAmD;;CAE3C,CAAC;AAEX,MAAM,MAAM,yBAAyB,GACnC,CAAC,OAAO,4BAA4B,CAAC,CAAC,MAAM,OAAO,4BAA4B,CAAC,CAAC;AAEnF;;GAEG;AACH,wBAAgB,gCAAgC,CAC9C,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,OAAO,EAAE,CAAC,GAAG,CAAC,CAQf"}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __moduleCache = /* @__PURE__ */ new WeakMap;
|
|
7
|
+
var __toCommonJS = (from) => {
|
|
8
|
+
var entry = __moduleCache.get(from), desc;
|
|
9
|
+
if (entry)
|
|
10
|
+
return entry;
|
|
11
|
+
entry = __defProp({}, "__esModule", { value: true });
|
|
12
|
+
if (from && typeof from === "object" || typeof from === "function")
|
|
13
|
+
__getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
|
|
14
|
+
get: () => from[key],
|
|
15
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
16
|
+
}));
|
|
17
|
+
__moduleCache.set(from, entry);
|
|
18
|
+
return entry;
|
|
19
|
+
};
|
|
20
|
+
var __export = (target, all) => {
|
|
21
|
+
for (var name in all)
|
|
22
|
+
__defProp(target, name, {
|
|
23
|
+
get: all[name],
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
set: (newValue) => all[name] = () => newValue
|
|
27
|
+
});
|
|
28
|
+
};
|
|
29
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
30
|
+
var __require = import.meta.require;
|
|
31
|
+
|
|
32
|
+
// src/ownership.ts
|
|
33
|
+
var StabilityEnum = {
|
|
34
|
+
Idea: "idea",
|
|
35
|
+
InCreation: "in_creation",
|
|
36
|
+
Experimental: "experimental",
|
|
37
|
+
Beta: "beta",
|
|
38
|
+
Stable: "stable",
|
|
39
|
+
Deprecated: "deprecated"
|
|
40
|
+
};
|
|
41
|
+
var OwnersEnum = {
|
|
42
|
+
PlatformCore: "platform.core",
|
|
43
|
+
PlatformSigil: "platform.sigil",
|
|
44
|
+
PlatformMarketplace: "platform.marketplace",
|
|
45
|
+
PlatformMessaging: "platform.messaging",
|
|
46
|
+
PlatformContent: "platform.content",
|
|
47
|
+
PlatformFeatureFlags: "platform.featureflags",
|
|
48
|
+
PlatformFinance: "platform.finance"
|
|
49
|
+
};
|
|
50
|
+
var Owners = OwnersEnum;
|
|
51
|
+
var TagsEnum = {
|
|
52
|
+
Spots: "spots",
|
|
53
|
+
Collectivity: "collectivity",
|
|
54
|
+
Marketplace: "marketplace",
|
|
55
|
+
Sellers: "sellers",
|
|
56
|
+
Auth: "auth",
|
|
57
|
+
Login: "login",
|
|
58
|
+
Signup: "signup",
|
|
59
|
+
Guide: "guide",
|
|
60
|
+
Docs: "docs",
|
|
61
|
+
I18n: "i18n",
|
|
62
|
+
Incident: "incident",
|
|
63
|
+
Automation: "automation",
|
|
64
|
+
Hygiene: "hygiene"
|
|
65
|
+
};
|
|
66
|
+
var Tags = TagsEnum;
|
|
67
|
+
|
|
68
|
+
// src/integrations/providers/posthog-llm-telemetry.ts
|
|
69
|
+
var posthogLLMTelemetrySpec = {
|
|
70
|
+
meta: {
|
|
71
|
+
key: "analytics.posthog.llm",
|
|
72
|
+
version: "1.0.0",
|
|
73
|
+
title: "PostHog LLM Analytics Telemetry",
|
|
74
|
+
description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
|
|
75
|
+
domain: "analytics",
|
|
76
|
+
owners: ["@platform.integrations"],
|
|
77
|
+
tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
|
|
78
|
+
stability: StabilityEnum.Beta
|
|
79
|
+
},
|
|
80
|
+
events: [
|
|
81
|
+
{
|
|
82
|
+
key: "posthog.llm.generation",
|
|
83
|
+
version: "1.0.0",
|
|
84
|
+
semantics: {
|
|
85
|
+
who: "AI agent or LLM consumer",
|
|
86
|
+
what: "Captures an LLM generation call with full observability metadata",
|
|
87
|
+
why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
|
|
88
|
+
},
|
|
89
|
+
properties: {
|
|
90
|
+
$ai_model: {
|
|
91
|
+
type: "string",
|
|
92
|
+
required: true,
|
|
93
|
+
description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
|
|
94
|
+
},
|
|
95
|
+
$ai_provider: {
|
|
96
|
+
type: "string",
|
|
97
|
+
required: true,
|
|
98
|
+
description: "The LLM provider (e.g., openai, anthropic, contractspec)"
|
|
99
|
+
},
|
|
100
|
+
$ai_latency: {
|
|
101
|
+
type: "number",
|
|
102
|
+
required: false,
|
|
103
|
+
description: "LLM call latency in seconds"
|
|
104
|
+
},
|
|
105
|
+
$ai_input_tokens: {
|
|
106
|
+
type: "number",
|
|
107
|
+
required: false,
|
|
108
|
+
description: "Number of input/prompt tokens consumed"
|
|
109
|
+
},
|
|
110
|
+
$ai_output_tokens: {
|
|
111
|
+
type: "number",
|
|
112
|
+
required: false,
|
|
113
|
+
description: "Number of output/completion tokens generated"
|
|
114
|
+
},
|
|
115
|
+
$ai_total_cost_usd: {
|
|
116
|
+
type: "number",
|
|
117
|
+
required: false,
|
|
118
|
+
description: "Total cost in USD (input + output)"
|
|
119
|
+
},
|
|
120
|
+
$ai_is_error: {
|
|
121
|
+
type: "boolean",
|
|
122
|
+
required: false,
|
|
123
|
+
description: "Whether the generation resulted in an error"
|
|
124
|
+
},
|
|
125
|
+
$ai_trace_id: {
|
|
126
|
+
type: "string",
|
|
127
|
+
required: false,
|
|
128
|
+
description: "Trace ID for grouping related generations"
|
|
129
|
+
},
|
|
130
|
+
$ai_stream: {
|
|
131
|
+
type: "boolean",
|
|
132
|
+
required: false,
|
|
133
|
+
description: "Whether the response was streamed"
|
|
134
|
+
},
|
|
135
|
+
$ai_time_to_first_token: {
|
|
136
|
+
type: "number",
|
|
137
|
+
required: false,
|
|
138
|
+
description: "Time to first token in seconds (streaming only)"
|
|
139
|
+
},
|
|
140
|
+
$ai_tools: {
|
|
141
|
+
type: "json",
|
|
142
|
+
required: false,
|
|
143
|
+
description: "Tools/functions available to the LLM"
|
|
144
|
+
},
|
|
145
|
+
$ai_input: {
|
|
146
|
+
type: "json",
|
|
147
|
+
required: false,
|
|
148
|
+
pii: true,
|
|
149
|
+
redact: true,
|
|
150
|
+
description: "Messages sent to the LLM (may contain PII)"
|
|
151
|
+
},
|
|
152
|
+
$ai_output_choices: {
|
|
153
|
+
type: "json",
|
|
154
|
+
required: false,
|
|
155
|
+
pii: true,
|
|
156
|
+
redact: true,
|
|
157
|
+
description: "Response choices from the LLM (may contain PII)"
|
|
158
|
+
},
|
|
159
|
+
contractspec_operation: {
|
|
160
|
+
type: "string",
|
|
161
|
+
required: false,
|
|
162
|
+
description: "ContractSpec operation name"
|
|
163
|
+
},
|
|
164
|
+
contractspec_version: {
|
|
165
|
+
type: "string",
|
|
166
|
+
required: false,
|
|
167
|
+
description: "ContractSpec operation version"
|
|
168
|
+
},
|
|
169
|
+
contractspec_agent_id: {
|
|
170
|
+
type: "string",
|
|
171
|
+
required: false,
|
|
172
|
+
description: "ContractSpec agent identifier"
|
|
173
|
+
},
|
|
174
|
+
contractspec_finish_reason: {
|
|
175
|
+
type: "string",
|
|
176
|
+
required: false,
|
|
177
|
+
description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
|
|
178
|
+
},
|
|
179
|
+
contractspec_tool_count: {
|
|
180
|
+
type: "number",
|
|
181
|
+
required: false,
|
|
182
|
+
description: "Number of tool calls made in this step"
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
privacy: "internal",
|
|
186
|
+
tags: ["llm", "generation", "posthog"]
|
|
187
|
+
}
|
|
188
|
+
],
|
|
189
|
+
config: {
|
|
190
|
+
defaultRetentionDays: 90,
|
|
191
|
+
defaultSamplingRate: 1,
|
|
192
|
+
providers: [
|
|
193
|
+
{
|
|
194
|
+
type: "posthog",
|
|
195
|
+
config: {
|
|
196
|
+
eventName: "$ai_generation",
|
|
197
|
+
enableEvaluations: true,
|
|
198
|
+
evaluationTemplates: [
|
|
199
|
+
"relevance",
|
|
200
|
+
"helpfulness",
|
|
201
|
+
"jailbreak",
|
|
202
|
+
"hallucination",
|
|
203
|
+
"toxicity"
|
|
204
|
+
]
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
]
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
var POSTHOG_LLM_PII_FIELDS = [
|
|
211
|
+
"$ai_input",
|
|
212
|
+
"$ai_output_choices"
|
|
213
|
+
];
|
|
214
|
+
var POSTHOG_LLM_TELEMETRY_EVENTS = {
|
|
215
|
+
generation: "$ai_generation",
|
|
216
|
+
span: "$ai_span",
|
|
217
|
+
evaluation: "$ai_evaluation"
|
|
218
|
+
};
|
|
219
|
+
var POSTHOG_EVALUATION_TEMPLATES = {
|
|
220
|
+
relevance: "relevance",
|
|
221
|
+
helpfulness: "helpfulness",
|
|
222
|
+
jailbreak: "jailbreak",
|
|
223
|
+
hallucination: "hallucination",
|
|
224
|
+
toxicity: "toxicity"
|
|
225
|
+
};
|
|
226
|
+
function redactPostHogLLMTelemetryPayload(payload) {
|
|
227
|
+
const redacted = { ...payload };
|
|
228
|
+
for (const field of POSTHOG_LLM_PII_FIELDS) {
|
|
229
|
+
if (field in redacted) {
|
|
230
|
+
redacted[field] = "[REDACTED]";
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return redacted;
|
|
234
|
+
}
|
|
235
|
+
export {
|
|
236
|
+
redactPostHogLLMTelemetryPayload,
|
|
237
|
+
posthogLLMTelemetrySpec,
|
|
238
|
+
POSTHOG_LLM_TELEMETRY_EVENTS,
|
|
239
|
+
POSTHOG_LLM_PII_FIELDS,
|
|
240
|
+
POSTHOG_EVALUATION_TEMPLATES
|
|
241
|
+
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"posthog.d.ts","sourceRoot":"","sources":["../../../src/integrations/providers/posthog.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,uBAAuB,EAAE,MAAM,SAAS,CAAC;AAErE,eAAO,MAAM,sBAAsB,
|
|
1
|
+
{"version":3,"file":"posthog.d.ts","sourceRoot":"","sources":["../../../src/integrations/providers/posthog.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,uBAAuB,EAAE,MAAM,SAAS,CAAC;AAErE,eAAO,MAAM,sBAAsB,8BAmFjC,CAAC;AAEH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,uBAAuB,GAChC,uBAAuB,CAEzB"}
|
|
@@ -266,13 +266,13 @@ var defineIntegration = (spec) => spec;
|
|
|
266
266
|
var posthogIntegrationSpec = defineIntegration({
|
|
267
267
|
meta: {
|
|
268
268
|
key: "analytics.posthog",
|
|
269
|
-
version: "1.
|
|
269
|
+
version: "1.1.0",
|
|
270
270
|
category: "analytics",
|
|
271
271
|
title: "PostHog",
|
|
272
|
-
description: "PostHog integration for product analytics, feature flags, and
|
|
272
|
+
description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
|
|
273
273
|
domain: "analytics",
|
|
274
274
|
owners: ["@platform.integrations"],
|
|
275
|
-
tags: ["analytics", "posthog"],
|
|
275
|
+
tags: ["analytics", "posthog", "llm", "ai"],
|
|
276
276
|
stability: StabilityEnum.Beta
|
|
277
277
|
},
|
|
278
278
|
supportedModes: ["managed", "byok"],
|
|
@@ -285,7 +285,9 @@ var posthogIntegrationSpec = defineIntegration({
|
|
|
285
285
|
{ key: "analytics.persons", version: "1.0.0" },
|
|
286
286
|
{ key: "analytics.insights", version: "1.0.0" },
|
|
287
287
|
{ key: "analytics.cohorts", version: "1.0.0" },
|
|
288
|
-
{ key: "analytics.annotations", version: "1.0.0" }
|
|
288
|
+
{ key: "analytics.annotations", version: "1.0.0" },
|
|
289
|
+
{ key: "analytics.llm-tracing", version: "1.0.0" },
|
|
290
|
+
{ key: "analytics.llm-evaluations", version: "1.0.0" }
|
|
289
291
|
]
|
|
290
292
|
},
|
|
291
293
|
configSchema: {
|
|
@@ -1371,13 +1371,13 @@ function registerPowensIntegration(registry) {
|
|
|
1371
1371
|
var posthogIntegrationSpec = defineIntegration({
|
|
1372
1372
|
meta: {
|
|
1373
1373
|
key: "analytics.posthog",
|
|
1374
|
-
version: "1.
|
|
1374
|
+
version: "1.1.0",
|
|
1375
1375
|
category: "analytics",
|
|
1376
1376
|
title: "PostHog",
|
|
1377
|
-
description: "PostHog integration for product analytics, feature flags, and
|
|
1377
|
+
description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
|
|
1378
1378
|
domain: "analytics",
|
|
1379
1379
|
owners: ["@platform.integrations"],
|
|
1380
|
-
tags: ["analytics", "posthog"],
|
|
1380
|
+
tags: ["analytics", "posthog", "llm", "ai"],
|
|
1381
1381
|
stability: StabilityEnum.Beta
|
|
1382
1382
|
},
|
|
1383
1383
|
supportedModes: ["managed", "byok"],
|
|
@@ -1390,7 +1390,9 @@ var posthogIntegrationSpec = defineIntegration({
|
|
|
1390
1390
|
{ key: "analytics.persons", version: "1.0.0" },
|
|
1391
1391
|
{ key: "analytics.insights", version: "1.0.0" },
|
|
1392
1392
|
{ key: "analytics.cohorts", version: "1.0.0" },
|
|
1393
|
-
{ key: "analytics.annotations", version: "1.0.0" }
|
|
1393
|
+
{ key: "analytics.annotations", version: "1.0.0" },
|
|
1394
|
+
{ key: "analytics.llm-tracing", version: "1.0.0" },
|
|
1395
|
+
{ key: "analytics.llm-evaluations", version: "1.0.0" }
|
|
1394
1396
|
]
|
|
1395
1397
|
},
|
|
1396
1398
|
configSchema: {
|
package/dist/node/index.js
CHANGED
|
@@ -11964,13 +11964,13 @@ function registerPowensIntegration(registry6) {
|
|
|
11964
11964
|
var posthogIntegrationSpec = defineIntegration({
|
|
11965
11965
|
meta: {
|
|
11966
11966
|
key: "analytics.posthog",
|
|
11967
|
-
version: "1.
|
|
11967
|
+
version: "1.1.0",
|
|
11968
11968
|
category: "analytics",
|
|
11969
11969
|
title: "PostHog",
|
|
11970
|
-
description: "PostHog integration for product analytics, feature flags, and
|
|
11970
|
+
description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
|
|
11971
11971
|
domain: "analytics",
|
|
11972
11972
|
owners: ["@platform.integrations"],
|
|
11973
|
-
tags: ["analytics", "posthog"],
|
|
11973
|
+
tags: ["analytics", "posthog", "llm", "ai"],
|
|
11974
11974
|
stability: StabilityEnum.Beta
|
|
11975
11975
|
},
|
|
11976
11976
|
supportedModes: ["managed", "byok"],
|
|
@@ -11983,7 +11983,9 @@ var posthogIntegrationSpec = defineIntegration({
|
|
|
11983
11983
|
{ key: "analytics.persons", version: "1.0.0" },
|
|
11984
11984
|
{ key: "analytics.insights", version: "1.0.0" },
|
|
11985
11985
|
{ key: "analytics.cohorts", version: "1.0.0" },
|
|
11986
|
-
{ key: "analytics.annotations", version: "1.0.0" }
|
|
11986
|
+
{ key: "analytics.annotations", version: "1.0.0" },
|
|
11987
|
+
{ key: "analytics.llm-tracing", version: "1.0.0" },
|
|
11988
|
+
{ key: "analytics.llm-evaluations", version: "1.0.0" }
|
|
11987
11989
|
]
|
|
11988
11990
|
},
|
|
11989
11991
|
configSchema: {
|
|
@@ -12042,6 +12044,174 @@ function registerPosthogIntegration(registry6) {
|
|
|
12042
12044
|
return registry6.register(posthogIntegrationSpec);
|
|
12043
12045
|
}
|
|
12044
12046
|
|
|
12047
|
+
// src/integrations/providers/posthog-llm-telemetry.ts
|
|
12048
|
+
var posthogLLMTelemetrySpec = {
|
|
12049
|
+
meta: {
|
|
12050
|
+
key: "analytics.posthog.llm",
|
|
12051
|
+
version: "1.0.0",
|
|
12052
|
+
title: "PostHog LLM Analytics Telemetry",
|
|
12053
|
+
description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
|
|
12054
|
+
domain: "analytics",
|
|
12055
|
+
owners: ["@platform.integrations"],
|
|
12056
|
+
tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
|
|
12057
|
+
stability: StabilityEnum.Beta
|
|
12058
|
+
},
|
|
12059
|
+
events: [
|
|
12060
|
+
{
|
|
12061
|
+
key: "posthog.llm.generation",
|
|
12062
|
+
version: "1.0.0",
|
|
12063
|
+
semantics: {
|
|
12064
|
+
who: "AI agent or LLM consumer",
|
|
12065
|
+
what: "Captures an LLM generation call with full observability metadata",
|
|
12066
|
+
why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
|
|
12067
|
+
},
|
|
12068
|
+
properties: {
|
|
12069
|
+
$ai_model: {
|
|
12070
|
+
type: "string",
|
|
12071
|
+
required: true,
|
|
12072
|
+
description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
|
|
12073
|
+
},
|
|
12074
|
+
$ai_provider: {
|
|
12075
|
+
type: "string",
|
|
12076
|
+
required: true,
|
|
12077
|
+
description: "The LLM provider (e.g., openai, anthropic, contractspec)"
|
|
12078
|
+
},
|
|
12079
|
+
$ai_latency: {
|
|
12080
|
+
type: "number",
|
|
12081
|
+
required: false,
|
|
12082
|
+
description: "LLM call latency in seconds"
|
|
12083
|
+
},
|
|
12084
|
+
$ai_input_tokens: {
|
|
12085
|
+
type: "number",
|
|
12086
|
+
required: false,
|
|
12087
|
+
description: "Number of input/prompt tokens consumed"
|
|
12088
|
+
},
|
|
12089
|
+
$ai_output_tokens: {
|
|
12090
|
+
type: "number",
|
|
12091
|
+
required: false,
|
|
12092
|
+
description: "Number of output/completion tokens generated"
|
|
12093
|
+
},
|
|
12094
|
+
$ai_total_cost_usd: {
|
|
12095
|
+
type: "number",
|
|
12096
|
+
required: false,
|
|
12097
|
+
description: "Total cost in USD (input + output)"
|
|
12098
|
+
},
|
|
12099
|
+
$ai_is_error: {
|
|
12100
|
+
type: "boolean",
|
|
12101
|
+
required: false,
|
|
12102
|
+
description: "Whether the generation resulted in an error"
|
|
12103
|
+
},
|
|
12104
|
+
$ai_trace_id: {
|
|
12105
|
+
type: "string",
|
|
12106
|
+
required: false,
|
|
12107
|
+
description: "Trace ID for grouping related generations"
|
|
12108
|
+
},
|
|
12109
|
+
$ai_stream: {
|
|
12110
|
+
type: "boolean",
|
|
12111
|
+
required: false,
|
|
12112
|
+
description: "Whether the response was streamed"
|
|
12113
|
+
},
|
|
12114
|
+
$ai_time_to_first_token: {
|
|
12115
|
+
type: "number",
|
|
12116
|
+
required: false,
|
|
12117
|
+
description: "Time to first token in seconds (streaming only)"
|
|
12118
|
+
},
|
|
12119
|
+
$ai_tools: {
|
|
12120
|
+
type: "json",
|
|
12121
|
+
required: false,
|
|
12122
|
+
description: "Tools/functions available to the LLM"
|
|
12123
|
+
},
|
|
12124
|
+
$ai_input: {
|
|
12125
|
+
type: "json",
|
|
12126
|
+
required: false,
|
|
12127
|
+
pii: true,
|
|
12128
|
+
redact: true,
|
|
12129
|
+
description: "Messages sent to the LLM (may contain PII)"
|
|
12130
|
+
},
|
|
12131
|
+
$ai_output_choices: {
|
|
12132
|
+
type: "json",
|
|
12133
|
+
required: false,
|
|
12134
|
+
pii: true,
|
|
12135
|
+
redact: true,
|
|
12136
|
+
description: "Response choices from the LLM (may contain PII)"
|
|
12137
|
+
},
|
|
12138
|
+
contractspec_operation: {
|
|
12139
|
+
type: "string",
|
|
12140
|
+
required: false,
|
|
12141
|
+
description: "ContractSpec operation name"
|
|
12142
|
+
},
|
|
12143
|
+
contractspec_version: {
|
|
12144
|
+
type: "string",
|
|
12145
|
+
required: false,
|
|
12146
|
+
description: "ContractSpec operation version"
|
|
12147
|
+
},
|
|
12148
|
+
contractspec_agent_id: {
|
|
12149
|
+
type: "string",
|
|
12150
|
+
required: false,
|
|
12151
|
+
description: "ContractSpec agent identifier"
|
|
12152
|
+
},
|
|
12153
|
+
contractspec_finish_reason: {
|
|
12154
|
+
type: "string",
|
|
12155
|
+
required: false,
|
|
12156
|
+
description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
|
|
12157
|
+
},
|
|
12158
|
+
contractspec_tool_count: {
|
|
12159
|
+
type: "number",
|
|
12160
|
+
required: false,
|
|
12161
|
+
description: "Number of tool calls made in this step"
|
|
12162
|
+
}
|
|
12163
|
+
},
|
|
12164
|
+
privacy: "internal",
|
|
12165
|
+
tags: ["llm", "generation", "posthog"]
|
|
12166
|
+
}
|
|
12167
|
+
],
|
|
12168
|
+
config: {
|
|
12169
|
+
defaultRetentionDays: 90,
|
|
12170
|
+
defaultSamplingRate: 1,
|
|
12171
|
+
providers: [
|
|
12172
|
+
{
|
|
12173
|
+
type: "posthog",
|
|
12174
|
+
config: {
|
|
12175
|
+
eventName: "$ai_generation",
|
|
12176
|
+
enableEvaluations: true,
|
|
12177
|
+
evaluationTemplates: [
|
|
12178
|
+
"relevance",
|
|
12179
|
+
"helpfulness",
|
|
12180
|
+
"jailbreak",
|
|
12181
|
+
"hallucination",
|
|
12182
|
+
"toxicity"
|
|
12183
|
+
]
|
|
12184
|
+
}
|
|
12185
|
+
}
|
|
12186
|
+
]
|
|
12187
|
+
}
|
|
12188
|
+
};
|
|
12189
|
+
var POSTHOG_LLM_PII_FIELDS = [
|
|
12190
|
+
"$ai_input",
|
|
12191
|
+
"$ai_output_choices"
|
|
12192
|
+
];
|
|
12193
|
+
var POSTHOG_LLM_TELEMETRY_EVENTS = {
|
|
12194
|
+
generation: "$ai_generation",
|
|
12195
|
+
span: "$ai_span",
|
|
12196
|
+
evaluation: "$ai_evaluation"
|
|
12197
|
+
};
|
|
12198
|
+
var POSTHOG_EVALUATION_TEMPLATES = {
|
|
12199
|
+
relevance: "relevance",
|
|
12200
|
+
helpfulness: "helpfulness",
|
|
12201
|
+
jailbreak: "jailbreak",
|
|
12202
|
+
hallucination: "hallucination",
|
|
12203
|
+
toxicity: "toxicity"
|
|
12204
|
+
};
|
|
12205
|
+
function redactPostHogLLMTelemetryPayload(payload) {
|
|
12206
|
+
const redacted = { ...payload };
|
|
12207
|
+
for (const field of POSTHOG_LLM_PII_FIELDS) {
|
|
12208
|
+
if (field in redacted) {
|
|
12209
|
+
redacted[field] = "[REDACTED]";
|
|
12210
|
+
}
|
|
12211
|
+
}
|
|
12212
|
+
return redacted;
|
|
12213
|
+
}
|
|
12214
|
+
|
|
12045
12215
|
// src/integrations/providers/linear.ts
|
|
12046
12216
|
var linearIntegrationSpec = defineIntegration({
|
|
12047
12217
|
meta: {
|
|
@@ -17850,6 +18020,7 @@ export {
|
|
|
17850
18020
|
registerDefaultReactRenderer,
|
|
17851
18021
|
registerContractsOnBuilder,
|
|
17852
18022
|
registerBasicValidation,
|
|
18023
|
+
redactPostHogLLMTelemetryPayload,
|
|
17853
18024
|
redactOpenBankingTelemetryPayload,
|
|
17854
18025
|
redactMeetingRecorderTelemetryPayload,
|
|
17855
18026
|
qdrantIntegrationSpec,
|
|
@@ -17857,6 +18028,7 @@ export {
|
|
|
17857
18028
|
presentationToMarkdown,
|
|
17858
18029
|
powensIntegrationSpec,
|
|
17859
18030
|
postmarkIntegrationSpec,
|
|
18031
|
+
posthogLLMTelemetrySpec,
|
|
17860
18032
|
posthogIntegrationSpec,
|
|
17861
18033
|
parseVersionedRefKey,
|
|
17862
18034
|
parseVersionStrict,
|
|
@@ -18098,6 +18270,9 @@ export {
|
|
|
18098
18270
|
PolicyRegistry,
|
|
18099
18271
|
PolicyEngine,
|
|
18100
18272
|
PatchChangeModel,
|
|
18273
|
+
POSTHOG_LLM_TELEMETRY_EVENTS,
|
|
18274
|
+
POSTHOG_LLM_PII_FIELDS,
|
|
18275
|
+
POSTHOG_EVALUATION_TEMPLATES,
|
|
18101
18276
|
OwnersEnum,
|
|
18102
18277
|
Owners,
|
|
18103
18278
|
OpportunityBriefModel,
|