@contractspec/lib.contracts 1.61.0 → 1.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +179 -4
- package/dist/integrations/index.js +179 -4
- package/dist/integrations/providers/index.d.ts +1 -0
- package/dist/integrations/providers/index.d.ts.map +1 -1
- package/dist/integrations/providers/index.js +180 -5
- package/dist/integrations/providers/posthog-llm-telemetry.d.ts +52 -0
- package/dist/integrations/providers/posthog-llm-telemetry.d.ts.map +1 -0
- package/dist/integrations/providers/posthog-llm-telemetry.js +241 -0
- package/dist/integrations/providers/posthog.d.ts.map +1 -1
- package/dist/integrations/providers/posthog.js +6 -4
- package/dist/integrations/providers/registry.js +6 -4
- package/dist/node/index.js +179 -4
- package/dist/node/integrations/index.js +179 -4
- package/dist/node/integrations/providers/index.js +180 -5
- package/dist/node/integrations/providers/posthog-llm-telemetry.js +241 -0
- package/dist/node/integrations/providers/posthog.js +6 -4
- package/dist/node/integrations/providers/registry.js +6 -4
- package/package.json +22 -10
package/dist/index.js
CHANGED
|
@@ -11964,13 +11964,13 @@ function registerPowensIntegration(registry6) {
|
|
|
11964
11964
|
var posthogIntegrationSpec = defineIntegration({
|
|
11965
11965
|
meta: {
|
|
11966
11966
|
key: "analytics.posthog",
|
|
11967
|
-
version: "1.
|
|
11967
|
+
version: "1.1.0",
|
|
11968
11968
|
category: "analytics",
|
|
11969
11969
|
title: "PostHog",
|
|
11970
|
-
description: "PostHog integration for product analytics, feature flags, and
|
|
11970
|
+
description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
|
|
11971
11971
|
domain: "analytics",
|
|
11972
11972
|
owners: ["@platform.integrations"],
|
|
11973
|
-
tags: ["analytics", "posthog"],
|
|
11973
|
+
tags: ["analytics", "posthog", "llm", "ai"],
|
|
11974
11974
|
stability: StabilityEnum.Beta
|
|
11975
11975
|
},
|
|
11976
11976
|
supportedModes: ["managed", "byok"],
|
|
@@ -11983,7 +11983,9 @@ var posthogIntegrationSpec = defineIntegration({
|
|
|
11983
11983
|
{ key: "analytics.persons", version: "1.0.0" },
|
|
11984
11984
|
{ key: "analytics.insights", version: "1.0.0" },
|
|
11985
11985
|
{ key: "analytics.cohorts", version: "1.0.0" },
|
|
11986
|
-
{ key: "analytics.annotations", version: "1.0.0" }
|
|
11986
|
+
{ key: "analytics.annotations", version: "1.0.0" },
|
|
11987
|
+
{ key: "analytics.llm-tracing", version: "1.0.0" },
|
|
11988
|
+
{ key: "analytics.llm-evaluations", version: "1.0.0" }
|
|
11987
11989
|
]
|
|
11988
11990
|
},
|
|
11989
11991
|
configSchema: {
|
|
@@ -12042,6 +12044,174 @@ function registerPosthogIntegration(registry6) {
|
|
|
12042
12044
|
return registry6.register(posthogIntegrationSpec);
|
|
12043
12045
|
}
|
|
12044
12046
|
|
|
12047
|
+
// src/integrations/providers/posthog-llm-telemetry.ts
|
|
12048
|
+
var posthogLLMTelemetrySpec = {
|
|
12049
|
+
meta: {
|
|
12050
|
+
key: "analytics.posthog.llm",
|
|
12051
|
+
version: "1.0.0",
|
|
12052
|
+
title: "PostHog LLM Analytics Telemetry",
|
|
12053
|
+
description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
|
|
12054
|
+
domain: "analytics",
|
|
12055
|
+
owners: ["@platform.integrations"],
|
|
12056
|
+
tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
|
|
12057
|
+
stability: StabilityEnum.Beta
|
|
12058
|
+
},
|
|
12059
|
+
events: [
|
|
12060
|
+
{
|
|
12061
|
+
key: "posthog.llm.generation",
|
|
12062
|
+
version: "1.0.0",
|
|
12063
|
+
semantics: {
|
|
12064
|
+
who: "AI agent or LLM consumer",
|
|
12065
|
+
what: "Captures an LLM generation call with full observability metadata",
|
|
12066
|
+
why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
|
|
12067
|
+
},
|
|
12068
|
+
properties: {
|
|
12069
|
+
$ai_model: {
|
|
12070
|
+
type: "string",
|
|
12071
|
+
required: true,
|
|
12072
|
+
description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
|
|
12073
|
+
},
|
|
12074
|
+
$ai_provider: {
|
|
12075
|
+
type: "string",
|
|
12076
|
+
required: true,
|
|
12077
|
+
description: "The LLM provider (e.g., openai, anthropic, contractspec)"
|
|
12078
|
+
},
|
|
12079
|
+
$ai_latency: {
|
|
12080
|
+
type: "number",
|
|
12081
|
+
required: false,
|
|
12082
|
+
description: "LLM call latency in seconds"
|
|
12083
|
+
},
|
|
12084
|
+
$ai_input_tokens: {
|
|
12085
|
+
type: "number",
|
|
12086
|
+
required: false,
|
|
12087
|
+
description: "Number of input/prompt tokens consumed"
|
|
12088
|
+
},
|
|
12089
|
+
$ai_output_tokens: {
|
|
12090
|
+
type: "number",
|
|
12091
|
+
required: false,
|
|
12092
|
+
description: "Number of output/completion tokens generated"
|
|
12093
|
+
},
|
|
12094
|
+
$ai_total_cost_usd: {
|
|
12095
|
+
type: "number",
|
|
12096
|
+
required: false,
|
|
12097
|
+
description: "Total cost in USD (input + output)"
|
|
12098
|
+
},
|
|
12099
|
+
$ai_is_error: {
|
|
12100
|
+
type: "boolean",
|
|
12101
|
+
required: false,
|
|
12102
|
+
description: "Whether the generation resulted in an error"
|
|
12103
|
+
},
|
|
12104
|
+
$ai_trace_id: {
|
|
12105
|
+
type: "string",
|
|
12106
|
+
required: false,
|
|
12107
|
+
description: "Trace ID for grouping related generations"
|
|
12108
|
+
},
|
|
12109
|
+
$ai_stream: {
|
|
12110
|
+
type: "boolean",
|
|
12111
|
+
required: false,
|
|
12112
|
+
description: "Whether the response was streamed"
|
|
12113
|
+
},
|
|
12114
|
+
$ai_time_to_first_token: {
|
|
12115
|
+
type: "number",
|
|
12116
|
+
required: false,
|
|
12117
|
+
description: "Time to first token in seconds (streaming only)"
|
|
12118
|
+
},
|
|
12119
|
+
$ai_tools: {
|
|
12120
|
+
type: "json",
|
|
12121
|
+
required: false,
|
|
12122
|
+
description: "Tools/functions available to the LLM"
|
|
12123
|
+
},
|
|
12124
|
+
$ai_input: {
|
|
12125
|
+
type: "json",
|
|
12126
|
+
required: false,
|
|
12127
|
+
pii: true,
|
|
12128
|
+
redact: true,
|
|
12129
|
+
description: "Messages sent to the LLM (may contain PII)"
|
|
12130
|
+
},
|
|
12131
|
+
$ai_output_choices: {
|
|
12132
|
+
type: "json",
|
|
12133
|
+
required: false,
|
|
12134
|
+
pii: true,
|
|
12135
|
+
redact: true,
|
|
12136
|
+
description: "Response choices from the LLM (may contain PII)"
|
|
12137
|
+
},
|
|
12138
|
+
contractspec_operation: {
|
|
12139
|
+
type: "string",
|
|
12140
|
+
required: false,
|
|
12141
|
+
description: "ContractSpec operation name"
|
|
12142
|
+
},
|
|
12143
|
+
contractspec_version: {
|
|
12144
|
+
type: "string",
|
|
12145
|
+
required: false,
|
|
12146
|
+
description: "ContractSpec operation version"
|
|
12147
|
+
},
|
|
12148
|
+
contractspec_agent_id: {
|
|
12149
|
+
type: "string",
|
|
12150
|
+
required: false,
|
|
12151
|
+
description: "ContractSpec agent identifier"
|
|
12152
|
+
},
|
|
12153
|
+
contractspec_finish_reason: {
|
|
12154
|
+
type: "string",
|
|
12155
|
+
required: false,
|
|
12156
|
+
description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
|
|
12157
|
+
},
|
|
12158
|
+
contractspec_tool_count: {
|
|
12159
|
+
type: "number",
|
|
12160
|
+
required: false,
|
|
12161
|
+
description: "Number of tool calls made in this step"
|
|
12162
|
+
}
|
|
12163
|
+
},
|
|
12164
|
+
privacy: "internal",
|
|
12165
|
+
tags: ["llm", "generation", "posthog"]
|
|
12166
|
+
}
|
|
12167
|
+
],
|
|
12168
|
+
config: {
|
|
12169
|
+
defaultRetentionDays: 90,
|
|
12170
|
+
defaultSamplingRate: 1,
|
|
12171
|
+
providers: [
|
|
12172
|
+
{
|
|
12173
|
+
type: "posthog",
|
|
12174
|
+
config: {
|
|
12175
|
+
eventName: "$ai_generation",
|
|
12176
|
+
enableEvaluations: true,
|
|
12177
|
+
evaluationTemplates: [
|
|
12178
|
+
"relevance",
|
|
12179
|
+
"helpfulness",
|
|
12180
|
+
"jailbreak",
|
|
12181
|
+
"hallucination",
|
|
12182
|
+
"toxicity"
|
|
12183
|
+
]
|
|
12184
|
+
}
|
|
12185
|
+
}
|
|
12186
|
+
]
|
|
12187
|
+
}
|
|
12188
|
+
};
|
|
12189
|
+
var POSTHOG_LLM_PII_FIELDS = [
|
|
12190
|
+
"$ai_input",
|
|
12191
|
+
"$ai_output_choices"
|
|
12192
|
+
];
|
|
12193
|
+
var POSTHOG_LLM_TELEMETRY_EVENTS = {
|
|
12194
|
+
generation: "$ai_generation",
|
|
12195
|
+
span: "$ai_span",
|
|
12196
|
+
evaluation: "$ai_evaluation"
|
|
12197
|
+
};
|
|
12198
|
+
var POSTHOG_EVALUATION_TEMPLATES = {
|
|
12199
|
+
relevance: "relevance",
|
|
12200
|
+
helpfulness: "helpfulness",
|
|
12201
|
+
jailbreak: "jailbreak",
|
|
12202
|
+
hallucination: "hallucination",
|
|
12203
|
+
toxicity: "toxicity"
|
|
12204
|
+
};
|
|
12205
|
+
function redactPostHogLLMTelemetryPayload(payload) {
|
|
12206
|
+
const redacted = { ...payload };
|
|
12207
|
+
for (const field of POSTHOG_LLM_PII_FIELDS) {
|
|
12208
|
+
if (field in redacted) {
|
|
12209
|
+
redacted[field] = "[REDACTED]";
|
|
12210
|
+
}
|
|
12211
|
+
}
|
|
12212
|
+
return redacted;
|
|
12213
|
+
}
|
|
12214
|
+
|
|
12045
12215
|
// src/integrations/providers/linear.ts
|
|
12046
12216
|
var linearIntegrationSpec = defineIntegration({
|
|
12047
12217
|
meta: {
|
|
@@ -17850,6 +18020,7 @@ export {
|
|
|
17850
18020
|
registerDefaultReactRenderer,
|
|
17851
18021
|
registerContractsOnBuilder,
|
|
17852
18022
|
registerBasicValidation,
|
|
18023
|
+
redactPostHogLLMTelemetryPayload,
|
|
17853
18024
|
redactOpenBankingTelemetryPayload,
|
|
17854
18025
|
redactMeetingRecorderTelemetryPayload,
|
|
17855
18026
|
qdrantIntegrationSpec,
|
|
@@ -17857,6 +18028,7 @@ export {
|
|
|
17857
18028
|
presentationToMarkdown,
|
|
17858
18029
|
powensIntegrationSpec,
|
|
17859
18030
|
postmarkIntegrationSpec,
|
|
18031
|
+
posthogLLMTelemetrySpec,
|
|
17860
18032
|
posthogIntegrationSpec,
|
|
17861
18033
|
parseVersionedRefKey,
|
|
17862
18034
|
parseVersionStrict,
|
|
@@ -18098,6 +18270,9 @@ export {
|
|
|
18098
18270
|
PolicyRegistry,
|
|
18099
18271
|
PolicyEngine,
|
|
18100
18272
|
PatchChangeModel,
|
|
18273
|
+
POSTHOG_LLM_TELEMETRY_EVENTS,
|
|
18274
|
+
POSTHOG_LLM_PII_FIELDS,
|
|
18275
|
+
POSTHOG_EVALUATION_TEMPLATES,
|
|
18101
18276
|
OwnersEnum,
|
|
18102
18277
|
Owners,
|
|
18103
18278
|
OpportunityBriefModel,
|
|
@@ -1891,13 +1891,13 @@ function registerPowensIntegration(registry2) {
|
|
|
1891
1891
|
var posthogIntegrationSpec = defineIntegration({
|
|
1892
1892
|
meta: {
|
|
1893
1893
|
key: "analytics.posthog",
|
|
1894
|
-
version: "1.
|
|
1894
|
+
version: "1.1.0",
|
|
1895
1895
|
category: "analytics",
|
|
1896
1896
|
title: "PostHog",
|
|
1897
|
-
description: "PostHog integration for product analytics, feature flags, and
|
|
1897
|
+
description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
|
|
1898
1898
|
domain: "analytics",
|
|
1899
1899
|
owners: ["@platform.integrations"],
|
|
1900
|
-
tags: ["analytics", "posthog"],
|
|
1900
|
+
tags: ["analytics", "posthog", "llm", "ai"],
|
|
1901
1901
|
stability: StabilityEnum.Beta
|
|
1902
1902
|
},
|
|
1903
1903
|
supportedModes: ["managed", "byok"],
|
|
@@ -1910,7 +1910,9 @@ var posthogIntegrationSpec = defineIntegration({
|
|
|
1910
1910
|
{ key: "analytics.persons", version: "1.0.0" },
|
|
1911
1911
|
{ key: "analytics.insights", version: "1.0.0" },
|
|
1912
1912
|
{ key: "analytics.cohorts", version: "1.0.0" },
|
|
1913
|
-
{ key: "analytics.annotations", version: "1.0.0" }
|
|
1913
|
+
{ key: "analytics.annotations", version: "1.0.0" },
|
|
1914
|
+
{ key: "analytics.llm-tracing", version: "1.0.0" },
|
|
1915
|
+
{ key: "analytics.llm-evaluations", version: "1.0.0" }
|
|
1914
1916
|
]
|
|
1915
1917
|
},
|
|
1916
1918
|
configSchema: {
|
|
@@ -1969,6 +1971,174 @@ function registerPosthogIntegration(registry2) {
|
|
|
1969
1971
|
return registry2.register(posthogIntegrationSpec);
|
|
1970
1972
|
}
|
|
1971
1973
|
|
|
1974
|
+
// src/integrations/providers/posthog-llm-telemetry.ts
|
|
1975
|
+
var posthogLLMTelemetrySpec = {
|
|
1976
|
+
meta: {
|
|
1977
|
+
key: "analytics.posthog.llm",
|
|
1978
|
+
version: "1.0.0",
|
|
1979
|
+
title: "PostHog LLM Analytics Telemetry",
|
|
1980
|
+
description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
|
|
1981
|
+
domain: "analytics",
|
|
1982
|
+
owners: ["@platform.integrations"],
|
|
1983
|
+
tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
|
|
1984
|
+
stability: StabilityEnum.Beta
|
|
1985
|
+
},
|
|
1986
|
+
events: [
|
|
1987
|
+
{
|
|
1988
|
+
key: "posthog.llm.generation",
|
|
1989
|
+
version: "1.0.0",
|
|
1990
|
+
semantics: {
|
|
1991
|
+
who: "AI agent or LLM consumer",
|
|
1992
|
+
what: "Captures an LLM generation call with full observability metadata",
|
|
1993
|
+
why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
|
|
1994
|
+
},
|
|
1995
|
+
properties: {
|
|
1996
|
+
$ai_model: {
|
|
1997
|
+
type: "string",
|
|
1998
|
+
required: true,
|
|
1999
|
+
description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
|
|
2000
|
+
},
|
|
2001
|
+
$ai_provider: {
|
|
2002
|
+
type: "string",
|
|
2003
|
+
required: true,
|
|
2004
|
+
description: "The LLM provider (e.g., openai, anthropic, contractspec)"
|
|
2005
|
+
},
|
|
2006
|
+
$ai_latency: {
|
|
2007
|
+
type: "number",
|
|
2008
|
+
required: false,
|
|
2009
|
+
description: "LLM call latency in seconds"
|
|
2010
|
+
},
|
|
2011
|
+
$ai_input_tokens: {
|
|
2012
|
+
type: "number",
|
|
2013
|
+
required: false,
|
|
2014
|
+
description: "Number of input/prompt tokens consumed"
|
|
2015
|
+
},
|
|
2016
|
+
$ai_output_tokens: {
|
|
2017
|
+
type: "number",
|
|
2018
|
+
required: false,
|
|
2019
|
+
description: "Number of output/completion tokens generated"
|
|
2020
|
+
},
|
|
2021
|
+
$ai_total_cost_usd: {
|
|
2022
|
+
type: "number",
|
|
2023
|
+
required: false,
|
|
2024
|
+
description: "Total cost in USD (input + output)"
|
|
2025
|
+
},
|
|
2026
|
+
$ai_is_error: {
|
|
2027
|
+
type: "boolean",
|
|
2028
|
+
required: false,
|
|
2029
|
+
description: "Whether the generation resulted in an error"
|
|
2030
|
+
},
|
|
2031
|
+
$ai_trace_id: {
|
|
2032
|
+
type: "string",
|
|
2033
|
+
required: false,
|
|
2034
|
+
description: "Trace ID for grouping related generations"
|
|
2035
|
+
},
|
|
2036
|
+
$ai_stream: {
|
|
2037
|
+
type: "boolean",
|
|
2038
|
+
required: false,
|
|
2039
|
+
description: "Whether the response was streamed"
|
|
2040
|
+
},
|
|
2041
|
+
$ai_time_to_first_token: {
|
|
2042
|
+
type: "number",
|
|
2043
|
+
required: false,
|
|
2044
|
+
description: "Time to first token in seconds (streaming only)"
|
|
2045
|
+
},
|
|
2046
|
+
$ai_tools: {
|
|
2047
|
+
type: "json",
|
|
2048
|
+
required: false,
|
|
2049
|
+
description: "Tools/functions available to the LLM"
|
|
2050
|
+
},
|
|
2051
|
+
$ai_input: {
|
|
2052
|
+
type: "json",
|
|
2053
|
+
required: false,
|
|
2054
|
+
pii: true,
|
|
2055
|
+
redact: true,
|
|
2056
|
+
description: "Messages sent to the LLM (may contain PII)"
|
|
2057
|
+
},
|
|
2058
|
+
$ai_output_choices: {
|
|
2059
|
+
type: "json",
|
|
2060
|
+
required: false,
|
|
2061
|
+
pii: true,
|
|
2062
|
+
redact: true,
|
|
2063
|
+
description: "Response choices from the LLM (may contain PII)"
|
|
2064
|
+
},
|
|
2065
|
+
contractspec_operation: {
|
|
2066
|
+
type: "string",
|
|
2067
|
+
required: false,
|
|
2068
|
+
description: "ContractSpec operation name"
|
|
2069
|
+
},
|
|
2070
|
+
contractspec_version: {
|
|
2071
|
+
type: "string",
|
|
2072
|
+
required: false,
|
|
2073
|
+
description: "ContractSpec operation version"
|
|
2074
|
+
},
|
|
2075
|
+
contractspec_agent_id: {
|
|
2076
|
+
type: "string",
|
|
2077
|
+
required: false,
|
|
2078
|
+
description: "ContractSpec agent identifier"
|
|
2079
|
+
},
|
|
2080
|
+
contractspec_finish_reason: {
|
|
2081
|
+
type: "string",
|
|
2082
|
+
required: false,
|
|
2083
|
+
description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
|
|
2084
|
+
},
|
|
2085
|
+
contractspec_tool_count: {
|
|
2086
|
+
type: "number",
|
|
2087
|
+
required: false,
|
|
2088
|
+
description: "Number of tool calls made in this step"
|
|
2089
|
+
}
|
|
2090
|
+
},
|
|
2091
|
+
privacy: "internal",
|
|
2092
|
+
tags: ["llm", "generation", "posthog"]
|
|
2093
|
+
}
|
|
2094
|
+
],
|
|
2095
|
+
config: {
|
|
2096
|
+
defaultRetentionDays: 90,
|
|
2097
|
+
defaultSamplingRate: 1,
|
|
2098
|
+
providers: [
|
|
2099
|
+
{
|
|
2100
|
+
type: "posthog",
|
|
2101
|
+
config: {
|
|
2102
|
+
eventName: "$ai_generation",
|
|
2103
|
+
enableEvaluations: true,
|
|
2104
|
+
evaluationTemplates: [
|
|
2105
|
+
"relevance",
|
|
2106
|
+
"helpfulness",
|
|
2107
|
+
"jailbreak",
|
|
2108
|
+
"hallucination",
|
|
2109
|
+
"toxicity"
|
|
2110
|
+
]
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
]
|
|
2114
|
+
}
|
|
2115
|
+
};
|
|
2116
|
+
var POSTHOG_LLM_PII_FIELDS = [
|
|
2117
|
+
"$ai_input",
|
|
2118
|
+
"$ai_output_choices"
|
|
2119
|
+
];
|
|
2120
|
+
var POSTHOG_LLM_TELEMETRY_EVENTS = {
|
|
2121
|
+
generation: "$ai_generation",
|
|
2122
|
+
span: "$ai_span",
|
|
2123
|
+
evaluation: "$ai_evaluation"
|
|
2124
|
+
};
|
|
2125
|
+
var POSTHOG_EVALUATION_TEMPLATES = {
|
|
2126
|
+
relevance: "relevance",
|
|
2127
|
+
helpfulness: "helpfulness",
|
|
2128
|
+
jailbreak: "jailbreak",
|
|
2129
|
+
hallucination: "hallucination",
|
|
2130
|
+
toxicity: "toxicity"
|
|
2131
|
+
};
|
|
2132
|
+
function redactPostHogLLMTelemetryPayload(payload) {
|
|
2133
|
+
const redacted = { ...payload };
|
|
2134
|
+
for (const field of POSTHOG_LLM_PII_FIELDS) {
|
|
2135
|
+
if (field in redacted) {
|
|
2136
|
+
redacted[field] = "[REDACTED]";
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
return redacted;
|
|
2140
|
+
}
|
|
2141
|
+
|
|
1972
2142
|
// src/integrations/providers/linear.ts
|
|
1973
2143
|
var linearIntegrationSpec = defineIntegration({
|
|
1974
2144
|
meta: {
|
|
@@ -3755,11 +3925,13 @@ export {
|
|
|
3755
3925
|
registerFathomIntegration,
|
|
3756
3926
|
registerFalIntegration,
|
|
3757
3927
|
registerElevenLabsIntegration,
|
|
3928
|
+
redactPostHogLLMTelemetryPayload,
|
|
3758
3929
|
redactOpenBankingTelemetryPayload,
|
|
3759
3930
|
redactMeetingRecorderTelemetryPayload,
|
|
3760
3931
|
qdrantIntegrationSpec,
|
|
3761
3932
|
powensIntegrationSpec,
|
|
3762
3933
|
postmarkIntegrationSpec,
|
|
3934
|
+
posthogLLMTelemetrySpec,
|
|
3763
3935
|
posthogIntegrationSpec,
|
|
3764
3936
|
notionIntegrationSpec,
|
|
3765
3937
|
mistralIntegrationSpec,
|
|
@@ -3779,6 +3951,9 @@ export {
|
|
|
3779
3951
|
defineIntegration,
|
|
3780
3952
|
createDefaultIntegrationSpecRegistry,
|
|
3781
3953
|
assertPrimaryOpenBankingReady,
|
|
3954
|
+
POSTHOG_LLM_TELEMETRY_EVENTS,
|
|
3955
|
+
POSTHOG_LLM_PII_FIELDS,
|
|
3956
|
+
POSTHOG_EVALUATION_TEMPLATES,
|
|
3782
3957
|
OpenBankingSyncTransactions,
|
|
3783
3958
|
OpenBankingSyncAccounts,
|
|
3784
3959
|
OpenBankingRefreshBalances,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/integrations/providers/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,OAAO,CAAC;AACtB,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,SAAS,CAAC;AACxB,cAAc,YAAY,CAAC;AAC3B,cAAc,OAAO,CAAC;AACtB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC;AAC1B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,OAAO,CAAC;AACtB,cAAc,SAAS,CAAC;AACxB,cAAc,mBAAmB,CAAC;AAClC,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,QAAQ,CAAC;AACvB,cAAc,aAAa,CAAC;AAC5B,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/integrations/providers/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,OAAO,CAAC;AACtB,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,SAAS,CAAC;AACxB,cAAc,YAAY,CAAC;AAC3B,cAAc,OAAO,CAAC;AACtB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC;AAC1B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,OAAO,CAAC;AACtB,cAAc,SAAS,CAAC;AACxB,cAAc,mBAAmB,CAAC;AAClC,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,yBAAyB,CAAC;AACxC,cAAc,UAAU,CAAC;AACzB,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,QAAQ,CAAC;AACvB,cAAc,aAAa,CAAC;AAC5B,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC"}
|
|
@@ -1370,13 +1370,13 @@ function registerPowensIntegration(registry) {
|
|
|
1370
1370
|
var posthogIntegrationSpec = defineIntegration({
|
|
1371
1371
|
meta: {
|
|
1372
1372
|
key: "analytics.posthog",
|
|
1373
|
-
version: "1.
|
|
1373
|
+
version: "1.1.0",
|
|
1374
1374
|
category: "analytics",
|
|
1375
1375
|
title: "PostHog",
|
|
1376
|
-
description: "PostHog integration for product analytics, feature flags, and
|
|
1376
|
+
description: "PostHog integration for product analytics, feature flags, HogQL queries, and LLM analytics with evaluations.",
|
|
1377
1377
|
domain: "analytics",
|
|
1378
1378
|
owners: ["@platform.integrations"],
|
|
1379
|
-
tags: ["analytics", "posthog"],
|
|
1379
|
+
tags: ["analytics", "posthog", "llm", "ai"],
|
|
1380
1380
|
stability: StabilityEnum.Beta
|
|
1381
1381
|
},
|
|
1382
1382
|
supportedModes: ["managed", "byok"],
|
|
@@ -1389,7 +1389,9 @@ var posthogIntegrationSpec = defineIntegration({
|
|
|
1389
1389
|
{ key: "analytics.persons", version: "1.0.0" },
|
|
1390
1390
|
{ key: "analytics.insights", version: "1.0.0" },
|
|
1391
1391
|
{ key: "analytics.cohorts", version: "1.0.0" },
|
|
1392
|
-
{ key: "analytics.annotations", version: "1.0.0" }
|
|
1392
|
+
{ key: "analytics.annotations", version: "1.0.0" },
|
|
1393
|
+
{ key: "analytics.llm-tracing", version: "1.0.0" },
|
|
1394
|
+
{ key: "analytics.llm-evaluations", version: "1.0.0" }
|
|
1393
1395
|
]
|
|
1394
1396
|
},
|
|
1395
1397
|
configSchema: {
|
|
@@ -1448,6 +1450,174 @@ function registerPosthogIntegration(registry) {
|
|
|
1448
1450
|
return registry.register(posthogIntegrationSpec);
|
|
1449
1451
|
}
|
|
1450
1452
|
|
|
1453
|
+
// src/integrations/providers/posthog-llm-telemetry.ts
|
|
1454
|
+
var posthogLLMTelemetrySpec = {
|
|
1455
|
+
meta: {
|
|
1456
|
+
key: "analytics.posthog.llm",
|
|
1457
|
+
version: "1.0.0",
|
|
1458
|
+
title: "PostHog LLM Analytics Telemetry",
|
|
1459
|
+
description: "Telemetry events for PostHog LLM Analytics: generation tracing, token usage, and evaluation-ready metadata.",
|
|
1460
|
+
domain: "analytics",
|
|
1461
|
+
owners: ["@platform.integrations"],
|
|
1462
|
+
tags: ["analytics", "posthog", "llm", "ai", "telemetry"],
|
|
1463
|
+
stability: StabilityEnum.Beta
|
|
1464
|
+
},
|
|
1465
|
+
events: [
|
|
1466
|
+
{
|
|
1467
|
+
key: "posthog.llm.generation",
|
|
1468
|
+
version: "1.0.0",
|
|
1469
|
+
semantics: {
|
|
1470
|
+
who: "AI agent or LLM consumer",
|
|
1471
|
+
what: "Captures an LLM generation call with full observability metadata",
|
|
1472
|
+
why: "Enable LLM performance monitoring, cost tracking, and automated quality evaluations via PostHog"
|
|
1473
|
+
},
|
|
1474
|
+
properties: {
|
|
1475
|
+
$ai_model: {
|
|
1476
|
+
type: "string",
|
|
1477
|
+
required: true,
|
|
1478
|
+
description: "The LLM model identifier (e.g., gpt-4o, claude-sonnet-4-20250514)"
|
|
1479
|
+
},
|
|
1480
|
+
$ai_provider: {
|
|
1481
|
+
type: "string",
|
|
1482
|
+
required: true,
|
|
1483
|
+
description: "The LLM provider (e.g., openai, anthropic, contractspec)"
|
|
1484
|
+
},
|
|
1485
|
+
$ai_latency: {
|
|
1486
|
+
type: "number",
|
|
1487
|
+
required: false,
|
|
1488
|
+
description: "LLM call latency in seconds"
|
|
1489
|
+
},
|
|
1490
|
+
$ai_input_tokens: {
|
|
1491
|
+
type: "number",
|
|
1492
|
+
required: false,
|
|
1493
|
+
description: "Number of input/prompt tokens consumed"
|
|
1494
|
+
},
|
|
1495
|
+
$ai_output_tokens: {
|
|
1496
|
+
type: "number",
|
|
1497
|
+
required: false,
|
|
1498
|
+
description: "Number of output/completion tokens generated"
|
|
1499
|
+
},
|
|
1500
|
+
$ai_total_cost_usd: {
|
|
1501
|
+
type: "number",
|
|
1502
|
+
required: false,
|
|
1503
|
+
description: "Total cost in USD (input + output)"
|
|
1504
|
+
},
|
|
1505
|
+
$ai_is_error: {
|
|
1506
|
+
type: "boolean",
|
|
1507
|
+
required: false,
|
|
1508
|
+
description: "Whether the generation resulted in an error"
|
|
1509
|
+
},
|
|
1510
|
+
$ai_trace_id: {
|
|
1511
|
+
type: "string",
|
|
1512
|
+
required: false,
|
|
1513
|
+
description: "Trace ID for grouping related generations"
|
|
1514
|
+
},
|
|
1515
|
+
$ai_stream: {
|
|
1516
|
+
type: "boolean",
|
|
1517
|
+
required: false,
|
|
1518
|
+
description: "Whether the response was streamed"
|
|
1519
|
+
},
|
|
1520
|
+
$ai_time_to_first_token: {
|
|
1521
|
+
type: "number",
|
|
1522
|
+
required: false,
|
|
1523
|
+
description: "Time to first token in seconds (streaming only)"
|
|
1524
|
+
},
|
|
1525
|
+
$ai_tools: {
|
|
1526
|
+
type: "json",
|
|
1527
|
+
required: false,
|
|
1528
|
+
description: "Tools/functions available to the LLM"
|
|
1529
|
+
},
|
|
1530
|
+
$ai_input: {
|
|
1531
|
+
type: "json",
|
|
1532
|
+
required: false,
|
|
1533
|
+
pii: true,
|
|
1534
|
+
redact: true,
|
|
1535
|
+
description: "Messages sent to the LLM (may contain PII)"
|
|
1536
|
+
},
|
|
1537
|
+
$ai_output_choices: {
|
|
1538
|
+
type: "json",
|
|
1539
|
+
required: false,
|
|
1540
|
+
pii: true,
|
|
1541
|
+
redact: true,
|
|
1542
|
+
description: "Response choices from the LLM (may contain PII)"
|
|
1543
|
+
},
|
|
1544
|
+
contractspec_operation: {
|
|
1545
|
+
type: "string",
|
|
1546
|
+
required: false,
|
|
1547
|
+
description: "ContractSpec operation name"
|
|
1548
|
+
},
|
|
1549
|
+
contractspec_version: {
|
|
1550
|
+
type: "string",
|
|
1551
|
+
required: false,
|
|
1552
|
+
description: "ContractSpec operation version"
|
|
1553
|
+
},
|
|
1554
|
+
contractspec_agent_id: {
|
|
1555
|
+
type: "string",
|
|
1556
|
+
required: false,
|
|
1557
|
+
description: "ContractSpec agent identifier"
|
|
1558
|
+
},
|
|
1559
|
+
contractspec_finish_reason: {
|
|
1560
|
+
type: "string",
|
|
1561
|
+
required: false,
|
|
1562
|
+
description: "AI SDK finish reason (stop, tool-calls, error, etc.)"
|
|
1563
|
+
},
|
|
1564
|
+
contractspec_tool_count: {
|
|
1565
|
+
type: "number",
|
|
1566
|
+
required: false,
|
|
1567
|
+
description: "Number of tool calls made in this step"
|
|
1568
|
+
}
|
|
1569
|
+
},
|
|
1570
|
+
privacy: "internal",
|
|
1571
|
+
tags: ["llm", "generation", "posthog"]
|
|
1572
|
+
}
|
|
1573
|
+
],
|
|
1574
|
+
config: {
|
|
1575
|
+
defaultRetentionDays: 90,
|
|
1576
|
+
defaultSamplingRate: 1,
|
|
1577
|
+
providers: [
|
|
1578
|
+
{
|
|
1579
|
+
type: "posthog",
|
|
1580
|
+
config: {
|
|
1581
|
+
eventName: "$ai_generation",
|
|
1582
|
+
enableEvaluations: true,
|
|
1583
|
+
evaluationTemplates: [
|
|
1584
|
+
"relevance",
|
|
1585
|
+
"helpfulness",
|
|
1586
|
+
"jailbreak",
|
|
1587
|
+
"hallucination",
|
|
1588
|
+
"toxicity"
|
|
1589
|
+
]
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
]
|
|
1593
|
+
}
|
|
1594
|
+
};
|
|
1595
|
+
var POSTHOG_LLM_PII_FIELDS = [
|
|
1596
|
+
"$ai_input",
|
|
1597
|
+
"$ai_output_choices"
|
|
1598
|
+
];
|
|
1599
|
+
var POSTHOG_LLM_TELEMETRY_EVENTS = {
|
|
1600
|
+
generation: "$ai_generation",
|
|
1601
|
+
span: "$ai_span",
|
|
1602
|
+
evaluation: "$ai_evaluation"
|
|
1603
|
+
};
|
|
1604
|
+
var POSTHOG_EVALUATION_TEMPLATES = {
|
|
1605
|
+
relevance: "relevance",
|
|
1606
|
+
helpfulness: "helpfulness",
|
|
1607
|
+
jailbreak: "jailbreak",
|
|
1608
|
+
hallucination: "hallucination",
|
|
1609
|
+
toxicity: "toxicity"
|
|
1610
|
+
};
|
|
1611
|
+
function redactPostHogLLMTelemetryPayload(payload) {
|
|
1612
|
+
const redacted = { ...payload };
|
|
1613
|
+
for (const field of POSTHOG_LLM_PII_FIELDS) {
|
|
1614
|
+
if (field in redacted) {
|
|
1615
|
+
redacted[field] = "[REDACTED]";
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
return redacted;
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1451
1621
|
// src/integrations/providers/linear.ts
|
|
1452
1622
|
var linearIntegrationSpec = defineIntegration({
|
|
1453
1623
|
meta: {
|
|
@@ -2121,9 +2291,11 @@ export {
|
|
|
2121
2291
|
registerFathomIntegration,
|
|
2122
2292
|
registerFalIntegration,
|
|
2123
2293
|
registerElevenLabsIntegration,
|
|
2294
|
+
redactPostHogLLMTelemetryPayload,
|
|
2124
2295
|
qdrantIntegrationSpec,
|
|
2125
2296
|
powensIntegrationSpec,
|
|
2126
2297
|
postmarkIntegrationSpec,
|
|
2298
|
+
posthogLLMTelemetrySpec,
|
|
2127
2299
|
posthogIntegrationSpec,
|
|
2128
2300
|
notionIntegrationSpec,
|
|
2129
2301
|
mistralIntegrationSpec,
|
|
@@ -2138,5 +2310,8 @@ export {
|
|
|
2138
2310
|
fathomIntegrationSpec,
|
|
2139
2311
|
falIntegrationSpec,
|
|
2140
2312
|
elevenLabsIntegrationSpec,
|
|
2141
|
-
createDefaultIntegrationSpecRegistry
|
|
2313
|
+
createDefaultIntegrationSpecRegistry,
|
|
2314
|
+
POSTHOG_LLM_TELEMETRY_EVENTS,
|
|
2315
|
+
POSTHOG_LLM_PII_FIELDS,
|
|
2316
|
+
POSTHOG_EVALUATION_TEMPLATES
|
|
2142
2317
|
};
|