@copilotkit/shared 1.72.0 → 1.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/finalize-events.cjs +92 -71
- package/dist/finalize-events.cjs.map +1 -1
- package/dist/finalize-events.d.cts +20 -1
- package/dist/finalize-events.d.cts.map +1 -1
- package/dist/finalize-events.d.mts +20 -1
- package/dist/finalize-events.d.mts.map +1 -1
- package/dist/finalize-events.mjs +92 -72
- package/dist/finalize-events.mjs.map +1 -1
- package/dist/index.cjs +7 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +4 -3
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.mts +4 -3
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +4 -3
- package/dist/index.mjs.map +1 -1
- package/dist/index.umd.js +218 -72
- package/dist/index.umd.js.map +1 -1
- package/dist/package.cjs +1 -1
- package/dist/package.mjs +1 -1
- package/dist/telemetry/events.d.cts +33 -0
- package/dist/telemetry/events.d.cts.map +1 -1
- package/dist/telemetry/events.d.mts +33 -0
- package/dist/telemetry/events.d.mts.map +1 -1
- package/dist/telemetry/index.cjs +3 -0
- package/dist/telemetry/index.d.cts +2 -2
- package/dist/telemetry/index.d.mts +2 -2
- package/dist/telemetry/index.mjs +2 -2
- package/dist/telemetry/model-host.cjs +104 -0
- package/dist/telemetry/model-host.cjs.map +1 -0
- package/dist/telemetry/model-host.d.cts +26 -0
- package/dist/telemetry/model-host.d.cts.map +1 -0
- package/dist/telemetry/model-host.d.mts +26 -0
- package/dist/telemetry/model-host.d.mts.map +1 -0
- package/dist/telemetry/model-host.mjs +102 -0
- package/dist/telemetry/model-host.mjs.map +1 -0
- package/dist/telemetry/sampling.cjs +22 -0
- package/dist/telemetry/sampling.cjs.map +1 -1
- package/dist/telemetry/sampling.d.cts +22 -2
- package/dist/telemetry/sampling.d.cts.map +1 -1
- package/dist/telemetry/sampling.d.mts +22 -2
- package/dist/telemetry/sampling.d.mts.map +1 -1
- package/dist/telemetry/sampling.mjs +20 -1
- package/dist/telemetry/sampling.mjs.map +1 -1
- package/dist/telemetry/telemetry-client.cjs +18 -11
- package/dist/telemetry/telemetry-client.cjs.map +1 -1
- package/dist/telemetry/telemetry-client.d.cts +3 -3
- package/dist/telemetry/telemetry-client.d.cts.map +1 -1
- package/dist/telemetry/telemetry-client.d.mts +3 -3
- package/dist/telemetry/telemetry-client.d.mts.map +1 -1
- package/dist/telemetry/telemetry-client.mjs +19 -12
- package/dist/telemetry/telemetry-client.mjs.map +1 -1
- package/package.json +1 -1
- package/src/finalize-events.test.ts +146 -0
- package/src/finalize-events.ts +124 -124
- package/src/index.ts +9 -1
- package/src/telemetry/events.ts +33 -0
- package/src/telemetry/model-host.test.ts +145 -0
- package/src/telemetry/model-host.ts +155 -0
- package/src/telemetry/sampling.ts +39 -6
- package/src/telemetry/telemetry-blank-license-identity.test.ts +11 -4
- package/src/telemetry/telemetry-client.test.ts +107 -39
- package/src/telemetry/telemetry-client.ts +48 -27
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// Which endpoint the runtime sends completions to, as a closed vocabulary.
|
|
2
|
+
//
|
|
3
|
+
// The AI SDK's provider label cannot answer this. `createOpenAI` stamps
|
|
4
|
+
// `openai.responses` on every model it builds, whether the base URL points at
|
|
5
|
+
// api.openai.com, Azure, OpenRouter or a laptop running Ollama — and Azure's
|
|
6
|
+
// own migration guide tells customers to use exactly that client. So the
|
|
7
|
+
// provider field reports the wire protocol, and the host reports the vendor.
|
|
8
|
+
//
|
|
9
|
+
// Only the classification travels. A base URL like
|
|
10
|
+
// `myresource.openai.azure.com` carries the customer's Azure resource name,
|
|
11
|
+
// which identifies them, so the raw host must never reach telemetry. That is
|
|
12
|
+
// why this returns a member of a fixed list rather than anything derived from
|
|
13
|
+
// the input.
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Every value `classifyModelHost` can return.
|
|
17
|
+
*
|
|
18
|
+
* `other` means a host we observed and did not recognize: a gateway, a proxy,
|
|
19
|
+
* or a vendor we have no rule for. `unknown` means we never saw a host at all,
|
|
20
|
+
* which is the case when a caller hands us an already-built LanguageModel.
|
|
21
|
+
* The two are not interchangeable — one is an unrecognized answer, the other
|
|
22
|
+
* is a missing one, and they need different follow-up.
|
|
23
|
+
*/
|
|
24
|
+
export const MODEL_HOST_CLASSES = [
|
|
25
|
+
"openai",
|
|
26
|
+
"azure",
|
|
27
|
+
"openrouter",
|
|
28
|
+
"anthropic",
|
|
29
|
+
"google",
|
|
30
|
+
"vertex",
|
|
31
|
+
"bedrock",
|
|
32
|
+
"groq",
|
|
33
|
+
"unify",
|
|
34
|
+
"minimax",
|
|
35
|
+
"local",
|
|
36
|
+
"other",
|
|
37
|
+
"unknown",
|
|
38
|
+
] as const;
|
|
39
|
+
|
|
40
|
+
export type ModelHostClass = (typeof MODEL_HOST_CLASSES)[number];
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Host suffixes mapped to their vendor.
|
|
44
|
+
*
|
|
45
|
+
* Matched on label boundaries, never as substrings, so
|
|
46
|
+
* `myresource.openai.azure.com.attacker.test` does not read as Azure.
|
|
47
|
+
*/
|
|
48
|
+
const HOST_SUFFIXES: ReadonlyArray<readonly [string, ModelHostClass]> = [
|
|
49
|
+
// Azure exposes the same models under three host families: the classic
|
|
50
|
+
// Azure OpenAI host, the Foundry Models host, and the general Cognitive
|
|
51
|
+
// Services host. All three are Azure for our purposes.
|
|
52
|
+
["openai.azure.com", "azure"],
|
|
53
|
+
["services.ai.azure.com", "azure"],
|
|
54
|
+
["cognitiveservices.azure.com", "azure"],
|
|
55
|
+
// API Management fronting an Azure OpenAI deployment, a common enterprise
|
|
56
|
+
// pattern. Reported as azure because that is what sits behind it.
|
|
57
|
+
["azure-api.net", "azure"],
|
|
58
|
+
|
|
59
|
+
["api.openai.com", "openai"],
|
|
60
|
+
["openrouter.ai", "openrouter"],
|
|
61
|
+
["api.anthropic.com", "anthropic"],
|
|
62
|
+
["generativelanguage.googleapis.com", "google"],
|
|
63
|
+
// Vertex regional hosts are `<region>-aiplatform.googleapis.com`, which is
|
|
64
|
+
// not a suffix of a single label, so both spellings are listed.
|
|
65
|
+
["aiplatform.googleapis.com", "vertex"],
|
|
66
|
+
["api.groq.com", "groq"],
|
|
67
|
+
["api.unify.ai", "unify"],
|
|
68
|
+
["api.minimax.io", "minimax"],
|
|
69
|
+
["api.minimaxi.com", "minimax"],
|
|
70
|
+
];
|
|
71
|
+
|
|
72
|
+
/** Suffixes that mean the model runs on the same machine or private network. */
|
|
73
|
+
const LOCAL_SUFFIXES = ["localhost", "local", "internal", "docker.internal"];
|
|
74
|
+
|
|
75
|
+
/** Matches `<region>-aiplatform.googleapis.com` and the Bedrock regional hosts. */
|
|
76
|
+
const REGIONAL_HOST_PATTERNS: ReadonlyArray<readonly [RegExp, ModelHostClass]> =
|
|
77
|
+
[
|
|
78
|
+
[/^[a-z0-9-]+-aiplatform\.googleapis\.com$/, "vertex"],
|
|
79
|
+
[/^bedrock[a-z0-9.-]*\.[a-z0-9-]+\.amazonaws\.com$/, "bedrock"],
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
/** True when `host` equals `suffix` or sits directly beneath it. */
|
|
83
|
+
function matchesSuffix(host: string, suffix: string): boolean {
|
|
84
|
+
return host === suffix || host.endsWith(`.${suffix}`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** True for loopback, unspecified, and RFC 1918 private addresses. */
|
|
88
|
+
function isPrivateAddress(host: string): boolean {
|
|
89
|
+
if (host === "127.0.0.1" || host === "0.0.0.0" || host === "::1") {
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
// 127.0.0.0/8 is entirely loopback.
|
|
93
|
+
if (/^127\./.test(host)) return true;
|
|
94
|
+
// 10.0.0.0/8 and 192.168.0.0/16.
|
|
95
|
+
if (/^10\./.test(host) || /^192\.168\./.test(host)) return true;
|
|
96
|
+
// 172.16.0.0/12 is 172.16 through 172.31 only. 172.32 is public, so the
|
|
97
|
+
// second octet is range-checked rather than prefix-matched.
|
|
98
|
+
const privateClassB = /^172\.(\d{1,3})\./.exec(host);
|
|
99
|
+
if (privateClassB) {
|
|
100
|
+
const secondOctet = Number(privateClassB[1]);
|
|
101
|
+
return secondOctet >= 16 && secondOctet <= 31;
|
|
102
|
+
}
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Reduce a model endpoint to one of {@link MODEL_HOST_CLASSES}.
|
|
108
|
+
*
|
|
109
|
+
* @param baseUrl The endpoint the provider was configured with. Absent when
|
|
110
|
+
* the provider was left on its own default, which is why `providerDefault`
|
|
111
|
+
* exists: "no base URL, provider is openai" means api.openai.com.
|
|
112
|
+
* @param providerDefault What to report when no base URL was supplied. Callers
|
|
113
|
+
* that genuinely cannot observe a host — a pre-built LanguageModel handed in
|
|
114
|
+
* by the developer — leave this alone and get `unknown`.
|
|
115
|
+
*/
|
|
116
|
+
export function classifyModelHost(
|
|
117
|
+
baseUrl?: string | null,
|
|
118
|
+
providerDefault: ModelHostClass = "unknown",
|
|
119
|
+
): ModelHostClass {
|
|
120
|
+
const trimmed = baseUrl?.trim();
|
|
121
|
+
if (!trimmed) {
|
|
122
|
+
return providerDefault;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let host: string;
|
|
126
|
+
try {
|
|
127
|
+
// `hostname` drops the port, the path, the query and any credentials, so
|
|
128
|
+
// nothing but the host can survive into the return value below.
|
|
129
|
+
host = new URL(trimmed).hostname.toLowerCase();
|
|
130
|
+
} catch {
|
|
131
|
+
// We were given something and could not read it. That is a different fact
|
|
132
|
+
// from having been given nothing, so it does not fall back to the default.
|
|
133
|
+
return "other";
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// IPv6 hostnames arrive bracketed from the URL parser.
|
|
137
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
138
|
+
|
|
139
|
+
if (!host) return "other";
|
|
140
|
+
|
|
141
|
+
if (isPrivateAddress(host)) return "local";
|
|
142
|
+
for (const suffix of LOCAL_SUFFIXES) {
|
|
143
|
+
if (matchesSuffix(host, suffix)) return "local";
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
for (const [suffix, hostClass] of HOST_SUFFIXES) {
|
|
147
|
+
if (matchesSuffix(host, suffix)) return hostClass;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
for (const [pattern, hostClass] of REGIONAL_HOST_PATTERNS) {
|
|
151
|
+
if (pattern.test(host)) return hostClass;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return "other";
|
|
155
|
+
}
|
|
@@ -1,17 +1,25 @@
|
|
|
1
1
|
// Per-event telemetry metadata shared by both TelemetryClients.
|
|
2
2
|
//
|
|
3
3
|
// Two clients emit `oss.runtime.*`: the v1 client in this package and the
|
|
4
|
-
// v2 client in @copilotkit/runtime.
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
4
|
+
// v2 client in @copilotkit/runtime. Neither samples the lambda sink any
|
|
5
|
+
// more — that sink is ours, so anonymous volume costs us nothing per
|
|
6
|
+
// event and full fidelity beats an extrapolation. One wire is still
|
|
7
|
+
// gated: the Segment copy the v1 client sends, which is billed per event.
|
|
8
|
+
//
|
|
9
|
+
// Every event therefore still carries a sampling block, because the two
|
|
10
|
+
// populations remain distinguishable and a Segment-sourced count still
|
|
11
|
+
// needs sum(sampleWeight) to mean anything. On the lambda wire the block
|
|
12
|
+
// reads sampleRate=1/sampleWeight=1, which is the honest answer rather
|
|
13
|
+
// than a placeholder.
|
|
9
14
|
//
|
|
10
15
|
// The two clients computed this independently and drifted: v2 sampled but
|
|
11
16
|
// stamped nothing, leaving ~24% of runtime volume unweightable from the
|
|
12
17
|
// data alone (OSS-1017). Computing it here is what stops them drifting
|
|
13
18
|
// again.
|
|
14
19
|
|
|
20
|
+
/** The rate a sink that takes every event is "gated" at. */
|
|
21
|
+
export const UNSAMPLED_RATE = 1;
|
|
22
|
+
|
|
15
23
|
/** Identifies which client emitted an event. */
|
|
16
24
|
export const TELEMETRY_EMITTER_V1 = "v1-shared";
|
|
17
25
|
export const TELEMETRY_EMITTER_V2 = "v2-runtime";
|
|
@@ -20,6 +28,26 @@ export type TelemetryEmitter =
|
|
|
20
28
|
| typeof TELEMETRY_EMITTER_V1
|
|
21
29
|
| typeof TELEMETRY_EMITTER_V2;
|
|
22
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Which public API surface the developer built against.
|
|
33
|
+
*
|
|
34
|
+
* Separate from {@link TelemetryEmitter} on purpose. The emitter names a
|
|
35
|
+
* client library, and there are six of them across four languages; the
|
|
36
|
+
* surface answers the question anyone actually asks of this data, which
|
|
37
|
+
* is how much traffic still arrives through the deprecated v1 API. The
|
|
38
|
+
* native runtimes (Go, Python, Ruby, .NET) expose v2 only, so they report
|
|
39
|
+
* `v2` without having a v1 notion of their own.
|
|
40
|
+
*/
|
|
41
|
+
// `as const` so the values keep their literal types when they land in a
|
|
42
|
+
// mutable object property, which is how the v1 shim passes the surface
|
|
43
|
+
// into the V2 runtime's options.
|
|
44
|
+
export const TELEMETRY_SURFACE_V1 = "v1" as const;
|
|
45
|
+
export const TELEMETRY_SURFACE_V2 = "v2" as const;
|
|
46
|
+
|
|
47
|
+
export type TelemetrySurface =
|
|
48
|
+
| typeof TELEMETRY_SURFACE_V1
|
|
49
|
+
| typeof TELEMETRY_SURFACE_V2;
|
|
50
|
+
|
|
23
51
|
/**
|
|
24
52
|
* Which wire an event copy travelled on. The v1 client sends each capture
|
|
25
53
|
* to both, so this is what tells the two copies apart downstream
|
|
@@ -28,7 +56,7 @@ export type TelemetryEmitter =
|
|
|
28
56
|
export type TelemetryTransport = "segment" | "lambda";
|
|
29
57
|
|
|
30
58
|
export interface SamplingMeta {
|
|
31
|
-
/** The rate this event was actually gated at: 1 when identified. */
|
|
59
|
+
/** The rate this event was actually gated at: 1 when unsampled or identified. */
|
|
32
60
|
sampleRate: number;
|
|
33
61
|
sampleRateAdjustmentFactor: number;
|
|
34
62
|
/** Multiply by this to extrapolate the population the event stands for. */
|
|
@@ -43,6 +71,11 @@ export interface SamplingMeta {
|
|
|
43
71
|
* `telemetryId` is the caller's parsed license telemetry_id, or null when
|
|
44
72
|
* anonymous — it decides the branch, and is deliberately not returned:
|
|
45
73
|
* only the non-PII shape below travels on the event.
|
|
74
|
+
*
|
|
75
|
+
* `sampleRate` is the rate of the wire this copy travels on, so an
|
|
76
|
+
* unsampled sink passes {@link UNSAMPLED_RATE}. Call it once per wire:
|
|
77
|
+
* the v1 client sends one capture to two wires gated differently, and a
|
|
78
|
+
* shared block would misweight one of them.
|
|
46
79
|
*/
|
|
47
80
|
export function computeSamplingMeta({
|
|
48
81
|
telemetryId,
|
|
@@ -58,16 +58,21 @@ function setupSharedCapture(randomValue: number, telemetryId: string = " \t ") {
|
|
|
58
58
|
};
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
test("Shared whitespace-only legacy claim stays subject to
|
|
61
|
+
test("Shared whitespace-only legacy claim stays subject to the Segment gate", async () => {
|
|
62
62
|
const sinkSpy = vi.spyOn(lambdaClient, "send").mockResolvedValue(undefined);
|
|
63
63
|
const { client, randomSpy, teardown } = setupSharedCapture(0.99);
|
|
64
64
|
|
|
65
65
|
try {
|
|
66
66
|
await client.capture("oss.runtime.instance_created", instanceCreatedEvent);
|
|
67
67
|
|
|
68
|
+
// An unusable claim must not buy identified status. The gate it now
|
|
69
|
+
// fails is Segment's alone: the lambda copy goes either way, and what
|
|
70
|
+
// proves the claim was rejected is that it goes marked anonymous.
|
|
68
71
|
expect(randomSpy).toHaveBeenCalledTimes(1);
|
|
69
|
-
expect(sinkSpy).not.toHaveBeenCalled();
|
|
70
72
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
73
|
+
expect(sinkSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
74
|
+
telemetry_identified: false,
|
|
75
|
+
});
|
|
71
76
|
} finally {
|
|
72
77
|
sinkSpy.mockRestore();
|
|
73
78
|
teardown();
|
|
@@ -96,7 +101,7 @@ test("Shared sampled whitespace-only legacy claim sends no identity header", asy
|
|
|
96
101
|
});
|
|
97
102
|
|
|
98
103
|
test.each(["bad\nid", "bad\u0000id", "tenant-🚀"])(
|
|
99
|
-
"Shared header-invalid legacy claim %j stays subject to
|
|
104
|
+
"Shared header-invalid legacy claim %j stays subject to the Segment gate",
|
|
100
105
|
async (invalidTelemetryId) => {
|
|
101
106
|
const sinkSpy = vi.spyOn(lambdaClient, "send").mockResolvedValue(undefined);
|
|
102
107
|
const { client, randomSpy, teardown } = setupSharedCapture(
|
|
@@ -111,8 +116,10 @@ test.each(["bad\nid", "bad\u0000id", "tenant-🚀"])(
|
|
|
111
116
|
);
|
|
112
117
|
|
|
113
118
|
expect(randomSpy).toHaveBeenCalledTimes(1);
|
|
114
|
-
expect(sinkSpy).not.toHaveBeenCalled();
|
|
115
119
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
120
|
+
expect(sinkSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
121
|
+
telemetry_identified: false,
|
|
122
|
+
});
|
|
116
123
|
} finally {
|
|
117
124
|
sinkSpy.mockRestore();
|
|
118
125
|
teardown();
|
|
@@ -122,12 +122,23 @@ describe("v1 TelemetryClient", () => {
|
|
|
122
122
|
"cloud.api_key_provided": false,
|
|
123
123
|
} as const;
|
|
124
124
|
|
|
125
|
+
/** The block every unsampled wire carries, whatever the configured rate. */
|
|
126
|
+
const unsampledMeta = {
|
|
127
|
+
sampleRate: 1,
|
|
128
|
+
sampleRateAdjustmentFactor: 0,
|
|
129
|
+
sampleWeight: 1,
|
|
130
|
+
} as const;
|
|
131
|
+
|
|
125
132
|
/**
|
|
126
|
-
* Verifies the shared event shape and sampling metadata
|
|
133
|
+
* Verifies the shared event shape and per-wire sampling metadata.
|
|
134
|
+
*
|
|
135
|
+
* The two wires are gated separately, so they carry different sampling
|
|
136
|
+
* blocks for the same capture: the lambda copy is never sampled, while the
|
|
137
|
+
* Segment copy reports the rate its own gate used.
|
|
127
138
|
*
|
|
128
139
|
* @param expectedIdentity - Identity fields expected only on the Lambda send.
|
|
129
|
-
* @param
|
|
130
|
-
* @param telemetryIdentified - Whether a license-derived identity
|
|
140
|
+
* @param segmentSamplingMeta - Sampling metadata expected on the Segment copy.
|
|
141
|
+
* @param telemetryIdentified - Whether a license-derived identity was present.
|
|
131
142
|
* @param identityValues - Raw identity values that must not leak into payloads.
|
|
132
143
|
*/
|
|
133
144
|
function expectBothSinksReceivedEvent(
|
|
@@ -135,7 +146,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
135
146
|
Parameters<typeof lambdaClient.send>[0],
|
|
136
147
|
"licenseToken" | "telemetryId"
|
|
137
148
|
>,
|
|
138
|
-
|
|
149
|
+
segmentSamplingMeta: {
|
|
139
150
|
sampleRate: number;
|
|
140
151
|
sampleRateAdjustmentFactor: number;
|
|
141
152
|
sampleWeight: number;
|
|
@@ -147,14 +158,15 @@ describe("v1 TelemetryClient", () => {
|
|
|
147
158
|
const lambdaEvent = lambdaSpy.mock.calls[0][0];
|
|
148
159
|
expect(lambdaEvent).toMatchObject({
|
|
149
160
|
...expectedIdentity,
|
|
150
|
-
globalProperties:
|
|
161
|
+
globalProperties: unsampledMeta,
|
|
151
162
|
});
|
|
152
163
|
expect(lambdaEvent.properties).toEqual(flattenedBaseInstanceEvent);
|
|
153
164
|
expect(lambdaEvent.globalProperties).toEqual({
|
|
154
165
|
"copilotkit.package.name": "@copilotkit/shared",
|
|
155
166
|
"copilotkit.package.version": "1.0.0",
|
|
156
|
-
...
|
|
167
|
+
...unsampledMeta,
|
|
157
168
|
telemetry_emitter: "v1-shared",
|
|
169
|
+
telemetry_surface: "v1",
|
|
158
170
|
telemetry_event_id: expect.any(String),
|
|
159
171
|
telemetry_identified: telemetryIdentified,
|
|
160
172
|
telemetry_transport: "lambda",
|
|
@@ -169,8 +181,9 @@ describe("v1 TelemetryClient", () => {
|
|
|
169
181
|
...flattenedBaseInstanceEvent,
|
|
170
182
|
"copilotkit.package.name": "@copilotkit/shared",
|
|
171
183
|
"copilotkit.package.version": "1.0.0",
|
|
172
|
-
...
|
|
184
|
+
...segmentSamplingMeta,
|
|
173
185
|
telemetry_emitter: "v1-shared",
|
|
186
|
+
telemetry_surface: "v1",
|
|
174
187
|
telemetry_event_id: expect.any(String),
|
|
175
188
|
telemetry_identified: telemetryIdentified,
|
|
176
189
|
telemetry_transport: "segment",
|
|
@@ -188,7 +201,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
188
201
|
}
|
|
189
202
|
}
|
|
190
203
|
|
|
191
|
-
test("capture sends to both sinks when
|
|
204
|
+
test("capture sends to both sinks when the anonymous Segment gate opens", async () => {
|
|
192
205
|
vi.spyOn(Math, "random").mockReturnValue(0);
|
|
193
206
|
const client = makeClient({ sampleRate: 0.05 });
|
|
194
207
|
|
|
@@ -201,15 +214,53 @@ describe("v1 TelemetryClient", () => {
|
|
|
201
214
|
});
|
|
202
215
|
});
|
|
203
216
|
|
|
204
|
-
test("
|
|
205
|
-
// Math.random=0.99 vs sampleRate=0.05
|
|
206
|
-
//
|
|
217
|
+
test("an anonymous event sampled out of Segment still reaches the lambda sink", async () => {
|
|
218
|
+
// Math.random=0.99 vs sampleRate=0.05. The gate is Segment's alone now:
|
|
219
|
+
// the lambda sink is ours and takes every event, so a rate that drops
|
|
220
|
+
// 95% of the Segment copies must drop none of the lambda copies. This is
|
|
221
|
+
// the whole point of the change — if it ever reverts, this fails first.
|
|
207
222
|
vi.spyOn(Math, "random").mockReturnValue(0.99);
|
|
208
223
|
const client = makeClient({ sampleRate: 0.05 });
|
|
209
224
|
|
|
210
225
|
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
211
226
|
|
|
212
|
-
expect(lambdaSpy).
|
|
227
|
+
expect(lambdaSpy).toHaveBeenCalledTimes(1);
|
|
228
|
+
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
test("the lambda copy reports rate 1 while the Segment copy of the same capture reports the gated rate", async () => {
|
|
232
|
+
// One capture, two wires, two different weights. A shared block would
|
|
233
|
+
// make the unsampled lambda copy claim it stands for 20 events.
|
|
234
|
+
vi.spyOn(Math, "random").mockReturnValue(0);
|
|
235
|
+
const client = makeClient({ sampleRate: 0.05 });
|
|
236
|
+
|
|
237
|
+
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
238
|
+
|
|
239
|
+
expect(lambdaSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
240
|
+
sampleRate: 1,
|
|
241
|
+
sampleRateAdjustmentFactor: 0,
|
|
242
|
+
sampleWeight: 1,
|
|
243
|
+
});
|
|
244
|
+
expect(segmentTrackMock.mock.calls[0][0]).toMatchObject({
|
|
245
|
+
properties: expect.objectContaining({
|
|
246
|
+
sampleRate: 0.05,
|
|
247
|
+
sampleRateAdjustmentFactor: 0.95,
|
|
248
|
+
sampleWeight: 20,
|
|
249
|
+
}),
|
|
250
|
+
});
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
test("COPILOTKIT_TELEMETRY_SAMPLE_RATE=0 silences Segment without silencing the lambda sink", async () => {
|
|
254
|
+
// The env var survives as a Segment-only lever. Setting it to 0 is the
|
|
255
|
+
// strongest form of that lever and still must not be mistaken for an
|
|
256
|
+
// opt-out: COPILOTKIT_TELEMETRY_DISABLED is the opt-out.
|
|
257
|
+
vi.spyOn(Math, "random").mockReturnValue(0);
|
|
258
|
+
process.env.COPILOTKIT_TELEMETRY_SAMPLE_RATE = "0";
|
|
259
|
+
const client = makeClient();
|
|
260
|
+
|
|
261
|
+
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
262
|
+
|
|
263
|
+
expect(lambdaSpy).toHaveBeenCalledTimes(1);
|
|
213
264
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
214
265
|
});
|
|
215
266
|
|
|
@@ -220,7 +271,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
220
271
|
priorLicenseToken: jwtWith({ telemetry_id: "legacy-telemetry-id" }),
|
|
221
272
|
},
|
|
222
273
|
])(
|
|
223
|
-
"standalone identity
|
|
274
|
+
"standalone identity stays behind the Segment gate when $label",
|
|
224
275
|
async ({ priorLicenseToken }) => {
|
|
225
276
|
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.99);
|
|
226
277
|
const client = makeClient({ sampleRate: 0.05 });
|
|
@@ -234,12 +285,12 @@ describe("v1 TelemetryClient", () => {
|
|
|
234
285
|
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
235
286
|
|
|
236
287
|
expect(randomSpy).toHaveBeenCalledTimes(1);
|
|
237
|
-
expect(lambdaSpy).
|
|
288
|
+
expect(lambdaSpy).toHaveBeenCalledTimes(1);
|
|
238
289
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
239
290
|
},
|
|
240
291
|
);
|
|
241
292
|
|
|
242
|
-
test("
|
|
293
|
+
test("standalone identity remains a transport claim with anonymous Segment sampling metadata", async () => {
|
|
243
294
|
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
|
244
295
|
const client = makeClient({ sampleRate: 0.05 });
|
|
245
296
|
client.setTelemetryIdentity({
|
|
@@ -301,7 +352,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
301
352
|
},
|
|
302
353
|
);
|
|
303
354
|
|
|
304
|
-
test("legacy license identity bypasses
|
|
355
|
+
test("legacy license identity bypasses the Segment gate and weighs 1 on both wires", async () => {
|
|
305
356
|
const telemetryId = "legacy-telemetry-id";
|
|
306
357
|
const licenseToken = jwtWith({ telemetry_id: telemetryId });
|
|
307
358
|
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.99);
|
|
@@ -323,7 +374,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
323
374
|
);
|
|
324
375
|
});
|
|
325
376
|
|
|
326
|
-
test("clearing telemetry identity restores
|
|
377
|
+
test("clearing telemetry identity restores the anonymous Segment decision", async () => {
|
|
327
378
|
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
|
328
379
|
const client = makeClient({ sampleRate: 0.05 });
|
|
329
380
|
client.setTelemetryIdentity({ telemetryId: "standalone-telemetry-id" });
|
|
@@ -336,11 +387,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
336
387
|
expect(lambdaSpy.mock.calls[0][0]).toMatchObject({
|
|
337
388
|
licenseToken: undefined,
|
|
338
389
|
telemetryId: undefined,
|
|
339
|
-
globalProperties:
|
|
340
|
-
sampleRate: 0.05,
|
|
341
|
-
sampleRateAdjustmentFactor: 0.95,
|
|
342
|
-
sampleWeight: 20,
|
|
343
|
-
},
|
|
390
|
+
globalProperties: unsampledMeta,
|
|
344
391
|
});
|
|
345
392
|
expect(segmentTrackMock).toHaveBeenCalledTimes(1);
|
|
346
393
|
expect(segmentTrackMock.mock.calls[0][0]).toMatchObject({
|
|
@@ -377,7 +424,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
377
424
|
);
|
|
378
425
|
|
|
379
426
|
test.each(["", " \t "])(
|
|
380
|
-
"blank standalone identity %j without a legacy identity
|
|
427
|
+
"blank standalone identity %j without a legacy identity stays anonymous",
|
|
381
428
|
async (blankTelemetryId) => {
|
|
382
429
|
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
|
383
430
|
const client = makeClient({ sampleRate: 0.05 });
|
|
@@ -390,13 +437,19 @@ describe("v1 TelemetryClient", () => {
|
|
|
390
437
|
expect.objectContaining({
|
|
391
438
|
licenseToken: undefined,
|
|
392
439
|
telemetryId: undefined,
|
|
393
|
-
globalProperties: expect.objectContaining({
|
|
440
|
+
globalProperties: expect.objectContaining({
|
|
441
|
+
sampleRate: 1,
|
|
442
|
+
telemetry_identified: false,
|
|
443
|
+
}),
|
|
394
444
|
}),
|
|
395
445
|
);
|
|
446
|
+
expect(segmentTrackMock.mock.calls[0][0]).toMatchObject({
|
|
447
|
+
properties: expect.objectContaining({ sampleRate: 0.05 }),
|
|
448
|
+
});
|
|
396
449
|
},
|
|
397
450
|
);
|
|
398
451
|
|
|
399
|
-
test("identified callers bypass the
|
|
452
|
+
test("identified callers bypass the Segment gate (both wires fire even when Math.random would fail)", async () => {
|
|
400
453
|
vi.spyOn(Math, "random").mockReturnValue(0.99);
|
|
401
454
|
const client = makeClient({ sampleRate: 0.05 });
|
|
402
455
|
client.setLicenseToken(jwtWith({ telemetry_id: "abc-123" }));
|
|
@@ -480,17 +533,16 @@ describe("v1 TelemetryClient", () => {
|
|
|
480
533
|
expect(warn).not.toHaveBeenCalled();
|
|
481
534
|
});
|
|
482
535
|
|
|
483
|
-
test("
|
|
484
|
-
//
|
|
485
|
-
//
|
|
486
|
-
//
|
|
487
|
-
// so sampleWeight must be 1 — not the population's 1/sampleRate.
|
|
536
|
+
test("Segment keeps sampleWeight=1/sampleRate for anonymous events and 1 for identified ones", async () => {
|
|
537
|
+
// On the Segment wire the weight still has work to do: anonymous events
|
|
538
|
+
// stand for 1 / sampleRate of them, identified events stand for
|
|
539
|
+
// themselves. The lambda wire is unsampled, so its weight is always 1.
|
|
488
540
|
vi.spyOn(Math, "random").mockReturnValue(0);
|
|
489
541
|
const anonClient = makeClient({ sampleRate: 0.05 });
|
|
490
542
|
await anonClient.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
491
543
|
expect(lambdaSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
492
|
-
sampleRate:
|
|
493
|
-
sampleWeight:
|
|
544
|
+
sampleRate: 1,
|
|
545
|
+
sampleWeight: 1,
|
|
494
546
|
});
|
|
495
547
|
expect(segmentTrackMock.mock.calls[0][0]).toMatchObject({
|
|
496
548
|
properties: expect.objectContaining({
|
|
@@ -514,7 +566,7 @@ describe("v1 TelemetryClient", () => {
|
|
|
514
566
|
});
|
|
515
567
|
});
|
|
516
568
|
|
|
517
|
-
test("malformed license token stays anonymous and
|
|
569
|
+
test("malformed license token stays anonymous and stays behind the Segment gate", async () => {
|
|
518
570
|
// parseTelemetryIdFromLicense returns null for any of: empty token,
|
|
519
571
|
// wrong-shape (not three dot-separated segments), base64/JSON parse
|
|
520
572
|
// failure. A misconfigured customer must not flip to identified-bypass.
|
|
@@ -525,11 +577,13 @@ describe("v1 TelemetryClient", () => {
|
|
|
525
577
|
client.setLicenseToken("not-a-jwt");
|
|
526
578
|
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
527
579
|
|
|
528
|
-
expect(lambdaSpy).not.toHaveBeenCalled();
|
|
529
580
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
581
|
+
expect(lambdaSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
582
|
+
telemetry_identified: false,
|
|
583
|
+
});
|
|
530
584
|
});
|
|
531
585
|
|
|
532
|
-
test("illegal base64url license payload cannot bypass sampleRate 0", async () => {
|
|
586
|
+
test("illegal base64url license payload cannot bypass Segment sampleRate 0", async () => {
|
|
533
587
|
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
|
534
588
|
vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
|
535
589
|
const client = makeClient({ sampleRate: 0 });
|
|
@@ -538,8 +592,12 @@ describe("v1 TelemetryClient", () => {
|
|
|
538
592
|
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
539
593
|
|
|
540
594
|
expect(randomSpy).toHaveBeenCalledTimes(1);
|
|
541
|
-
expect(lambdaSpy).not.toHaveBeenCalled();
|
|
542
595
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
596
|
+
// The lambda copy still goes, and still says it is anonymous: an
|
|
597
|
+
// unparseable token must not buy identified status on either wire.
|
|
598
|
+
expect(lambdaSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
599
|
+
telemetry_identified: false,
|
|
600
|
+
});
|
|
543
601
|
});
|
|
544
602
|
|
|
545
603
|
test("setLicenseToken cache is overwritable (good token replaced by bad → back to anonymous gate)", async () => {
|
|
@@ -554,8 +612,10 @@ describe("v1 TelemetryClient", () => {
|
|
|
554
612
|
client.setLicenseToken(jwtWith({ license_id: "no-telemetry-id" }));
|
|
555
613
|
await client.capture("oss.runtime.instance_created", baseInstanceEvent);
|
|
556
614
|
|
|
557
|
-
expect(lambdaSpy).not.toHaveBeenCalled();
|
|
558
615
|
expect(segmentTrackMock).not.toHaveBeenCalled();
|
|
616
|
+
expect(lambdaSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
617
|
+
telemetry_identified: false,
|
|
618
|
+
});
|
|
559
619
|
});
|
|
560
620
|
|
|
561
621
|
test("setCloudConfiguration writes cloud keys into globalProperties for both sinks", async () => {
|
|
@@ -647,7 +707,11 @@ describe("v1 TelemetryClient", () => {
|
|
|
647
707
|
expect(first).not.toBe(second);
|
|
648
708
|
});
|
|
649
709
|
|
|
650
|
-
test("both copies are stamped as emitted by the v1 client", async () => {
|
|
710
|
+
test("both copies are stamped as emitted by the v1 client, on the v1 surface", async () => {
|
|
711
|
+
// telemetry_emitter names the client library; telemetry_surface names
|
|
712
|
+
// the API the developer built against. They are separate because six
|
|
713
|
+
// emitters across four languages report only two surfaces, and the
|
|
714
|
+
// question anyone asks of this data is how much traffic is still v1.
|
|
651
715
|
vi.spyOn(Math, "random").mockReturnValue(0);
|
|
652
716
|
const client = makeClient({ sampleRate: 0.05 });
|
|
653
717
|
|
|
@@ -655,9 +719,13 @@ describe("v1 TelemetryClient", () => {
|
|
|
655
719
|
|
|
656
720
|
expect(lambdaSpy.mock.calls[0][0].globalProperties).toMatchObject({
|
|
657
721
|
telemetry_emitter: "v1-shared",
|
|
722
|
+
telemetry_surface: "v1",
|
|
658
723
|
});
|
|
659
724
|
expect(segmentTrackMock.mock.calls[0][0]).toMatchObject({
|
|
660
|
-
properties: expect.objectContaining({
|
|
725
|
+
properties: expect.objectContaining({
|
|
726
|
+
telemetry_emitter: "v1-shared",
|
|
727
|
+
telemetry_surface: "v1",
|
|
728
|
+
}),
|
|
661
729
|
});
|
|
662
730
|
});
|
|
663
731
|
|