@copilotkit/shared 1.72.0 → 1.72.1-canary.sp-longthread-perf

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/LICENSE +21 -0
  2. package/dist/index.cjs +6 -0
  3. package/dist/index.cjs.map +1 -1
  4. package/dist/index.d.cts +3 -2
  5. package/dist/index.d.cts.map +1 -1
  6. package/dist/index.d.mts +3 -2
  7. package/dist/index.d.mts.map +1 -1
  8. package/dist/index.mjs +3 -2
  9. package/dist/index.mjs.map +1 -1
  10. package/dist/index.umd.js +126 -2
  11. package/dist/index.umd.js.map +1 -1
  12. package/dist/package.cjs +1 -1
  13. package/dist/package.mjs +1 -1
  14. package/dist/telemetry/events.d.cts +33 -0
  15. package/dist/telemetry/events.d.cts.map +1 -1
  16. package/dist/telemetry/events.d.mts +33 -0
  17. package/dist/telemetry/events.d.mts.map +1 -1
  18. package/dist/telemetry/index.cjs +3 -0
  19. package/dist/telemetry/index.d.cts +2 -2
  20. package/dist/telemetry/index.d.mts +2 -2
  21. package/dist/telemetry/index.mjs +2 -2
  22. package/dist/telemetry/model-host.cjs +104 -0
  23. package/dist/telemetry/model-host.cjs.map +1 -0
  24. package/dist/telemetry/model-host.d.cts +26 -0
  25. package/dist/telemetry/model-host.d.cts.map +1 -0
  26. package/dist/telemetry/model-host.d.mts +26 -0
  27. package/dist/telemetry/model-host.d.mts.map +1 -0
  28. package/dist/telemetry/model-host.mjs +102 -0
  29. package/dist/telemetry/model-host.mjs.map +1 -0
  30. package/dist/telemetry/sampling.cjs +22 -0
  31. package/dist/telemetry/sampling.cjs.map +1 -1
  32. package/dist/telemetry/sampling.d.cts +22 -2
  33. package/dist/telemetry/sampling.d.cts.map +1 -1
  34. package/dist/telemetry/sampling.d.mts +22 -2
  35. package/dist/telemetry/sampling.d.mts.map +1 -1
  36. package/dist/telemetry/sampling.mjs +20 -1
  37. package/dist/telemetry/sampling.mjs.map +1 -1
  38. package/dist/telemetry/telemetry-client.cjs +18 -11
  39. package/dist/telemetry/telemetry-client.cjs.map +1 -1
  40. package/dist/telemetry/telemetry-client.d.cts +3 -3
  41. package/dist/telemetry/telemetry-client.d.cts.map +1 -1
  42. package/dist/telemetry/telemetry-client.d.mts +3 -3
  43. package/dist/telemetry/telemetry-client.d.mts.map +1 -1
  44. package/dist/telemetry/telemetry-client.mjs +19 -12
  45. package/dist/telemetry/telemetry-client.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/src/index.ts +3 -0
  48. package/src/telemetry/events.ts +33 -0
  49. package/src/telemetry/model-host.test.ts +145 -0
  50. package/src/telemetry/model-host.ts +155 -0
  51. package/src/telemetry/sampling.ts +39 -6
  52. package/src/telemetry/telemetry-blank-license-identity.test.ts +11 -4
  53. package/src/telemetry/telemetry-client.test.ts +107 -39
  54. package/src/telemetry/telemetry-client.ts +48 -27
@@ -0,0 +1,145 @@
1
+ import { describe, it, expect } from "vitest";
2
+ import {
3
+ classifyModelHost,
4
+ MODEL_HOST_CLASSES,
5
+ type ModelHostClass,
6
+ } from "./model-host";
7
+
8
+ describe("classifyModelHost", () => {
9
+ describe("the case this exists for: an OpenAI-shaped client aimed elsewhere", () => {
10
+ // Every row below reports `provider: "openai.responses"` from the AI SDK,
11
+ // so the provider label cannot tell them apart. The host can.
12
+ it.each([
13
+ ["https://api.openai.com/v1", "openai"],
14
+ ["https://myresource.openai.azure.com/openai/v1", "azure"],
15
+ ["https://myresource.services.ai.azure.com/models", "azure"],
16
+ ["https://myresource.cognitiveservices.azure.com/", "azure"],
17
+ ["https://openrouter.ai/api/v1", "openrouter"],
18
+ ["http://localhost:11434/v1", "local"],
19
+ ["https://llm.internal.acme.example/v1", "other"],
20
+ ])("classifies %s as %s", (baseUrl, expected) => {
21
+ expect(classifyModelHost(baseUrl, "openai")).toBe(expected);
22
+ });
23
+ });
24
+
25
+ describe("absent base URL falls back to the provider default", () => {
26
+ it("returns the provider default when the base URL is undefined", () => {
27
+ expect(classifyModelHost(undefined, "openai")).toBe("openai");
28
+ expect(classifyModelHost(undefined, "anthropic")).toBe("anthropic");
29
+ });
30
+
31
+ it("returns the provider default for null and empty string", () => {
32
+ expect(classifyModelHost(null, "vertex")).toBe("vertex");
33
+ expect(classifyModelHost("", "vertex")).toBe("vertex");
34
+ expect(classifyModelHost(" ", "vertex")).toBe("vertex");
35
+ });
36
+
37
+ it("defaults to unknown when the caller supplies no provider default", () => {
38
+ // This is the pre-built LanguageModel case: nobody knows the host.
39
+ expect(classifyModelHost(undefined)).toBe("unknown");
40
+ });
41
+ });
42
+
43
+ describe("known vendor hosts", () => {
44
+ it.each([
45
+ ["https://api.anthropic.com", "anthropic"],
46
+ ["https://generativelanguage.googleapis.com/v1beta", "google"],
47
+ ["https://us-central1-aiplatform.googleapis.com/v1", "vertex"],
48
+ ["https://bedrock-runtime.us-east-1.amazonaws.com", "bedrock"],
49
+ ["https://api.groq.com/openai/v1", "groq"],
50
+ ["https://api.unify.ai/v0/", "unify"],
51
+ ["https://api.minimax.io/v1", "minimax"],
52
+ ])("classifies %s as %s", (baseUrl, expected) => {
53
+ expect(classifyModelHost(baseUrl)).toBe(expected);
54
+ });
55
+ });
56
+
57
+ describe("local and private endpoints", () => {
58
+ it.each([
59
+ "http://localhost:1234/v1",
60
+ "http://127.0.0.1:8000/v1",
61
+ "http://[::1]:11434/v1",
62
+ "http://0.0.0.0:11434/v1",
63
+ "http://10.1.2.3:8000/v1",
64
+ "http://192.168.1.50:8000/v1",
65
+ "http://172.16.0.9:8000/v1",
66
+ "http://172.31.255.254:8000/v1",
67
+ "http://host.docker.internal:11434/v1",
68
+ "http://ollama.local:11434/v1",
69
+ ])("treats %s as local", (baseUrl) => {
70
+ expect(classifyModelHost(baseUrl, "openai")).toBe("local");
71
+ });
72
+
73
+ it("does not mistake a public 172.x address for a private one", () => {
74
+ // 172.32.x is outside the 172.16.0.0/12 private block.
75
+ expect(classifyModelHost("http://172.32.0.1:8000/v1", "openai")).toBe(
76
+ "other",
77
+ );
78
+ });
79
+ });
80
+
81
+ describe("it never leaks the host", () => {
82
+ it.each([
83
+ "https://myresource.openai.azure.com/openai/v1",
84
+ "https://acme-secret-gateway.internal.example/v1",
85
+ "not a url at all",
86
+ "://malformed",
87
+ "ftp://weird.example/v1",
88
+ ])("returns a closed-vocabulary value for %s", (baseUrl) => {
89
+ const result = classifyModelHost(baseUrl, "openai");
90
+ expect(MODEL_HOST_CLASSES).toContain(result);
91
+ // The customer's resource name must never reach telemetry.
92
+ expect(result).not.toContain("myresource");
93
+ expect(result).not.toContain("acme");
94
+ });
95
+
96
+ it("classifies an unparseable base URL as other, not unknown", () => {
97
+ // `unknown` means "we could not observe a host". A malformed string
98
+ // means we did observe one and failed to read it. Those differ.
99
+ expect(classifyModelHost("not a url at all", "openai")).toBe("other");
100
+ });
101
+ });
102
+
103
+ describe("normalization", () => {
104
+ it("ignores case and trailing whitespace", () => {
105
+ expect(
106
+ classifyModelHost(" HTTPS://MyResource.OpenAI.Azure.COM/openai/v1 "),
107
+ ).toBe("azure");
108
+ });
109
+
110
+ it("ignores port, path and query", () => {
111
+ expect(
112
+ classifyModelHost("https://api.openai.com:443/v1/chat?x=1#frag"),
113
+ ).toBe("openai");
114
+ });
115
+
116
+ it("does not match a lookalike suffix", () => {
117
+ // `evil-openai.azure.com.attacker.example` must not read as azure.
118
+ expect(
119
+ classifyModelHost("https://myresource.openai.azure.com.attacker.test/"),
120
+ ).toBe("other");
121
+ // A bare substring is not a suffix match either.
122
+ expect(classifyModelHost("https://notapi.openai.com.example/")).toBe(
123
+ "other",
124
+ );
125
+ });
126
+ });
127
+
128
+ describe("the vocabulary is closed", () => {
129
+ it("exposes unknown and other as members", () => {
130
+ expect(MODEL_HOST_CLASSES).toContain("unknown");
131
+ expect(MODEL_HOST_CLASSES).toContain("other");
132
+ });
133
+
134
+ it("has no duplicate members", () => {
135
+ expect(new Set(MODEL_HOST_CLASSES).size).toBe(MODEL_HOST_CLASSES.length);
136
+ });
137
+
138
+ it("accepts every member as a provider default", () => {
139
+ for (const member of MODEL_HOST_CLASSES) {
140
+ const value: ModelHostClass = classifyModelHost(undefined, member);
141
+ expect(value).toBe(member);
142
+ }
143
+ });
144
+ });
145
+ });
@@ -0,0 +1,155 @@
1
+ // Which endpoint the runtime sends completions to, as a closed vocabulary.
2
+ //
3
+ // The AI SDK's provider label cannot answer this. `createOpenAI` stamps
4
+ // `openai.responses` on every model it builds, whether the base URL points at
5
+ // api.openai.com, Azure, OpenRouter or a laptop running Ollama — and Azure's
6
+ // own migration guide tells customers to use exactly that client. So the
7
+ // provider field reports the wire protocol, and the host reports the vendor.
8
+ //
9
+ // Only the classification travels. A base URL like
10
+ // `myresource.openai.azure.com` carries the customer's Azure resource name,
11
+ // which identifies them, so the raw host must never reach telemetry. That is
12
+ // why this returns a member of a fixed list rather than anything derived from
13
+ // the input.
14
+
15
+ /**
16
+ * Every value `classifyModelHost` can return.
17
+ *
18
+ * `other` means a host we observed and did not recognize: a gateway, a proxy,
19
+ * or a vendor we have no rule for. `unknown` means we never saw a host at all,
20
+ * which is the case when a caller hands us an already-built LanguageModel.
21
+ * The two are not interchangeable — one is an unrecognized answer, the other
22
+ * is a missing one, and they need different follow-up.
23
+ */
24
+ export const MODEL_HOST_CLASSES = [
25
+ "openai",
26
+ "azure",
27
+ "openrouter",
28
+ "anthropic",
29
+ "google",
30
+ "vertex",
31
+ "bedrock",
32
+ "groq",
33
+ "unify",
34
+ "minimax",
35
+ "local",
36
+ "other",
37
+ "unknown",
38
+ ] as const;
39
+
40
+ export type ModelHostClass = (typeof MODEL_HOST_CLASSES)[number];
41
+
42
+ /**
43
+ * Host suffixes mapped to their vendor.
44
+ *
45
+ * Matched on label boundaries, never as substrings, so
46
+ * `myresource.openai.azure.com.attacker.test` does not read as Azure.
47
+ */
48
+ const HOST_SUFFIXES: ReadonlyArray<readonly [string, ModelHostClass]> = [
49
+ // Azure exposes the same models under three host families: the classic
50
+ // Azure OpenAI host, the Foundry Models host, and the general Cognitive
51
+ // Services host. All three are Azure for our purposes.
52
+ ["openai.azure.com", "azure"],
53
+ ["services.ai.azure.com", "azure"],
54
+ ["cognitiveservices.azure.com", "azure"],
55
+ // API Management fronting an Azure OpenAI deployment, a common enterprise
56
+ // pattern. Reported as azure because that is what sits behind it.
57
+ ["azure-api.net", "azure"],
58
+
59
+ ["api.openai.com", "openai"],
60
+ ["openrouter.ai", "openrouter"],
61
+ ["api.anthropic.com", "anthropic"],
62
+ ["generativelanguage.googleapis.com", "google"],
63
+ // Vertex regional hosts are `<region>-aiplatform.googleapis.com`, which is
64
+ // not a suffix of a single label, so both spellings are listed.
65
+ ["aiplatform.googleapis.com", "vertex"],
66
+ ["api.groq.com", "groq"],
67
+ ["api.unify.ai", "unify"],
68
+ ["api.minimax.io", "minimax"],
69
+ ["api.minimaxi.com", "minimax"],
70
+ ];
71
+
72
+ /** Suffixes that mean the model runs on the same machine or private network. */
73
+ const LOCAL_SUFFIXES = ["localhost", "local", "internal", "docker.internal"];
74
+
75
+ /** Matches `<region>-aiplatform.googleapis.com` and the Bedrock regional hosts. */
76
+ const REGIONAL_HOST_PATTERNS: ReadonlyArray<readonly [RegExp, ModelHostClass]> =
77
+ [
78
+ [/^[a-z0-9-]+-aiplatform\.googleapis\.com$/, "vertex"],
79
+ [/^bedrock[a-z0-9.-]*\.[a-z0-9-]+\.amazonaws\.com$/, "bedrock"],
80
+ ];
81
+
82
+ /** True when `host` equals `suffix` or sits directly beneath it. */
83
+ function matchesSuffix(host: string, suffix: string): boolean {
84
+ return host === suffix || host.endsWith(`.${suffix}`);
85
+ }
86
+
87
+ /** True for loopback, unspecified, and RFC 1918 private addresses. */
88
+ function isPrivateAddress(host: string): boolean {
89
+ if (host === "127.0.0.1" || host === "0.0.0.0" || host === "::1") {
90
+ return true;
91
+ }
92
+ // 127.0.0.0/8 is entirely loopback.
93
+ if (/^127\./.test(host)) return true;
94
+ // 10.0.0.0/8 and 192.168.0.0/16.
95
+ if (/^10\./.test(host) || /^192\.168\./.test(host)) return true;
96
+ // 172.16.0.0/12 is 172.16 through 172.31 only. 172.32 is public, so the
97
+ // second octet is range-checked rather than prefix-matched.
98
+ const privateClassB = /^172\.(\d{1,3})\./.exec(host);
99
+ if (privateClassB) {
100
+ const secondOctet = Number(privateClassB[1]);
101
+ return secondOctet >= 16 && secondOctet <= 31;
102
+ }
103
+ return false;
104
+ }
105
+
106
+ /**
107
+ * Reduce a model endpoint to one of {@link MODEL_HOST_CLASSES}.
108
+ *
109
+ * @param baseUrl The endpoint the provider was configured with. Absent when
110
+ * the provider was left on its own default, which is why `providerDefault`
111
+ * exists: "no base URL, provider is openai" means api.openai.com.
112
+ * @param providerDefault What to report when no base URL was supplied. Callers
113
+ * that genuinely cannot observe a host — a pre-built LanguageModel handed in
114
+ * by the developer — leave this alone and get `unknown`.
115
+ */
116
+ export function classifyModelHost(
117
+ baseUrl?: string | null,
118
+ providerDefault: ModelHostClass = "unknown",
119
+ ): ModelHostClass {
120
+ const trimmed = baseUrl?.trim();
121
+ if (!trimmed) {
122
+ return providerDefault;
123
+ }
124
+
125
+ let host: string;
126
+ try {
127
+ // `hostname` drops the port, the path, the query and any credentials, so
128
+ // nothing but the host can survive into the return value below.
129
+ host = new URL(trimmed).hostname.toLowerCase();
130
+ } catch {
131
+ // We were given something and could not read it. That is a different fact
132
+ // from having been given nothing, so it does not fall back to the default.
133
+ return "other";
134
+ }
135
+
136
+ // IPv6 hostnames arrive bracketed from the URL parser.
137
+ host = host.replace(/^\[|\]$/g, "");
138
+
139
+ if (!host) return "other";
140
+
141
+ if (isPrivateAddress(host)) return "local";
142
+ for (const suffix of LOCAL_SUFFIXES) {
143
+ if (matchesSuffix(host, suffix)) return "local";
144
+ }
145
+
146
+ for (const [suffix, hostClass] of HOST_SUFFIXES) {
147
+ if (matchesSuffix(host, suffix)) return hostClass;
148
+ }
149
+
150
+ for (const [pattern, hostClass] of REGIONAL_HOST_PATTERNS) {
151
+ if (pattern.test(host)) return hostClass;
152
+ }
153
+
154
+ return "other";
155
+ }
@@ -1,17 +1,25 @@
1
1
  // Per-event telemetry metadata shared by both TelemetryClients.
2
2
  //
3
3
  // Two clients emit `oss.runtime.*`: the v1 client in this package and the
4
- // v2 client in @copilotkit/runtime. Both gate anonymous events at
5
- // `sampleRate` and let identified callers (a license token that yielded a
6
- // telemetry_id) through at 100%, so real volume can only be recovered
7
- // downstream as sum(sampleWeight) — a flat multiplier is wrong whenever
8
- // the identified share moves, which it does as Intelligence keys roll out.
4
+ // v2 client in @copilotkit/runtime. Neither samples the lambda sink any
5
+ // more — that sink is ours, so anonymous volume costs us nothing per
6
+ // event and full fidelity beats an extrapolation. One wire is still
7
+ // gated: the Segment copy the v1 client sends, which is billed per event.
8
+ //
9
+ // Every event therefore still carries a sampling block, because the two
10
+ // populations remain distinguishable and a Segment-sourced count still
11
+ // needs sum(sampleWeight) to mean anything. On the lambda wire the block
12
+ // reads sampleRate=1/sampleWeight=1, which is the honest answer rather
13
+ // than a placeholder.
9
14
  //
10
15
  // The two clients computed this independently and drifted: v2 sampled but
11
16
  // stamped nothing, leaving ~24% of runtime volume unweightable from the
12
17
  // data alone (OSS-1017). Computing it here is what stops them drifting
13
18
  // again.
14
19
 
20
+ /** The rate a sink that takes every event is "gated" at. */
21
+ export const UNSAMPLED_RATE = 1;
22
+
15
23
  /** Identifies which client emitted an event. */
16
24
  export const TELEMETRY_EMITTER_V1 = "v1-shared";
17
25
  export const TELEMETRY_EMITTER_V2 = "v2-runtime";
@@ -20,6 +28,26 @@ export type TelemetryEmitter =
20
28
  | typeof TELEMETRY_EMITTER_V1
21
29
  | typeof TELEMETRY_EMITTER_V2;
22
30
 
31
+ /**
32
+ * Which public API surface the developer built against.
33
+ *
34
+ * Separate from {@link TelemetryEmitter} on purpose. The emitter names a
35
+ * client library, and there are six of them across four languages; the
36
+ * surface answers the question anyone actually asks of this data, which
37
+ * is how much traffic still arrives through the deprecated v1 API. The
38
+ * native runtimes (Go, Python, Ruby, .NET) expose v2 only, so they report
39
+ * `v2` without having a v1 notion of their own.
40
+ */
41
+ // `as const` so the values keep their literal types when they land in a
42
+ // mutable object property, which is how the v1 shim passes the surface
43
+ // into the V2 runtime's options.
44
+ export const TELEMETRY_SURFACE_V1 = "v1" as const;
45
+ export const TELEMETRY_SURFACE_V2 = "v2" as const;
46
+
47
+ export type TelemetrySurface =
48
+ | typeof TELEMETRY_SURFACE_V1
49
+ | typeof TELEMETRY_SURFACE_V2;
50
+
23
51
  /**
24
52
  * Which wire an event copy travelled on. The v1 client sends each capture
25
53
  * to both, so this is what tells the two copies apart downstream
@@ -28,7 +56,7 @@ export type TelemetryEmitter =
28
56
  export type TelemetryTransport = "segment" | "lambda";
29
57
 
30
58
  export interface SamplingMeta {
31
- /** The rate this event was actually gated at: 1 when identified. */
59
+ /** The rate this event was actually gated at: 1 when unsampled or identified. */
32
60
  sampleRate: number;
33
61
  sampleRateAdjustmentFactor: number;
34
62
  /** Multiply by this to extrapolate the population the event stands for. */
@@ -43,6 +71,11 @@ export interface SamplingMeta {
43
71
  * `telemetryId` is the caller's parsed license telemetry_id, or null when
44
72
  * anonymous — it decides the branch, and is deliberately not returned:
45
73
  * only the non-PII shape below travels on the event.
74
+ *
75
+ * `sampleRate` is the rate of the wire this copy travels on, so an
76
+ * unsampled sink passes {@link UNSAMPLED_RATE}. Call it once per wire:
77
+ * the v1 client sends one capture to two wires gated differently, and a
78
+ * shared block would misweight one of them.
46
79
  */
47
80
  export function computeSamplingMeta({
48
81
  telemetryId,
@@ -58,16 +58,21 @@ function setupSharedCapture(randomValue: number, telemetryId: string = " \t ") {
58
58
  };
59
59
  }
60
60
 
61
- test("Shared whitespace-only legacy claim stays subject to sampling", async () => {
61
+ test("Shared whitespace-only legacy claim stays subject to the Segment gate", async () => {
62
62
  const sinkSpy = vi.spyOn(lambdaClient, "send").mockResolvedValue(undefined);
63
63
  const { client, randomSpy, teardown } = setupSharedCapture(0.99);
64
64
 
65
65
  try {
66
66
  await client.capture("oss.runtime.instance_created", instanceCreatedEvent);
67
67
 
68
+ // An unusable claim must not buy identified status. The gate it now
69
+ // fails is Segment's alone: the lambda copy goes either way, and what
70
+ // proves the claim was rejected is that it goes marked anonymous.
68
71
  expect(randomSpy).toHaveBeenCalledTimes(1);
69
- expect(sinkSpy).not.toHaveBeenCalled();
70
72
  expect(segmentTrackMock).not.toHaveBeenCalled();
73
+ expect(sinkSpy.mock.calls[0][0].globalProperties).toMatchObject({
74
+ telemetry_identified: false,
75
+ });
71
76
  } finally {
72
77
  sinkSpy.mockRestore();
73
78
  teardown();
@@ -96,7 +101,7 @@ test("Shared sampled whitespace-only legacy claim sends no identity header", asy
96
101
  });
97
102
 
98
103
  test.each(["bad\nid", "bad\u0000id", "tenant-🚀"])(
99
- "Shared header-invalid legacy claim %j stays subject to sampling",
104
+ "Shared header-invalid legacy claim %j stays subject to the Segment gate",
100
105
  async (invalidTelemetryId) => {
101
106
  const sinkSpy = vi.spyOn(lambdaClient, "send").mockResolvedValue(undefined);
102
107
  const { client, randomSpy, teardown } = setupSharedCapture(
@@ -111,8 +116,10 @@ test.each(["bad\nid", "bad\u0000id", "tenant-🚀"])(
111
116
  );
112
117
 
113
118
  expect(randomSpy).toHaveBeenCalledTimes(1);
114
- expect(sinkSpy).not.toHaveBeenCalled();
115
119
  expect(segmentTrackMock).not.toHaveBeenCalled();
120
+ expect(sinkSpy.mock.calls[0][0].globalProperties).toMatchObject({
121
+ telemetry_identified: false,
122
+ });
116
123
  } finally {
117
124
  sinkSpy.mockRestore();
118
125
  teardown();