@saguiitay/eventhouse-ai-observability 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +125 -18
- package/dist/adapters/openai.d.ts +25 -0
- package/dist/adapters/vercel.d.ts +25 -0
- package/dist/{chunk-XQFUPGYU.js → chunk-XGCHW72B.js} +11 -55
- package/dist/{chunk-XQFUPGYU.js.map → chunk-XGCHW72B.js.map} +1 -1
- package/dist/copilot.js +5 -3
- package/dist/copilot.js.map +1 -1
- package/dist/index.d.ts +1 -6
- package/dist/index.js +3 -11
- package/dist/openai.d.ts +12 -0
- package/dist/openai.js +408 -0
- package/dist/openai.js.map +1 -0
- package/dist/validate.d.ts +2 -0
- package/dist/validate.js +237 -0
- package/dist/validate.js.map +1 -0
- package/dist/validation/fabric.d.ts +29 -0
- package/dist/vercel.d.ts +15 -0
- package/dist/vercel.js +369 -0
- package/dist/vercel.js.map +1 -0
- package/docs/architecture.md +19 -3
- package/docs/copilot-adapter.md +1 -1
- package/docs/extending.md +45 -1
- package/docs/fabric-setup.md +24 -0
- package/docs/implementation-report.md +62 -18
- package/docs/openai-adapter.md +137 -0
- package/docs/quickstart.md +75 -6
- package/docs/research-eventstream.md +8 -6
- package/docs/research.md +1 -1
- package/docs/security.md +1 -1
- package/docs/telemetry-schema.md +18 -0
- package/docs/vercel-adapter.md +201 -0
- package/examples/README.md +75 -6
- package/examples/copilot-basic/package.json +1 -1
- package/examples/openai-basic/env.example +8 -0
- package/examples/openai-basic/index.ts +46 -0
- package/examples/shared.ts +3 -62
- package/examples/telemetry.ts +124 -0
- package/examples/vercel-basic/env.example +8 -0
- package/examples/vercel-basic/index.ts +60 -0
- package/fabric-setup.zip +0 -0
- package/package.json +39 -5
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# Vercel AI SDK adapter
|
|
2
|
+
|
|
3
|
+
`observeVercelAI` supplies a per-call `telemetry` configuration for
|
|
4
|
+
Vercel AI SDK **7.0.165**, using its native **`@ai-sdk/otel@1.0.165`** integration.
|
|
5
|
+
Supported APIs are `generateText` and
|
|
6
|
+
`streamText`, including multiple model steps and local tool execution.
|
|
7
|
+
The SDK is an optional peer; importing the observability package or its
|
|
8
|
+
`/vercel` entry point does not load `ai` at runtime.
|
|
9
|
+
|
|
10
|
+
Install a compatible SDK in the application:
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
pnpm add ai@7.0.165 @ai-sdk/otel@1.0.165 zod@4.6.7
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
SDK 7 moved tracer injection from `experimental_telemetry.tracer` into the
|
|
17
|
+
`OpenTelemetry` integration constructor. This adapter uses the replacement
|
|
18
|
+
API; no global provider registration is required. SDK 6's telemetry settings
|
|
19
|
+
are not supported by this version of the adapter.
|
|
20
|
+
|
|
21
|
+
## Per-call setup
|
|
22
|
+
|
|
23
|
+
```ts
|
|
24
|
+
import { generateText, streamText } from "ai";
|
|
25
|
+
import { OpenTelemetry } from "@ai-sdk/otel";
|
|
26
|
+
import { observeVercelAI } from "@saguiitay/eventhouse-ai-observability/vercel";
|
|
27
|
+
|
|
28
|
+
// `model` is an existing AI SDK language model supplied by your application.
|
|
29
|
+
const observer = observeVercelAI({
|
|
30
|
+
connectionString: process.env.FABRIC_EVENTSTREAM_CONNECTION_STRING!,
|
|
31
|
+
serviceName: "my-ai-app",
|
|
32
|
+
integration: (tracer) => new OpenTelemetry({ tracer }),
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
try {
|
|
36
|
+
const generated = await generateText({
|
|
37
|
+
model,
|
|
38
|
+
prompt: "Summarize the report",
|
|
39
|
+
telemetry: observer.telemetry,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const streamed = streamText({
|
|
43
|
+
model,
|
|
44
|
+
prompt: generated.text,
|
|
45
|
+
telemetry: observer.telemetry,
|
|
46
|
+
});
|
|
47
|
+
// Consume the stream before flushing or shutting down.
|
|
48
|
+
await streamed.consumeStream();
|
|
49
|
+
await observer.flush();
|
|
50
|
+
console.log(observer.getDiagnostics());
|
|
51
|
+
} finally {
|
|
52
|
+
await observer.shutdown();
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Pass `observer.telemetry` on **every SDK call**. The required `integration`
|
|
57
|
+
factory receives the adapter's private tracer and constructs the SDK's native
|
|
58
|
+
`OpenTelemetry` integration. The library does not import either `ai` or
|
|
59
|
+
`@ai-sdk/otel` at runtime.
|
|
60
|
+
|
|
61
|
+
This is not a global provider
|
|
62
|
+
registration or a wrapper around `generateText`/`streamText`. Calls without this
|
|
63
|
+
configuration are not observed by this adapter. Do not combine it with another
|
|
64
|
+
adapter that instruments the same provider request, or token totals may be
|
|
65
|
+
counted twice.
|
|
66
|
+
|
|
67
|
+
The normal engine transport, sampling, queue limits, resource attributes,
|
|
68
|
+
redaction and diagnostics options are accepted when the observer creates its
|
|
69
|
+
own engine. Fabric Eventstream configuration works the same way as at the root
|
|
70
|
+
entry point.
|
|
71
|
+
|
|
72
|
+
## Native spans and analytics
|
|
73
|
+
|
|
74
|
+
The SDK's native integration creates root, step, model and tool spans.
|
|
75
|
+
A tracer bridge routes them
|
|
76
|
+
through the engine's private provider and local asynchronous context. It does
|
|
77
|
+
not register a global OTel tracer provider or context manager. Application
|
|
78
|
+
parents, explicit native parents and unsampled parent contexts are preserved.
|
|
79
|
+
|
|
80
|
+
The bridge retains `gen_ai.operation.name` (`invoke_agent`, `agent_step`, `chat`,
|
|
81
|
+
`execute_tool`),
|
|
82
|
+
provider/model and tool attributes. `vercel.span.type` distinguishes `turn`,
|
|
83
|
+
`step`, `model_request` and `tool`. Span names are fixed to those operation
|
|
84
|
+
names rather than carrying arbitrary function identifiers or content.
|
|
85
|
+
Model usage remains `gen_ai.usage.input_tokens` and
|
|
86
|
+
`gen_ai.usage.output_tokens`; native root aggregate usage is discarded. Each
|
|
87
|
+
provider operation contributes usage once, even in a multistep tool loop.
|
|
88
|
+
SDK model switching retains each request's own model identity. The public
|
|
89
|
+
`executeLanguageModelCall` and `executeTool` integration hooks are bound to the
|
|
90
|
+
engine's local context, so telemetry created inside provider calls or nested
|
|
91
|
+
tool work inherits the native model/tool parent without a global context
|
|
92
|
+
manager.
|
|
93
|
+
|
|
94
|
+
Some error-only streams omit a native model-call completion event. The bridge
|
|
95
|
+
ends that existing model span with `observability.incomplete: true` when its
|
|
96
|
+
step finishes, retains the failure classification, and leaves unknown usage
|
|
97
|
+
unset. It does not copy tokens from the parent.
|
|
98
|
+
|
|
99
|
+
The SDK's outer text span covers the complete SDK call, not a separate adapter
|
|
100
|
+
execution. Retries retain the native SDK operation structure. Embeddings,
|
|
101
|
+
speech, transcription, legacy object-generation APIs and durable workflows
|
|
102
|
+
are outside this adapter's supported contract.
|
|
103
|
+
|
|
104
|
+
## Privacy
|
|
105
|
+
|
|
106
|
+
Inputs and outputs are not recorded by default. The returned configuration has
|
|
107
|
+
both `recordInputs: false` and `recordOutputs: false`, even if the engine permits
|
|
108
|
+
content capture.
|
|
109
|
+
|
|
110
|
+
SDK 7 filters content before telemetry integrations receive events, but native
|
|
111
|
+
integration options and enrichment callbacks can still add request headers,
|
|
112
|
+
arbitrary metadata, provider metadata or exception details.
|
|
113
|
+
The tracer bridge uses an attribute allowlist before
|
|
114
|
+
the engine receives mutations. It drops those fields, span/link attributes
|
|
115
|
+
outside the allowlist, custom span names, native events and status messages.
|
|
116
|
+
Failures keep an error status and a fixed `provider_error` classification; raw
|
|
117
|
+
errors still reach the application's normal SDK error handling unchanged.
|
|
118
|
+
This filtering also applies to settings supplied by spreading the returned
|
|
119
|
+
configuration.
|
|
120
|
+
|
|
121
|
+
To record text or tool content, opt in on both the engine and the relevant
|
|
122
|
+
direction:
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
const observer = observeVercelAI({
|
|
126
|
+
exporter,
|
|
127
|
+
captureContent: true,
|
|
128
|
+
recordInputs: true,
|
|
129
|
+
recordOutputs: true,
|
|
130
|
+
integration: (tracer) => new OpenTelemetry({ tracer }),
|
|
131
|
+
});
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Captured content is mapped to `gen_ai.input.messages`,
|
|
135
|
+
`gen_ai.output.messages`, and standard tool content attributes. Strings are
|
|
136
|
+
bounded to 4,096 characters and sanitized by the engine's secret redactor and
|
|
137
|
+
your optional `redact` callback. Provider metadata, headers and raw errors stay
|
|
138
|
+
excluded even with capture enabled. Secret redaction is not a general PII
|
|
139
|
+
detector: use your own redactor for application-specific sensitive content.
|
|
140
|
+
Model identifiers and tool names/IDs are retained as bounded operational
|
|
141
|
+
metadata; do not place user content or secrets in those identifiers.
|
|
142
|
+
|
|
143
|
+
The adapter cannot control other telemetry integrations, exporters or
|
|
144
|
+
application callbacks. Configure their privacy separately. Native error
|
|
145
|
+
objects are replaced with a fixed classification before this integration sees
|
|
146
|
+
them, including errors used as metric dimensions. The native integration may
|
|
147
|
+
also emit operational latency metrics to an application's existing global
|
|
148
|
+
meter provider; those metrics are not exported or counted by this tracing
|
|
149
|
+
engine.
|
|
150
|
+
|
|
151
|
+
## Shared engines and shutdown
|
|
152
|
+
|
|
153
|
+
```ts
|
|
154
|
+
import { createObservability } from "@saguiitay/eventhouse-ai-observability";
|
|
155
|
+
|
|
156
|
+
const engine = createObservability({ exporter });
|
|
157
|
+
const observer = observeVercelAI({
|
|
158
|
+
engine,
|
|
159
|
+
integration: (tracer) => new OpenTelemetry({ tracer }),
|
|
160
|
+
});
|
|
161
|
+
try {
|
|
162
|
+
await generateText({
|
|
163
|
+
model,
|
|
164
|
+
prompt: "Hello",
|
|
165
|
+
telemetry: observer.telemetry,
|
|
166
|
+
});
|
|
167
|
+
} finally {
|
|
168
|
+
await observer.shutdown(); // Ends this adapter's live spans and flushes.
|
|
169
|
+
await engine.shutdown(); // The application closes its shared engine once.
|
|
170
|
+
}
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
An externally supplied engine remains application-owned by default. The
|
|
174
|
+
integration instance belongs to this attachment; return a fresh
|
|
175
|
+
`OpenTelemetry` instance from each factory rather than sharing one between
|
|
176
|
+
observers.
|
|
177
|
+
`shutdownEngine: true` explicitly transfers its shutdown ownership to the
|
|
178
|
+
adapter. For engines created by `observeVercelAI`, shutdown ownership is always
|
|
179
|
+
the adapter's; `shutdownEngine: false` requires an external engine.
|
|
180
|
+
|
|
181
|
+
`flush()` exports ended spans without ending ongoing generation.
|
|
182
|
+
`shutdown()` is idempotent, disables this configuration and ends outstanding
|
|
183
|
+
adapter spans with `observability.incomplete: true`. It never cancels a model
|
|
184
|
+
request, closes a provider client, or consumes an unfinished stream. Wait for
|
|
185
|
+
your application work before shutdown if complete spans are required.
|
|
186
|
+
|
|
187
|
+
## Verification and references
|
|
188
|
+
|
|
189
|
+
`tests/vercel.test.ts` runs the real SDK with `MockLanguageModelV3` from the
|
|
190
|
+
SDK's public `ai/test` entry point and the real native `OpenTelemetry`
|
|
191
|
+
integration. No provider credentials or external
|
|
192
|
+
inference calls are used.
|
|
193
|
+
The verified versions are `ai@7.0.165`, `@ai-sdk/otel@1.0.165` and
|
|
194
|
+
`zod@4.6.7`.
|
|
195
|
+
|
|
196
|
+
The [official telemetry guide](https://ai-sdk.dev/docs/ai-sdk-core/telemetry)
|
|
197
|
+
documents per-call integrations and custom tracer injection. The adapter also
|
|
198
|
+
checks the official
|
|
199
|
+
[native integration](https://github.com/vercel/ai/blob/ai%407.0.165/packages/otel/src/open-telemetry.ts),
|
|
200
|
+
[native span/error recording](https://github.com/vercel/ai/blob/ai%407.0.165/packages/otel/src/record-span.ts),
|
|
201
|
+
and [supplemental attribute options](https://github.com/vercel/ai/blob/ai%407.0.165/packages/otel/src/supplemental-attributes.ts).
|
package/examples/README.md
CHANGED
|
@@ -1,23 +1,45 @@
|
|
|
1
|
-
#
|
|
1
|
+
# SDK examples
|
|
2
|
+
|
|
3
|
+
Every adapter has a runnable sample:
|
|
4
|
+
|
|
5
|
+
| Adapter | Sample command | SDK authentication |
|
|
6
|
+
| --------- | ---------------------------------------------------------------------- | ----------------------------------------------- |
|
|
7
|
+
| Copilot | `pnpm example:basic`, `pnpm example:tools`, `pnpm example:multi-agent` | Authenticated Copilot CLI |
|
|
8
|
+
| OpenAI | `pnpm example:openai` | `OPENAI_API_KEY` |
|
|
9
|
+
| Vercel AI | `pnpm example:vercel` | `OPENAI_API_KEY` for the direct OpenAI provider |
|
|
10
|
+
|
|
11
|
+
Examples keep telemetry metadata-only, default to a local console exporter,
|
|
12
|
+
and flush/shut down telemetry before exiting. Console telemetry means no spans
|
|
13
|
+
were exported remotely; real SDK model calls still use the network and consume
|
|
14
|
+
quota or incur charges.
|
|
15
|
+
|
|
16
|
+
Fabric is the package's built-in transport. For OTLP, the checkout helper
|
|
17
|
+
explicitly constructs OpenTelemetry's official exporter from a development
|
|
18
|
+
dependency and passes it through `exporter`. The library itself does not read
|
|
19
|
+
OTLP endpoint variables or include that exporter as a runtime dependency.
|
|
20
|
+
|
|
21
|
+
## Copilot SDK
|
|
2
22
|
|
|
3
23
|
These are real Copilot SDK 1.0.31 applications, not simulated event emitters.
|
|
4
24
|
Use Node.js 22.15+ (validated with 24.22.3), the root package dependencies, and an
|
|
5
25
|
authenticated Copilot CLI. Model calls consume your Copilot quota.
|
|
6
26
|
|
|
7
27
|
The **basic** example consumes published
|
|
8
|
-
`@saguiitay/eventhouse-ai-observability@0.
|
|
28
|
+
`@saguiitay/eventhouse-ai-observability@0.2.0` once that release is published.
|
|
9
29
|
Version 0.1.0 was successfully installed and used against a live
|
|
10
30
|
Fabric deployment on 2026-11-12. The example has its own
|
|
11
31
|
dependency manifest and disables the root TypeScript source aliases, so it
|
|
12
32
|
cannot silently use the checkout's runtime. The tools and multi-agent examples
|
|
13
33
|
continue to use the locally built package.
|
|
14
34
|
|
|
15
|
-
Install the basic example's dependency once. To use npmjs
|
|
16
|
-
|
|
35
|
+
Install the basic example's dependency once. To use npmjs or your organization's
|
|
36
|
+
approved mirror, override the example's existing GitHub scope mapping without
|
|
37
|
+
committing organization-specific registry configuration:
|
|
17
38
|
|
|
18
39
|
```powershell
|
|
19
40
|
Push-Location .\examples\copilot-basic
|
|
20
|
-
|
|
41
|
+
$registry = Read-Host "Your approved npm registry URL"
|
|
42
|
+
npm install --package-lock=false "--@saguiitay:registry=$registry"
|
|
21
43
|
Pop-Location
|
|
22
44
|
```
|
|
23
45
|
|
|
@@ -25,7 +47,7 @@ Mirror availability can lag publication. Alternatively, after the pinned version
|
|
|
25
47
|
has also been published to GitHub Packages, use the existing scope mapping.
|
|
26
48
|
GitHub Packages requires a classic
|
|
27
49
|
personal access token with `read:packages`, even for public
|
|
28
|
-
packages.
|
|
50
|
+
packages. Other dependencies continue to use your configured, approved registry:
|
|
29
51
|
|
|
30
52
|
```powershell
|
|
31
53
|
Push-Location .\examples\copilot-basic
|
|
@@ -93,3 +115,50 @@ timeout, throttling, and size problems without printing endpoints or keys.
|
|
|
93
115
|
|
|
94
116
|
If a model cannot delegate with the runtime's `task` tool, choose another
|
|
95
117
|
available model with `COPILOT_MODEL`; do not widen the permission policy.
|
|
118
|
+
|
|
119
|
+
## OpenAI and Vercel AI SDKs
|
|
120
|
+
|
|
121
|
+
Both examples use the locally built package and the root development
|
|
122
|
+
dependencies. From the checkout:
|
|
123
|
+
|
|
124
|
+
```powershell
|
|
125
|
+
pnpm install --frozen-lockfile
|
|
126
|
+
pnpm build
|
|
127
|
+
foreach ($example in 'openai-basic', 'vercel-basic') {
|
|
128
|
+
if (-not (Test-Path "examples\$example\.env")) {
|
|
129
|
+
Copy-Item "examples\$example\env.example" "examples\$example\.env"
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
pnpm example:openai
|
|
133
|
+
pnpm example:vercel
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Set `OPENAI_API_KEY` securely in each example's ignored `.env` or the process
|
|
137
|
+
environment. `OPENAI_MODEL` defaults to `gpt-4.1-mini`; choose a model available
|
|
138
|
+
to your account. No key is supplied by the templates. A root `.env` is not loaded.
|
|
139
|
+
|
|
140
|
+
The OpenAI sample performs a nonstreaming Responses call and a streaming Chat
|
|
141
|
+
Completions call. It consumes the stream and requests the provider's usage
|
|
142
|
+
chunk; it does not estimate tokens. The original SDK client remains unchanged.
|
|
143
|
+
|
|
144
|
+
The Vercel sample uses `@ai-sdk/openai` directly, not the AI Gateway. It
|
|
145
|
+
demonstrates generation with a bounded, side-effect-free arithmetic tool and
|
|
146
|
+
a separate streaming call. Model-driven tool selection is not guaranteed;
|
|
147
|
+
the tool loop is limited to two steps. Native telemetry is passed on every
|
|
148
|
+
call through `@ai-sdk/otel` with a private tracer.
|
|
149
|
+
|
|
150
|
+
The OpenAI sample makes two paid model calls. Vercel makes up to three,
|
|
151
|
+
depending on tool selection. Each call caps output at 256 tokens.
|
|
152
|
+
Samples print the model's response for
|
|
153
|
+
demonstration; this application output is separate from metadata-only
|
|
154
|
+
telemetry. They do not print API keys or raw errors.
|
|
155
|
+
|
|
156
|
+
For remote telemetry, optionally configure
|
|
157
|
+
`FABRIC_EVENTSTREAM_CONNECTION_STRING` and, if needed,
|
|
158
|
+
`FABRIC_EVENTSTREAM_ENTITY_NAME`, or an OTLP endpoint such as
|
|
159
|
+
`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. Export failures/drops cause the sample to
|
|
160
|
+
fail after telemetry cleanup. Producer acknowledgements do not establish
|
|
161
|
+
Eventhouse ingestion.
|
|
162
|
+
|
|
163
|
+
See [OpenAI adapter](../docs/openai-adapter.md) and
|
|
164
|
+
[Vercel adapter](../docs/vercel-adapter.md) for supported APIs and lifecycle.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Copy to .env and supply your own OpenAI API key. Both calls incur API usage.
|
|
2
|
+
OPENAI_API_KEY=
|
|
3
|
+
OPENAI_MODEL=gpt-4.1-mini
|
|
4
|
+
# Telemetry is metadata-only and goes to the console by default.
|
|
5
|
+
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
|
|
6
|
+
# FABRIC_EVENTSTREAM_CONNECTION_STRING=<Custom endpoint SOURCE producer string>
|
|
7
|
+
# FABRIC_EVENTSTREAM_ENTITY_NAME=<only if EntityPath is absent>
|
|
8
|
+
# Keep all service credentials in the process environment or an untracked .env.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import OpenAI from "openai";
|
|
2
|
+
import { observeOpenAI } from "@saguiitay/eventhouse-ai-observability/openai";
|
|
3
|
+
import { runTelemetryExample } from "../telemetry.js";
|
|
4
|
+
|
|
5
|
+
if (!process.env.OPENAI_API_KEY?.trim())
|
|
6
|
+
throw new Error("OPENAI_API_KEY is required to run the OpenAI example.");
|
|
7
|
+
|
|
8
|
+
try {
|
|
9
|
+
const client = new OpenAI();
|
|
10
|
+
const model = process.env.OPENAI_MODEL ?? "gpt-4.1-mini";
|
|
11
|
+
|
|
12
|
+
await runTelemetryExample(
|
|
13
|
+
"openai-basic",
|
|
14
|
+
(options) => observeOpenAI(client, options),
|
|
15
|
+
async (observer) => {
|
|
16
|
+
const response = await observer.client.responses.create({
|
|
17
|
+
model,
|
|
18
|
+
max_output_tokens: 256,
|
|
19
|
+
input:
|
|
20
|
+
"Explain in one sentence why application observability is useful.",
|
|
21
|
+
});
|
|
22
|
+
console.log("Responses:", response.output_text);
|
|
23
|
+
|
|
24
|
+
const stream = await observer.client.chat.completions.create({
|
|
25
|
+
model,
|
|
26
|
+
max_completion_tokens: 256,
|
|
27
|
+
messages: [
|
|
28
|
+
{
|
|
29
|
+
role: "user",
|
|
30
|
+
content: "Explain in one sentence what a distributed trace shows.",
|
|
31
|
+
},
|
|
32
|
+
],
|
|
33
|
+
stream: true,
|
|
34
|
+
stream_options: { include_usage: true },
|
|
35
|
+
});
|
|
36
|
+
let text = "";
|
|
37
|
+
for await (const chunk of stream)
|
|
38
|
+
text += chunk.choices[0]?.delta.content ?? "";
|
|
39
|
+
console.log("Chat Completions:", text);
|
|
40
|
+
},
|
|
41
|
+
);
|
|
42
|
+
} catch {
|
|
43
|
+
throw new Error(
|
|
44
|
+
"OpenAI example failed; check provider configuration and telemetry diagnostics.",
|
|
45
|
+
);
|
|
46
|
+
}
|
package/examples/shared.ts
CHANGED
|
@@ -4,51 +4,14 @@ import {
|
|
|
4
4
|
type PermissionHandler,
|
|
5
5
|
type SessionConfig,
|
|
6
6
|
} from "@github/copilot-sdk";
|
|
7
|
-
import type {
|
|
8
|
-
EventhouseObservabilityOptions,
|
|
9
|
-
Diagnostics,
|
|
10
|
-
SpanExporter,
|
|
11
|
-
} from "@saguiitay/eventhouse-ai-observability";
|
|
7
|
+
import type { Diagnostics } from "@saguiitay/eventhouse-ai-observability";
|
|
12
8
|
import type { observeCopilot } from "@saguiitay/eventhouse-ai-observability/copilot";
|
|
9
|
+
import { reportTelemetry, telemetryOptions } from "./telemetry.js";
|
|
13
10
|
|
|
14
11
|
export const denyPermissions: PermissionHandler = () => ({
|
|
15
12
|
kind: "denied-no-approval-rule-and-could-not-request-from-user",
|
|
16
13
|
});
|
|
17
14
|
|
|
18
|
-
const localExporter: SpanExporter = {
|
|
19
|
-
export(spans, callback) {
|
|
20
|
-
for (const span of spans) {
|
|
21
|
-
console.log(
|
|
22
|
-
JSON.stringify({
|
|
23
|
-
name: span.name,
|
|
24
|
-
traceId: span.spanContext().traceId,
|
|
25
|
-
spanId: span.spanContext().spanId,
|
|
26
|
-
parentSpanId: span.parentSpanContext?.spanId,
|
|
27
|
-
durationMs: span.duration[0] * 1000 + span.duration[1] / 1e6,
|
|
28
|
-
attributes: span.attributes,
|
|
29
|
-
}),
|
|
30
|
-
);
|
|
31
|
-
}
|
|
32
|
-
callback({ code: 0 });
|
|
33
|
-
},
|
|
34
|
-
async shutdown() {},
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
function telemetryOptions(serviceName: string) {
|
|
38
|
-
const mode = process.env.FABRIC_EVENTSTREAM_CONNECTION_STRING
|
|
39
|
-
? "eventstream"
|
|
40
|
-
: process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||
|
|
41
|
-
process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
42
|
-
? "otlp"
|
|
43
|
-
: "console";
|
|
44
|
-
const options: EventhouseObservabilityOptions = {
|
|
45
|
-
serviceName,
|
|
46
|
-
captureContent: false,
|
|
47
|
-
...(mode === "console" ? { exporter: localExporter } : { transport: mode }),
|
|
48
|
-
};
|
|
49
|
-
return { mode, options };
|
|
50
|
-
}
|
|
51
|
-
|
|
52
15
|
export function exampleIterations(
|
|
53
16
|
raw = process.env.COPILOT_EXAMPLE_ITERATIONS,
|
|
54
17
|
): number {
|
|
@@ -61,28 +24,6 @@ export function exampleIterations(
|
|
|
61
24
|
return count;
|
|
62
25
|
}
|
|
63
26
|
|
|
64
|
-
function reportTelemetry(
|
|
65
|
-
mode: string,
|
|
66
|
-
diagnostics: Readonly<Diagnostics>,
|
|
67
|
-
): void {
|
|
68
|
-
console.info(
|
|
69
|
-
`[telemetry] ${mode}: generated=${diagnostics.eventsGenerated}, exported=${diagnostics.eventsExported}, dropped=${diagnostics.eventsDropped}, failures=${diagnostics.exportFailures}`,
|
|
70
|
-
);
|
|
71
|
-
if (diagnostics.lastError) {
|
|
72
|
-
const { category, message } = diagnostics.lastError;
|
|
73
|
-
console.warn(`[telemetry] ${category}: ${message}`);
|
|
74
|
-
}
|
|
75
|
-
if (mode === "eventstream") {
|
|
76
|
-
console.info(
|
|
77
|
-
"[telemetry] Exported means producer acknowledgement, not Eventhouse ingestion. Check Eventstream incoming messages and Eventhouse rows.",
|
|
78
|
-
);
|
|
79
|
-
} else if (mode === "console") {
|
|
80
|
-
console.info(
|
|
81
|
-
"[telemetry] Console only; no data was sent to Fabric or OTLP.",
|
|
82
|
-
);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
27
|
export async function runExample(
|
|
87
28
|
name: string,
|
|
88
29
|
configuration: SessionConfig,
|
|
@@ -97,7 +38,7 @@ export async function runExample(
|
|
|
97
38
|
let session: CopilotSession | undefined;
|
|
98
39
|
let observer: ReturnType<typeof observeCopilot> | undefined;
|
|
99
40
|
let diagnostics: Readonly<Diagnostics> | undefined;
|
|
100
|
-
const { mode, options } = telemetryOptions(name);
|
|
41
|
+
const { mode, options } = await telemetryOptions(name);
|
|
101
42
|
console.info(`[telemetry] transport=${mode}`);
|
|
102
43
|
try {
|
|
103
44
|
await client.start();
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Diagnostics,
|
|
3
|
+
EventhouseObservabilityOptions,
|
|
4
|
+
SpanExporter,
|
|
5
|
+
} from "@saguiitay/eventhouse-ai-observability";
|
|
6
|
+
|
|
7
|
+
const localExporter: SpanExporter = {
|
|
8
|
+
export(spans, callback) {
|
|
9
|
+
for (const span of spans) {
|
|
10
|
+
console.log(
|
|
11
|
+
JSON.stringify({
|
|
12
|
+
name: span.name,
|
|
13
|
+
traceId: span.spanContext().traceId,
|
|
14
|
+
spanId: span.spanContext().spanId,
|
|
15
|
+
parentSpanId: span.parentSpanContext?.spanId,
|
|
16
|
+
durationMs: span.duration[0] * 1000 + span.duration[1] / 1e6,
|
|
17
|
+
attributes: span.attributes,
|
|
18
|
+
}),
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
callback({ code: 0 });
|
|
22
|
+
},
|
|
23
|
+
async shutdown() {},
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export async function telemetryOptions(serviceName: string) {
|
|
27
|
+
const mode = process.env.FABRIC_EVENTSTREAM_CONNECTION_STRING
|
|
28
|
+
? "eventstream"
|
|
29
|
+
: process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||
|
|
30
|
+
process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
31
|
+
? "otlp"
|
|
32
|
+
: "console";
|
|
33
|
+
let exporter: SpanExporter | undefined;
|
|
34
|
+
if (mode === "console") exporter = localExporter;
|
|
35
|
+
if (mode === "otlp") {
|
|
36
|
+
const base = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
37
|
+
const endpoint =
|
|
38
|
+
process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ??
|
|
39
|
+
`${base?.replace(/\/$/, "")}/v1/traces`;
|
|
40
|
+
try {
|
|
41
|
+
const url = new URL(endpoint);
|
|
42
|
+
if (
|
|
43
|
+
!["http:", "https:"].includes(url.protocol) ||
|
|
44
|
+
url.username ||
|
|
45
|
+
url.password
|
|
46
|
+
)
|
|
47
|
+
throw new Error();
|
|
48
|
+
} catch {
|
|
49
|
+
throw new Error("Invalid example OTLP endpoint configuration.");
|
|
50
|
+
}
|
|
51
|
+
const { OTLPTraceExporter } =
|
|
52
|
+
await import("@opentelemetry/exporter-trace-otlp-http");
|
|
53
|
+
exporter = new OTLPTraceExporter({ url: endpoint });
|
|
54
|
+
}
|
|
55
|
+
const options: EventhouseObservabilityOptions = {
|
|
56
|
+
serviceName,
|
|
57
|
+
captureContent: false,
|
|
58
|
+
...(exporter ? { exporter } : {}),
|
|
59
|
+
};
|
|
60
|
+
return { mode, options };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function reportTelemetry(
|
|
64
|
+
mode: string,
|
|
65
|
+
diagnostics: Readonly<Diagnostics>,
|
|
66
|
+
): void {
|
|
67
|
+
console.info(
|
|
68
|
+
`[telemetry] ${mode}: generated=${diagnostics.eventsGenerated}, exported=${diagnostics.eventsExported}, dropped=${diagnostics.eventsDropped}, failures=${diagnostics.exportFailures}`,
|
|
69
|
+
);
|
|
70
|
+
if (diagnostics.lastError) {
|
|
71
|
+
const { category, message } = diagnostics.lastError;
|
|
72
|
+
console.warn(`[telemetry] ${category}: ${message}`);
|
|
73
|
+
}
|
|
74
|
+
if (mode === "eventstream") {
|
|
75
|
+
console.info(
|
|
76
|
+
"[telemetry] Exported means producer acknowledgement, not Eventhouse ingestion. Check Eventstream incoming messages and Eventhouse rows.",
|
|
77
|
+
);
|
|
78
|
+
} else if (mode === "console") {
|
|
79
|
+
console.info(
|
|
80
|
+
"[telemetry] Console only; no data was sent to Fabric or OTLP.",
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
interface ExampleObserver {
|
|
86
|
+
flush(): Promise<void>;
|
|
87
|
+
shutdown(): Promise<void>;
|
|
88
|
+
getDiagnostics(): Readonly<Diagnostics>;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function runTelemetryExample<O extends ExampleObserver>(
|
|
92
|
+
name: string,
|
|
93
|
+
observe: (options: EventhouseObservabilityOptions) => O,
|
|
94
|
+
run: (observer: O) => Promise<void>,
|
|
95
|
+
): Promise<void> {
|
|
96
|
+
const { mode, options } = await telemetryOptions(name);
|
|
97
|
+
let observer: O | undefined;
|
|
98
|
+
let diagnostics: Readonly<Diagnostics> | undefined;
|
|
99
|
+
console.info(`[telemetry] transport=${mode}`);
|
|
100
|
+
try {
|
|
101
|
+
observer = observe(options);
|
|
102
|
+
await run(observer);
|
|
103
|
+
} finally {
|
|
104
|
+
try {
|
|
105
|
+
await observer?.flush();
|
|
106
|
+
} finally {
|
|
107
|
+
try {
|
|
108
|
+
await observer?.shutdown();
|
|
109
|
+
} finally {
|
|
110
|
+
if (observer) {
|
|
111
|
+
diagnostics = observer.getDiagnostics();
|
|
112
|
+
reportTelemetry(mode, diagnostics);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (
|
|
118
|
+
diagnostics &&
|
|
119
|
+
(diagnostics.exportFailures > 0 || diagnostics.eventsDropped > 0)
|
|
120
|
+
)
|
|
121
|
+
throw new Error(
|
|
122
|
+
"Telemetry export failed or dropped spans; see the telemetry summary above.",
|
|
123
|
+
);
|
|
124
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Set OPENAI_API_KEY in your environment. Real calls consume OpenAI quota.
|
|
2
|
+
OPENAI_API_KEY=
|
|
3
|
+
OPENAI_MODEL=gpt-4.1-mini
|
|
4
|
+
# Uncomment ONE transport, or leave both unset for console-only telemetry.
|
|
5
|
+
# Content capture is disabled; credentials are never telemetry content.
|
|
6
|
+
# FABRIC_EVENTSTREAM_CONNECTION_STRING=<Custom endpoint SOURCE producer string>
|
|
7
|
+
# FABRIC_EVENTSTREAM_ENTITY_NAME=<only if EntityPath is absent>
|
|
8
|
+
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
2
|
+
import { OpenTelemetry } from "@ai-sdk/otel";
|
|
3
|
+
import { generateText, stepCountIs, streamText, tool } from "ai";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { observeVercelAI } from "@saguiitay/eventhouse-ai-observability/vercel";
|
|
6
|
+
import { runTelemetryExample } from "../telemetry.js";
|
|
7
|
+
|
|
8
|
+
const apiKey = process.env.OPENAI_API_KEY?.trim();
|
|
9
|
+
if (!apiKey) throw new Error("Set OPENAI_API_KEY before running this example");
|
|
10
|
+
|
|
11
|
+
const square = tool({
|
|
12
|
+
description:
|
|
13
|
+
"Square a finite number using pure local arithmetic, without external access.",
|
|
14
|
+
inputSchema: z.object({ value: z.number().finite().min(-1e6).max(1e6) }),
|
|
15
|
+
execute: async ({ value }) => {
|
|
16
|
+
if (!Number.isFinite(value) || Math.abs(value) > 1e6)
|
|
17
|
+
throw new Error("Number outside demo range");
|
|
18
|
+
return value * value;
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
try {
|
|
23
|
+
const openai = createOpenAI({ apiKey });
|
|
24
|
+
const model = openai(process.env.OPENAI_MODEL?.trim() || "gpt-4.1-mini");
|
|
25
|
+
await runTelemetryExample(
|
|
26
|
+
"vercel-basic",
|
|
27
|
+
(options) =>
|
|
28
|
+
observeVercelAI({
|
|
29
|
+
...options,
|
|
30
|
+
integration: (tracer) => new OpenTelemetry({ tracer }),
|
|
31
|
+
}),
|
|
32
|
+
async (observer) => {
|
|
33
|
+
const generated = await generateText({
|
|
34
|
+
model,
|
|
35
|
+
maxOutputTokens: 256,
|
|
36
|
+
prompt: "Use the square tool to square 12, then report the result.",
|
|
37
|
+
tools: { square },
|
|
38
|
+
stopWhen: stepCountIs(2),
|
|
39
|
+
telemetry: observer.telemetry,
|
|
40
|
+
});
|
|
41
|
+
console.log("Generated:", generated.text);
|
|
42
|
+
|
|
43
|
+
const streamed = streamText({
|
|
44
|
+
model,
|
|
45
|
+
maxOutputTokens: 256,
|
|
46
|
+
prompt: "Explain in one short sentence what squaring a number means.",
|
|
47
|
+
telemetry: observer.telemetry,
|
|
48
|
+
// Prevent the SDK's default error logger from exposing provider details.
|
|
49
|
+
onError: () => {},
|
|
50
|
+
});
|
|
51
|
+
for await (const chunk of streamed.textStream)
|
|
52
|
+
console.log("Stream chunk:", chunk);
|
|
53
|
+
console.log("Streamed:", await streamed.text);
|
|
54
|
+
},
|
|
55
|
+
);
|
|
56
|
+
} catch {
|
|
57
|
+
throw new Error(
|
|
58
|
+
"Vercel example failed; check provider configuration and telemetry diagnostics.",
|
|
59
|
+
);
|
|
60
|
+
}
|
package/fabric-setup.zip
ADDED
|
Binary file
|