langwatch 0.3.0-prerelease.1 → 0.3.0-prerelease.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-4BZATFKJ.mjs +181 -0
- package/dist/chunk-4BZATFKJ.mjs.map +1 -0
- package/dist/chunk-CSC3CMIT.mjs +118 -0
- package/dist/chunk-CSC3CMIT.mjs.map +1 -0
- package/dist/chunk-F63YKTXA.mjs +47 -0
- package/dist/chunk-F63YKTXA.mjs.map +1 -0
- package/dist/chunk-G3AUABT7.js +4 -0
- package/dist/chunk-G3AUABT7.js.map +1 -0
- package/dist/chunk-HPC6Z7J4.js +118 -0
- package/dist/chunk-HPC6Z7J4.js.map +1 -0
- package/dist/chunk-KGDAENGD.js +50 -0
- package/dist/chunk-KGDAENGD.js.map +1 -0
- package/dist/chunk-LD74LVRU.js +47 -0
- package/dist/chunk-LD74LVRU.js.map +1 -0
- package/dist/chunk-OM7VY3XT.mjs +4 -0
- package/dist/chunk-OM7VY3XT.mjs.map +1 -0
- package/dist/chunk-PCQVQ7SB.js +45 -0
- package/dist/chunk-PCQVQ7SB.js.map +1 -0
- package/dist/chunk-PMBEK6YE.mjs +424 -0
- package/dist/chunk-PMBEK6YE.mjs.map +1 -0
- package/dist/chunk-PR3JDWC3.mjs +50 -0
- package/dist/chunk-PR3JDWC3.mjs.map +1 -0
- package/dist/chunk-PTJ6AAI7.js +360 -0
- package/dist/chunk-PTJ6AAI7.js.map +1 -0
- package/dist/chunk-QEWDG5QE.mjs +45 -0
- package/dist/chunk-QEWDG5QE.mjs.map +1 -0
- package/dist/chunk-REUCVT7A.mjs +39 -0
- package/dist/chunk-REUCVT7A.mjs.map +1 -0
- package/dist/chunk-SVJ7SCGB.js +424 -0
- package/dist/chunk-SVJ7SCGB.js.map +1 -0
- package/dist/chunk-VJSOCNPA.js +181 -0
- package/dist/chunk-VJSOCNPA.js.map +1 -0
- package/dist/chunk-WM2GRSRW.js +39 -0
- package/dist/chunk-WM2GRSRW.js.map +1 -0
- package/dist/chunk-Z5J5UI5E.mjs +360 -0
- package/dist/chunk-Z5J5UI5E.mjs.map +1 -0
- package/dist/client-B2HqIKg6.d.ts +51 -0
- package/dist/client-XyCqclCi.d.mts +51 -0
- package/dist/client-browser.d.mts +8 -0
- package/dist/client-browser.d.ts +8 -0
- package/dist/client-browser.js +83 -0
- package/dist/client-browser.js.map +1 -0
- package/dist/client-browser.mjs +83 -0
- package/dist/client-browser.mjs.map +1 -0
- package/dist/client-node.d.mts +8 -0
- package/dist/client-node.d.ts +8 -0
- package/dist/client-node.js +90 -0
- package/dist/client-node.js.map +1 -0
- package/dist/client-node.mjs +90 -0
- package/dist/client-node.mjs.map +1 -0
- package/dist/evaluation/index.d.mts +897 -0
- package/dist/evaluation/index.d.ts +897 -0
- package/dist/evaluation/index.js +13 -0
- package/dist/evaluation/index.js.map +1 -0
- package/dist/evaluation/index.mjs +13 -0
- package/dist/evaluation/index.mjs.map +1 -0
- package/dist/filterable-batch-span-processor-zO5kcjBY.d.mts +64 -0
- package/dist/filterable-batch-span-processor-zO5kcjBY.d.ts +64 -0
- package/dist/index.d.mts +48 -0
- package/{src/observability/exporters/langwatch-exporter.ts → dist/index.d.ts} +13 -18
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +30 -0
- package/dist/index.mjs.map +1 -0
- package/dist/observability/index.d.mts +260 -0
- package/dist/observability/index.d.ts +260 -0
- package/dist/observability/index.js +20 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/index.mjs +20 -0
- package/dist/observability/index.mjs.map +1 -0
- package/dist/observability/instrumentation/langchain/index.d.mts +40 -0
- package/dist/observability/instrumentation/langchain/index.d.ts +40 -0
- package/dist/observability/instrumentation/langchain/index.js +666 -0
- package/dist/observability/instrumentation/langchain/index.js.map +1 -0
- package/dist/observability/instrumentation/langchain/index.mjs +666 -0
- package/dist/observability/instrumentation/langchain/index.mjs.map +1 -0
- package/dist/prompt/index.d.mts +10 -0
- package/dist/prompt/index.d.ts +10 -0
- package/dist/prompt/index.js +18 -0
- package/dist/prompt/index.js.map +1 -0
- package/dist/prompt/index.mjs +18 -0
- package/dist/prompt/index.mjs.map +1 -0
- package/dist/prompt-BXJWdbQp.d.mts +1967 -0
- package/dist/prompt-BXJWdbQp.d.ts +1967 -0
- package/dist/record-evaluation-CmxMXa-3.d.mts +25 -0
- package/dist/record-evaluation-CmxMXa-3.d.ts +25 -0
- package/dist/trace-D-bZOuqb.d.mts +622 -0
- package/dist/trace-G2312klE.d.ts +622 -0
- package/package.json +9 -4
- package/.editorconfig +0 -16
- package/.eslintrc.cjs +0 -37
- package/copy-types.sh +0 -28
- package/examples/langchain/.env.example +0 -2
- package/examples/langchain/README.md +0 -42
- package/examples/langchain/package-lock.json +0 -2930
- package/examples/langchain/package.json +0 -27
- package/examples/langchain/src/cli-markdown.d.ts +0 -137
- package/examples/langchain/src/index.ts +0 -109
- package/examples/langchain/tsconfig.json +0 -25
- package/examples/langgraph/.env.example +0 -2
- package/examples/langgraph/README.md +0 -42
- package/examples/langgraph/package-lock.json +0 -3031
- package/examples/langgraph/package.json +0 -28
- package/examples/langgraph/src/cli-markdown.d.ts +0 -137
- package/examples/langgraph/src/index.ts +0 -196
- package/examples/langgraph/tsconfig.json +0 -25
- package/examples/mastra/.env.example +0 -2
- package/examples/mastra/README.md +0 -57
- package/examples/mastra/package-lock.json +0 -5296
- package/examples/mastra/package.json +0 -32
- package/examples/mastra/src/cli-markdown.d.ts +0 -137
- package/examples/mastra/src/index.ts +0 -120
- package/examples/mastra/src/mastra/agents/weather-agent.ts +0 -30
- package/examples/mastra/src/mastra/index.ts +0 -21
- package/examples/mastra/src/mastra/tools/weather-tool.ts +0 -102
- package/examples/mastra/tsconfig.json +0 -25
- package/examples/vercel-ai/.env.example +0 -2
- package/examples/vercel-ai/README.md +0 -38
- package/examples/vercel-ai/package-lock.json +0 -2571
- package/examples/vercel-ai/package.json +0 -27
- package/examples/vercel-ai/src/cli-markdown.d.ts +0 -137
- package/examples/vercel-ai/src/index.ts +0 -110
- package/examples/vercel-ai/src/instrumentation.ts +0 -9
- package/examples/vercel-ai/tsconfig.json +0 -25
- package/src/__tests__/client-browser.test.ts +0 -92
- package/src/__tests__/client-node.test.ts +0 -76
- package/src/__tests__/client.test.ts +0 -71
- package/src/__tests__/integration/client-browser.test.ts +0 -46
- package/src/__tests__/integration/client-node.test.ts +0 -46
- package/src/client-browser.ts +0 -70
- package/src/client-node.ts +0 -82
- package/src/client-shared.ts +0 -72
- package/src/client.ts +0 -119
- package/src/evaluation/__tests__/record-evaluation.test.ts +0 -112
- package/src/evaluation/__tests__/run-evaluation.test.ts +0 -171
- package/src/evaluation/index.ts +0 -2
- package/src/evaluation/record-evaluation.ts +0 -101
- package/src/evaluation/run-evaluation.ts +0 -133
- package/src/evaluation/tracer.ts +0 -3
- package/src/evaluation/types.ts +0 -23
- package/src/index.ts +0 -13
- package/src/internal/api/__tests__/errors.test.ts +0 -98
- package/src/internal/api/client.ts +0 -30
- package/src/internal/api/errors.ts +0 -32
- package/src/internal/generated/openapi/.gitkeep +0 -0
- package/src/internal/generated/types/.gitkeep +0 -0
- package/src/observability/__tests__/integration/base.test.ts +0 -74
- package/src/observability/__tests__/integration/browser-setup-ordering.test.ts +0 -60
- package/src/observability/__tests__/integration/complex-nested-spans.test.ts +0 -29
- package/src/observability/__tests__/integration/error-handling.test.ts +0 -24
- package/src/observability/__tests__/integration/langwatch-disabled-otel.test.ts +0 -24
- package/src/observability/__tests__/integration/langwatch-first-then-vercel.test.ts +0 -24
- package/src/observability/__tests__/integration/multiple-setup-attempts.test.ts +0 -27
- package/src/observability/__tests__/integration/otel-ordering.test.ts +0 -27
- package/src/observability/__tests__/integration/vercel-configurations.test.ts +0 -20
- package/src/observability/__tests__/integration/vercel-first-then-langwatch.test.ts +0 -27
- package/src/observability/__tests__/span.test.ts +0 -214
- package/src/observability/__tests__/trace.test.ts +0 -180
- package/src/observability/exporters/index.ts +0 -1
- package/src/observability/index.ts +0 -4
- package/src/observability/instrumentation/langchain/__tests__/integration/langchain-chatbot.test.ts +0 -112
- package/src/observability/instrumentation/langchain/__tests__/langchain.test.ts +0 -284
- package/src/observability/instrumentation/langchain/index.ts +0 -624
- package/src/observability/processors/__tests__/filterable-batch-span-exporter.test.ts +0 -98
- package/src/observability/processors/filterable-batch-span-processor.ts +0 -99
- package/src/observability/processors/index.ts +0 -1
- package/src/observability/semconv/attributes.ts +0 -185
- package/src/observability/semconv/events.ts +0 -42
- package/src/observability/semconv/index.ts +0 -16
- package/src/observability/semconv/values.ts +0 -159
- package/src/observability/span.ts +0 -728
- package/src/observability/trace.ts +0 -301
- package/src/prompt/__tests__/prompt.test.ts +0 -139
- package/src/prompt/get-prompt-version.ts +0 -49
- package/src/prompt/get-prompt.ts +0 -44
- package/src/prompt/index.ts +0 -3
- package/src/prompt/prompt.ts +0 -133
- package/src/prompt/service.ts +0 -221
- package/src/prompt/tracer.ts +0 -3
- package/src/prompt/types.ts +0 -0
- package/ts-to-zod.config.js +0 -35
- package/tsconfig.json +0 -26
- package/tsup.config.ts +0 -20
- package/vitest.config.ts +0 -9
package/src/client-browser.ts
DELETED
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
import { setConfig, SetupOptions, getApiKey, getEndpoint } from "./client";
|
|
2
|
-
import { SpanProcessor, WebTracerProvider } from '@opentelemetry/sdk-trace-web';
|
|
3
|
-
import { ZoneContextManager } from '@opentelemetry/context-zone';
|
|
4
|
-
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
|
5
|
-
import { version } from "../package.json";
|
|
6
|
-
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
7
|
-
import * as intSemconv from "./observability/semconv";
|
|
8
|
-
import { FilterableBatchSpanProcessor } from "./observability/processors";
|
|
9
|
-
import { LangWatchExporter } from "./observability/exporters";
|
|
10
|
-
import { addSpanProcessorToExistingTracerProvider, isOtelInitialized, mergeResourceIntoExistingTracerProvider } from "./client-shared";
|
|
11
|
-
|
|
12
|
-
let managedSpanProcessors: SpanProcessor[] = [];
|
|
13
|
-
let provider: WebTracerProvider | null = null;
|
|
14
|
-
let browserSetupCalled: boolean = false;
|
|
15
|
-
|
|
16
|
-
export async function setupLangWatch(options: SetupOptions = {}) {
|
|
17
|
-
if (browserSetupCalled) {
|
|
18
|
-
throw new Error("LangWatch setup has already been called in this process. Setup can only be called once, if you need to modify OpenTelemetry setup then use the OpenTelemetry API directly.");
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
setConfig(options);
|
|
22
|
-
|
|
23
|
-
if (options.skipOpenTelemetrySetup) return;
|
|
24
|
-
|
|
25
|
-
const endpointURL = new URL("/api/otel/v1/traces", getEndpoint());
|
|
26
|
-
const langwatchSpanProcessor = new FilterableBatchSpanProcessor(
|
|
27
|
-
new LangWatchExporter(getApiKey(), endpointURL.toString()),
|
|
28
|
-
options.otelSpanProcessingExcludeRules ?? [],
|
|
29
|
-
);
|
|
30
|
-
|
|
31
|
-
const langwatchResource = resourceFromAttributes({
|
|
32
|
-
...options.baseAttributes,
|
|
33
|
-
[intSemconv.ATTR_LANGWATCH_SDK_LANGUAGE]: "typescript-browser",
|
|
34
|
-
[intSemconv.ATTR_LANGWATCH_SDK_VERSION]: version,
|
|
35
|
-
[intSemconv.ATTR_LANGWATCH_SDK_NAME]: "langwatch-observability-sdk",
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
if (isOtelInitialized()) {
|
|
39
|
-
mergeResourceIntoExistingTracerProvider(langwatchResource);
|
|
40
|
-
addSpanProcessorToExistingTracerProvider(langwatchSpanProcessor);
|
|
41
|
-
for (const spanProcessor of options.otelSpanProcessors ?? []) {
|
|
42
|
-
addSpanProcessorToExistingTracerProvider(spanProcessor);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
managedSpanProcessors = [langwatchSpanProcessor];
|
|
46
|
-
} else {
|
|
47
|
-
provider = new WebTracerProvider({
|
|
48
|
-
resource: resourceFromAttributes({
|
|
49
|
-
[intSemconv.ATTR_LANGWATCH_SDK_LANGUAGE]: "typescript-browser",
|
|
50
|
-
[intSemconv.ATTR_LANGWATCH_SDK_VERSION]: version,
|
|
51
|
-
[intSemconv.ATTR_LANGWATCH_SDK_NAME]: "langwatch-observability-sdk",
|
|
52
|
-
}),
|
|
53
|
-
spanProcessors: [langwatchSpanProcessor, ...(options.otelSpanProcessors ?? [])],
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
provider.register({
|
|
57
|
-
contextManager: new ZoneContextManager(),
|
|
58
|
-
propagator: new W3CTraceContextPropagator(),
|
|
59
|
-
});
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// This is not guaranteed to be called, but it's a good nice to have.
|
|
63
|
-
window.addEventListener("beforeunload", async () => {
|
|
64
|
-
if (provider) {
|
|
65
|
-
await provider.shutdown();
|
|
66
|
-
} else {
|
|
67
|
-
await Promise.all(managedSpanProcessors.map(p => p.shutdown()));
|
|
68
|
-
}
|
|
69
|
-
});
|
|
70
|
-
}
|
package/src/client-node.ts
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { SpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
2
|
-
import { getApiKey, getEndpoint, setConfig, SetupOptions } from "./client";
|
|
3
|
-
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
4
|
-
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
5
|
-
import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
|
|
6
|
-
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
7
|
-
import { version } from "../package.json";
|
|
8
|
-
import * as intSemconv from "./observability/semconv";
|
|
9
|
-
import { addSpanProcessorToExistingTracerProvider, isOtelInitialized, mergeResourceIntoExistingTracerProvider } from "./client-shared";
|
|
10
|
-
import { FilterableBatchSpanProcessor } from "./observability";
|
|
11
|
-
import { LangWatchExporter } from "./observability/exporters";
|
|
12
|
-
|
|
13
|
-
let managedSpanProcessors: SpanProcessor[] = [];
|
|
14
|
-
let nodeSetupCalled: boolean = false;
|
|
15
|
-
let sdk: NodeSDK | null = null;
|
|
16
|
-
|
|
17
|
-
export async function setupLangWatch(options: SetupOptions = {}) {
|
|
18
|
-
if (nodeSetupCalled) {
|
|
19
|
-
throw new Error("LangWatch setup has already been called in this process. Setup can only be called once, if you need to modify OpenTelemetry setup then use the OpenTelemetry API directly.");
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
setConfig(options);
|
|
23
|
-
nodeSetupCalled = true;
|
|
24
|
-
|
|
25
|
-
if (options.skipOpenTelemetrySetup) return;
|
|
26
|
-
|
|
27
|
-
const endpointURL = new URL("/api/otel/v1/traces", getEndpoint());
|
|
28
|
-
const langwatchSpanProcessor = new FilterableBatchSpanProcessor(
|
|
29
|
-
new LangWatchExporter(getApiKey(), endpointURL.toString()),
|
|
30
|
-
options.otelSpanProcessingExcludeRules ?? [],
|
|
31
|
-
);
|
|
32
|
-
|
|
33
|
-
const langwatchResource = resourceFromAttributes({
|
|
34
|
-
...options.baseAttributes,
|
|
35
|
-
[intSemconv.ATTR_LANGWATCH_SDK_LANGUAGE]: "typescript-node",
|
|
36
|
-
[intSemconv.ATTR_LANGWATCH_SDK_VERSION]: version,
|
|
37
|
-
[intSemconv.ATTR_LANGWATCH_SDK_NAME]: "langwatch-observability-sdk",
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
if (isOtelInitialized()) {
|
|
41
|
-
mergeResourceIntoExistingTracerProvider(langwatchResource);
|
|
42
|
-
addSpanProcessorToExistingTracerProvider(langwatchSpanProcessor);
|
|
43
|
-
for (const spanProcessor of options.otelSpanProcessors ?? []) {
|
|
44
|
-
addSpanProcessorToExistingTracerProvider(spanProcessor);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
managedSpanProcessors = [langwatchSpanProcessor];
|
|
48
|
-
} else {
|
|
49
|
-
sdk = new NodeSDK({
|
|
50
|
-
resource: langwatchResource,
|
|
51
|
-
spanProcessors: [langwatchSpanProcessor, ...(options.otelSpanProcessors ?? [])],
|
|
52
|
-
contextManager: new AsyncLocalStorageContextManager(),
|
|
53
|
-
textMapPropagator: new W3CTraceContextPropagator(),
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
sdk.start();
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
// If we detect interrupt, termination, or test beforeExit signals, then we attempt
|
|
60
|
-
// to shutdown.
|
|
61
|
-
// - If an SDK exists, then we just attempt to shutdown the SDK.
|
|
62
|
-
// - If no SDK exists, then we attempt to shutdown ONLY the SpanProcessors that are
|
|
63
|
-
// managed by this LangWatch SDK.
|
|
64
|
-
["SIGINT", "SIGTERM", "beforeExit"].forEach((signal) => {
|
|
65
|
-
process.on(signal as any, async () => {
|
|
66
|
-
try {
|
|
67
|
-
if (sdk) {
|
|
68
|
-
await sdk.shutdown();
|
|
69
|
-
} else {
|
|
70
|
-
await Promise.all(managedSpanProcessors.map(p => p.shutdown()));
|
|
71
|
-
}
|
|
72
|
-
} catch (error) {
|
|
73
|
-
// eslint-disable-next-line no-console
|
|
74
|
-
console.error("Error shutting down OpenTelemetry SDK:", error);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
if (signal !== "beforeExit") {
|
|
78
|
-
process.exit();
|
|
79
|
-
}
|
|
80
|
-
});
|
|
81
|
-
});
|
|
82
|
-
}
|
package/src/client-shared.ts
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
trace,
|
|
3
|
-
ProxyTracerProvider,
|
|
4
|
-
} from "@opentelemetry/api";
|
|
5
|
-
import { Resource } from "@opentelemetry/resources";
|
|
6
|
-
import { SpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Gets the actual tracer provider, handling the proxy delegate pattern.
|
|
10
|
-
*
|
|
11
|
-
* @returns The actual tracer provider or undefined if not available
|
|
12
|
-
*/
|
|
13
|
-
function getActualTracerProvider(): any {
|
|
14
|
-
const potentiallyProxyTracerProvider = trace.getTracerProvider() as unknown;
|
|
15
|
-
|
|
16
|
-
// Attempt to get the delegate if it's a ProxyTracerProvider
|
|
17
|
-
const delegate = (potentiallyProxyTracerProvider as ProxyTracerProvider | undefined)?.getDelegate?.();
|
|
18
|
-
|
|
19
|
-
// Return the delegate if available, otherwise return the original provider
|
|
20
|
-
return delegate ?? (potentiallyProxyTracerProvider as any);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* Checks if the OpenTelemetry SDK has been initialized anywhere in the process.
|
|
25
|
-
*
|
|
26
|
-
* @returns true if the OpenTelemetry SDK has been initialized, false otherwise.
|
|
27
|
-
*/
|
|
28
|
-
export function isOtelInitialized() {
|
|
29
|
-
const provider = getActualTracerProvider();
|
|
30
|
-
|
|
31
|
-
// Check if the provider has the addSpanProcessor method, which indicates SDK initialization
|
|
32
|
-
return provider && typeof provider.addSpanProcessor === "function";
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Merges a resource into the existing tracer provider.
|
|
37
|
-
*
|
|
38
|
-
* @param resource - The resource to merge into the existing tracer provider.
|
|
39
|
-
*/
|
|
40
|
-
export function mergeResourceIntoExistingTracerProvider(resource: Resource) {
|
|
41
|
-
if (!isOtelInitialized()) {
|
|
42
|
-
throw new Error("OpenTelemetry SDK is not initialized, cannot merge resource into existing tracer provider.");
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
const provider = getActualTracerProvider();
|
|
46
|
-
|
|
47
|
-
if (!provider?.resource) {
|
|
48
|
-
throw new Error("OpenTelemetry SDK is not initialized, provider does not have a resource.");
|
|
49
|
-
}
|
|
50
|
-
if (typeof resource !== "object") {
|
|
51
|
-
throw new Error("OpenTelemetry SDK is not initialized, provider resource is not an object.");
|
|
52
|
-
}
|
|
53
|
-
if (typeof provider.resource.merge !== "function") {
|
|
54
|
-
throw new Error("OpenTelemetry SDK is not initialized, provider resource does not have a merge method.");
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
provider.resource = provider.resource.merge(resource);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export function addSpanProcessorToExistingTracerProvider(spanProcessor: SpanProcessor) {
|
|
61
|
-
if (!isOtelInitialized()) {
|
|
62
|
-
throw new Error("OpenTelemetry SDK is not initialized, cannot add span processor to existing tracer provider.");
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const provider = getActualTracerProvider();
|
|
66
|
-
|
|
67
|
-
if (!provider?.addSpanProcessor) {
|
|
68
|
-
throw new Error("OpenTelemetry SDK is not initialized, provider does not have a addSpanProcessor method.");
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
provider.addSpanProcessor(spanProcessor);
|
|
72
|
-
}
|
package/src/client.ts
DELETED
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
import { SpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
2
|
-
import { SpanProcessingExcludeRule } from "./observability";
|
|
3
|
-
import { Attributes } from "@opentelemetry/api";
|
|
4
|
-
|
|
5
|
-
export interface SetupOptions {
|
|
6
|
-
/**
|
|
7
|
-
* The API key to use for the LangWatch API.
|
|
8
|
-
*/
|
|
9
|
-
apiKey?: string;
|
|
10
|
-
|
|
11
|
-
/**
|
|
12
|
-
* The endpoint to use for the LangWatch API.
|
|
13
|
-
*/
|
|
14
|
-
endpoint?: string;
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* The span processors to use for the OpenTelemetry SDK.
|
|
18
|
-
*
|
|
19
|
-
* If provided, these will be added to the OpenTelemetry SDK after the LangWatch SDK has
|
|
20
|
-
* been initialized.
|
|
21
|
-
*/
|
|
22
|
-
otelSpanProcessors?: SpanProcessor[];
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* The span processing exclude rules to use for the OpenTelemetry SDK.
|
|
26
|
-
*
|
|
27
|
-
* If provided, these will be added to the OpenTelemetry SDK after the LangWatch SDK has
|
|
28
|
-
* been initialized.
|
|
29
|
-
*
|
|
30
|
-
* If you are using the `otelSpanProcessors` option, then these will be ignored.
|
|
31
|
-
*/
|
|
32
|
-
otelSpanProcessingExcludeRules?: SpanProcessingExcludeRule[];
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Whether to skip the automatic setup of the OpenTelemetry SDK. If this is set, then
|
|
36
|
-
* the LangWatch SDK will not attempt to setup the OpenTelemetry SDK. You will need to
|
|
37
|
-
* setup the OpenTelemetry yourself, and ensure that a SpanProcessor is added to the
|
|
38
|
-
* OpenTelemetry SDK that will send traces to the LangWatch API.
|
|
39
|
-
*/
|
|
40
|
-
skipOpenTelemetrySetup?: boolean;
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Whether to disable the automatic capture of input.
|
|
44
|
-
*/
|
|
45
|
-
disableAutomaticInputCapture?: boolean;
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Whether to disable the automatic capture of output.
|
|
49
|
-
*/
|
|
50
|
-
disableAutomaticOutputCapture?: boolean;
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
* The base attributes to use for the OpenTelemetry SDK.
|
|
54
|
-
*/
|
|
55
|
-
baseAttributes?: Attributes;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
interface InternalConfig {
|
|
59
|
-
apiKey: string;
|
|
60
|
-
endpoint: string;
|
|
61
|
-
setupCalled: boolean;
|
|
62
|
-
skipOpenTelemetrySetup: boolean;
|
|
63
|
-
disableAutomaticInputCapture: boolean;
|
|
64
|
-
disableAutomaticOutputCapture: boolean;
|
|
65
|
-
|
|
66
|
-
baseAttributes: Attributes;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
const config: InternalConfig = {
|
|
70
|
-
apiKey: process.env.LANGWATCH_API_KEY ?? "",
|
|
71
|
-
endpoint: process.env.LANGWATCH_ENDPOINT ?? "https://app.langwatch.ai",
|
|
72
|
-
setupCalled: false,
|
|
73
|
-
skipOpenTelemetrySetup: false,
|
|
74
|
-
disableAutomaticInputCapture: false,
|
|
75
|
-
disableAutomaticOutputCapture: false,
|
|
76
|
-
baseAttributes: {},
|
|
77
|
-
};
|
|
78
|
-
|
|
79
|
-
export function setConfig(options: SetupOptions) {
|
|
80
|
-
config.setupCalled = true;
|
|
81
|
-
|
|
82
|
-
config.apiKey = options.apiKey !== void 0
|
|
83
|
-
? options.apiKey
|
|
84
|
-
: (process.env.LANGWATCH_API_KEY ?? config.apiKey);
|
|
85
|
-
|
|
86
|
-
config.endpoint = options.endpoint !== void 0
|
|
87
|
-
? options.endpoint
|
|
88
|
-
: (process.env.LANGWATCH_ENDPOINT ?? config.endpoint);
|
|
89
|
-
|
|
90
|
-
if (config.apiKey === "") {
|
|
91
|
-
console.warn("[langwatch setup] No API key provided. Please set the LANGWATCH_API_KEY environment variable or pass it to the setup function. The SDK will perform no operations.");
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
config.skipOpenTelemetrySetup = options.skipOpenTelemetrySetup ?? config.skipOpenTelemetrySetup;
|
|
95
|
-
config.disableAutomaticInputCapture = options.disableAutomaticInputCapture ?? config.disableAutomaticInputCapture;
|
|
96
|
-
config.disableAutomaticOutputCapture = options.disableAutomaticOutputCapture ?? config.disableAutomaticOutputCapture;
|
|
97
|
-
|
|
98
|
-
config.baseAttributes = options.baseAttributes ?? config.baseAttributes;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
export function getApiKey(): string {
|
|
102
|
-
return config.apiKey;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
export function getEndpoint(): string {
|
|
106
|
-
return config.endpoint;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
export function canAutomaticallyCaptureInput(): boolean {
|
|
110
|
-
return !config.disableAutomaticInputCapture;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
export function canAutomaticallyCaptureOutput(): boolean {
|
|
114
|
-
return !config.disableAutomaticOutputCapture;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
export function isSetupCalled(): boolean {
|
|
118
|
-
return config.setupCalled;
|
|
119
|
-
}
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
// --- Mock setup (must be at the top for Vitest hoisting) ---
|
|
2
|
-
const { mockStartActiveSpan } = vi.hoisted(() => ({
|
|
3
|
-
mockStartActiveSpan: vi.fn((name, fn) => fn({
|
|
4
|
-
setType: vi.fn(),
|
|
5
|
-
addEvent: vi.fn(),
|
|
6
|
-
setOutput: vi.fn(),
|
|
7
|
-
setAttributes: vi.fn(),
|
|
8
|
-
setMetrics: vi.fn(),
|
|
9
|
-
setStatus: vi.fn(),
|
|
10
|
-
recordException: vi.fn(),
|
|
11
|
-
end: vi.fn(),
|
|
12
|
-
})),
|
|
13
|
-
}));
|
|
14
|
-
|
|
15
|
-
vi.mock('../tracer', () => ({ tracer: { startActiveSpan: mockStartActiveSpan } }));
|
|
16
|
-
vi.mock('../../observability/semconv', () => ({
|
|
17
|
-
ATTR_LANGWATCH_EVALUATION_CUSTOM: 'custom_event',
|
|
18
|
-
}));
|
|
19
|
-
|
|
20
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
21
|
-
import { recordEvaluation } from '../record-evaluation';
|
|
22
|
-
|
|
23
|
-
const baseDetails: import('../record-evaluation').RecordedEvaluationDetails = {
|
|
24
|
-
evaluationId: 'eval1',
|
|
25
|
-
name: 'test',
|
|
26
|
-
type: 'custom',
|
|
27
|
-
isGuardrail: false,
|
|
28
|
-
status: 'processed',
|
|
29
|
-
passed: true,
|
|
30
|
-
score: 1,
|
|
31
|
-
label: 'label',
|
|
32
|
-
details: 'ok',
|
|
33
|
-
cost: { currency: 'USD', amount: 0.1 },
|
|
34
|
-
error: undefined,
|
|
35
|
-
timestamps: { startedAtUnixMs: 1, finishedAtUnixMs: 2 },
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
describe('recordEvaluation', () => {
|
|
39
|
-
beforeEach(() => {
|
|
40
|
-
vi.clearAllMocks();
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
it('records processed evaluation', () => {
|
|
44
|
-
const span = {
|
|
45
|
-
setType: vi.fn(),
|
|
46
|
-
addEvent: vi.fn(),
|
|
47
|
-
setOutput: vi.fn(),
|
|
48
|
-
setAttributes: vi.fn(),
|
|
49
|
-
setMetrics: vi.fn(),
|
|
50
|
-
setOutputEvaluation: vi.fn(),
|
|
51
|
-
recordException: vi.fn(),
|
|
52
|
-
end: vi.fn(),
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
mockStartActiveSpan.mockImplementationOnce((name, fn) => fn(span));
|
|
56
|
-
recordEvaluation({ ...baseDetails });
|
|
57
|
-
|
|
58
|
-
expect(mockStartActiveSpan).toHaveBeenCalledWith(
|
|
59
|
-
'record evaluation',
|
|
60
|
-
expect.any(Function)
|
|
61
|
-
);
|
|
62
|
-
expect(span.setType).toHaveBeenCalledWith('evaluation');
|
|
63
|
-
expect(span.addEvent).toHaveBeenCalledWith('custom_event', expect.objectContaining({
|
|
64
|
-
json_encoded_event: expect.stringContaining('"name":"test"')
|
|
65
|
-
}));
|
|
66
|
-
expect(span.setOutput).toHaveBeenCalledWith(expect.objectContaining({
|
|
67
|
-
status: 'processed',
|
|
68
|
-
passed: true,
|
|
69
|
-
score: 1,
|
|
70
|
-
label: 'label',
|
|
71
|
-
details: 'ok',
|
|
72
|
-
cost: { currency: 'USD', amount: 0.1 },
|
|
73
|
-
}));
|
|
74
|
-
expect(span.end).toHaveBeenCalled();
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
it('records skipped evaluation', () => {
|
|
78
|
-
recordEvaluation({ ...baseDetails, status: 'skipped', details: 'skipped' });
|
|
79
|
-
expect(mockStartActiveSpan).toHaveBeenCalled();
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
it('records error evaluation', () => {
|
|
83
|
-
recordEvaluation({ ...baseDetails, status: 'error', error: new Error('fail'), details: 'fail' });
|
|
84
|
-
expect(mockStartActiveSpan).toHaveBeenCalled();
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
it('sets cost metric if cost is present', () => {
|
|
88
|
-
recordEvaluation({ ...baseDetails, cost: { currency: 'USD', amount: 42 } });
|
|
89
|
-
// No assertion needed, just ensure no error
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
it('sets attributes if provided', () => {
|
|
93
|
-
const attrs = { foo: 'bar' };
|
|
94
|
-
recordEvaluation({ ...baseDetails }, attrs);
|
|
95
|
-
// No assertion needed, just ensure no error
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
it('handles error in span', () => {
|
|
99
|
-
const errorSpan = {
|
|
100
|
-
setType: vi.fn(() => { throw new Error('fail in span'); }),
|
|
101
|
-
addEvent: vi.fn(),
|
|
102
|
-
setOutput: vi.fn(),
|
|
103
|
-
setAttributes: vi.fn(),
|
|
104
|
-
setMetrics: vi.fn(),
|
|
105
|
-
recordException: vi.fn(),
|
|
106
|
-
end: vi.fn(),
|
|
107
|
-
};
|
|
108
|
-
expect(() => {
|
|
109
|
-
mockStartActiveSpan.mock.calls[0]?.[1]?.(errorSpan);
|
|
110
|
-
}).not.toThrow();
|
|
111
|
-
});
|
|
112
|
-
});
|
|
@@ -1,171 +0,0 @@
|
|
|
1
|
-
// --- Mock setup (must be at the top for Vitest hoisting) ---
|
|
2
|
-
const { mockStartActiveSpan } = vi.hoisted(() => ({
|
|
3
|
-
mockStartActiveSpan: vi.fn((name, fn) => fn({
|
|
4
|
-
setType: vi.fn(),
|
|
5
|
-
setInput: vi.fn(),
|
|
6
|
-
setMetrics: vi.fn(),
|
|
7
|
-
setStatus: vi.fn(),
|
|
8
|
-
setOutputEvaluation: vi.fn(),
|
|
9
|
-
recordException: vi.fn(),
|
|
10
|
-
end: vi.fn(),
|
|
11
|
-
spanContext: () => ({ traceId: 'trace', spanId: 'span' }),
|
|
12
|
-
})),
|
|
13
|
-
}));
|
|
14
|
-
|
|
15
|
-
vi.mock('../tracer', () => ({ tracer: { startActiveSpan: mockStartActiveSpan } }));
|
|
16
|
-
|
|
17
|
-
const mockFetch = vi.fn();
|
|
18
|
-
globalThis.fetch = mockFetch;
|
|
19
|
-
|
|
20
|
-
vi.mock('../../client', () => ({
|
|
21
|
-
canAutomaticallyCaptureInput: () => true,
|
|
22
|
-
getApiKey: () => 'test-key',
|
|
23
|
-
getEndpoint: () => 'https://api',
|
|
24
|
-
}));
|
|
25
|
-
|
|
26
|
-
// --- Imports (must be after mocks for Vitest hoisting) ---
|
|
27
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
28
|
-
import { runEvaluation } from '../run-evaluation';
|
|
29
|
-
import { LangWatchApiError } from '../../internal/api/errors';
|
|
30
|
-
|
|
31
|
-
const baseProcessed = {
|
|
32
|
-
status: 'processed',
|
|
33
|
-
passed: true,
|
|
34
|
-
score: 1,
|
|
35
|
-
details: 'ok',
|
|
36
|
-
label: 'label',
|
|
37
|
-
cost: { currency: 'USD', amount: 0.1 },
|
|
38
|
-
};
|
|
39
|
-
const baseSkipped = { status: 'skipped', details: 'skipped' };
|
|
40
|
-
const baseError = { status: 'error', details: 'fail', error_type: 'EvalError', traceback: ['trace'] };
|
|
41
|
-
|
|
42
|
-
const details = {
|
|
43
|
-
name: 'test',
|
|
44
|
-
data: { input: 'foo', output: 'bar' },
|
|
45
|
-
evaluator: 'test-eval',
|
|
46
|
-
};
|
|
47
|
-
|
|
48
|
-
describe('runEvaluation', () => {
|
|
49
|
-
beforeEach(() => {
|
|
50
|
-
vi.clearAllMocks();
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
it('returns processed result', async () => {
|
|
54
|
-
mockFetch.mockResolvedValueOnce({
|
|
55
|
-
ok: true,
|
|
56
|
-
json: async () => ({ ...baseProcessed }),
|
|
57
|
-
});
|
|
58
|
-
const result = await runEvaluation(details as any);
|
|
59
|
-
expect(result.status).toBe('processed');
|
|
60
|
-
if (result.status === 'processed') {
|
|
61
|
-
expect(result.passed).toBe(true);
|
|
62
|
-
expect(result.score).toBe(1);
|
|
63
|
-
expect(result.details).toBe('ok');
|
|
64
|
-
expect(result.label).toBe('label');
|
|
65
|
-
expect(result.cost).toEqual({ currency: 'USD', amount: 0.1 });
|
|
66
|
-
} else {
|
|
67
|
-
throw new Error('Expected processed result');
|
|
68
|
-
}
|
|
69
|
-
expect(mockFetch).toHaveBeenCalledWith(
|
|
70
|
-
expect.stringContaining('/api/evaluations/test-eval/evaluate'),
|
|
71
|
-
expect.objectContaining({ method: 'POST' })
|
|
72
|
-
);
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
it('returns skipped result', async () => {
|
|
76
|
-
mockFetch.mockResolvedValueOnce({
|
|
77
|
-
ok: true,
|
|
78
|
-
json: async () => ({ ...baseSkipped }),
|
|
79
|
-
});
|
|
80
|
-
const result = await runEvaluation(details as any);
|
|
81
|
-
expect(result.status).toBe('skipped');
|
|
82
|
-
expect(result.details).toBe('skipped');
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
it('returns error result', async () => {
|
|
86
|
-
mockFetch.mockResolvedValueOnce({
|
|
87
|
-
ok: true,
|
|
88
|
-
json: async () => ({ ...baseError }),
|
|
89
|
-
});
|
|
90
|
-
const result = await runEvaluation(details as any);
|
|
91
|
-
expect(result.status).toBe('error');
|
|
92
|
-
if (result.status === 'error') {
|
|
93
|
-
expect(result.details).toBe('fail');
|
|
94
|
-
expect(result.error_type).toBe('EvalError');
|
|
95
|
-
expect(result.traceback).toEqual(['trace']);
|
|
96
|
-
} else {
|
|
97
|
-
throw new Error('Expected error result');
|
|
98
|
-
}
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
it('returns unknown status as error', async () => {
|
|
102
|
-
mockFetch.mockResolvedValueOnce({
|
|
103
|
-
ok: true,
|
|
104
|
-
json: async () => ({ status: 'weird' }),
|
|
105
|
-
});
|
|
106
|
-
const result = await runEvaluation(details as any);
|
|
107
|
-
expect(result.status).toBe('error');
|
|
108
|
-
if (result.status === 'error') {
|
|
109
|
-
expect(result.error_type).toBe('UnknownStatus');
|
|
110
|
-
expect(result.details).toContain('Unknown evaluation status');
|
|
111
|
-
} else {
|
|
112
|
-
throw new Error('Expected error result');
|
|
113
|
-
}
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
it('throws LangWatchApiError on non-ok response', async () => {
|
|
117
|
-
mockFetch.mockResolvedValueOnce({ ok: false, json: async () => ({}), status: 400, statusText: 'Bad', headers: { get: () => 'application/json' } });
|
|
118
|
-
await expect(runEvaluation(details as any)).rejects.toBeInstanceOf(LangWatchApiError);
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
it('propagates fetch errors', async () => {
|
|
122
|
-
mockFetch.mockRejectedValueOnce(new Error('network fail'));
|
|
123
|
-
await expect(runEvaluation(details as any)).rejects.toThrow('network fail');
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
it('calls setInput if canAutomaticallyCaptureInput is true', async () => {
|
|
127
|
-
vi.resetModules();
|
|
128
|
-
vi.doMock('../../client', () => ({
|
|
129
|
-
canAutomaticallyCaptureInput: () => true,
|
|
130
|
-
getApiKey: () => 'test-key',
|
|
131
|
-
getEndpoint: () => 'https://api',
|
|
132
|
-
}));
|
|
133
|
-
const span = {
|
|
134
|
-
setType: vi.fn(),
|
|
135
|
-
setInput: vi.fn(),
|
|
136
|
-
setMetrics: vi.fn(),
|
|
137
|
-
setOutputEvaluation: vi.fn(),
|
|
138
|
-
recordException: vi.fn(),
|
|
139
|
-
end: vi.fn(),
|
|
140
|
-
spanContext: () => ({ traceId: 'trace', spanId: 'span' }),
|
|
141
|
-
};
|
|
142
|
-
mockStartActiveSpan.mockImplementationOnce((name, fn) => fn(span));
|
|
143
|
-
mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ ...baseProcessed }) });
|
|
144
|
-
const { runEvaluation: runEval } = await import('../run-evaluation.js');
|
|
145
|
-
await runEval(details as any);
|
|
146
|
-
expect(span.setInput).toHaveBeenCalledWith(expect.objectContaining({ trace_id: 'trace' }));
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
it('does not call setInput if canAutomaticallyCaptureInput is false', async () => {
|
|
150
|
-
vi.resetModules();
|
|
151
|
-
vi.doMock('../../client', () => ({
|
|
152
|
-
canAutomaticallyCaptureInput: () => false,
|
|
153
|
-
getApiKey: () => 'test-key',
|
|
154
|
-
getEndpoint: () => 'https://api',
|
|
155
|
-
}));
|
|
156
|
-
const span = {
|
|
157
|
-
setType: vi.fn(),
|
|
158
|
-
setInput: vi.fn(),
|
|
159
|
-
setMetrics: vi.fn(),
|
|
160
|
-
setOutputEvaluation: vi.fn(),
|
|
161
|
-
recordException: vi.fn(),
|
|
162
|
-
end: vi.fn(),
|
|
163
|
-
spanContext: () => ({ traceId: 'trace', spanId: 'span' }),
|
|
164
|
-
};
|
|
165
|
-
mockStartActiveSpan.mockImplementationOnce((name, fn) => fn(span));
|
|
166
|
-
mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ ...baseProcessed }) });
|
|
167
|
-
const { runEvaluation: runEval } = await import('../run-evaluation.js');
|
|
168
|
-
await runEval(details as any);
|
|
169
|
-
expect(span.setInput).not.toHaveBeenCalled();
|
|
170
|
-
});
|
|
171
|
-
});
|
package/src/evaluation/index.ts
DELETED