@hue-run/sdk 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ENVIRONMENTS.md +182 -0
- package/EVALUATIONS.md +12 -0
- package/README.md +194 -18
- package/dist/ai-sdk.d.ts +9 -1
- package/dist/ai-sdk.js +34 -8
- package/dist/client.d.ts +121 -6
- package/dist/client.js +329 -56
- package/dist/config.d.ts +11 -2
- package/dist/config.js +36 -7
- package/dist/environment/client.d.ts +73 -0
- package/dist/environment/client.js +209 -0
- package/dist/environment/tools.d.ts +30 -0
- package/dist/environment/tools.js +24 -0
- package/dist/environment/types.d.ts +429 -0
- package/dist/environment/types.js +1 -0
- package/dist/environment.d.ts +5 -0
- package/dist/environment.js +2 -0
- package/dist/evals/attempt.d.ts +454 -0
- package/dist/evals/attempt.js +687 -0
- package/dist/evals/client.d.ts +99 -5
- package/dist/evals/client.js +136 -7
- package/dist/evals/environment-evidence.d.ts +6 -0
- package/dist/evals/environment-evidence.js +123 -0
- package/dist/evals/environment-json.d.ts +3 -0
- package/dist/evals/environment-json.js +76 -0
- package/dist/evals/json.d.ts +9 -1
- package/dist/evals/json.js +14 -6
- package/dist/evals/runner.d.ts +61 -2
- package/dist/evals/runner.js +71 -9
- package/dist/evals/scorer-publication.d.ts +2 -0
- package/dist/evals/scorer-publication.js +84 -0
- package/dist/evals/scorers.d.ts +11 -0
- package/dist/evals/scorers.js +56 -5
- package/dist/evals/simulation.d.ts +184 -0
- package/dist/evals/simulation.js +603 -0
- package/dist/evals/types.d.ts +304 -0
- package/dist/evals.d.ts +5 -1
- package/dist/evals.js +3 -1
- package/dist/experimental-telemetry.d.ts +8 -0
- package/dist/experimental-telemetry.js +13 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +2 -0
- package/dist/managed.d.ts +51 -1
- package/dist/managed.js +11 -1
- package/dist/privacy.d.ts +2 -0
- package/dist/privacy.js +16 -1
- package/dist/receipt.d.ts +12 -1
- package/dist/receipt.js +10 -1
- package/dist/safety.d.ts +1 -2
- package/dist/snapshot.js +4 -0
- package/dist/transport.d.ts +41 -9
- package/dist/transport.js +80 -22
- package/dist/types.d.ts +144 -8
- package/dist/version.d.ts +2 -0
- package/dist/version.js +3 -0
- package/package.json +51 -15
package/dist/client.d.ts
CHANGED
|
@@ -1,19 +1,45 @@
|
|
|
1
1
|
import { type Context, type Span, type Tracer } from "@opentelemetry/api";
|
|
2
2
|
import { HueTransport } from "./transport.js";
|
|
3
|
-
import type { ExportReport, FlushableLoggerProvider, FlushableTracerProvider, HueOptions, HueSpan,
|
|
3
|
+
import type { ExportReport, FlushableLoggerProvider, FlushableTracerProvider, HueOptions, HueSpan, ModelOptions, ProjectConnection, SpanOptions, VerifyTraceOptions, TraceVerification, SafeLifecycleOptions, SafeLifecycleResult } from "./types.js";
|
|
4
|
+
/**
|
|
5
|
+
* Attach mode: the application owns its OpenTelemetry providers and passes the transport whose
|
|
6
|
+
* processors it attached to them. The client flushes these providers but never shuts them down.
|
|
7
|
+
*/
|
|
4
8
|
export interface ExistingHueProviders {
|
|
9
|
+
/** Transport from {@link createHueTransport} whose processors are attached to the providers below. */
|
|
5
10
|
transport: HueTransport;
|
|
11
|
+
/** Application-owned tracer provider; must support `forceFlush()`. */
|
|
6
12
|
tracerProvider: FlushableTracerProvider;
|
|
13
|
+
/** Application-owned logger provider; must support `forceFlush()`. */
|
|
7
14
|
loggerProvider: FlushableLoggerProvider;
|
|
8
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* Thrown by {@link HueClient.checkConnection} and {@link HueClient.verifyTrace} when Hue cannot be
|
|
18
|
+
* reached, rejects the project key or answers unexpectedly. The message is fixed and safe to log;
|
|
19
|
+
* the underlying network, timeout or parsing error, when there is one, is available as `cause`.
|
|
20
|
+
*/
|
|
9
21
|
export declare class HueConnectionError extends Error {
|
|
22
|
+
/** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
|
|
10
23
|
readonly status?: number | undefined;
|
|
11
|
-
constructor(message: string,
|
|
24
|
+
constructor(message: string,
|
|
25
|
+
/** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
|
|
26
|
+
status?: number | undefined, options?: {
|
|
27
|
+
cause?: unknown;
|
|
28
|
+
});
|
|
12
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* Hue tracing client. Helpers create spans through a private tracer and local async context, never
|
|
32
|
+
* through global OpenTelemetry registration, and they fail open: capture problems are counted in
|
|
33
|
+
* the export report while application code runs and returns unchanged.
|
|
34
|
+
*/
|
|
13
35
|
export declare class HueClient {
|
|
36
|
+
/** Export pipeline: counters, issue history and the processors that feed Hue. */
|
|
14
37
|
readonly transport: HueTransport;
|
|
38
|
+
/** Hue's tracer for other instrumentations; spans parent under `withSpan` and inherit identifiers. */
|
|
15
39
|
readonly tracer: Tracer;
|
|
40
|
+
/** Whether helpers record content, as configured. */
|
|
16
41
|
readonly captureContent: boolean;
|
|
42
|
+
/** False for `enabled: false` clients and for `createHueSafe` fallbacks; helpers then only run callbacks. */
|
|
17
43
|
readonly enabled: boolean;
|
|
18
44
|
private logger;
|
|
19
45
|
private storage;
|
|
@@ -26,26 +52,115 @@ export declare class HueClient {
|
|
|
26
52
|
private safeFlushPromise?;
|
|
27
53
|
private safeShutdownPromise?;
|
|
28
54
|
constructor(options: HueOptions | ExistingHueProviders);
|
|
55
|
+
/**
|
|
56
|
+
* Verifies that Hue stored a trace by ID, optionally waiting for expected span IDs and normalized
|
|
57
|
+
* fields within the budget. Read-only; it does not flush or inspect content.
|
|
58
|
+
*
|
|
59
|
+
* @throws HueConnectionError when the client is disabled.
|
|
60
|
+
* @throws HueTraceVerificationError for authentication, unsupported endpoint, transport or invalid response failures.
|
|
61
|
+
* @throws TypeError for an invalid trace ID or options.
|
|
62
|
+
*/
|
|
29
63
|
verifyTrace(traceId: string, options?: VerifyTraceOptions): Promise<TraceVerification>;
|
|
64
|
+
/** The helper's current context: the innermost active Hue span, else the OpenTelemetry active context. */
|
|
30
65
|
getContext(): Context;
|
|
66
|
+
/**
|
|
67
|
+
* Runs `callback` inside a new span that nests under the active Hue span. Application errors are
|
|
68
|
+
* recorded on the span and rethrown unchanged; the span always ends when the callback settles.
|
|
69
|
+
*/
|
|
31
70
|
withSpan<T>(name: string, callback: (span: HueSpan) => Promise<T> | T, options?: SpanOptions): Promise<T>;
|
|
32
|
-
|
|
71
|
+
/**
|
|
72
|
+
* Runs `execute` inside an `execute_tool` span named after the tool. `input` and a defined result
|
|
73
|
+
* are recorded as `gen_ai.tool.call.arguments` / `gen_ai.tool.call.result` when `captureContent`
|
|
74
|
+
* is true; values that are not JSON-encodable are omitted with an instrumentation failure.
|
|
75
|
+
* `options.callId` is recorded as `gen_ai.tool.call.id`, like the Python `call_id=` keyword.
|
|
76
|
+
*/
|
|
77
|
+
tool<T>(name: string, input: unknown, execute: () => Promise<T> | T, options?: Pick<SpanOptions, "parentContext"> & {
|
|
78
|
+
/** Provider-issued identifier of this tool call, recorded as `gen_ai.tool.call.id`. */
|
|
79
|
+
callId?: string;
|
|
80
|
+
}): Promise<T>;
|
|
81
|
+
/**
|
|
82
|
+
* Runs `callback` inside a GenAI client span for one direct provider call, named
|
|
83
|
+
* `{operation} {model}` unless `options.name` is given and carrying `gen_ai.operation.name`,
|
|
84
|
+
* `gen_ai.request.model` and `gen_ai.provider.name`. The argument order matches `withSpan`. The
|
|
85
|
+
* handle's `setInput`/`setOutput` record `gen_ai.input.messages` / `gen_ai.output.messages`,
|
|
86
|
+
* which should use the GenAI semantic-convention message shape; `recordMessages` inside the
|
|
87
|
+
* callback inherits the request metadata.
|
|
88
|
+
*/
|
|
89
|
+
model<T>(model: string, callback: (span: HueSpan) => Promise<T> | T, options: ModelOptions): Promise<T>;
|
|
90
|
+
private setUsage;
|
|
91
|
+
/**
|
|
92
|
+
* Writes W3C `traceparent` for the active span into a carrier; never the API key or baggage.
|
|
93
|
+
* Propagation also runs for a disabled or closed client so downstream tracing stays connected.
|
|
94
|
+
*/
|
|
95
|
+
inject(carrier: Record<string, string>, activeContext?: Context): void;
|
|
96
|
+
/** Reads W3C trace context from a carrier for use as `parentContext`. */
|
|
97
|
+
extract(carrier: Record<string, string | string[] | undefined>): Context;
|
|
98
|
+
/**
|
|
99
|
+
* Marks a span failed the same way in every helper: `error.type`, an ERROR status without a
|
|
100
|
+
* description and an `exception` event carrying only the type. Exception messages and stack
|
|
101
|
+
* traces are never recorded, whatever `captureContent` is, matching the Python SDK.
|
|
102
|
+
*/
|
|
33
103
|
recordError(span: Span, error: unknown): void;
|
|
104
|
+
/**
|
|
105
|
+
* Emits a `gen_ai.client.inference.operation.details` log record correlated with the active (or
|
|
106
|
+
* given) span, carrying the messages in its body. `gen_ai.operation.name`, `gen_ai.provider.name`
|
|
107
|
+
* and `gen_ai.request.model` are set as record attributes from the caller's values or the
|
|
108
|
+
* enclosing {@link model} span, and `gen_ai.conversation.id` from the active session. Nothing is
|
|
109
|
+
* emitted when `captureContent` is false; capture failures are counted, never thrown.
|
|
110
|
+
*/
|
|
34
111
|
recordMessages(messages: {
|
|
35
|
-
|
|
36
|
-
|
|
112
|
+
/** Input messages, ideally in the GenAI semantic-convention shape; any JSON-encodable value. */
|
|
113
|
+
input?: unknown;
|
|
114
|
+
/** Output messages, ideally in the GenAI semantic-convention shape; any JSON-encodable value. */
|
|
115
|
+
output?: unknown;
|
|
116
|
+
/** `gen_ai.operation.name` for the record; defaults to the enclosing `model()` span's value. */
|
|
117
|
+
operation?: string;
|
|
118
|
+
/** `gen_ai.provider.name` for the record; defaults to the enclosing `model()` span's value. */
|
|
119
|
+
provider?: string;
|
|
120
|
+
/** `gen_ai.request.model` for the record; defaults to the enclosing `model()` span's value. */
|
|
121
|
+
model?: string;
|
|
37
122
|
}, explicitContext?: Context): void;
|
|
38
123
|
private setContent;
|
|
124
|
+
/**
|
|
125
|
+
* Confirms the key and origin by reading the current project; a setup and CI diagnostic, not a
|
|
126
|
+
* readiness gate. Redirects are refused and the response is bounded.
|
|
127
|
+
*
|
|
128
|
+
* @throws HueConnectionError when the client is disabled, Hue is unreachable (the network or
|
|
129
|
+
* timeout error is the `cause`), the key is rejected (`status` is set) or the response is invalid.
|
|
130
|
+
*/
|
|
39
131
|
checkConnection(): Promise<ProjectConnection>;
|
|
40
132
|
/** Production lifecycle path: never rejects; timeouts do not cancel borrowed provider work. */
|
|
41
133
|
flushSafe(options?: SafeLifecycleOptions): Promise<SafeLifecycleResult>;
|
|
42
134
|
/** Safe for finally blocks; preserves the application's result or original exception. */
|
|
43
135
|
shutdownSafe(options?: SafeLifecycleOptions): Promise<SafeLifecycleResult>;
|
|
44
136
|
private safeLifecycle;
|
|
137
|
+
/**
|
|
138
|
+
* Drains the trace and log providers and waits for Hue's acknowledgements. Each caller gets a
|
|
139
|
+
* fresh serialized drain that includes records emitted before its call.
|
|
140
|
+
*
|
|
141
|
+
* @throws HueExportError when this drain observed a new rejection, delivery failure, drop or
|
|
142
|
+
* invalid record; its `issues` and `report` are sanitized counts, never server text.
|
|
143
|
+
*/
|
|
45
144
|
flush(): Promise<ExportReport>;
|
|
46
145
|
private flushOnce;
|
|
146
|
+
/**
|
|
147
|
+
* Flushes, then shuts down the providers this client owns; borrowed providers are left running.
|
|
148
|
+
* Idempotent: later calls return the same promise, and later helper calls only run their callbacks.
|
|
149
|
+
*
|
|
150
|
+
* @throws HueExportError when the final flush observed new failures; owned providers are still released.
|
|
151
|
+
*/
|
|
47
152
|
shutdown(): Promise<ExportReport>;
|
|
48
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* Creates a client that owns its providers ({@link HueOptions}) or attaches to the application's
|
|
156
|
+
* ({@link ExistingHueProviders}). Strict: use it for setup and CI, `createHueSafe` in serving code.
|
|
157
|
+
*
|
|
158
|
+
* @throws TypeError for invalid options, including a missing `captureContent` choice, an invalid
|
|
159
|
+
* key or `serviceName`, a non-origin or insecure `baseUrl`, or out-of-range budgets.
|
|
160
|
+
*/
|
|
49
161
|
export declare function createHue(options: HueOptions | ExistingHueProviders): HueClient;
|
|
50
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* Fail-open initialization for production: invalid options return a disabled client with one
|
|
164
|
+
* instrumentation failure recorded instead of throwing. Strict {@link createHue} remains for setup/CI.
|
|
165
|
+
*/
|
|
51
166
|
export declare function createHueSafe(options: HueOptions | ExistingHueProviders): HueClient;
|