netra-sdk 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +209 -0
- package/dist/index.cjs +9300 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +1243 -0
- package/dist/index.d.ts +1243 -0
- package/dist/index.js +9238 -0
- package/dist/index.js.map +1 -0
- package/package.json +105 -0
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,1243 @@
|
|
|
1
|
+
import { Span, Context, TracerProvider } from '@opentelemetry/api';
|
|
2
|
+
import { SpanProcessor, ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base';
|
|
3
|
+
import { ExportResultCode, ExportResult } from '@opentelemetry/core';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Configuration management for Netra SDK
|
|
7
|
+
*/
|
|
8
|
+
interface NetraConfig {
|
|
9
|
+
appName?: string;
|
|
10
|
+
headers?: string | Record<string, string>;
|
|
11
|
+
disableBatch?: boolean;
|
|
12
|
+
traceContent?: boolean;
|
|
13
|
+
debugMode?: boolean;
|
|
14
|
+
enableRootSpan?: boolean;
|
|
15
|
+
resourceAttributes?: Record<string, any>;
|
|
16
|
+
environment?: string;
|
|
17
|
+
enableScrubbing?: boolean;
|
|
18
|
+
blockedSpans?: string[];
|
|
19
|
+
instruments?: Set<NetraInstruments>;
|
|
20
|
+
blockInstruments?: Set<NetraInstruments>;
|
|
21
|
+
}
|
|
22
|
+
declare enum NetraInstruments {
|
|
23
|
+
OPENAI = "openai",
|
|
24
|
+
GOOGLE_GENERATIVE_AI = "google_genai",
|
|
25
|
+
MISTRAL = "mistral",
|
|
26
|
+
GROQ = "groq",
|
|
27
|
+
VERTEX_AI = "vertexai",
|
|
28
|
+
TOGETHER = "together",
|
|
29
|
+
ANTHROPIC = "anthropic",
|
|
30
|
+
LANGCHAIN = "langchain",
|
|
31
|
+
LANGGRAPH = "langgraph",
|
|
32
|
+
LLAMA_INDEX = "llama_index",
|
|
33
|
+
PINECONE = "pinecone",
|
|
34
|
+
QDRANT = "qdrant",
|
|
35
|
+
CHROMADB = "chromadb",
|
|
36
|
+
HTTP = "http",
|
|
37
|
+
HTTPS = "https",
|
|
38
|
+
FETCH = "fetch",
|
|
39
|
+
PRISMA = "prisma",
|
|
40
|
+
TYPEORM = "typeorm",
|
|
41
|
+
MONGODB = "mongodb",
|
|
42
|
+
POSTGRES = "postgres",
|
|
43
|
+
MYSQL = "mysql",
|
|
44
|
+
REDIS = "redis",
|
|
45
|
+
EXPRESS = "express",
|
|
46
|
+
FASTIFY = "fastify",
|
|
47
|
+
NESTJS = "nestjs",
|
|
48
|
+
KAFKA = "kafka",
|
|
49
|
+
RABBITMQ = "rabbitmq"
|
|
50
|
+
}
|
|
51
|
+
declare class Config {
|
|
52
|
+
static readonly SDK_NAME = "netra";
|
|
53
|
+
static readonly LIBRARY_NAME = "netra";
|
|
54
|
+
static readonly LIBRARY_VERSION = "1.0.0";
|
|
55
|
+
static readonly TRIAL_BLOCK_DURATION_SECONDS = 900;
|
|
56
|
+
static readonly ATTRIBUTE_MAX_LEN: number;
|
|
57
|
+
static readonly CONVERSATION_MAX_LEN: number;
|
|
58
|
+
appName: string;
|
|
59
|
+
otlpEndpoint?: string;
|
|
60
|
+
apiKey?: string;
|
|
61
|
+
headers: Record<string, string>;
|
|
62
|
+
disableBatch: boolean;
|
|
63
|
+
traceContent: boolean;
|
|
64
|
+
debugMode: boolean;
|
|
65
|
+
enableRootSpan: boolean;
|
|
66
|
+
enableScrubbing: boolean;
|
|
67
|
+
environment: string;
|
|
68
|
+
resourceAttributes: Record<string, any>;
|
|
69
|
+
blockedSpans?: string[];
|
|
70
|
+
constructor(config?: NetraConfig);
|
|
71
|
+
private _getAppName;
|
|
72
|
+
private _getOtlpEndpoint;
|
|
73
|
+
private _parseHeaders;
|
|
74
|
+
private _parseEnvHeaders;
|
|
75
|
+
private _validateApiKey;
|
|
76
|
+
private _setupAuthentication;
|
|
77
|
+
private _getBoolConfig;
|
|
78
|
+
private _getResourceAttributes;
|
|
79
|
+
private _setTraceContentEnv;
|
|
80
|
+
/**
|
|
81
|
+
* Format the OTLP endpoint URL by appending /v1/traces if not already present
|
|
82
|
+
*/
|
|
83
|
+
formatOtlpEndpoint(): any;
|
|
84
|
+
/**
|
|
85
|
+
* Set Traceloop environment variables based on Netra config.
|
|
86
|
+
* This ensures the Traceloop SDK picks up our configuration.
|
|
87
|
+
*/
|
|
88
|
+
setTraceloopEnv(): void;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Dashboard API Models
|
|
93
|
+
*/
|
|
94
|
+
declare enum Scope {
|
|
95
|
+
SPANS = "Spans",
|
|
96
|
+
TRACES = "Traces"
|
|
97
|
+
}
|
|
98
|
+
declare enum ChartType {
|
|
99
|
+
LINE_TIME_SERIES = "Line Time Series",
|
|
100
|
+
BAR_TIME_SERIES = "Bar Time Series",
|
|
101
|
+
HORIZONTAL_BAR = "Horizontal Bar",
|
|
102
|
+
VERTICAL_BAR = "Vertical Bar",
|
|
103
|
+
PIE = "Pie",
|
|
104
|
+
NUMBER = "Number"
|
|
105
|
+
}
|
|
106
|
+
declare enum Measure {
|
|
107
|
+
LATENCY = "Latency",
|
|
108
|
+
ERROR_RATE = "Error Rate",
|
|
109
|
+
PII_COUNT = "PII Count",
|
|
110
|
+
REQUEST_COUNT = "Request Count",
|
|
111
|
+
TOTAL_COST = "Total Cost",
|
|
112
|
+
VIOLATIONS = "Violations",
|
|
113
|
+
TOTAL_TOKENS = "Total Tokens"
|
|
114
|
+
}
|
|
115
|
+
declare enum Aggregation {
|
|
116
|
+
AVERAGE = "Average",
|
|
117
|
+
P50 = "p50",
|
|
118
|
+
P90 = "p90",
|
|
119
|
+
P95 = "p95",
|
|
120
|
+
P99 = "p99",
|
|
121
|
+
MEDIAN = "Median (p50)",
|
|
122
|
+
PERCENTAGE = "Percentage",
|
|
123
|
+
TOTAL_COUNT = "Total Count"
|
|
124
|
+
}
|
|
125
|
+
declare enum GroupBy {
|
|
126
|
+
DAY = "day",
|
|
127
|
+
HOUR = "hour",
|
|
128
|
+
MINUTE = "minute"
|
|
129
|
+
}
|
|
130
|
+
declare enum DimensionField {
|
|
131
|
+
ENVIRONMENT = "environment",
|
|
132
|
+
SERVICE = "service",
|
|
133
|
+
MODEL_NAME = "model_name"
|
|
134
|
+
}
|
|
135
|
+
declare enum Operator {
|
|
136
|
+
EQUALS = "equals",
|
|
137
|
+
NOT_EQUALS = "not_equals",
|
|
138
|
+
CONTAINS = "contains",
|
|
139
|
+
NOT_CONTAINS = "not_contains",
|
|
140
|
+
STARTS_WITH = "starts_with",
|
|
141
|
+
ENDS_WITH = "ends_with",
|
|
142
|
+
GREATER_THAN = "greater_than",
|
|
143
|
+
LESS_THAN = "less_than",
|
|
144
|
+
GREATER_EQUAL_TO = "greater_equal_to",
|
|
145
|
+
LESS_EQUAL_TO = "less_equal_to",
|
|
146
|
+
ANY_OF = "any_of",
|
|
147
|
+
NONE_OF = "none_of"
|
|
148
|
+
}
|
|
149
|
+
declare enum FilterType {
|
|
150
|
+
STRING = "string",
|
|
151
|
+
NUMBER = "number",
|
|
152
|
+
BOOLEAN = "boolean",
|
|
153
|
+
ARRAY_OPTIONS = "arrayOptions",
|
|
154
|
+
OBJECT = "object"
|
|
155
|
+
}
|
|
156
|
+
declare enum FilterField {
|
|
157
|
+
TOTAL_COST = "total_cost",
|
|
158
|
+
SERVICE = "service",
|
|
159
|
+
TENANT_ID = "tenant_id",
|
|
160
|
+
USER_ID = "user_id",
|
|
161
|
+
SESSION_ID = "session_id",
|
|
162
|
+
ENVIRONMENT = "environment",
|
|
163
|
+
LATENCY = "latency",
|
|
164
|
+
MODEL_NAME = "model_name",
|
|
165
|
+
MODELS = "models",
|
|
166
|
+
METADATA = "metadata"
|
|
167
|
+
}
|
|
168
|
+
declare enum SessionFilterField {
|
|
169
|
+
TENANT_ID = "tenant_id"
|
|
170
|
+
}
|
|
171
|
+
declare enum SessionOperator {
|
|
172
|
+
ANY_OF = "any_of"
|
|
173
|
+
}
|
|
174
|
+
declare enum SessionFilterType {
|
|
175
|
+
ARRAY = "arrayOptions"
|
|
176
|
+
}
|
|
177
|
+
declare enum SortField {
|
|
178
|
+
SESSION_ID = "session_id",
|
|
179
|
+
START_TIME = "start_time",
|
|
180
|
+
TOTAL_REQUESTS = "totalRequests",
|
|
181
|
+
TOTAL_COST = "totalCost"
|
|
182
|
+
}
|
|
183
|
+
declare enum SortOrder {
|
|
184
|
+
ASC = "asc",
|
|
185
|
+
DESC = "desc"
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Create a metadata filter field
|
|
189
|
+
* @param key The metadata key to filter on
|
|
190
|
+
* @returns The formatted metadata field string
|
|
191
|
+
* @example
|
|
192
|
+
* metadataField("customer_tier") // returns "metadata['customer_tier']"
|
|
193
|
+
*/
|
|
194
|
+
declare function metadataField(key: string): string;
|
|
195
|
+
interface Filter {
|
|
196
|
+
field: FilterField | string;
|
|
197
|
+
operator: Operator;
|
|
198
|
+
type: FilterType;
|
|
199
|
+
value: any;
|
|
200
|
+
key?: string;
|
|
201
|
+
}
|
|
202
|
+
interface Metrics {
|
|
203
|
+
measure: Measure;
|
|
204
|
+
aggregation: Aggregation;
|
|
205
|
+
}
|
|
206
|
+
interface Dimension {
|
|
207
|
+
field: DimensionField;
|
|
208
|
+
}
|
|
209
|
+
interface FilterConfig {
|
|
210
|
+
startTime: string;
|
|
211
|
+
endTime: string;
|
|
212
|
+
groupBy: GroupBy;
|
|
213
|
+
filters?: Filter[];
|
|
214
|
+
}
|
|
215
|
+
interface TimeRange {
|
|
216
|
+
startTime: string;
|
|
217
|
+
endTime: string;
|
|
218
|
+
}
|
|
219
|
+
interface TimeSeriesDataPoint {
|
|
220
|
+
date: string;
|
|
221
|
+
value: number;
|
|
222
|
+
}
|
|
223
|
+
interface DimensionValue {
|
|
224
|
+
dimension: string;
|
|
225
|
+
value: number;
|
|
226
|
+
}
|
|
227
|
+
interface TimeSeriesWithDimension {
|
|
228
|
+
date: string;
|
|
229
|
+
values: DimensionValue[];
|
|
230
|
+
}
|
|
231
|
+
interface TimeSeriesResponse {
|
|
232
|
+
timeSeries: TimeSeriesWithDimension[];
|
|
233
|
+
dimensions: string[];
|
|
234
|
+
}
|
|
235
|
+
interface CategoricalDataPoint {
|
|
236
|
+
dimension: string;
|
|
237
|
+
value: number;
|
|
238
|
+
}
|
|
239
|
+
interface NumberResponse {
|
|
240
|
+
value: number;
|
|
241
|
+
}
|
|
242
|
+
type DashboardData = TimeSeriesDataPoint[] | TimeSeriesResponse | CategoricalDataPoint[] | NumberResponse | Record<string, any>;
|
|
243
|
+
interface QueryResponse {
|
|
244
|
+
timeRange: TimeRange;
|
|
245
|
+
data: DashboardData;
|
|
246
|
+
}
|
|
247
|
+
interface QueryDataParams {
|
|
248
|
+
scope: Scope;
|
|
249
|
+
chartType: ChartType;
|
|
250
|
+
metrics: Metrics;
|
|
251
|
+
filter: FilterConfig;
|
|
252
|
+
dimension?: Dimension;
|
|
253
|
+
}
|
|
254
|
+
interface SessionFilter {
|
|
255
|
+
field: SessionFilterField;
|
|
256
|
+
operator: SessionOperator;
|
|
257
|
+
type: SessionFilterType;
|
|
258
|
+
value: string[];
|
|
259
|
+
}
|
|
260
|
+
interface SessionStatsResult {
|
|
261
|
+
data: SessionStatsData[];
|
|
262
|
+
hasNextPage: boolean;
|
|
263
|
+
nextCursor?: string;
|
|
264
|
+
}
|
|
265
|
+
interface SessionFilterConfig {
|
|
266
|
+
startTime: string;
|
|
267
|
+
endTime: string;
|
|
268
|
+
filters?: SessionFilter[];
|
|
269
|
+
}
|
|
270
|
+
interface SessionStatsData {
|
|
271
|
+
session_id: string;
|
|
272
|
+
session_start_time: string;
|
|
273
|
+
totalRequests: number;
|
|
274
|
+
totalCost: number;
|
|
275
|
+
session_duration: string;
|
|
276
|
+
cursor: string;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Public Dashboard API
|
|
281
|
+
* Exposed as Netra.dashboard
|
|
282
|
+
*/
|
|
283
|
+
|
|
284
|
+
declare class Dashboard {
|
|
285
|
+
private config;
|
|
286
|
+
private client;
|
|
287
|
+
constructor(config: Config);
|
|
288
|
+
/**
|
|
289
|
+
* Execute a dynamic query for dashboards to retrieve metrics and time-series data
|
|
290
|
+
* @param params Query parameters
|
|
291
|
+
* @returns Query response containing timeRange and data, or null on error
|
|
292
|
+
*/
|
|
293
|
+
queryData(params: QueryDataParams): Promise<QueryResponse | null>;
|
|
294
|
+
getSessionStats(startTime: string, endTime: string, limit?: number, filters?: SessionFilter[], cursor?: string, sortField?: SortField, sortOrder?: SortOrder): Promise<SessionStatsResult | null>;
|
|
295
|
+
iterSessionStats(startTime: string, endTime: string, filters?: SessionFilter[], sortField?: SortField, sortOrder?: SortOrder): AsyncGenerator<SessionStatsData, void, unknown>;
|
|
296
|
+
getSessionSummary(filter: SessionFilterConfig): Promise<any | null>;
|
|
297
|
+
private isValidScope;
|
|
298
|
+
private isValidChartType;
|
|
299
|
+
private isValidMetrics;
|
|
300
|
+
private isValidFilterConfig;
|
|
301
|
+
private isValidDimension;
|
|
302
|
+
private isSessionFilterConfig;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Evaluation API Models
|
|
307
|
+
*/
|
|
308
|
+
interface DatasetEntry {
|
|
309
|
+
id: string;
|
|
310
|
+
input: string;
|
|
311
|
+
datasetId: string;
|
|
312
|
+
expectedOutput?: any;
|
|
313
|
+
}
|
|
314
|
+
interface DatasetItem {
|
|
315
|
+
input: any;
|
|
316
|
+
expectedOutput?: any;
|
|
317
|
+
metadata?: Record<string, any>;
|
|
318
|
+
tags?: string[];
|
|
319
|
+
}
|
|
320
|
+
interface DatasetRecord {
|
|
321
|
+
id: string;
|
|
322
|
+
input: any;
|
|
323
|
+
datasetId: string;
|
|
324
|
+
expectedOutput?: any;
|
|
325
|
+
}
|
|
326
|
+
interface Dataset {
|
|
327
|
+
items: DatasetItem[] | DatasetRecord[];
|
|
328
|
+
}
|
|
329
|
+
interface Run {
|
|
330
|
+
id: string;
|
|
331
|
+
datasetId: string;
|
|
332
|
+
name?: string;
|
|
333
|
+
testEntries: DatasetItem[];
|
|
334
|
+
}
|
|
335
|
+
interface EvaluationScore {
|
|
336
|
+
metricType: string;
|
|
337
|
+
score: number;
|
|
338
|
+
}
|
|
339
|
+
declare enum EntryStatus {
|
|
340
|
+
AGENT_TRIGGERED = "agent_triggered",
|
|
341
|
+
AGENT_COMPLETED = "agent_completed",
|
|
342
|
+
FAILED = "failed"
|
|
343
|
+
}
|
|
344
|
+
declare enum RunStatus {
|
|
345
|
+
COMPLETED = "completed"
|
|
346
|
+
}
|
|
347
|
+
interface CreateDatasetParams {
|
|
348
|
+
name: string;
|
|
349
|
+
tags?: string[];
|
|
350
|
+
}
|
|
351
|
+
interface TestSuiteResult {
|
|
352
|
+
success: boolean;
|
|
353
|
+
runId?: string;
|
|
354
|
+
error?: string;
|
|
355
|
+
results?: Array<{
|
|
356
|
+
id: string;
|
|
357
|
+
input: string;
|
|
358
|
+
output: any;
|
|
359
|
+
status: string;
|
|
360
|
+
error?: string;
|
|
361
|
+
}>;
|
|
362
|
+
}
|
|
363
|
+
declare enum ScoreType {
|
|
364
|
+
BOOLEAN = "boolean",
|
|
365
|
+
NUMERICAL = "numerical",
|
|
366
|
+
CATEGORICAL = "categorical"
|
|
367
|
+
}
|
|
368
|
+
interface EvaluatorConfig {
|
|
369
|
+
name: string;
|
|
370
|
+
label: string;
|
|
371
|
+
scoreType: ScoreType;
|
|
372
|
+
}
|
|
373
|
+
interface AddDatasetItemResponse {
|
|
374
|
+
datasetId: string;
|
|
375
|
+
projectId: string;
|
|
376
|
+
organizationId: string;
|
|
377
|
+
source: string;
|
|
378
|
+
input: any;
|
|
379
|
+
expectedOutput?: any;
|
|
380
|
+
isActive: boolean;
|
|
381
|
+
tags?: string[];
|
|
382
|
+
createdBy: string;
|
|
383
|
+
updatedBy: string;
|
|
384
|
+
updatedAt: string;
|
|
385
|
+
sourceId?: string;
|
|
386
|
+
metadata?: Record<string, any>;
|
|
387
|
+
id: string;
|
|
388
|
+
createdAt?: string;
|
|
389
|
+
deletedAt?: string;
|
|
390
|
+
}
|
|
391
|
+
interface CreateDatasetResponse {
|
|
392
|
+
projectId: string;
|
|
393
|
+
organizationId: string;
|
|
394
|
+
name: string;
|
|
395
|
+
tags?: string[];
|
|
396
|
+
createdBy: string;
|
|
397
|
+
updatedBy: string;
|
|
398
|
+
updatedAt: string;
|
|
399
|
+
id: string;
|
|
400
|
+
createdAt: string;
|
|
401
|
+
deletedAt?: string | null;
|
|
402
|
+
}
|
|
403
|
+
interface GetDatasetItemsResponse {
|
|
404
|
+
items: DatasetRecord[];
|
|
405
|
+
}
|
|
406
|
+
type EvaluatorFunction = (params: {
|
|
407
|
+
input: string;
|
|
408
|
+
output: any;
|
|
409
|
+
expectedOutput?: any;
|
|
410
|
+
}) => EvaluationScore | EvaluationScore[] | Promise<EvaluationScore | EvaluationScore[]>;
|
|
411
|
+
type TaskFunction = (input: any) => any | Promise<any>;
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* Public Evaluation API
|
|
415
|
+
* Exposed as Netra.evaluation
|
|
416
|
+
*/
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Public entry-point exposed as Netra.evaluation
|
|
420
|
+
*/
|
|
421
|
+
declare class Evaluation {
|
|
422
|
+
private config;
|
|
423
|
+
private client;
|
|
424
|
+
constructor(config: Config);
|
|
425
|
+
/**
|
|
426
|
+
* Create an empty dataset and return its id on success, else null
|
|
427
|
+
* @param name The name of the dataset
|
|
428
|
+
* @param tags Optional list of tags to associate with the dataset
|
|
429
|
+
* @returns A backend JSON response containing dataset info (id, name, tags, etc.) on success
|
|
430
|
+
*/
|
|
431
|
+
createDataset(name: string, tags?: string[]): Promise<CreateDatasetResponse | null>;
|
|
432
|
+
/**
|
|
433
|
+
* Add a single item to an existing dataset
|
|
434
|
+
* @param datasetId The id of the dataset to which the item will be added
|
|
435
|
+
* @param item The dataset item to add
|
|
436
|
+
* @returns A backend JSON response containing dataset item info (id, input, expectedOutput, etc.) on success
|
|
437
|
+
*/
|
|
438
|
+
addDatasetItem(datasetId: string, item: DatasetEntry): Promise<AddDatasetItemResponse | null>;
|
|
439
|
+
/**
|
|
440
|
+
* Get a dataset by ID
|
|
441
|
+
* @param datasetId The id of the dataset to retrieve
|
|
442
|
+
* @returns A backend JSON response containing dataset info (id, input, expectedOutput etc.) on success
|
|
443
|
+
*/
|
|
444
|
+
getDataset(datasetId: string): Promise<GetDatasetItemsResponse | null>;
|
|
445
|
+
/**
|
|
446
|
+
* Create a new run for the given dataset and evaluators
|
|
447
|
+
* @param name The name of the run
|
|
448
|
+
* @param datasetId The id of the dataset to which the run will be associated
|
|
449
|
+
* @param evaluatorsConfig Optional list of evaluators to be used for the run
|
|
450
|
+
* @returns runId: The id of the created run
|
|
451
|
+
*/
|
|
452
|
+
createRun(name: string, datasetId?: string, evaluatorsConfig?: EvaluatorConfig[]): Promise<string | null>;
|
|
453
|
+
/**
|
|
454
|
+
* Netra evaluation function to initiate a test suite
|
|
455
|
+
* @param name The name of the run
|
|
456
|
+
* @param data The dataset to be used for the test suite
|
|
457
|
+
* @param task The task to be executed for each item in the dataset
|
|
458
|
+
* @param evaluators Optional list of evaluators to be used for the test suite
|
|
459
|
+
* @param maxConcurrency The maximum number of concurrent tasks to be executed
|
|
460
|
+
* @returns A dictionary containing the run id and the results of the test suite
|
|
461
|
+
*/
|
|
462
|
+
runTestSuite(name: string, data: Dataset, task: TaskFunction, evaluators?: any[], maxConcurrency?: number): Promise<Record<string, any> | null>;
|
|
463
|
+
/**
|
|
464
|
+
* Async implementation of runTestSuite
|
|
465
|
+
* @param name The name of the run
|
|
466
|
+
* @param data The dataset to be used for the test suite
|
|
467
|
+
* @param task The task to be executed for each item in the dataset
|
|
468
|
+
* @param evaluators Optional list of evaluators to be used for the test suite
|
|
469
|
+
* @param maxConcurrency The maximum number of concurrent tasks to be executed
|
|
470
|
+
* @returns items: The results of the test suite
|
|
471
|
+
*/
|
|
472
|
+
private runTestSuiteAsync;
|
|
473
|
+
/**
|
|
474
|
+
* Create an ItemContext from a dataset item
|
|
475
|
+
* @param idx The index of the item
|
|
476
|
+
* @param item The dataset item
|
|
477
|
+
* @returns ItemContext: The created ItemContext
|
|
478
|
+
*/
|
|
479
|
+
private createItemContext;
|
|
480
|
+
/**
|
|
481
|
+
* Type guard for DatasetRecord
|
|
482
|
+
*/
|
|
483
|
+
private isDatasetRecord;
|
|
484
|
+
/**
|
|
485
|
+
* Execute the full pipeline for a single item
|
|
486
|
+
* @param runId The run ID
|
|
487
|
+
* @param runName The name of the run
|
|
488
|
+
* @param ctx The item context
|
|
489
|
+
* @param task The task function to execute
|
|
490
|
+
* @param evaluators Optional list of evaluators
|
|
491
|
+
* @param results List to append results to
|
|
492
|
+
* @param bgEvalTasks List to append background evaluation tasks to
|
|
493
|
+
*/
|
|
494
|
+
private executeItemPipeline;
|
|
495
|
+
/**
|
|
496
|
+
* Post agent_triggered status and return testRunItemId
|
|
497
|
+
* @param runId The run ID
|
|
498
|
+
* @param ctx The item context
|
|
499
|
+
* @returns str: The testRunItemId
|
|
500
|
+
*/
|
|
501
|
+
private postTriggeredStatus;
|
|
502
|
+
/**
|
|
503
|
+
* Post completed/failed status with task output
|
|
504
|
+
* @param runId The run ID
|
|
505
|
+
* @param ctx The item context
|
|
506
|
+
*/
|
|
507
|
+
private postCompletedStatus;
|
|
508
|
+
/**
|
|
509
|
+
* Run all evaluators for a single item after span ingestion
|
|
510
|
+
* @param runId The run ID
|
|
511
|
+
* @param ctx The item context
|
|
512
|
+
* @param evaluators List of evaluators
|
|
513
|
+
*/
|
|
514
|
+
private runEvaluatorsForItem;
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Base HTTP client for Netra API calls
|
|
519
|
+
*/
|
|
520
|
+
|
|
521
|
+
interface HttpResponse<T = any> {
|
|
522
|
+
data: T;
|
|
523
|
+
status: number;
|
|
524
|
+
ok: boolean;
|
|
525
|
+
}
|
|
526
|
+
declare class NetraHttpClient {
|
|
527
|
+
private baseUrl;
|
|
528
|
+
private headers;
|
|
529
|
+
private timeout;
|
|
530
|
+
private initialized;
|
|
531
|
+
constructor(config: Config, timeoutEnvVar: string, defaultTimeout: number);
|
|
532
|
+
private resolveBaseUrl;
|
|
533
|
+
private buildHeaders;
|
|
534
|
+
private getTimeout;
|
|
535
|
+
isInitialized(): boolean;
|
|
536
|
+
get<T = any>(path: string, params?: Record<string, string | number | undefined>): Promise<HttpResponse<T>>;
|
|
537
|
+
post<T = any>(path: string, body?: Record<string, any>): Promise<HttpResponse<T>>;
|
|
538
|
+
/**
|
|
539
|
+
* Extract data from nested response format
|
|
540
|
+
* Netra API typically returns { data: { data: actualData } }
|
|
541
|
+
*/
|
|
542
|
+
extractData<T>(response: HttpResponse<any>, defaultValue: T): T;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Internal HTTP client for Evaluation APIs
|
|
547
|
+
*/
|
|
548
|
+
|
|
549
|
+
declare class EvaluationHttpClient extends NetraHttpClient {
|
|
550
|
+
constructor(config: Config);
|
|
551
|
+
/**
|
|
552
|
+
* Fetch dataset items for a dataset id
|
|
553
|
+
*/
|
|
554
|
+
getDataset(datasetId: string): Promise<any[]>;
|
|
555
|
+
/**
|
|
556
|
+
* Create a run for a dataset
|
|
557
|
+
*/
|
|
558
|
+
createRun(name: string, datasetId?: string, evaluatorsConfig?: Record<string, any>[]): Promise<any>;
|
|
559
|
+
/**
|
|
560
|
+
* Create an empty dataset and return backend data
|
|
561
|
+
*/
|
|
562
|
+
createDataset(name: string, tags?: string[]): Promise<any>;
|
|
563
|
+
/**
|
|
564
|
+
* Add a single item to an existing dataset
|
|
565
|
+
*/
|
|
566
|
+
addDatasetItem(datasetId: string, itemPayload: Record<string, any>): Promise<any>;
|
|
567
|
+
/**
|
|
568
|
+
* Submit a new run item to the backend
|
|
569
|
+
*/
|
|
570
|
+
postRunItem(runId: string, payload: Record<string, any>): Promise<any>;
|
|
571
|
+
/**
|
|
572
|
+
* Submit local evaluations result
|
|
573
|
+
*/
|
|
574
|
+
submitLocalEvaluations(runId: string, testRunItemId: string, evaluatorResults: Array<Record<string, any>>): Promise<any>;
|
|
575
|
+
/**
|
|
576
|
+
* Submit the run status
|
|
577
|
+
*/
|
|
578
|
+
postRunStatus(runId: string, status: string): Promise<any>;
|
|
579
|
+
/**
|
|
580
|
+
* Check if a span exists in the backend
|
|
581
|
+
*/
|
|
582
|
+
getSpanById(spanId: string): Promise<any>;
|
|
583
|
+
/**
|
|
584
|
+
* Wait until a span is available in the backend
|
|
585
|
+
* Polls the GET /spans/:id endpoint to verify span availability before running evaluators
|
|
586
|
+
*/
|
|
587
|
+
waitForSpanIngestion(spanId?: string, timeoutSeconds?: number, pollIntervalSeconds?: number, initialDelaySeconds?: number): Promise<boolean>;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Run Entry Context for Evaluation
|
|
592
|
+
* Provides context management for evaluation run entries
|
|
593
|
+
*/
|
|
594
|
+
|
|
595
|
+
declare class RunEntryContext {
|
|
596
|
+
private client;
|
|
597
|
+
private config;
|
|
598
|
+
readonly evaluationRun: Run;
|
|
599
|
+
readonly entry: DatasetEntry;
|
|
600
|
+
traceId?: string;
|
|
601
|
+
private span?;
|
|
602
|
+
private startTime?;
|
|
603
|
+
constructor(client: EvaluationHttpClient, config: Config, run: Run, entry: DatasetEntry);
|
|
604
|
+
/**
|
|
605
|
+
* Get the run associated with this context
|
|
606
|
+
* Alias for evaluationRun for backwards compatibility
|
|
607
|
+
*/
|
|
608
|
+
get run(): Run;
|
|
609
|
+
/**
|
|
610
|
+
* Start the run entry context
|
|
611
|
+
* Creates a span and posts agent_triggered status
|
|
612
|
+
*/
|
|
613
|
+
start(): Promise<void>;
|
|
614
|
+
/**
|
|
615
|
+
* End the run entry context
|
|
616
|
+
* @param success Whether the entry completed successfully
|
|
617
|
+
* @param scores Optional scores to record
|
|
618
|
+
*/
|
|
619
|
+
end(success?: boolean, scores?: EvaluationScore[]): Promise<void>;
|
|
620
|
+
/**
|
|
621
|
+
* Get the current span
|
|
622
|
+
*/
|
|
623
|
+
getSpan(): Span | undefined;
|
|
624
|
+
/**
|
|
625
|
+
* Execute a function within this context
|
|
626
|
+
* Automatically handles start/end and error handling
|
|
627
|
+
*/
|
|
628
|
+
execute<T>(fn: (ctx: RunEntryContext) => Promise<T>, scores?: EvaluationScore[]): Promise<T>;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Usage API Models
|
|
633
|
+
*/
|
|
634
|
+
interface SessionUsageData {
|
|
635
|
+
sessionId: string;
|
|
636
|
+
tokenCount: number;
|
|
637
|
+
requestCount: number;
|
|
638
|
+
totalCost: number;
|
|
639
|
+
}
|
|
640
|
+
interface TenantUsageData {
|
|
641
|
+
tenantId: string;
|
|
642
|
+
organisationId: string;
|
|
643
|
+
tokenCount: number;
|
|
644
|
+
requestCount: number;
|
|
645
|
+
sessionCount: number;
|
|
646
|
+
totalCost: number;
|
|
647
|
+
}
|
|
648
|
+
interface TraceSummary {
|
|
649
|
+
id: string;
|
|
650
|
+
name: string;
|
|
651
|
+
kind: string;
|
|
652
|
+
latencyMs: number;
|
|
653
|
+
startTime: string;
|
|
654
|
+
endTime: string;
|
|
655
|
+
cursor: string;
|
|
656
|
+
organisationId: string;
|
|
657
|
+
projectId: string;
|
|
658
|
+
sessionId?: string;
|
|
659
|
+
tenantId?: string;
|
|
660
|
+
userId?: string;
|
|
661
|
+
environment?: string;
|
|
662
|
+
models?: string[];
|
|
663
|
+
promptTokens?: number;
|
|
664
|
+
completionTokens?: number;
|
|
665
|
+
cachedTokens?: number;
|
|
666
|
+
totalTokens?: number;
|
|
667
|
+
promptTokensCost?: number;
|
|
668
|
+
completionTokensCost?: number;
|
|
669
|
+
cachedTokensCost?: number;
|
|
670
|
+
totalCost?: number;
|
|
671
|
+
hasPii?: boolean;
|
|
672
|
+
piiEntities?: any[];
|
|
673
|
+
hasViolation?: boolean;
|
|
674
|
+
violations?: any[];
|
|
675
|
+
hasError?: boolean;
|
|
676
|
+
service?: string;
|
|
677
|
+
}
|
|
678
|
+
interface TracesPage {
|
|
679
|
+
traces: TraceSummary[];
|
|
680
|
+
hasNextPage: boolean;
|
|
681
|
+
nextCursor?: string;
|
|
682
|
+
}
|
|
683
|
+
interface TraceSpan {
|
|
684
|
+
id: string;
|
|
685
|
+
traceId: string;
|
|
686
|
+
name: string;
|
|
687
|
+
kind: string;
|
|
688
|
+
parentSpanId?: string;
|
|
689
|
+
latencyMs?: number;
|
|
690
|
+
startTimeMs?: string;
|
|
691
|
+
endTimeMs?: string;
|
|
692
|
+
organisationId: string;
|
|
693
|
+
projectId: string;
|
|
694
|
+
sessionId?: string;
|
|
695
|
+
tenantId?: string;
|
|
696
|
+
userId?: string;
|
|
697
|
+
environment?: string;
|
|
698
|
+
modelName?: string;
|
|
699
|
+
promptTokens?: number;
|
|
700
|
+
completionTokens?: number;
|
|
701
|
+
cachedTokens?: number;
|
|
702
|
+
totalTokens?: number;
|
|
703
|
+
promptTokensCost?: number;
|
|
704
|
+
completionTokensCost?: number;
|
|
705
|
+
cachedTokensCost?: number;
|
|
706
|
+
totalCost?: number;
|
|
707
|
+
hasPii?: boolean;
|
|
708
|
+
piiEntities?: any[];
|
|
709
|
+
piiActions?: Record<string, any>;
|
|
710
|
+
hasViolation?: boolean;
|
|
711
|
+
violations?: any[];
|
|
712
|
+
violationActions?: Record<string, any>;
|
|
713
|
+
status?: string;
|
|
714
|
+
attributes?: string;
|
|
715
|
+
events?: string;
|
|
716
|
+
links?: string;
|
|
717
|
+
resources?: string;
|
|
718
|
+
metadata?: Record<string, any>;
|
|
719
|
+
hasError?: boolean;
|
|
720
|
+
createdAt: string;
|
|
721
|
+
cursor: string;
|
|
722
|
+
}
|
|
723
|
+
interface SpansPage {
|
|
724
|
+
spans: TraceSpan[];
|
|
725
|
+
hasNextPage: boolean;
|
|
726
|
+
nextCursor?: string;
|
|
727
|
+
}
|
|
728
|
+
interface ListTracesParams {
|
|
729
|
+
startTime: string;
|
|
730
|
+
endTime: string;
|
|
731
|
+
traceId?: string;
|
|
732
|
+
sessionId?: string;
|
|
733
|
+
userId?: string;
|
|
734
|
+
tenantId?: string;
|
|
735
|
+
limit?: number;
|
|
736
|
+
cursor?: string;
|
|
737
|
+
direction?: "up" | "down";
|
|
738
|
+
sortField?: string;
|
|
739
|
+
sortOrder?: "asc" | "desc";
|
|
740
|
+
}
|
|
741
|
+
interface ListSpansParams {
|
|
742
|
+
traceId: string;
|
|
743
|
+
cursor?: string;
|
|
744
|
+
direction?: "up" | "down";
|
|
745
|
+
limit?: number;
|
|
746
|
+
spanName?: string;
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/**
|
|
750
|
+
* Public Usage API
|
|
751
|
+
* Exposed as Netra.usage
|
|
752
|
+
*/
|
|
753
|
+
|
|
754
|
+
declare class Usage {
|
|
755
|
+
private config;
|
|
756
|
+
private client;
|
|
757
|
+
constructor(config: Config);
|
|
758
|
+
/**
|
|
759
|
+
* Get session usage data
|
|
760
|
+
* @param sessionId Session identifier
|
|
761
|
+
* @param startTime Start time for the usage data (in ISO 8601 UTC format)
|
|
762
|
+
* @param endTime End time for the usage data (in ISO 8601 UTC format)
|
|
763
|
+
* @returns Session usage data or null on error
|
|
764
|
+
*/
|
|
765
|
+
getSessionUsage(sessionId: string, startTime: string, endTime: string): Promise<SessionUsageData | null>;
|
|
766
|
+
/**
|
|
767
|
+
* Get tenant usage data
|
|
768
|
+
* @param tenantId Tenant identifier
|
|
769
|
+
* @param startTime Start time for the usage data (in ISO 8601 UTC format)
|
|
770
|
+
* @param endTime End time for the usage data (in ISO 8601 UTC format)
|
|
771
|
+
* @returns Tenant usage data or null on error
|
|
772
|
+
*/
|
|
773
|
+
getTenantUsage(tenantId: string, startTime: string, endTime: string): Promise<TenantUsageData | null>;
|
|
774
|
+
/**
|
|
775
|
+
* List all traces
|
|
776
|
+
* @param params List traces parameters
|
|
777
|
+
* @returns Traces page or null on error
|
|
778
|
+
*/
|
|
779
|
+
listTraces(params: ListTracesParams): Promise<TracesPage | null>;
|
|
780
|
+
/**
|
|
781
|
+
* List all spans for a given trace
|
|
782
|
+
* @param params List spans parameters
|
|
783
|
+
* @returns Spans page or null on error
|
|
784
|
+
*/
|
|
785
|
+
listSpansByTraceId(params: ListSpansParams): Promise<SpansPage | null>;
|
|
786
|
+
/**
|
|
787
|
+
* Iterate over traces using cursor-based pagination
|
|
788
|
+
* @param params List traces parameters
|
|
789
|
+
* @returns Async generator yielding TraceSummary items
|
|
790
|
+
*/
|
|
791
|
+
iterTraces(params: ListTracesParams): AsyncGenerator<TraceSummary, void, unknown>;
|
|
792
|
+
/**
|
|
793
|
+
* Iterate over spans for a given trace using cursor-based pagination
|
|
794
|
+
* @param params List spans parameters
|
|
795
|
+
* @returns Async generator yielding TraceSpan items
|
|
796
|
+
*/
|
|
797
|
+
iterSpansByTraceId(params: ListSpansParams): AsyncGenerator<TraceSpan, void, unknown>;
|
|
798
|
+
private mapToTraceSummary;
|
|
799
|
+
private mapToTraceSpan;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Session Manager for tracking user sessions and context
|
|
804
|
+
*
|
|
805
|
+
* Uses AsyncLocalStorage for entity stacks (workflow, task, agent)
|
|
806
|
+
* and OpenTelemetry's baggage API for session context (session_id, user_id, tenant_id).
|
|
807
|
+
* This provides proper concurrency support for multi-request environments.
|
|
808
|
+
*/
|
|
809
|
+
|
|
810
|
+
declare enum ConversationType {
|
|
811
|
+
INPUT = "input",
|
|
812
|
+
OUTPUT = "output"
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
/**
|
|
816
|
+
* Type definitions for Netra SDK
|
|
817
|
+
*/
|
|
818
|
+
declare enum SpanType {
|
|
819
|
+
SPAN = "SPAN",
|
|
820
|
+
GENERATION = "GENERATION",
|
|
821
|
+
TOOL = "TOOL",
|
|
822
|
+
EMBEDDING = "EMBEDDING",
|
|
823
|
+
AGENT = "AGENT"
|
|
824
|
+
}
|
|
825
|
+
interface UsageModel {
|
|
826
|
+
model: string;
|
|
827
|
+
usage_type: string;
|
|
828
|
+
units_used?: number;
|
|
829
|
+
cost_in_usd?: number;
|
|
830
|
+
}
|
|
831
|
+
interface ActionModel {
|
|
832
|
+
start_time: string;
|
|
833
|
+
action: string;
|
|
834
|
+
action_type: string;
|
|
835
|
+
success: boolean;
|
|
836
|
+
affected_records?: Array<{
|
|
837
|
+
record_id: string;
|
|
838
|
+
record_type: string;
|
|
839
|
+
}>;
|
|
840
|
+
metadata?: Record<string, string>;
|
|
841
|
+
}
|
|
842
|
+
interface DecoratorOptions {
|
|
843
|
+
name?: string;
|
|
844
|
+
asType?: SpanType;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/**
|
|
848
|
+
* Span Wrapper for custom span tracking
|
|
849
|
+
*/
|
|
850
|
+
|
|
851
|
+
declare class SpanWrapper {
|
|
852
|
+
private name;
|
|
853
|
+
private attributes;
|
|
854
|
+
private moduleName;
|
|
855
|
+
private startTime?;
|
|
856
|
+
private endTime?;
|
|
857
|
+
private status;
|
|
858
|
+
private errorMessage?;
|
|
859
|
+
private span?;
|
|
860
|
+
private activeContext?;
|
|
861
|
+
private tracer?;
|
|
862
|
+
constructor(name: string, attributes?: Record<string, string>, moduleName?: string, asType?: SpanType, tracer?: any);
|
|
863
|
+
start(): this;
|
|
864
|
+
end(): this;
|
|
865
|
+
setAttribute(key: string, value: string): this;
|
|
866
|
+
/**
|
|
867
|
+
* Run a function with this span set as the active span in the OTel context.
|
|
868
|
+
* Useful for nesting spans.
|
|
869
|
+
*/
|
|
870
|
+
withActive<T>(fn: () => T): T;
|
|
871
|
+
/**
|
|
872
|
+
* Async version of withActive(). Keeps the context active across async work.
|
|
873
|
+
*/
|
|
874
|
+
withActiveAsync<T>(fn: () => Promise<T>): Promise<T>;
|
|
875
|
+
setPrompt(prompt: string): this;
|
|
876
|
+
setNegativePrompt(negativePrompt: string): this;
|
|
877
|
+
setUsage(usage: UsageModel[]): this;
|
|
878
|
+
setAction(action: ActionModel[]): this;
|
|
879
|
+
setModel(model: string): this;
|
|
880
|
+
setLlmSystem(system: string): this;
|
|
881
|
+
setError(errorMessage: string): this;
|
|
882
|
+
setSuccess(): this;
|
|
883
|
+
addEvent(name: string, attributes?: Record<string, string>): this;
|
|
884
|
+
getCurrentSpan(): Span | undefined;
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
declare function withBlockedSpansLocal<T>(patterns: readonly string[], fn: () => Promise<T> | T): Promise<T>;
|
|
888
|
+
|
|
889
|
+
/**
|
|
890
|
+
* Decorators for easy instrumentation
|
|
891
|
+
*/
|
|
892
|
+
|
|
893
|
+
type AnyFunction = (...args: any[]) => any;
|
|
894
|
+
declare function workflow<T extends AnyFunction>(target: T, options?: DecoratorOptions): T;
|
|
895
|
+
declare function workflow<T extends AnyFunction>(options?: DecoratorOptions): (target: T) => T;
|
|
896
|
+
declare function agent<T extends AnyFunction>(target: T, options?: DecoratorOptions): T;
|
|
897
|
+
declare function agent<T extends AnyFunction>(options?: DecoratorOptions): (target: T) => T;
|
|
898
|
+
declare function task<T extends AnyFunction>(target: T, options?: DecoratorOptions): T;
|
|
899
|
+
declare function task<T extends AnyFunction>(options?: DecoratorOptions): (target: T) => T;
|
|
900
|
+
declare function span<T extends AnyFunction>(target: T, options?: DecoratorOptions): T;
|
|
901
|
+
declare function span<T extends AnyFunction>(options?: DecoratorOptions): (target: T) => T;
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* Session Span Processor
|
|
905
|
+
*
|
|
906
|
+
* OpenTelemetry span processor that automatically adds session attributes to spans.
|
|
907
|
+
* This includes session_id, user_id, tenant_id from OpenTelemetry baggage, and entity context
|
|
908
|
+
* (workflow, task, agent names) from SessionManager.
|
|
909
|
+
*
|
|
910
|
+
* Uses OpenTelemetry's baggage API for automatic context propagation across async boundaries.
|
|
911
|
+
*/
|
|
912
|
+
|
|
913
|
+
declare class SessionSpanProcessor implements SpanProcessor {
|
|
914
|
+
/**
|
|
915
|
+
* Called when a span starts. Adds session and entity context attributes.
|
|
916
|
+
*/
|
|
917
|
+
onStart(span: Span, parentContext: Context): void;
|
|
918
|
+
/**
|
|
919
|
+
* Called when a span ends. No-op for this processor.
|
|
920
|
+
*/
|
|
921
|
+
onEnd(span: ReadableSpan): void;
|
|
922
|
+
/**
|
|
923
|
+
* Shuts down the processor.
|
|
924
|
+
*/
|
|
925
|
+
shutdown(): Promise<void>;
|
|
926
|
+
/**
|
|
927
|
+
* Forces a flush of any pending spans.
|
|
928
|
+
*/
|
|
929
|
+
forceFlush(): Promise<void>;
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/**
|
|
933
|
+
* Instrumentation Span Processor
|
|
934
|
+
*
|
|
935
|
+
* OpenTelemetry span processor that:
|
|
936
|
+
* 1. Records the raw instrumentation scope name for each span
|
|
937
|
+
* 2. Truncates attribute values to prevent oversized payloads
|
|
938
|
+
*/
|
|
939
|
+
|
|
940
|
+
declare class InstrumentationSpanProcessor implements SpanProcessor {
|
|
941
|
+
private maxAttributeLength;
|
|
942
|
+
constructor(maxAttributeLength?: number);
|
|
943
|
+
/**
|
|
944
|
+
* Detect the raw instrumentation name from the span's instrumentation scope.
|
|
945
|
+
*/
|
|
946
|
+
private detectRawInstrumentationName;
|
|
947
|
+
/**
|
|
948
|
+
* Truncate a value to the maximum allowed length.
|
|
949
|
+
* Handles strings, bytes, and recursively handles lists and objects.
|
|
950
|
+
*/
|
|
951
|
+
private truncateValue;
|
|
952
|
+
/**
|
|
953
|
+
* Called when a span starts. Wraps setAttribute to truncate values
|
|
954
|
+
* and records the instrumentation name.
|
|
955
|
+
*/
|
|
956
|
+
onStart(span: Span, parentContext: Context): void;
|
|
957
|
+
/**
|
|
958
|
+
* Called when a span ends. No-op for this processor.
|
|
959
|
+
*/
|
|
960
|
+
onEnd(span: ReadableSpan): void;
|
|
961
|
+
/**
|
|
962
|
+
* Shuts down the processor.
|
|
963
|
+
*/
|
|
964
|
+
shutdown(): Promise<void>;
|
|
965
|
+
/**
|
|
966
|
+
* Forces a flush of any pending spans.
|
|
967
|
+
*/
|
|
968
|
+
forceFlush(): Promise<void>;
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/**
|
|
972
|
+
* Scrubbing Span Processor
|
|
973
|
+
*
|
|
974
|
+
* OpenTelemetry span processor that scrubs sensitive data from span attributes.
|
|
975
|
+
* This includes API keys, emails, phone numbers, SSNs, credit cards, passwords,
|
|
976
|
+
* bearer tokens, and other sensitive information.
|
|
977
|
+
*/
|
|
978
|
+
|
|
979
|
+
declare class ScrubbingSpanProcessor implements SpanProcessor {
|
|
980
|
+
/**
|
|
981
|
+
* Check if a key is considered sensitive.
|
|
982
|
+
*/
|
|
983
|
+
private isSensitiveKey;
|
|
984
|
+
/**
|
|
985
|
+
* Scrub sensitive patterns from a string value.
|
|
986
|
+
*/
|
|
987
|
+
private scrubStringValue;
|
|
988
|
+
/**
|
|
989
|
+
* Recursively scrub sensitive data from a dictionary value.
|
|
990
|
+
*/
|
|
991
|
+
private scrubDictValue;
|
|
992
|
+
/**
|
|
993
|
+
* Recursively scrub sensitive data from a list value.
|
|
994
|
+
*/
|
|
995
|
+
private scrubListValue;
|
|
996
|
+
/**
|
|
997
|
+
* Scrub sensitive data from a key-value pair.
|
|
998
|
+
*/
|
|
999
|
+
private scrubKeyValue;
|
|
1000
|
+
/**
|
|
1001
|
+
* Called when a span starts. No-op for this processor.
|
|
1002
|
+
*/
|
|
1003
|
+
onStart(span: Span, parentContext: Context): void;
|
|
1004
|
+
/**
|
|
1005
|
+
* Called when a span ends. Scrubs sensitive data from span attributes.
|
|
1006
|
+
*/
|
|
1007
|
+
onEnd(span: ReadableSpan): void;
|
|
1008
|
+
/**
|
|
1009
|
+
* Shuts down the processor.
|
|
1010
|
+
*/
|
|
1011
|
+
shutdown(): Promise<void>;
|
|
1012
|
+
/**
|
|
1013
|
+
* Forces a flush of any pending spans.
|
|
1014
|
+
*/
|
|
1015
|
+
forceFlush(): Promise<void>;
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
/**
|
|
1019
|
+
* Custom MistralAI instrumentor for Netra SDK
|
|
1020
|
+
*
|
|
1021
|
+
* Note: '@mistralai/mistralai' is a peer dependency. The SDK dynamically imports it
|
|
1022
|
+
* to ensure we patch the same module instance the application uses.
|
|
1023
|
+
*/
|
|
1024
|
+
|
|
1025
|
+
interface InstrumentorOptions {
|
|
1026
|
+
tracerProvider?: TracerProvider;
|
|
1027
|
+
}
|
|
1028
|
+
/**
|
|
1029
|
+
* Custom MistralAI instrumentor for Netra SDK
|
|
1030
|
+
*/
|
|
1031
|
+
declare class NetraMistralAIInstrumentor {
|
|
1032
|
+
private tracer;
|
|
1033
|
+
private tracerProvider?;
|
|
1034
|
+
private resourceCtors;
|
|
1035
|
+
constructor();
|
|
1036
|
+
/**
|
|
1037
|
+
* Returns the list of instrumentation dependencies
|
|
1038
|
+
*/
|
|
1039
|
+
instrumentationDependencies(): string[];
|
|
1040
|
+
/**
|
|
1041
|
+
* Instrument MistralAI client methods (async version)
|
|
1042
|
+
* Uses dynamic import() to ensure we get the same ES module instance
|
|
1043
|
+
* that the application uses.
|
|
1044
|
+
*/
|
|
1045
|
+
instrumentAsync(options?: InstrumentorOptions): Promise<NetraMistralAIInstrumentor>;
|
|
1046
|
+
/**
|
|
1047
|
+
* Instrument MistralAI client methods (sync version - for backwards compatibility)
|
|
1048
|
+
* Note: This uses a cached Mistral class. Call instrumentAsync() for proper initialization.
|
|
1049
|
+
*/
|
|
1050
|
+
instrument(options?: InstrumentorOptions): NetraMistralAIInstrumentor;
|
|
1051
|
+
/**
|
|
1052
|
+
* Uninstrument MistralAI client methods
|
|
1053
|
+
*/
|
|
1054
|
+
uninstrument(): void;
|
|
1055
|
+
/**
|
|
1056
|
+
* Check if MistralAI is currently instrumented
|
|
1057
|
+
*/
|
|
1058
|
+
isInstrumented(): boolean;
|
|
1059
|
+
/**
|
|
1060
|
+
* Resolve public resource constructors from the exported Mistral client.
|
|
1061
|
+
* This avoids relying on internal "@mistralai/mistralai/sdk/*" paths.
|
|
1062
|
+
*
|
|
1063
|
+
* We create a temporary client instance purely for discovery; it should not
|
|
1064
|
+
* make network calls.
|
|
1065
|
+
*/
|
|
1066
|
+
private _ensureResourceCtors;
|
|
1067
|
+
private _getCtor;
|
|
1068
|
+
private _instrumentChat;
|
|
1069
|
+
private _instrumentEmbeddings;
|
|
1070
|
+
private _instrumentFIM;
|
|
1071
|
+
private _instrumentAgents;
|
|
1072
|
+
private _uninstrumentChat;
|
|
1073
|
+
private _uninstrumentEmbeddings;
|
|
1074
|
+
private _uninstrumentFIM;
|
|
1075
|
+
private _uninstrumentAgents;
|
|
1076
|
+
}
|
|
1077
|
+
declare const mistralAIInstrumentor: NetraMistralAIInstrumentor;
|
|
1078
|
+
|
|
1079
|
+
declare class FilteringSpanExporter implements SpanExporter {
|
|
1080
|
+
private readonly exporter;
|
|
1081
|
+
private exact;
|
|
1082
|
+
private prefixes;
|
|
1083
|
+
private suffixes;
|
|
1084
|
+
private rememberedBlockedParentMap;
|
|
1085
|
+
constructor(exporter: SpanExporter, patterns: string[]);
|
|
1086
|
+
export(spans: ReadableSpan[], resultCallback: (result: {
|
|
1087
|
+
code: ExportResultCode;
|
|
1088
|
+
}) => void): void;
|
|
1089
|
+
shutdown(): Promise<void>;
|
|
1090
|
+
forceFlush(): Promise<void>;
|
|
1091
|
+
private isBlocked;
|
|
1092
|
+
private matchesLocalPatterns;
|
|
1093
|
+
private hasLocalBlockFlag;
|
|
1094
|
+
private reparentBlockedChildren;
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
interface ExporterConfig {
|
|
1098
|
+
url: string;
|
|
1099
|
+
headers?: Record<string, string>;
|
|
1100
|
+
}
|
|
1101
|
+
declare class TrialAwareOTLPExporter implements SpanExporter {
|
|
1102
|
+
private _url;
|
|
1103
|
+
private _headers;
|
|
1104
|
+
private _isShutdown;
|
|
1105
|
+
constructor(config: ExporterConfig);
|
|
1106
|
+
export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void;
|
|
1107
|
+
private checkErrorBody;
|
|
1108
|
+
forceFlush(): Promise<void>;
|
|
1109
|
+
shutdown(): Promise<void>;
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
/**
|
|
1113
|
+
* Netra SDK - Main entry point
|
|
1114
|
+
* A comprehensive TypeScript/JavaScript SDK for AI application observability
|
|
1115
|
+
* Built on top of OpenTelemetry and Traceloop
|
|
1116
|
+
*/
|
|
1117
|
+
|
|
1118
|
+
declare class Netra {
|
|
1119
|
+
private static _initialized;
|
|
1120
|
+
private static _config;
|
|
1121
|
+
private static _tracer;
|
|
1122
|
+
/**
|
|
1123
|
+
* Usage API client for usage and traces
|
|
1124
|
+
* Available after calling Netra.init()
|
|
1125
|
+
*/
|
|
1126
|
+
static usage: Usage;
|
|
1127
|
+
/**
|
|
1128
|
+
* Evaluation API client for datasets, runs, and test suites
|
|
1129
|
+
* Available after calling Netra.init()
|
|
1130
|
+
*/
|
|
1131
|
+
static evaluation: Evaluation;
|
|
1132
|
+
/**
|
|
1133
|
+
* Dashboard API client for querying metrics and time-series data
|
|
1134
|
+
* Available after calling Netra.init()
|
|
1135
|
+
*/
|
|
1136
|
+
static dashboard: Dashboard;
|
|
1137
|
+
/**
|
|
1138
|
+
* Check if Netra has been initialized
|
|
1139
|
+
*/
|
|
1140
|
+
static isInitialized(): boolean;
|
|
1141
|
+
/**
|
|
1142
|
+
* Initialize the Netra SDK
|
|
1143
|
+
* Note: Custom instrumentations (OpenAI, Groq, MistralAI) are initialized
|
|
1144
|
+
* asynchronously. Use initAsync() or await Netra.ready() to ensure
|
|
1145
|
+
* instrumentations are complete before using instrumented modules.
|
|
1146
|
+
*/
|
|
1147
|
+
static init(config?: NetraConfig): void;
|
|
1148
|
+
/**
|
|
1149
|
+
* Initialize the Netra SDK and wait for all instrumentations to be ready.
|
|
1150
|
+
* This is the recommended way to initialize Netra when using ES modules,
|
|
1151
|
+
* as it ensures all async instrumentations (OpenAI, Groq, MistralAI) are
|
|
1152
|
+
* complete before the application starts using the instrumented modules.
|
|
1153
|
+
*
|
|
1154
|
+
* @example
|
|
1155
|
+
* await Netra.initAsync({ appName: 'my-app', instruments: new Set([NetraInstruments.OPENAI]) });
|
|
1156
|
+
* // Now OpenAI is fully instrumented
|
|
1157
|
+
* const openai = new OpenAI();
|
|
1158
|
+
*/
|
|
1159
|
+
static initAsync(config?: NetraConfig): Promise<void>;
|
|
1160
|
+
/**
|
|
1161
|
+
* Returns a promise that resolves when all async instrumentations are ready.
|
|
1162
|
+
* Can be called after init() to wait for instrumentations.
|
|
1163
|
+
*
|
|
1164
|
+
* @example
|
|
1165
|
+
* Netra.init({ appName: 'my-app' });
|
|
1166
|
+
* await Netra.ready();
|
|
1167
|
+
* // Now all instrumentations are complete
|
|
1168
|
+
*/
|
|
1169
|
+
static ready(): Promise<void>;
|
|
1170
|
+
/**
|
|
1171
|
+
* Optional cleanup to end the root span and uninstrument all.
|
|
1172
|
+
* Now async to ensure spans are flushed.
|
|
1173
|
+
*/
|
|
1174
|
+
static shutdown(): Promise<void>;
|
|
1175
|
+
/**
|
|
1176
|
+
* Run a function with the root span as the active parent context.
|
|
1177
|
+
* All spans created within this function will be children of the root span.
|
|
1178
|
+
*
|
|
1179
|
+
* @param fn The function to run within the root span context
|
|
1180
|
+
* @returns The result of the function
|
|
1181
|
+
*/
|
|
1182
|
+
static runWithRootSpan<T>(fn: () => T): T;
|
|
1183
|
+
/**
|
|
1184
|
+
* Run an async function with the root span as the active parent context.
|
|
1185
|
+
* All spans created within this function will be children of the root span.
|
|
1186
|
+
*
|
|
1187
|
+
* @param fn The async function to run within the root span context
|
|
1188
|
+
* @returns A promise that resolves with the result of the function
|
|
1189
|
+
*/
|
|
1190
|
+
static runWithRootSpanAsync<T>(fn: () => Promise<T>): Promise<T>;
|
|
1191
|
+
/**
|
|
1192
|
+
* Run a function within an isolated entity context.
|
|
1193
|
+
* This ensures that entity stacks (workflow, task, agent, span) are isolated
|
|
1194
|
+
* per request in concurrent environments.
|
|
1195
|
+
*
|
|
1196
|
+
* Note: Session context (session_id, user_id, tenant_id) is automatically
|
|
1197
|
+
* isolated via OpenTelemetry's baggage API and AsyncLocalStorage.
|
|
1198
|
+
* This method is primarily needed if you're using workflow/task/agent decorators
|
|
1199
|
+
* or span wrappers in concurrent environments.
|
|
1200
|
+
*
|
|
1201
|
+
* @param fn The function to run with isolated entity context
|
|
1202
|
+
* @returns The result of the function
|
|
1203
|
+
*
|
|
1204
|
+
*/
|
|
1205
|
+
static runWithContext<T>(fn: () => T): T;
|
|
1206
|
+
/**
|
|
1207
|
+
* Set session_id context attributes for all spans.
|
|
1208
|
+
* Uses OpenTelemetry baggage API for automatic context propagation.
|
|
1209
|
+
*
|
|
1210
|
+
* Context automatically propagates across async boundaries in concurrent environments
|
|
1211
|
+
* thanks to AsyncLocalStorage in @opentelemetry/sdk-node.
|
|
1212
|
+
*/
|
|
1213
|
+
static setSessionId(sessionId: string): void;
|
|
1214
|
+
/**
|
|
1215
|
+
* Set user_id context attributes for all spans.
|
|
1216
|
+
* Uses OpenTelemetry baggage API for automatic context propagation.
|
|
1217
|
+
*/
|
|
1218
|
+
static setUserId(userId: string): void;
|
|
1219
|
+
/**
|
|
1220
|
+
* Set tenant_id context attributes for all spans.
|
|
1221
|
+
* Uses OpenTelemetry baggage API for automatic context propagation.
|
|
1222
|
+
*/
|
|
1223
|
+
static setTenantId(tenantId: string): void;
|
|
1224
|
+
/**
|
|
1225
|
+
* Set a custom attribute on the current active span
|
|
1226
|
+
*/
|
|
1227
|
+
static setCustomAttributes(key: string, value: any): void;
|
|
1228
|
+
/**
|
|
1229
|
+
* Set custom event in the current active span
|
|
1230
|
+
*/
|
|
1231
|
+
static setCustomEvent(eventName: string, attributes: Record<string, any>): void;
|
|
1232
|
+
/**
|
|
1233
|
+
* Append a conversation entry to the current active span
|
|
1234
|
+
*/
|
|
1235
|
+
static addConversation(conversationType: ConversationType, role: string, content: string | Record<string, any>): void;
|
|
1236
|
+
/**
|
|
1237
|
+
* Start a new span
|
|
1238
|
+
*/
|
|
1239
|
+
static startSpan(name: string, attributes?: Record<string, string>, moduleName?: string, asType?: SpanType): SpanWrapper;
|
|
1240
|
+
static withBlockedSpansLocal: typeof withBlockedSpansLocal;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
export { type ActionModel, Aggregation, type CategoricalDataPoint, ChartType, Config, ConversationType, type CreateDatasetParams, Dashboard, type DashboardData, type Dataset, type DatasetEntry, type DatasetItem, type Dimension, DimensionField, type DimensionValue, EntryStatus, Evaluation, type EvaluationScore, type EvaluatorFunction, type Filter, type FilterConfig, FilterField, FilterType, FilteringSpanExporter, GroupBy, InstrumentationSpanProcessor, type ListSpansParams, type ListTracesParams, Measure, type Metrics, Netra, NetraInstruments, type NumberResponse, Operator, type QueryDataParams, type QueryResponse, type Run, RunEntryContext, RunStatus, Scope, ScrubbingSpanProcessor, SessionSpanProcessor, type SessionUsageData, SpanType, type SpansPage, type TaskFunction, type TenantUsageData, type TestSuiteResult, type TimeRange, type TimeSeriesDataPoint, type TimeSeriesResponse, type TimeSeriesWithDimension, type TraceSpan, type TraceSummary, type TracesPage, TrialAwareOTLPExporter, Usage, type UsageModel, agent, Netra as default, metadataField, mistralAIInstrumentor, span, task, workflow };
|