@sentrial/sdk 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +210 -137
- package/dist/index.cjs +1908 -46
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1043 -2
- package/dist/index.d.ts +1043 -2
- package/dist/index.js +1875 -45
- package/dist/index.js.map +1 -1
- package/package.json +15 -2
package/dist/index.d.cts
CHANGED
|
@@ -1,6 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PII Redaction Module
|
|
3
|
+
*
|
|
4
|
+
* Automatically detects and redacts personally identifiable information (PII)
|
|
5
|
+
* from SDK payloads before they are sent to the Sentrial API.
|
|
6
|
+
*/
|
|
7
|
+
/** How redacted values are replaced */
|
|
8
|
+
type PiiMode = 'label' | 'hash' | 'remove';
|
|
9
|
+
/** Fields that can be redacted in SDK payloads */
|
|
10
|
+
type PiiField = 'userInput' | 'assistantOutput' | 'toolInput' | 'toolOutput' | 'metadata' | 'reasoning' | 'failureReason';
|
|
11
|
+
/** A custom regex pattern with a human-readable label */
|
|
12
|
+
interface PiiCustomPattern {
|
|
13
|
+
pattern: RegExp;
|
|
14
|
+
label: string;
|
|
15
|
+
}
|
|
16
|
+
/** Toggle individual builtin pattern categories */
|
|
17
|
+
interface PiiBuiltinPatterns {
|
|
18
|
+
emails?: boolean;
|
|
19
|
+
phones?: boolean;
|
|
20
|
+
ssns?: boolean;
|
|
21
|
+
creditCards?: boolean;
|
|
22
|
+
ipAddresses?: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Full PII redaction configuration */
|
|
25
|
+
interface PiiConfig {
|
|
26
|
+
/** Enable PII redaction (default: false) */
|
|
27
|
+
enabled: boolean;
|
|
28
|
+
/** Which payload fields to redact (default: DEFAULT_FIELDS) */
|
|
29
|
+
fields?: PiiField[];
|
|
30
|
+
/** Replacement mode (default: 'label') */
|
|
31
|
+
mode?: PiiMode;
|
|
32
|
+
/** Enable enhanced detection heuristics (reserved for future use) */
|
|
33
|
+
enhancedDetection?: boolean;
|
|
34
|
+
/** Toggle builtin patterns (default: all enabled) */
|
|
35
|
+
builtinPatterns?: PiiBuiltinPatterns;
|
|
36
|
+
/** Additional custom patterns to apply */
|
|
37
|
+
customPatterns?: PiiCustomPattern[];
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Hash a value using SHA-256 and return the first 6 hex characters.
|
|
41
|
+
* Used in 'hash' mode to produce a short, deterministic pseudonym.
|
|
42
|
+
*/
|
|
43
|
+
declare function hashValue(value: string): string;
|
|
44
|
+
/**
|
|
45
|
+
* Produce the replacement string for a matched PII value.
|
|
46
|
+
*
|
|
47
|
+
* - 'label' mode: [LABEL]
|
|
48
|
+
* - 'hash' mode: [PII:abcdef]
|
|
49
|
+
* - 'remove' mode: '' (empty string)
|
|
50
|
+
*/
|
|
51
|
+
declare function replaceMatch(match: string, label: string, mode: PiiMode): string;
|
|
52
|
+
/**
|
|
53
|
+
* Run all enabled builtin and custom patterns against a single string,
|
|
54
|
+
* replacing every match according to the specified mode.
|
|
55
|
+
*/
|
|
56
|
+
declare function redactString(value: string, mode: PiiMode, builtinPatterns: Required<PiiBuiltinPatterns>, customPatterns: PiiCustomPattern[]): string;
|
|
57
|
+
/**
|
|
58
|
+
* Recursively redact PII from a value. Handles strings, arrays, plain objects,
|
|
59
|
+
* and passes through primitives (numbers, booleans, null, undefined) unchanged.
|
|
60
|
+
*/
|
|
61
|
+
declare function redactValue(value: unknown, mode: PiiMode, builtinPatterns: Required<PiiBuiltinPatterns>, customPatterns: PiiCustomPattern[]): unknown;
|
|
62
|
+
/**
|
|
63
|
+
* Main entry point: redact configured fields in a payload object.
|
|
64
|
+
*
|
|
65
|
+
* Only fields listed in the PiiConfig (or DEFAULT_FIELDS) are redacted.
|
|
66
|
+
* Other fields are returned unchanged.
|
|
67
|
+
*
|
|
68
|
+
* @param payload - The data payload (e.g., session params, tool call params)
|
|
69
|
+
* @param config - PII configuration from the client
|
|
70
|
+
* @returns A new payload object with PII redacted from the configured fields
|
|
71
|
+
*/
|
|
72
|
+
declare function redactPayload(payload: Record<string, unknown>, config: PiiConfig): Record<string, unknown>;
|
|
73
|
+
|
|
1
74
|
/**
|
|
2
75
|
* Type definitions for the Sentrial TypeScript SDK
|
|
3
76
|
*/
|
|
77
|
+
|
|
4
78
|
/**
|
|
5
79
|
* Event types that can be tracked in a session
|
|
6
80
|
*/
|
|
@@ -27,6 +101,12 @@ interface SentrialClientConfig {
|
|
|
27
101
|
* Set to false during development to see full errors.
|
|
28
102
|
*/
|
|
29
103
|
failSilently?: boolean;
|
|
104
|
+
/**
|
|
105
|
+
* PII redaction configuration.
|
|
106
|
+
* - Pass a PiiConfig object for full control.
|
|
107
|
+
* - Pass `true` to fetch the org's PII config from the server automatically.
|
|
108
|
+
*/
|
|
109
|
+
pii?: PiiConfig | boolean;
|
|
30
110
|
}
|
|
31
111
|
/**
|
|
32
112
|
* Parameters for creating a new session
|
|
@@ -38,6 +118,10 @@ interface CreateSessionParams {
|
|
|
38
118
|
agentName: string;
|
|
39
119
|
/** External user ID (for grouping sessions by end-user) */
|
|
40
120
|
userId: string;
|
|
121
|
+
/** Optional parent session ID for multi-agent hierarchies */
|
|
122
|
+
parentSessionId?: string;
|
|
123
|
+
/** Optional conversation ID to group related sessions into a thread */
|
|
124
|
+
convoId?: string;
|
|
41
125
|
/** Optional metadata */
|
|
42
126
|
metadata?: Record<string, unknown>;
|
|
43
127
|
}
|
|
@@ -67,6 +151,8 @@ interface CompleteSessionParams {
|
|
|
67
151
|
userInput?: string;
|
|
68
152
|
/** The final assistant response for this session */
|
|
69
153
|
assistantOutput?: string;
|
|
154
|
+
/** Alias for assistantOutput - the final output text */
|
|
155
|
+
output?: string;
|
|
70
156
|
}
|
|
71
157
|
/**
|
|
72
158
|
* Session data returned from the API
|
|
@@ -74,6 +160,8 @@ interface CompleteSessionParams {
|
|
|
74
160
|
interface Session {
|
|
75
161
|
id: string;
|
|
76
162
|
agentId?: string;
|
|
163
|
+
parentSessionId?: string;
|
|
164
|
+
convoId?: string;
|
|
77
165
|
name: string;
|
|
78
166
|
agentName: string;
|
|
79
167
|
userId: string;
|
|
@@ -363,12 +451,26 @@ declare class SentrialClient {
|
|
|
363
451
|
private readonly apiUrl;
|
|
364
452
|
private readonly apiKey?;
|
|
365
453
|
private readonly failSilently;
|
|
454
|
+
private piiConfig?;
|
|
455
|
+
private piiConfigNeedsHydration;
|
|
456
|
+
private piiHydrationPromise?;
|
|
366
457
|
private currentState;
|
|
367
458
|
constructor(config?: SentrialClientConfig);
|
|
368
459
|
/**
|
|
369
|
-
*
|
|
460
|
+
* Fetch the organization's PII config from the server.
|
|
461
|
+
*
|
|
462
|
+
* Called lazily on the first request when `pii: true` was passed to the constructor.
|
|
463
|
+
* Uses a single shared promise so concurrent requests don't trigger duplicate fetches.
|
|
464
|
+
*/
|
|
465
|
+
private hydratePiiConfig;
|
|
466
|
+
/**
|
|
467
|
+
* Make an HTTP request with retry logic and exponential backoff.
|
|
468
|
+
*
|
|
469
|
+
* Retries on transient failures (network errors, timeouts, 429/5xx).
|
|
470
|
+
* Up to MAX_RETRIES attempts with exponential backoff.
|
|
370
471
|
*/
|
|
371
472
|
private safeRequest;
|
|
473
|
+
private sleep;
|
|
372
474
|
/**
|
|
373
475
|
* Create a new session
|
|
374
476
|
*
|
|
@@ -404,6 +506,26 @@ declare class SentrialClient {
|
|
|
404
506
|
* @param value - State value
|
|
405
507
|
*/
|
|
406
508
|
updateState(key: string, value: unknown): void;
|
|
509
|
+
/**
|
|
510
|
+
* Set the user input for a session
|
|
511
|
+
*
|
|
512
|
+
* @param sessionId - Session ID
|
|
513
|
+
* @param input - User input text
|
|
514
|
+
* @returns Updated session or null on error
|
|
515
|
+
*/
|
|
516
|
+
setInput(sessionId: string, input: string): Promise<Session | null>;
|
|
517
|
+
/**
|
|
518
|
+
* Track a generic event
|
|
519
|
+
*
|
|
520
|
+
* @param params - Event parameters
|
|
521
|
+
* @returns Event data or null on error
|
|
522
|
+
*/
|
|
523
|
+
trackEvent(params: {
|
|
524
|
+
sessionId: string;
|
|
525
|
+
eventType: string;
|
|
526
|
+
eventData?: Record<string, unknown>;
|
|
527
|
+
metadata?: Record<string, unknown>;
|
|
528
|
+
}): Promise<Event | null>;
|
|
407
529
|
/**
|
|
408
530
|
* Complete a session with performance metrics
|
|
409
531
|
*
|
|
@@ -462,6 +584,7 @@ interface InteractionConfig {
|
|
|
462
584
|
eventId: string;
|
|
463
585
|
userId: string;
|
|
464
586
|
event: string;
|
|
587
|
+
userInput?: string;
|
|
465
588
|
}
|
|
466
589
|
/**
|
|
467
590
|
* Represents an in-progress interaction that can be finished.
|
|
@@ -485,6 +608,7 @@ declare class Interaction {
|
|
|
485
608
|
private success;
|
|
486
609
|
private failureReason?;
|
|
487
610
|
private output?;
|
|
611
|
+
private readonly userInput?;
|
|
488
612
|
private readonly degraded;
|
|
489
613
|
constructor(config: InteractionConfig);
|
|
490
614
|
/**
|
|
@@ -639,4 +763,921 @@ declare class ValidationError extends SentrialError {
|
|
|
639
763
|
constructor(message: string, details?: Record<string, unknown>);
|
|
640
764
|
}
|
|
641
765
|
|
|
642
|
-
|
|
766
|
+
/**
|
|
767
|
+
* Vercel AI SDK Integration for Sentrial
|
|
768
|
+
*
|
|
769
|
+
* Automatically traces AI SDK calls with full input/output logging, metrics, and tool execution.
|
|
770
|
+
* Supports Vercel AI SDK v3, v4, v5, and v6.
|
|
771
|
+
*
|
|
772
|
+
* @example Simple Usage
|
|
773
|
+
* ```ts
|
|
774
|
+
* import { configureVercel, wrapAISDK } from '@sentrial/sdk';
|
|
775
|
+
* import * as ai from 'ai';
|
|
776
|
+
* import { openai } from '@ai-sdk/openai';
|
|
777
|
+
*
|
|
778
|
+
* configureVercel({
|
|
779
|
+
* apiKey: process.env.SENTRIAL_API_KEY,
|
|
780
|
+
* defaultAgent: 'my-ai-agent',
|
|
781
|
+
* });
|
|
782
|
+
*
|
|
783
|
+
* const { generateText } = wrapAISDK(ai);
|
|
784
|
+
*
|
|
785
|
+
* // This automatically logs to Sentrial
|
|
786
|
+
* const { text } = await generateText({
|
|
787
|
+
* model: openai('gpt-4'),
|
|
788
|
+
* prompt: 'What is the capital of France?',
|
|
789
|
+
* });
|
|
790
|
+
* ```
|
|
791
|
+
*
|
|
792
|
+
* @example Multi-Turn Conversations
|
|
793
|
+
* ```ts
|
|
794
|
+
* const { generateText } = wrapAISDK(ai, { convoId: 'convo_123' });
|
|
795
|
+
*
|
|
796
|
+
* // All calls are linked to the same conversation thread
|
|
797
|
+
* const { text } = await generateText({
|
|
798
|
+
* model: openai('gpt-4o'),
|
|
799
|
+
* messages: conversationHistory,
|
|
800
|
+
* });
|
|
801
|
+
* ```
|
|
802
|
+
*
|
|
803
|
+
* @packageDocumentation
|
|
804
|
+
*/
|
|
805
|
+
|
|
806
|
+
type AIModule = {
|
|
807
|
+
generateText?: Function;
|
|
808
|
+
streamText?: Function;
|
|
809
|
+
generateObject?: Function;
|
|
810
|
+
streamObject?: Function;
|
|
811
|
+
experimental_generateText?: Function;
|
|
812
|
+
experimental_streamText?: Function;
|
|
813
|
+
};
|
|
814
|
+
type GenerateTextParams = {
|
|
815
|
+
model: {
|
|
816
|
+
modelId?: string;
|
|
817
|
+
provider?: string;
|
|
818
|
+
} & Record<string, unknown>;
|
|
819
|
+
prompt?: string;
|
|
820
|
+
messages?: Array<{
|
|
821
|
+
role: string;
|
|
822
|
+
content: string | unknown;
|
|
823
|
+
}>;
|
|
824
|
+
system?: string;
|
|
825
|
+
tools?: Record<string, {
|
|
826
|
+
execute?: Function;
|
|
827
|
+
} & Record<string, unknown>>;
|
|
828
|
+
maxTokens?: number;
|
|
829
|
+
maxSteps?: number;
|
|
830
|
+
temperature?: number;
|
|
831
|
+
[key: string]: unknown;
|
|
832
|
+
};
|
|
833
|
+
type UsageInfo = {
|
|
834
|
+
promptTokens?: number;
|
|
835
|
+
completionTokens?: number;
|
|
836
|
+
totalTokens?: number;
|
|
837
|
+
};
|
|
838
|
+
type StepResult = {
|
|
839
|
+
text?: string;
|
|
840
|
+
toolCalls?: Array<{
|
|
841
|
+
toolName: string;
|
|
842
|
+
args: unknown;
|
|
843
|
+
}>;
|
|
844
|
+
toolResults?: Array<{
|
|
845
|
+
toolName: string;
|
|
846
|
+
result: unknown;
|
|
847
|
+
}>;
|
|
848
|
+
usage?: UsageInfo;
|
|
849
|
+
finishReason?: string;
|
|
850
|
+
};
|
|
851
|
+
type GenerateTextResult = {
|
|
852
|
+
text: string;
|
|
853
|
+
usage?: UsageInfo;
|
|
854
|
+
finishReason?: string;
|
|
855
|
+
toolCalls?: Array<{
|
|
856
|
+
toolName: string;
|
|
857
|
+
args: unknown;
|
|
858
|
+
}>;
|
|
859
|
+
toolResults?: Array<{
|
|
860
|
+
toolName: string;
|
|
861
|
+
result: unknown;
|
|
862
|
+
}>;
|
|
863
|
+
response?: {
|
|
864
|
+
modelId?: string;
|
|
865
|
+
id?: string;
|
|
866
|
+
};
|
|
867
|
+
steps?: StepResult[];
|
|
868
|
+
reasoning?: unknown;
|
|
869
|
+
reasoningText?: string;
|
|
870
|
+
sources?: unknown[];
|
|
871
|
+
[key: string]: unknown;
|
|
872
|
+
};
|
|
873
|
+
type StreamTextResult = {
|
|
874
|
+
textStream: AsyncIterable<string>;
|
|
875
|
+
fullStream?: AsyncIterable<unknown>;
|
|
876
|
+
text?: string | Promise<string>;
|
|
877
|
+
usage?: Promise<UsageInfo>;
|
|
878
|
+
finishReason?: Promise<string>;
|
|
879
|
+
toolCalls?: Promise<Array<{
|
|
880
|
+
toolName: string;
|
|
881
|
+
args: unknown;
|
|
882
|
+
}>>;
|
|
883
|
+
toolResults?: Promise<Array<{
|
|
884
|
+
toolName: string;
|
|
885
|
+
result: unknown;
|
|
886
|
+
}>>;
|
|
887
|
+
steps?: Promise<StepResult[]>;
|
|
888
|
+
response?: Promise<{
|
|
889
|
+
modelId?: string;
|
|
890
|
+
id?: string;
|
|
891
|
+
}>;
|
|
892
|
+
[key: string]: unknown;
|
|
893
|
+
};
|
|
894
|
+
/** Per-instance config passed through closures (not global) */
|
|
895
|
+
type WrapperConfig = {
|
|
896
|
+
defaultAgent?: string;
|
|
897
|
+
userId?: string;
|
|
898
|
+
convoId?: string;
|
|
899
|
+
};
|
|
900
|
+
/**
|
|
901
|
+
* Configure the Sentrial SDK for Vercel AI SDK integration.
|
|
902
|
+
*
|
|
903
|
+
* @example
|
|
904
|
+
* ```ts
|
|
905
|
+
* import { configureVercel } from '@sentrial/sdk';
|
|
906
|
+
*
|
|
907
|
+
* configureVercel({
|
|
908
|
+
* apiKey: process.env.SENTRIAL_API_KEY,
|
|
909
|
+
* defaultAgent: 'my-chatbot',
|
|
910
|
+
* userId: 'user_123',
|
|
911
|
+
* });
|
|
912
|
+
* ```
|
|
913
|
+
*/
|
|
914
|
+
declare function configureVercel(config: {
|
|
915
|
+
apiKey?: string;
|
|
916
|
+
apiUrl?: string;
|
|
917
|
+
defaultAgent?: string;
|
|
918
|
+
userId?: string;
|
|
919
|
+
convoId?: string;
|
|
920
|
+
failSilently?: boolean;
|
|
921
|
+
}): void;
|
|
922
|
+
declare function wrapGenerateText(originalFn: Function, client: SentrialClient, config: WrapperConfig): (params: GenerateTextParams) => Promise<GenerateTextResult>;
|
|
923
|
+
declare function wrapStreamText(originalFn: Function, client: SentrialClient, config: WrapperConfig): (params: GenerateTextParams) => StreamTextResult;
|
|
924
|
+
declare function wrapGenerateObject(originalFn: Function, client: SentrialClient, config: WrapperConfig): (params: GenerateTextParams) => Promise<unknown>;
|
|
925
|
+
declare function wrapStreamObject(originalFn: Function, client: SentrialClient, config: WrapperConfig): (params: GenerateTextParams) => unknown;
|
|
926
|
+
/**
|
|
927
|
+
* Wrap the Vercel AI SDK to automatically trace all AI calls.
|
|
928
|
+
*
|
|
929
|
+
* @param ai - The imported `ai` module from the Vercel AI SDK
|
|
930
|
+
* @param options - Optional configuration (client, defaultAgent, userId, convoId)
|
|
931
|
+
* @returns Wrapped versions of the AI SDK functions
|
|
932
|
+
*
|
|
933
|
+
* @example Using with configureVercel
|
|
934
|
+
* ```ts
|
|
935
|
+
* configureVercel({ apiKey: process.env.SENTRIAL_API_KEY, defaultAgent: 'my-agent' });
|
|
936
|
+
* const { generateText, streamText } = wrapAISDK(ai);
|
|
937
|
+
* ```
|
|
938
|
+
*
|
|
939
|
+
* @example Multi-turn conversations
|
|
940
|
+
* ```ts
|
|
941
|
+
* const { generateText } = wrapAISDK(ai, { convoId: 'convo_123' });
|
|
942
|
+
* ```
|
|
943
|
+
*
|
|
944
|
+
* @example Using with existing SentrialClient
|
|
945
|
+
* ```ts
|
|
946
|
+
* const client = new SentrialClient({ apiKey: process.env.SENTRIAL_API_KEY });
|
|
947
|
+
* const { generateText } = wrapAISDK(ai, { client, defaultAgent: 'my-agent' });
|
|
948
|
+
* ```
|
|
949
|
+
*/
|
|
950
|
+
declare function wrapAISDK(ai: AIModule, options?: {
|
|
951
|
+
client?: SentrialClient;
|
|
952
|
+
defaultAgent?: string;
|
|
953
|
+
userId?: string;
|
|
954
|
+
convoId?: string;
|
|
955
|
+
}): {
|
|
956
|
+
generateText: ReturnType<typeof wrapGenerateText>;
|
|
957
|
+
streamText: ReturnType<typeof wrapStreamText>;
|
|
958
|
+
generateObject: ReturnType<typeof wrapGenerateObject>;
|
|
959
|
+
streamObject: ReturnType<typeof wrapStreamObject>;
|
|
960
|
+
};
|
|
961
|
+
|
|
962
|
+
/**
|
|
963
|
+
* Sentrial LLM Wrappers - Auto-instrument LLM provider SDKs
|
|
964
|
+
*
|
|
965
|
+
* These wrappers automatically track all LLM calls with:
|
|
966
|
+
* - Input messages
|
|
967
|
+
* - Output responses
|
|
968
|
+
* - Token counts
|
|
969
|
+
* - Cost estimation
|
|
970
|
+
* - Latency
|
|
971
|
+
*
|
|
972
|
+
* @example OpenAI
|
|
973
|
+
* ```ts
|
|
974
|
+
* import OpenAI from 'openai';
|
|
975
|
+
* import { wrapOpenAI, configure } from '@sentrial/sdk';
|
|
976
|
+
*
|
|
977
|
+
* configure({ apiKey: 'sentrial_live_xxx' });
|
|
978
|
+
*
|
|
979
|
+
* const openai = wrapOpenAI(new OpenAI());
|
|
980
|
+
*
|
|
981
|
+
* // All calls are now automatically tracked!
|
|
982
|
+
* const response = await openai.chat.completions.create({
|
|
983
|
+
* model: 'gpt-4o',
|
|
984
|
+
* messages: [{ role: 'user', content: 'Hello!' }],
|
|
985
|
+
* });
|
|
986
|
+
* ```
|
|
987
|
+
*
|
|
988
|
+
* @example Anthropic
|
|
989
|
+
* ```ts
|
|
990
|
+
* import Anthropic from '@anthropic-ai/sdk';
|
|
991
|
+
* import { wrapAnthropic, configure } from '@sentrial/sdk';
|
|
992
|
+
*
|
|
993
|
+
* configure({ apiKey: 'sentrial_live_xxx' });
|
|
994
|
+
*
|
|
995
|
+
* const anthropic = wrapAnthropic(new Anthropic());
|
|
996
|
+
*
|
|
997
|
+
* const response = await anthropic.messages.create({
|
|
998
|
+
* model: 'claude-3-5-sonnet-20241022',
|
|
999
|
+
* max_tokens: 1024,
|
|
1000
|
+
* messages: [{ role: 'user', content: 'Hello!' }],
|
|
1001
|
+
* });
|
|
1002
|
+
* ```
|
|
1003
|
+
*/
|
|
1004
|
+
|
|
1005
|
+
/**
|
|
1006
|
+
* Set the current session context for auto-tracking.
|
|
1007
|
+
*
|
|
1008
|
+
* Call this before making LLM calls to associate them with a session.
|
|
1009
|
+
* This is automatically called when using withSession() or decorators.
|
|
1010
|
+
*
|
|
1011
|
+
* @param sessionId - The session ID to track LLM calls under
|
|
1012
|
+
* @param client - Optional Sentrial client (uses default if not provided)
|
|
1013
|
+
*/
|
|
1014
|
+
declare function setSessionContext(sessionId: string, client?: SentrialClient): void;
|
|
1015
|
+
/**
|
|
1016
|
+
* Clear the current session context.
|
|
1017
|
+
*/
|
|
1018
|
+
declare function clearSessionContext(): void;
|
|
1019
|
+
/**
|
|
1020
|
+
* Get the current session ID.
|
|
1021
|
+
*/
|
|
1022
|
+
declare function getSessionContext(): string | null;
|
|
1023
|
+
/**
|
|
1024
|
+
* Set the default client for wrappers.
|
|
1025
|
+
*/
|
|
1026
|
+
declare function setDefaultClient(client: SentrialClient): void;
|
|
1027
|
+
/**
|
|
1028
|
+
* Wrap an OpenAI client to automatically track all LLM calls.
|
|
1029
|
+
*
|
|
1030
|
+
* @param client - OpenAI client instance
|
|
1031
|
+
* @param options - Wrapper options
|
|
1032
|
+
* @returns The same client, now with auto-tracking enabled
|
|
1033
|
+
*
|
|
1034
|
+
* @example
|
|
1035
|
+
* ```ts
|
|
1036
|
+
* import OpenAI from 'openai';
|
|
1037
|
+
* import { wrapOpenAI, configure } from '@sentrial/sdk';
|
|
1038
|
+
*
|
|
1039
|
+
* configure({ apiKey: 'sentrial_live_xxx' });
|
|
1040
|
+
* const openai = wrapOpenAI(new OpenAI());
|
|
1041
|
+
*
|
|
1042
|
+
* // Now use client normally - all calls are tracked!
|
|
1043
|
+
* const response = await openai.chat.completions.create({
|
|
1044
|
+
* model: 'gpt-4o',
|
|
1045
|
+
* messages: [{ role: 'user', content: 'Hello' }],
|
|
1046
|
+
* });
|
|
1047
|
+
* ```
|
|
1048
|
+
*/
|
|
1049
|
+
declare function wrapOpenAI<T extends object>(client: T, options?: {
|
|
1050
|
+
trackWithoutSession?: boolean;
|
|
1051
|
+
}): T;
|
|
1052
|
+
/**
|
|
1053
|
+
* Wrap an Anthropic client to automatically track all LLM calls.
|
|
1054
|
+
*
|
|
1055
|
+
* @param client - Anthropic client instance
|
|
1056
|
+
* @param options - Wrapper options
|
|
1057
|
+
* @returns The same client, now with auto-tracking enabled
|
|
1058
|
+
*
|
|
1059
|
+
* @example
|
|
1060
|
+
* ```ts
|
|
1061
|
+
* import Anthropic from '@anthropic-ai/sdk';
|
|
1062
|
+
* import { wrapAnthropic, configure } from '@sentrial/sdk';
|
|
1063
|
+
*
|
|
1064
|
+
* configure({ apiKey: 'sentrial_live_xxx' });
|
|
1065
|
+
* const anthropic = wrapAnthropic(new Anthropic());
|
|
1066
|
+
*
|
|
1067
|
+
* const response = await anthropic.messages.create({
|
|
1068
|
+
* model: 'claude-3-5-sonnet-20241022',
|
|
1069
|
+
* max_tokens: 1024,
|
|
1070
|
+
* messages: [{ role: 'user', content: 'Hello!' }],
|
|
1071
|
+
* });
|
|
1072
|
+
* ```
|
|
1073
|
+
*/
|
|
1074
|
+
declare function wrapAnthropic<T extends object>(client: T, options?: {
|
|
1075
|
+
trackWithoutSession?: boolean;
|
|
1076
|
+
}): T;
|
|
1077
|
+
/**
|
|
1078
|
+
* Wrap a Google GenerativeModel to automatically track all LLM calls.
|
|
1079
|
+
*
|
|
1080
|
+
* @param model - Google GenerativeModel instance
|
|
1081
|
+
* @param options - Wrapper options
|
|
1082
|
+
* @returns The same model, now with auto-tracking enabled
|
|
1083
|
+
*
|
|
1084
|
+
* @example
|
|
1085
|
+
* ```ts
|
|
1086
|
+
* import { GoogleGenerativeAI } from '@google/generative-ai';
|
|
1087
|
+
* import { wrapGoogle, configure } from '@sentrial/sdk';
|
|
1088
|
+
*
|
|
1089
|
+
* configure({ apiKey: 'sentrial_live_xxx' });
|
|
1090
|
+
*
|
|
1091
|
+
* const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY!);
|
|
1092
|
+
* const model = wrapGoogle(genAI.getGenerativeModel({ model: 'gemini-2.0-flash' }));
|
|
1093
|
+
*
|
|
1094
|
+
* const response = await model.generateContent('Hello!');
|
|
1095
|
+
* ```
|
|
1096
|
+
*/
|
|
1097
|
+
declare function wrapGoogle<T extends object>(model: T, options?: {
|
|
1098
|
+
trackWithoutSession?: boolean;
|
|
1099
|
+
}): T;
|
|
1100
|
+
/**
|
|
1101
|
+
* Auto-detect and wrap any supported LLM client.
|
|
1102
|
+
*
|
|
1103
|
+
* @param client - Any supported LLM client (OpenAI, Anthropic, Google)
|
|
1104
|
+
* @param provider - Optional provider hint
|
|
1105
|
+
* @returns The wrapped client
|
|
1106
|
+
*
|
|
1107
|
+
* @example
|
|
1108
|
+
* ```ts
|
|
1109
|
+
* import OpenAI from 'openai';
|
|
1110
|
+
* import { wrapLLM } from '@sentrial/sdk';
|
|
1111
|
+
*
|
|
1112
|
+
* const client = wrapLLM(new OpenAI()); // Auto-detected as OpenAI
|
|
1113
|
+
* ```
|
|
1114
|
+
*/
|
|
1115
|
+
declare function wrapLLM<T extends object>(client: T, provider?: 'openai' | 'anthropic' | 'google'): T;
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Sentrial Decorators - Easy instrumentation for AI agents
|
|
1119
|
+
*
|
|
1120
|
+
* Provides decorators and higher-order functions for automatic tracking:
|
|
1121
|
+
* - withTool: Track tool/function calls
|
|
1122
|
+
* - withSession: Create session boundaries around agent runs
|
|
1123
|
+
*
|
|
1124
|
+
* @example Using higher-order functions (works everywhere)
|
|
1125
|
+
* ```ts
|
|
1126
|
+
* import { withTool, withSession, configure } from '@sentrial/sdk';
|
|
1127
|
+
*
|
|
1128
|
+
* configure({ apiKey: 'sentrial_live_xxx' });
|
|
1129
|
+
*
|
|
1130
|
+
* // Wrap a tool function
|
|
1131
|
+
* const searchWeb = withTool('search', async (query: string) => {
|
|
1132
|
+
* return await fetch(`/search?q=${query}`).then(r => r.json());
|
|
1133
|
+
* });
|
|
1134
|
+
*
|
|
1135
|
+
* // Wrap an agent session
|
|
1136
|
+
* const handleRequest = withSession('support-agent', async (userId: string, message: string) => {
|
|
1137
|
+
* const results = await searchWeb(message);
|
|
1138
|
+
* return processResults(results);
|
|
1139
|
+
* });
|
|
1140
|
+
* ```
|
|
1141
|
+
*
|
|
1142
|
+
* @example Using TypeScript decorators (requires experimentalDecorators)
|
|
1143
|
+
* ```ts
|
|
1144
|
+
* import { Tool, Session } from '@sentrial/sdk';
|
|
1145
|
+
*
|
|
1146
|
+
* class MyAgent {
|
|
1147
|
+
* @Tool('search')
|
|
1148
|
+
* async searchWeb(query: string) {
|
|
1149
|
+
* return await fetch(`/search?q=${query}`).then(r => r.json());
|
|
1150
|
+
* }
|
|
1151
|
+
*
|
|
1152
|
+
* @Session('support-agent')
|
|
1153
|
+
* async handleRequest(userId: string, message: string) {
|
|
1154
|
+
* const results = await this.searchWeb(message);
|
|
1155
|
+
* return processResults(results);
|
|
1156
|
+
* }
|
|
1157
|
+
* }
|
|
1158
|
+
* ```
|
|
1159
|
+
*/
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* Set the default client for decorators
|
|
1163
|
+
*/
|
|
1164
|
+
declare function setClient(client: SentrialClient): void;
|
|
1165
|
+
/**
|
|
1166
|
+
* Get the current session ID (if inside a session context)
|
|
1167
|
+
*/
|
|
1168
|
+
declare function getCurrentSessionId(): string | null;
|
|
1169
|
+
/**
|
|
1170
|
+
* Get the current interaction (if inside a session context)
|
|
1171
|
+
*/
|
|
1172
|
+
declare function getCurrentInteraction(): Interaction | null;
|
|
1173
|
+
type AsyncFunction<TArgs extends unknown[], TReturn> = (...args: TArgs) => Promise<TReturn>;
|
|
1174
|
+
type SyncFunction<TArgs extends unknown[], TReturn> = (...args: TArgs) => TReturn;
|
|
1175
|
+
/**
|
|
1176
|
+
* Wrap a function to automatically track it as a tool call.
|
|
1177
|
+
*
|
|
1178
|
+
* When called within a session context, automatically tracks:
|
|
1179
|
+
* - tool_input: Function arguments
|
|
1180
|
+
* - tool_output: Return value
|
|
1181
|
+
* - duration: Execution time
|
|
1182
|
+
* - errors: Any exceptions raised
|
|
1183
|
+
*
|
|
1184
|
+
* @param name - Name of the tool
|
|
1185
|
+
* @param fn - The function to wrap
|
|
1186
|
+
* @returns Wrapped function with automatic tracking
|
|
1187
|
+
*
|
|
1188
|
+
* @example
|
|
1189
|
+
* ```ts
|
|
1190
|
+
* const searchWeb = withTool('search', async (query: string) => {
|
|
1191
|
+
* return await fetch(`/search?q=${query}`).then(r => r.json());
|
|
1192
|
+
* });
|
|
1193
|
+
*
|
|
1194
|
+
* // Use within a session - automatically tracked
|
|
1195
|
+
* const results = await searchWeb('AI agents');
|
|
1196
|
+
* ```
|
|
1197
|
+
*/
|
|
1198
|
+
declare function withTool<TArgs extends unknown[], TReturn>(name: string, fn: AsyncFunction<TArgs, TReturn>): AsyncFunction<TArgs, TReturn>;
|
|
1199
|
+
declare function withTool<TArgs extends unknown[], TReturn>(name: string, fn: SyncFunction<TArgs, TReturn>): SyncFunction<TArgs, TReturn>;
|
|
1200
|
+
interface SessionOptions {
|
|
1201
|
+
/** Parameter index or name for user ID (default: looks for 'userId' or first param) */
|
|
1202
|
+
userIdParam?: string | number;
|
|
1203
|
+
/** Parameter index or name for input (default: looks for 'input', 'message', 'query') */
|
|
1204
|
+
inputParam?: string | number;
|
|
1205
|
+
}
|
|
1206
|
+
/**
|
|
1207
|
+
* Wrap a function to automatically manage a session around it.
|
|
1208
|
+
*
|
|
1209
|
+
* Automatically:
|
|
1210
|
+
* - Creates a session when the function is called
|
|
1211
|
+
* - Sets up context for withTool and wrapped LLM calls
|
|
1212
|
+
* - Captures output from return value
|
|
1213
|
+
* - Completes the session when the function returns
|
|
1214
|
+
* - Marks as failed if an exception is raised
|
|
1215
|
+
*
|
|
1216
|
+
* @param agentName - Name of the agent
|
|
1217
|
+
* @param fn - The function to wrap
|
|
1218
|
+
* @param options - Session options
|
|
1219
|
+
* @returns Wrapped function with automatic session management
|
|
1220
|
+
*
|
|
1221
|
+
* @example
|
|
1222
|
+
* ```ts
|
|
1223
|
+
* const handleRequest = withSession('support-agent', async (userId: string, message: string) => {
|
|
1224
|
+
* // Session automatically created
|
|
1225
|
+
* // All withTool calls and wrapped LLM calls are tracked
|
|
1226
|
+
* const response = await processRequest(message);
|
|
1227
|
+
* return response; // Captured as output
|
|
1228
|
+
* });
|
|
1229
|
+
*
|
|
1230
|
+
* await handleRequest('user_123', 'Hello!');
|
|
1231
|
+
* ```
|
|
1232
|
+
*/
|
|
1233
|
+
declare function withSession<TArgs extends unknown[], TReturn>(agentName: string, fn: AsyncFunction<TArgs, TReturn>, options?: SessionOptions): AsyncFunction<TArgs, TReturn>;
|
|
1234
|
+
declare function withSession<TArgs extends unknown[], TReturn>(agentName: string, fn: SyncFunction<TArgs, TReturn>, options?: SessionOptions): AsyncFunction<TArgs, TReturn>;
|
|
1235
|
+
/**
|
|
1236
|
+
* Method decorator to track a method as a tool call.
|
|
1237
|
+
*
|
|
1238
|
+
* @param name - Name of the tool (optional, defaults to method name)
|
|
1239
|
+
*
|
|
1240
|
+
* @example
|
|
1241
|
+
* ```ts
|
|
1242
|
+
* class MyAgent {
|
|
1243
|
+
* @Tool('search')
|
|
1244
|
+
* async searchWeb(query: string) {
|
|
1245
|
+
* return await fetch(`/search?q=${query}`).then(r => r.json());
|
|
1246
|
+
* }
|
|
1247
|
+
* }
|
|
1248
|
+
* ```
|
|
1249
|
+
*/
|
|
1250
|
+
declare function Tool(name?: string): MethodDecorator;
|
|
1251
|
+
/**
|
|
1252
|
+
* Method decorator to wrap a method in a session.
|
|
1253
|
+
*
|
|
1254
|
+
* @param agentName - Name of the agent (optional, defaults to class name)
|
|
1255
|
+
* @param options - Session options
|
|
1256
|
+
*
|
|
1257
|
+
* @example
|
|
1258
|
+
* ```ts
|
|
1259
|
+
* class MyAgent {
|
|
1260
|
+
* @TrackSession('support-agent')
|
|
1261
|
+
* async handleRequest(userId: string, message: string) {
|
|
1262
|
+
* // Session automatically created
|
|
1263
|
+
* const results = await this.searchWeb(message);
|
|
1264
|
+
* return processResults(results);
|
|
1265
|
+
* }
|
|
1266
|
+
* }
|
|
1267
|
+
* ```
|
|
1268
|
+
*/
|
|
1269
|
+
declare function TrackSession(agentName?: string, options?: SessionOptions): MethodDecorator;
|
|
1270
|
+
/**
|
|
1271
|
+
* Manual session context for when you need more control.
|
|
1272
|
+
*
|
|
1273
|
+
* @example
|
|
1274
|
+
* ```ts
|
|
1275
|
+
* const ctx = new SessionContext({
|
|
1276
|
+
* userId: 'user_123',
|
|
1277
|
+
* agent: 'my-agent',
|
|
1278
|
+
* input: 'Hello!',
|
|
1279
|
+
* });
|
|
1280
|
+
*
|
|
1281
|
+
* await ctx.start();
|
|
1282
|
+
* try {
|
|
1283
|
+
* // All tool calls are tracked
|
|
1284
|
+
* const result = await myTool(...);
|
|
1285
|
+
* ctx.setOutput(result);
|
|
1286
|
+
* await ctx.finish();
|
|
1287
|
+
* } catch (error) {
|
|
1288
|
+
* await ctx.finish({ success: false, error: error.message });
|
|
1289
|
+
* }
|
|
1290
|
+
* ```
|
|
1291
|
+
*/
|
|
1292
|
+
declare class SessionContext {
|
|
1293
|
+
private readonly userId;
|
|
1294
|
+
private readonly agent;
|
|
1295
|
+
private readonly input?;
|
|
1296
|
+
private readonly client;
|
|
1297
|
+
private interaction;
|
|
1298
|
+
private output?;
|
|
1299
|
+
constructor(options: {
|
|
1300
|
+
userId: string;
|
|
1301
|
+
agent: string;
|
|
1302
|
+
input?: string;
|
|
1303
|
+
client?: SentrialClient;
|
|
1304
|
+
});
|
|
1305
|
+
/**
|
|
1306
|
+
* Start the session
|
|
1307
|
+
*/
|
|
1308
|
+
start(): Promise<this>;
|
|
1309
|
+
/**
|
|
1310
|
+
* Set the output for this session
|
|
1311
|
+
*/
|
|
1312
|
+
setOutput(output: string): void;
|
|
1313
|
+
/**
|
|
1314
|
+
* Finish the session
|
|
1315
|
+
*/
|
|
1316
|
+
finish(options?: {
|
|
1317
|
+
success?: boolean;
|
|
1318
|
+
error?: string;
|
|
1319
|
+
}): Promise<void>;
|
|
1320
|
+
/**
|
|
1321
|
+
* Get the session ID
|
|
1322
|
+
*/
|
|
1323
|
+
get sessionId(): string | null;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/**
|
|
1327
|
+
* Sentrial Experiment Context
|
|
1328
|
+
*
|
|
1329
|
+
* Provides context-aware configuration for experiments. When running experiments
|
|
1330
|
+
* via the CLI, the system prompt and other experiment config is automatically
|
|
1331
|
+
* available to your agent code via simple function calls.
|
|
1332
|
+
*
|
|
1333
|
+
* @example
|
|
1334
|
+
* ```ts
|
|
1335
|
+
* import { getSystemPrompt } from '@sentrial/sdk';
|
|
1336
|
+
*
|
|
1337
|
+
* async function myAgent(query: string): Promise<string> {
|
|
1338
|
+
* // When running normally: returns your default
|
|
1339
|
+
* // When running experiment: returns the variant's system prompt
|
|
1340
|
+
* const systemPrompt = getSystemPrompt('You are a helpful assistant.');
|
|
1341
|
+
*
|
|
1342
|
+
* const response = await openai.chat.completions.create({
|
|
1343
|
+
* model: 'gpt-4o',
|
|
1344
|
+
* messages: [
|
|
1345
|
+
* { role: 'system', content: systemPrompt },
|
|
1346
|
+
* { role: 'user', content: query },
|
|
1347
|
+
* ],
|
|
1348
|
+
* });
|
|
1349
|
+
*
|
|
1350
|
+
* return response.choices[0].message.content!;
|
|
1351
|
+
* }
|
|
1352
|
+
* ```
|
|
1353
|
+
*
|
|
1354
|
+
* That's it! No other changes needed. The experiment runner handles setting the context.
|
|
1355
|
+
*/
|
|
1356
|
+
/**
|
|
1357
|
+
* Context for the current experiment run.
|
|
1358
|
+
*/
|
|
1359
|
+
interface ExperimentContext {
|
|
1360
|
+
/** The system prompt for this variant */
|
|
1361
|
+
systemPrompt: string;
|
|
1362
|
+
/** Name of the variant being tested */
|
|
1363
|
+
variantName: string;
|
|
1364
|
+
/** The experiment ID */
|
|
1365
|
+
experimentId: string;
|
|
1366
|
+
/** The test case session ID (if applicable) */
|
|
1367
|
+
testCaseId?: string;
|
|
1368
|
+
/** Additional metadata */
|
|
1369
|
+
metadata?: Record<string, unknown>;
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Get the current system prompt.
|
|
1373
|
+
*
|
|
1374
|
+
* When running an experiment, this returns the variant's system prompt.
|
|
1375
|
+
* Otherwise, returns your default.
|
|
1376
|
+
*
|
|
1377
|
+
* @param defaultPrompt - The default system prompt to use when not in experiment mode.
|
|
1378
|
+
* This should be your normal production prompt.
|
|
1379
|
+
* @returns The system prompt to use.
|
|
1380
|
+
*
|
|
1381
|
+
* @example
|
|
1382
|
+
* ```ts
|
|
1383
|
+
* import { getSystemPrompt } from '@sentrial/sdk';
|
|
1384
|
+
*
|
|
1385
|
+
* async function myAgent(query: string) {
|
|
1386
|
+
* // Just wrap your existing prompt with getSystemPrompt()
|
|
1387
|
+
* const prompt = getSystemPrompt('You are a helpful weather assistant.');
|
|
1388
|
+
*
|
|
1389
|
+
* const response = await openai.chat.completions.create({
|
|
1390
|
+
* model: 'gpt-4o',
|
|
1391
|
+
* messages: [
|
|
1392
|
+
* { role: 'system', content: prompt },
|
|
1393
|
+
* { role: 'user', content: query },
|
|
1394
|
+
* ],
|
|
1395
|
+
* });
|
|
1396
|
+
*
|
|
1397
|
+
* return response.choices[0].message.content;
|
|
1398
|
+
* }
|
|
1399
|
+
* ```
|
|
1400
|
+
*/
|
|
1401
|
+
declare function getSystemPrompt(defaultPrompt?: string): string;
|
|
1402
|
+
/**
|
|
1403
|
+
* Get the full experiment context if running in experiment mode.
|
|
1404
|
+
*
|
|
1405
|
+
* Returns null if not running an experiment.
|
|
1406
|
+
*
|
|
1407
|
+
* @returns ExperimentContext with variant info, or null.
|
|
1408
|
+
*
|
|
1409
|
+
* @example
|
|
1410
|
+
* ```ts
|
|
1411
|
+
* import { getExperimentContext } from '@sentrial/sdk';
|
|
1412
|
+
*
|
|
1413
|
+
* const ctx = getExperimentContext();
|
|
1414
|
+
* if (ctx) {
|
|
1415
|
+
* console.log(`Running variant: ${ctx.variantName}`);
|
|
1416
|
+
* }
|
|
1417
|
+
* ```
|
|
1418
|
+
*/
|
|
1419
|
+
declare function getExperimentContext(): ExperimentContext | null;
|
|
1420
|
+
/**
|
|
1421
|
+
* Check if currently running in experiment mode.
|
|
1422
|
+
*
|
|
1423
|
+
* @returns True if running via experiment runner, false otherwise.
|
|
1424
|
+
*/
|
|
1425
|
+
declare function isExperimentMode(): boolean;
|
|
1426
|
+
/**
|
|
1427
|
+
* Get the current variant name if in experiment mode.
|
|
1428
|
+
*
|
|
1429
|
+
* @returns The variant name (e.g., "Control", "Variant A"), or null.
|
|
1430
|
+
*/
|
|
1431
|
+
declare function getVariantName(): string | null;
|
|
1432
|
+
/**
|
|
1433
|
+
* Get the current experiment ID if in experiment mode.
|
|
1434
|
+
*
|
|
1435
|
+
* @returns The experiment ID, or null.
|
|
1436
|
+
*/
|
|
1437
|
+
declare function getExperimentId(): string | null;
|
|
1438
|
+
/**
|
|
1439
|
+
* Set the experiment context. Called by the experiment runner before running agent.
|
|
1440
|
+
*
|
|
1441
|
+
* @internal This is an internal function - users don't need to call this directly.
|
|
1442
|
+
*/
|
|
1443
|
+
declare function setExperimentContext(context: ExperimentContext): void;
|
|
1444
|
+
/**
|
|
1445
|
+
* Clear the experiment context. Called by the experiment runner after agent completes.
|
|
1446
|
+
*
|
|
1447
|
+
* @internal This is an internal function - users don't need to call this directly.
|
|
1448
|
+
*/
|
|
1449
|
+
declare function clearExperimentContext(): void;
|
|
1450
|
+
|
|
1451
|
+
/**
|
|
1452
|
+
* Sentrial Experiments - Run prompt A/B tests with results synced to Sentrial
|
|
1453
|
+
*
|
|
1454
|
+
* @example
|
|
1455
|
+
* ```ts
|
|
1456
|
+
* import { Experiment } from '@sentrial/sdk';
|
|
1457
|
+
*
|
|
1458
|
+
* const experiment = new Experiment('exp_abc123');
|
|
1459
|
+
*
|
|
1460
|
+
* // Get config
|
|
1461
|
+
* await experiment.load();
|
|
1462
|
+
* console.log(`Testing ${experiment.variants.length} variants on ${experiment.testCases.length} inputs`);
|
|
1463
|
+
*
|
|
1464
|
+
* // Run with your agent function
|
|
1465
|
+
* await experiment.run(async (testCase, variant, tracker) => {
|
|
1466
|
+
* // Create a session for this run
|
|
1467
|
+
* const interaction = await client.begin({
|
|
1468
|
+
* userId: 'experiment',
|
|
1469
|
+
* event: 'my-agent',
|
|
1470
|
+
* input: testCase.userInput,
|
|
1471
|
+
* });
|
|
1472
|
+
* tracker.setResultSessionId(interaction.getSessionId()!);
|
|
1473
|
+
*
|
|
1474
|
+
* // Run your agent with the variant's system prompt
|
|
1475
|
+
* const response = await runAgent(testCase.userInput, variant.systemPrompt);
|
|
1476
|
+
*
|
|
1477
|
+
* await interaction.finish({ output: response, success: true });
|
|
1478
|
+
* });
|
|
1479
|
+
* ```
|
|
1480
|
+
*
|
|
1481
|
+
* @example Manual run tracking
|
|
1482
|
+
* ```ts
|
|
1483
|
+
* const experiment = new Experiment('exp_abc123');
|
|
1484
|
+
* await experiment.load();
|
|
1485
|
+
*
|
|
1486
|
+
* for (const variant of experiment.variants) {
|
|
1487
|
+
* for (const testCase of experiment.testCases) {
|
|
1488
|
+
* const tracker = await experiment.trackRun(variant.name, testCase.sessionId);
|
|
1489
|
+
* try {
|
|
1490
|
+
* const sessionId = await runAgent(testCase.userInput, variant.systemPrompt);
|
|
1491
|
+
* tracker.setResultSessionId(sessionId);
|
|
1492
|
+
* await tracker.complete();
|
|
1493
|
+
* } catch (error) {
|
|
1494
|
+
* await tracker.fail(error.message);
|
|
1495
|
+
* }
|
|
1496
|
+
* }
|
|
1497
|
+
* }
|
|
1498
|
+
* ```
|
|
1499
|
+
*/
|
|
1500
|
+
|
|
1501
|
+
/**
|
|
1502
|
+
* A prompt variant to test.
|
|
1503
|
+
*/
|
|
1504
|
+
interface ExperimentVariant {
|
|
1505
|
+
/** Name of the variant */
|
|
1506
|
+
name: string;
|
|
1507
|
+
/** The system prompt for this variant */
|
|
1508
|
+
systemPrompt: string;
|
|
1509
|
+
/** Optional description */
|
|
1510
|
+
description?: string;
|
|
1511
|
+
}
|
|
1512
|
+
/**
|
|
1513
|
+
* A test case (input) to run against each variant.
|
|
1514
|
+
*/
|
|
1515
|
+
interface ExperimentTestCase {
|
|
1516
|
+
/** Session ID of the original session */
|
|
1517
|
+
sessionId: string;
|
|
1518
|
+
/** The user input to test */
|
|
1519
|
+
userInput: string;
|
|
1520
|
+
}
|
|
1521
|
+
/**
|
|
1522
|
+
* Result of a single experiment run.
|
|
1523
|
+
*/
|
|
1524
|
+
interface ExperimentRunResult {
|
|
1525
|
+
/** Name of the variant */
|
|
1526
|
+
variantName: string;
|
|
1527
|
+
/** Session ID of the original test case */
|
|
1528
|
+
testCaseSessionId: string;
|
|
1529
|
+
/** Session ID of the result session (if created) */
|
|
1530
|
+
resultSessionId?: string;
|
|
1531
|
+
/** Whether the run succeeded */
|
|
1532
|
+
success: boolean;
|
|
1533
|
+
/** Error message if failed */
|
|
1534
|
+
errorMessage?: string;
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
* Aggregated experiment results.
|
|
1538
|
+
*/
|
|
1539
|
+
interface ExperimentResults {
|
|
1540
|
+
experimentId: string;
|
|
1541
|
+
status: string;
|
|
1542
|
+
variants: Array<{
|
|
1543
|
+
name: string;
|
|
1544
|
+
successRate: number;
|
|
1545
|
+
avgScore?: number;
|
|
1546
|
+
avgCost?: number;
|
|
1547
|
+
avgDuration?: number;
|
|
1548
|
+
runCount: number;
|
|
1549
|
+
}>;
|
|
1550
|
+
winner?: string;
|
|
1551
|
+
}
|
|
1552
|
+
/**
|
|
1553
|
+
* Tracks a single experiment run.
|
|
1554
|
+
*
|
|
1555
|
+
* Created by Experiment.trackRun() - use this to manually track runs.
|
|
1556
|
+
*/
|
|
1557
|
+
declare class ExperimentRunTracker {
|
|
1558
|
+
private readonly experiment;
|
|
1559
|
+
private readonly variantName;
|
|
1560
|
+
private readonly baseSessionId;
|
|
1561
|
+
private runId?;
|
|
1562
|
+
private resultSessionId?;
|
|
1563
|
+
private _success;
|
|
1564
|
+
private _errorMessage?;
|
|
1565
|
+
/** @internal */
|
|
1566
|
+
constructor(experiment: Experiment, variantName: string, baseSessionId: string);
|
|
1567
|
+
/**
|
|
1568
|
+
* Start the run - creates a run record via API.
|
|
1569
|
+
*/
|
|
1570
|
+
start(): Promise<this>;
|
|
1571
|
+
/**
|
|
1572
|
+
* Set the session ID of the result session.
|
|
1573
|
+
*/
|
|
1574
|
+
setResultSessionId(sessionId: string): void;
|
|
1575
|
+
/**
|
|
1576
|
+
* Mark the run as complete.
|
|
1577
|
+
*/
|
|
1578
|
+
complete(): Promise<void>;
|
|
1579
|
+
/**
|
|
1580
|
+
* Mark the run as failed.
|
|
1581
|
+
*/
|
|
1582
|
+
fail(errorMessage: string): Promise<void>;
|
|
1583
|
+
/**
|
|
1584
|
+
* Get the result of this run.
|
|
1585
|
+
*/
|
|
1586
|
+
getResult(): ExperimentRunResult;
|
|
1587
|
+
}
|
|
1588
|
+
type AgentFunction = (testCase: ExperimentTestCase, variant: ExperimentVariant, tracker: ExperimentRunTracker) => Promise<void>;
|
|
1589
|
+
/**
|
|
1590
|
+
* Run experiments with results synced to Sentrial.
|
|
1591
|
+
*
|
|
1592
|
+
* @example
|
|
1593
|
+
* ```ts
|
|
1594
|
+
* const experiment = new Experiment('exp_abc123');
|
|
1595
|
+
*
|
|
1596
|
+
* await experiment.load();
|
|
1597
|
+
* console.log(`Testing ${experiment.variants.length} variants`);
|
|
1598
|
+
*
|
|
1599
|
+
* await experiment.run(async (testCase, variant, tracker) => {
|
|
1600
|
+
* const sessionId = await runMyAgent(testCase.userInput, variant.systemPrompt);
|
|
1601
|
+
* tracker.setResultSessionId(sessionId);
|
|
1602
|
+
* });
|
|
1603
|
+
* ```
|
|
1604
|
+
*/
|
|
1605
|
+
declare class Experiment {
|
|
1606
|
+
/** The experiment ID */
|
|
1607
|
+
readonly experimentId: string;
|
|
1608
|
+
/** @internal */
|
|
1609
|
+
readonly client: SentrialClient;
|
|
1610
|
+
/** @internal */
|
|
1611
|
+
readonly apiUrl: string;
|
|
1612
|
+
/** @internal */
|
|
1613
|
+
readonly apiKey?: string;
|
|
1614
|
+
private config?;
|
|
1615
|
+
private _variants?;
|
|
1616
|
+
private _testCases?;
|
|
1617
|
+
/**
|
|
1618
|
+
* Create an experiment instance.
|
|
1619
|
+
*
|
|
1620
|
+
* @param experimentId - The experiment ID from Sentrial dashboard
|
|
1621
|
+
* @param options - Configuration options
|
|
1622
|
+
*/
|
|
1623
|
+
constructor(experimentId: string, options?: {
|
|
1624
|
+
apiKey?: string;
|
|
1625
|
+
apiUrl?: string;
|
|
1626
|
+
});
|
|
1627
|
+
/**
|
|
1628
|
+
* Make an HTTP request to the API
|
|
1629
|
+
* @internal
|
|
1630
|
+
*/
|
|
1631
|
+
request<T>(method: string, path: string, body?: unknown): Promise<T | null>;
|
|
1632
|
+
/**
|
|
1633
|
+
* Load the experiment configuration from the API.
|
|
1634
|
+
*/
|
|
1635
|
+
load(): Promise<this>;
|
|
1636
|
+
/**
|
|
1637
|
+
* Get experiment name.
|
|
1638
|
+
*/
|
|
1639
|
+
get name(): string;
|
|
1640
|
+
/**
|
|
1641
|
+
* Get experiment status.
|
|
1642
|
+
*/
|
|
1643
|
+
get status(): string;
|
|
1644
|
+
/**
|
|
1645
|
+
* Get list of variants to test.
|
|
1646
|
+
*/
|
|
1647
|
+
get variants(): ExperimentVariant[];
|
|
1648
|
+
/**
|
|
1649
|
+
* Get list of test cases.
|
|
1650
|
+
*/
|
|
1651
|
+
get testCases(): ExperimentTestCase[];
|
|
1652
|
+
/**
|
|
1653
|
+
* Get a specific variant by name.
|
|
1654
|
+
*/
|
|
1655
|
+
getVariant(name: string): ExperimentVariant | undefined;
|
|
1656
|
+
/**
|
|
1657
|
+
* Create a run tracker for manual experiment runs.
|
|
1658
|
+
*/
|
|
1659
|
+
trackRun(variantName: string, baseSessionId: string): Promise<ExperimentRunTracker>;
|
|
1660
|
+
/**
|
|
1661
|
+
* Run the experiment with all variants and test cases.
|
|
1662
|
+
*
|
|
1663
|
+
* @param agentFn - Function that runs your agent with a test case and variant
|
|
1664
|
+
* @param options - Run options
|
|
1665
|
+
* @returns List of run results
|
|
1666
|
+
*/
|
|
1667
|
+
run(agentFn: AgentFunction, options?: {
|
|
1668
|
+
/** Run test cases in parallel */
|
|
1669
|
+
parallel?: boolean;
|
|
1670
|
+
/** Max parallel workers */
|
|
1671
|
+
maxWorkers?: number;
|
|
1672
|
+
}): Promise<ExperimentRunResult[]>;
|
|
1673
|
+
/**
|
|
1674
|
+
* Reset all runs for this experiment (for re-running).
|
|
1675
|
+
*/
|
|
1676
|
+
reset(): Promise<boolean>;
|
|
1677
|
+
/**
|
|
1678
|
+
* Fetch aggregated results from the API.
|
|
1679
|
+
*/
|
|
1680
|
+
getResults(): Promise<ExperimentResults | null>;
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
export { ApiError, type ApiResponse, type BeginParams, type CompleteSessionParams, type CostParams, type CreateSessionParams, type Event, EventType, Experiment, type ExperimentContext, type ExperimentResults, type ExperimentRunResult, ExperimentRunTracker, type ExperimentTestCase, type ExperimentVariant, type FinishParams, type GenerateTextParams, type GenerateTextResult, Interaction, NetworkError, type PiiBuiltinPatterns, type PiiConfig, type PiiCustomPattern, type PiiField, type PiiMode, SentrialClient, type SentrialClientConfig, SentrialError, type Session, SessionContext, type SessionStatus, type StreamTextResult, Tool, type TrackDecisionParams, type TrackErrorParams, TrackSession, type TrackToolCallParams, ValidationError, begin, calculateAnthropicCost, calculateGoogleCost, calculateOpenAICost, clearExperimentContext, clearSessionContext, configure, configureVercel, getCurrentInteraction, getCurrentSessionId, getExperimentContext, getExperimentId, getSessionContext, getSystemPrompt, getVariantName, hashValue, isExperimentMode, redactPayload, redactString, redactValue, replaceMatch, sentrial, setClient, setDefaultClient, setExperimentContext, setSessionContext, withSession, withTool, wrapAISDK, wrapAnthropic, wrapGoogle, wrapLLM, wrapOpenAI };
|