@thinkhive/sdk 2.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +274 -0
- package/README.md +7 -22
- package/dist/api/calibration.d.ts +168 -0
- package/dist/api/calibration.js +176 -0
- package/dist/api/claims.d.ts +262 -0
- package/dist/api/claims.js +262 -0
- package/dist/api/runs.d.ts +200 -0
- package/dist/api/runs.js +262 -0
- package/dist/core/client.d.ts +29 -0
- package/dist/core/client.js +89 -0
- package/dist/core/config.d.ts +38 -0
- package/dist/core/config.js +76 -0
- package/dist/core/types.d.ts +354 -0
- package/dist/core/types.js +8 -0
- package/dist/index.d.ts +222 -512
- package/dist/index.js +169 -394
- package/dist/instrumentation/langchain.d.ts +194 -0
- package/dist/instrumentation/langchain.js +429 -0
- package/dist/instrumentation/openai.d.ts +141 -0
- package/dist/instrumentation/openai.js +279 -0
- package/dist/integrations/customer-context.d.ts +203 -0
- package/dist/integrations/customer-context.js +274 -0
- package/dist/integrations/ticket-linking.d.ts +217 -0
- package/dist/integrations/ticket-linking.js +259 -0
- package/package.json +61 -9
package/MIGRATION.md
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
# Migration Guide: v2.x to v3.0
|
|
2
|
+
|
|
3
|
+
This guide helps you migrate from ThinkHive SDK v2.x to v3.0.
|
|
4
|
+
|
|
5
|
+
## Breaking Changes
|
|
6
|
+
|
|
7
|
+
### Package Name Change
|
|
8
|
+
|
|
9
|
+
```diff
|
|
10
|
+
- npm install thinkhive-sdk
|
|
11
|
+
+ npm install @thinkhive/sdk
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Update your imports:
|
|
15
|
+
|
|
16
|
+
```diff
|
|
17
|
+
- import ThinkHive from 'thinkhive-sdk';
|
|
18
|
+
+ import ThinkHive from '@thinkhive/sdk';
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
### Minimum Node.js Version
|
|
22
|
+
|
|
23
|
+
- **v2.x**: Node.js 16+
|
|
24
|
+
- **v3.0**: Node.js 18+
|
|
25
|
+
|
|
26
|
+
### Core Concepts Change
|
|
27
|
+
|
|
28
|
+
v3 is **run-centric**, not trace-centric:
|
|
29
|
+
|
|
30
|
+
| v2 Concept | v3 Concept |
|
|
31
|
+
|------------|------------|
|
|
32
|
+
| `trace` | `run` (atomic unit) |
|
|
33
|
+
| `TraceOptions` | `RunOptions` |
|
|
34
|
+
| `explainer.analyze()` | `runs.create()` + `claims.getRunAnalysis()` |
|
|
35
|
+
| `businessContext` | `customerContext` (time-series snapshot) |
|
|
36
|
+
| Analysis results | Claims (facts vs inferences) |
|
|
37
|
+
|
|
38
|
+
## Migration Steps
|
|
39
|
+
|
|
40
|
+
### 1. Update Initialization
|
|
41
|
+
|
|
42
|
+
```typescript
|
|
43
|
+
// v2
|
|
44
|
+
import { init } from 'thinkhive-sdk';
|
|
45
|
+
|
|
46
|
+
init({
|
|
47
|
+
apiKey: 'th_xxx',
|
|
48
|
+
serviceName: 'my-agent',
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
// v3
|
|
52
|
+
import { init } from '@thinkhive/sdk';
|
|
53
|
+
|
|
54
|
+
init({
|
|
55
|
+
apiKey: 'th_xxx',
|
|
56
|
+
serviceName: 'my-agent',
|
|
57
|
+
apiVersion: 'v3', // Default is v3
|
|
58
|
+
});
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### 2. Migrate Trace Creation to Runs
|
|
62
|
+
|
|
63
|
+
```typescript
|
|
64
|
+
// v2 - Trace-based
|
|
65
|
+
import { explainer } from 'thinkhive-sdk';
|
|
66
|
+
|
|
67
|
+
const result = await explainer.analyze({
|
|
68
|
+
userMessage: 'Help me with my order',
|
|
69
|
+
agentResponse: 'I found your order...',
|
|
70
|
+
outcome: 'success',
|
|
71
|
+
businessContext: {
|
|
72
|
+
customerId: 'cust_123',
|
|
73
|
+
transactionValue: 500,
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
// v3 - Run-based
|
|
78
|
+
import { runs, claims } from '@thinkhive/sdk';
|
|
79
|
+
|
|
80
|
+
// Create a run
|
|
81
|
+
const run = await runs.create({
|
|
82
|
+
agentId: 'agent_123',
|
|
83
|
+
conversationMessages: [
|
|
84
|
+
{ role: 'user', content: 'Help me with my order' },
|
|
85
|
+
{ role: 'assistant', content: 'I found your order...' },
|
|
86
|
+
],
|
|
87
|
+
outcome: 'resolved',
|
|
88
|
+
customerContext: {
|
|
89
|
+
customerId: 'cust_123',
|
|
90
|
+
arr: 50000, // Customer ARR at run time
|
|
91
|
+
healthScore: 85, // Health score at run time
|
|
92
|
+
capturedAt: new Date().toISOString(),
|
|
93
|
+
},
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
// Get analysis with claims (facts vs inferences)
|
|
97
|
+
const analysis = await claims.getRunAnalysis(run.id);
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### 3. Migrate Business Context to Customer Context Snapshots
|
|
101
|
+
|
|
102
|
+
v3 uses **time-series snapshots** instead of current values:
|
|
103
|
+
|
|
104
|
+
```typescript
|
|
105
|
+
// v2 - Current values
|
|
106
|
+
const result = await explainer.analyze({
|
|
107
|
+
userMessage: '...',
|
|
108
|
+
agentResponse: '...',
|
|
109
|
+
businessContext: {
|
|
110
|
+
customerId: 'cust_123',
|
|
111
|
+
transactionValue: 500,
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
// v3 - Point-in-time snapshots
|
|
116
|
+
import { customerContext, runs } from '@thinkhive/sdk';
|
|
117
|
+
|
|
118
|
+
// First, capture customer metrics
|
|
119
|
+
const snapshot = await customerContext.captureSnapshot('cust_123', {
|
|
120
|
+
arr: 100000,
|
|
121
|
+
healthScore: 85,
|
|
122
|
+
segment: 'enterprise',
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// Use the snapshot in your run
|
|
126
|
+
const run = await runs.create({
|
|
127
|
+
agentId: 'agent_123',
|
|
128
|
+
conversationMessages: [...],
|
|
129
|
+
customerContext: {
|
|
130
|
+
customerId: 'cust_123',
|
|
131
|
+
arr: snapshot.arr,
|
|
132
|
+
healthScore: snapshot.healthScore,
|
|
133
|
+
capturedAt: snapshot.capturedAt,
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### 4. Migrate to Claims API (Facts vs Inferences)
|
|
139
|
+
|
|
140
|
+
v3 separates facts from inferences:
|
|
141
|
+
|
|
142
|
+
```typescript
|
|
143
|
+
// v2 - Single analysis result
|
|
144
|
+
const result = await explainer.analyze({...});
|
|
145
|
+
console.log(result.summary);
|
|
146
|
+
console.log(result.outcome.verdict);
|
|
147
|
+
|
|
148
|
+
// v3 - Claims with evidence
|
|
149
|
+
import { claims, isFact, isInference } from '@thinkhive/sdk';
|
|
150
|
+
|
|
151
|
+
const analysis = await claims.getRunAnalysis(run.id);
|
|
152
|
+
|
|
153
|
+
// Get all claims
|
|
154
|
+
for (const claim of analysis.claims) {
|
|
155
|
+
console.log(`[${claim.claimType}] ${claim.claimText}`);
|
|
156
|
+
console.log(`Confidence: ${claim.confidence}`);
|
|
157
|
+
|
|
158
|
+
if (isFact(claim)) {
|
|
159
|
+
console.log('This is an observed fact');
|
|
160
|
+
} else if (isInference(claim)) {
|
|
161
|
+
console.log('This is an LLM inference');
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Get facts vs inferences summary
|
|
166
|
+
const summary = await claims.summary({ runId: run.id });
|
|
167
|
+
console.log(`Facts: ${summary.observed.count}`);
|
|
168
|
+
console.log(`Inferences: ${summary.inferred.count}`);
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### 5. Add Ticket Linking (New in v3)
|
|
172
|
+
|
|
173
|
+
```typescript
|
|
174
|
+
import { runs, generateZendeskMarker, linkRunToZendeskTicket } from '@thinkhive/sdk';
|
|
175
|
+
|
|
176
|
+
// Method 1: Embed marker in agent response
|
|
177
|
+
const run = await runs.create({
|
|
178
|
+
agentId: 'agent_123',
|
|
179
|
+
conversationMessages: [...],
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
const marker = generateZendeskMarker(run.id);
|
|
183
|
+
const responseWithMarker = `Your order is on the way! ${marker}`;
|
|
184
|
+
// Send responseWithMarker to Zendesk
|
|
185
|
+
|
|
186
|
+
// Method 2: Explicit linking
|
|
187
|
+
await linkRunToZendeskTicket(run.id, '12345');
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### 6. Add Calibration Tracking (New in v3)
|
|
191
|
+
|
|
192
|
+
```typescript
|
|
193
|
+
import { calibration } from '@thinkhive/sdk';
|
|
194
|
+
|
|
195
|
+
// Check calibration status
|
|
196
|
+
const status = await calibration.status('agent_123', 'churn_risk');
|
|
197
|
+
console.log(`Brier score: ${status.brierScore}`);
|
|
198
|
+
console.log(`Is calibrated: ${status.isCalibrated}`);
|
|
199
|
+
|
|
200
|
+
// Record prediction outcomes
|
|
201
|
+
await calibration.recordOutcome({
|
|
202
|
+
runId: run.id,
|
|
203
|
+
predictionType: 'churn_risk',
|
|
204
|
+
predictedValue: 0.7, // We predicted 70% churn risk
|
|
205
|
+
actualOutcome: 1, // Customer did churn
|
|
206
|
+
});
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
## Deprecated APIs
|
|
210
|
+
|
|
211
|
+
The following v2 APIs still work but are deprecated:
|
|
212
|
+
|
|
213
|
+
```typescript
|
|
214
|
+
// Deprecated - use runs.create() + claims.getRunAnalysis()
|
|
215
|
+
import { explainer } from '@thinkhive/sdk';
|
|
216
|
+
const result = await explainer.analyze({...}); // Still works
|
|
217
|
+
|
|
218
|
+
// Deprecated types
|
|
219
|
+
import type { TraceOptions, BusinessContext } from '@thinkhive/sdk';
|
|
220
|
+
// Use RunOptions, CustomerContextSnapshot instead
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## New Instrumentation
|
|
224
|
+
|
|
225
|
+
### OpenAI Assistants
|
|
226
|
+
|
|
227
|
+
```typescript
|
|
228
|
+
import { wrapAssistantRun } from '@thinkhive/sdk/instrumentation/openai';
|
|
229
|
+
|
|
230
|
+
const run = await wrapAssistantRun(
|
|
231
|
+
() => openai.beta.threads.runs.create(threadId, { assistant_id: assistantId }),
|
|
232
|
+
{ assistantId, threadId }
|
|
233
|
+
);
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
### LangGraph
|
|
237
|
+
|
|
238
|
+
```typescript
|
|
239
|
+
import { wrapLangGraphNode, wrapLangGraphExecution } from '@thinkhive/sdk/instrumentation/langchain';
|
|
240
|
+
|
|
241
|
+
// Wrap individual nodes
|
|
242
|
+
workflow.addNode('agent', wrapLangGraphNode('agent', agentFunction));
|
|
243
|
+
|
|
244
|
+
// Wrap entire workflow
|
|
245
|
+
const result = await wrapLangGraphExecution('support_workflow', () =>
|
|
246
|
+
compiledGraph.invoke({ messages: [...] })
|
|
247
|
+
);
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
## TypeScript Changes
|
|
251
|
+
|
|
252
|
+
```typescript
|
|
253
|
+
// v2 types
|
|
254
|
+
import type { TraceOptions, SpanData, BusinessContext } from 'thinkhive-sdk';
|
|
255
|
+
|
|
256
|
+
// v3 types
|
|
257
|
+
import type {
|
|
258
|
+
RunOptions,
|
|
259
|
+
RunOutcome,
|
|
260
|
+
ConversationMessage,
|
|
261
|
+
CustomerContextSnapshot,
|
|
262
|
+
Claim,
|
|
263
|
+
ClaimType,
|
|
264
|
+
AnalysisResult,
|
|
265
|
+
LinkMethod,
|
|
266
|
+
CalibrationStatus,
|
|
267
|
+
} from '@thinkhive/sdk';
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
## Need Help?
|
|
271
|
+
|
|
272
|
+
- Documentation: https://docs.thinkhive.ai
|
|
273
|
+
- API Reference: https://api.thinkhive.ai/docs
|
|
274
|
+
- Support: support@thinkhive.ai
|
package/README.md
CHANGED
|
@@ -4,6 +4,7 @@ The official JavaScript/TypeScript SDK for [ThinkHive](https://thinkhive.ai) - A
|
|
|
4
4
|
|
|
5
5
|
## Features
|
|
6
6
|
|
|
7
|
+
- **25 Trace Format Support**: Automatic detection and normalization for LangSmith, Langfuse, Helicone, CrewAI, Opik, Braintrust, HoneyHive, Datadog, MLflow, AgentOps, Portkey, TruLens, Lunary, LangWatch, OpenLIT, Maxim AI, Galileo, PostHog, Keywords AI, Agenta, and more
|
|
7
8
|
- **Trace Analysis**: Analyze AI agent traces with detailed explainability
|
|
8
9
|
- **RAG Evaluation**: 8 quality metrics for RAG systems (groundedness, faithfulness, etc.)
|
|
9
10
|
- **Hallucination Detection**: 9 types of hallucination detection
|
|
@@ -188,28 +189,12 @@ autoInstrument(client, {
|
|
|
188
189
|
|
|
189
190
|
## Analysis Tiers
|
|
190
191
|
|
|
191
|
-
| Tier | Description |
|
|
192
|
-
|
|
193
|
-
| `rule_based` | Pattern matching, keyword extraction |
|
|
194
|
-
| `fast_llm` | Quick LLM analysis
|
|
195
|
-
| `full_llm` | Complete analysis
|
|
196
|
-
| `deep` | Multi-pass with validation |
|
|
197
|
-
|
|
198
|
-
## Supported Models (December 2025)
|
|
199
|
-
|
|
200
|
-
### Azure OpenAI (via Azure API)
|
|
201
|
-
- **GPT-5 Series**: gpt-5.2, gpt-5, gpt-5-pro, gpt-5-mini
|
|
202
|
-
- **GPT-4.1 Series**: gpt-4.1, gpt-4.1-mini, gpt-4.1-nano
|
|
203
|
-
- **O-Series Reasoning**: o4-mini, o3-pro, o3, o3-mini
|
|
204
|
-
- **Embeddings**: text-embedding-3-large, text-embedding-3-small
|
|
205
|
-
|
|
206
|
-
### Anthropic Claude (via Azure)
|
|
207
|
-
- **Claude 4.5 Series**: claude-opus-4.5, claude-sonnet-4.5, claude-haiku-4.5
|
|
208
|
-
|
|
209
|
-
### Google Gemini (Direct API)
|
|
210
|
-
- **Gemini 3**: gemini-3-flash, gemini-3-flash-lite
|
|
211
|
-
- **Gemini 2.5**: gemini-2.5-pro, gemini-2.5-flash
|
|
212
|
-
- **Gemini 2.0**: gemini-2.0-flash-lite
|
|
192
|
+
| Tier | Description | Latency | Cost |
|
|
193
|
+
|------|-------------|---------|------|
|
|
194
|
+
| `rule_based` | Pattern matching, keyword extraction | ~50ms | Free |
|
|
195
|
+
| `fast_llm` | Quick LLM analysis (GPT-3.5) | ~500ms | Low |
|
|
196
|
+
| `full_llm` | Complete analysis (GPT-4o) | ~3s | Standard |
|
|
197
|
+
| `deep` | Multi-pass with validation | ~15s | Premium |
|
|
213
198
|
|
|
214
199
|
## Environment Variables
|
|
215
200
|
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ThinkHive SDK v3.0 - Calibration API
|
|
3
|
+
*
|
|
4
|
+
* Prediction accuracy tracking with Brier scores and calibration metrics
|
|
5
|
+
*/
|
|
6
|
+
import type { CalibrationStatus, CalibrationBucket, PredictionType } from '../core/types';
|
|
7
|
+
/**
|
|
8
|
+
* Record outcome input
|
|
9
|
+
*/
|
|
10
|
+
export interface RecordOutcomeInput {
|
|
11
|
+
/** Run ID the prediction was made for */
|
|
12
|
+
runId: string;
|
|
13
|
+
/** Type of prediction */
|
|
14
|
+
predictionType: PredictionType;
|
|
15
|
+
/** The predicted value (0-1 for probabilities) */
|
|
16
|
+
predictedValue: number;
|
|
17
|
+
/** The actual outcome (0 or 1 for binary, or actual value) */
|
|
18
|
+
actualOutcome: number;
|
|
19
|
+
/** When the prediction was made */
|
|
20
|
+
predictedAt?: string;
|
|
21
|
+
/** When the outcome was observed */
|
|
22
|
+
observedAt?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Calibration metrics
|
|
26
|
+
*/
|
|
27
|
+
export interface CalibrationMetrics {
|
|
28
|
+
agentId: string;
|
|
29
|
+
predictionType: PredictionType;
|
|
30
|
+
/** Brier score (lower is better, <0.1 is good) */
|
|
31
|
+
brierScore: number;
|
|
32
|
+
/** Expected Calibration Error */
|
|
33
|
+
ece: number;
|
|
34
|
+
/** Maximum Calibration Error */
|
|
35
|
+
mce: number;
|
|
36
|
+
/** Sample count */
|
|
37
|
+
sampleCount: number;
|
|
38
|
+
/** Is the model well-calibrated */
|
|
39
|
+
isCalibrated: boolean;
|
|
40
|
+
/** Reliability diagram data */
|
|
41
|
+
reliabilityDiagram: CalibrationBucket[];
|
|
42
|
+
/** Last updated */
|
|
43
|
+
lastUpdated: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Calibration API client for prediction accuracy tracking
|
|
47
|
+
*/
|
|
48
|
+
export declare const calibration: {
|
|
49
|
+
/**
|
|
50
|
+
* Get calibration status for an agent
|
|
51
|
+
*
|
|
52
|
+
* @example
|
|
53
|
+
* ```typescript
|
|
54
|
+
* const status = await calibration.status('agent_123', 'churn_risk');
|
|
55
|
+
* console.log(`Brier score: ${status.brierScore}`);
|
|
56
|
+
* console.log(`Is calibrated: ${status.isCalibrated}`);
|
|
57
|
+
* ```
|
|
58
|
+
*/
|
|
59
|
+
status(agentId: string, predictionType: PredictionType): Promise<CalibrationStatus>;
|
|
60
|
+
/**
|
|
61
|
+
* Get all calibration metrics for an agent
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* ```typescript
|
|
65
|
+
* const metrics = await calibration.allMetrics('agent_123');
|
|
66
|
+
* for (const m of metrics) {
|
|
67
|
+
* console.log(`${m.predictionType}: Brier=${m.brierScore}`);
|
|
68
|
+
* }
|
|
69
|
+
* ```
|
|
70
|
+
*/
|
|
71
|
+
allMetrics(agentId: string): Promise<CalibrationMetrics[]>;
|
|
72
|
+
/**
|
|
73
|
+
* Record a prediction outcome for calibration tracking
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* ```typescript
|
|
77
|
+
* // Record a churn prediction outcome
|
|
78
|
+
* await calibration.recordOutcome({
|
|
79
|
+
* runId: 'run_abc123',
|
|
80
|
+
* predictionType: 'churn_risk',
|
|
81
|
+
* predictedValue: 0.7, // We predicted 70% churn risk
|
|
82
|
+
* actualOutcome: 1, // Customer did churn
|
|
83
|
+
* });
|
|
84
|
+
*
|
|
85
|
+
* // Record a resolution time prediction
|
|
86
|
+
* await calibration.recordOutcome({
|
|
87
|
+
* runId: 'run_abc123',
|
|
88
|
+
* predictionType: 'resolution_time',
|
|
89
|
+
* predictedValue: 15, // Predicted 15 minutes
|
|
90
|
+
* actualOutcome: 22, // Actual was 22 minutes
|
|
91
|
+
* });
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
recordOutcome(input: RecordOutcomeInput): Promise<{
|
|
95
|
+
recorded: boolean;
|
|
96
|
+
brierContribution: number;
|
|
97
|
+
message: string;
|
|
98
|
+
}>;
|
|
99
|
+
/**
|
|
100
|
+
* Trigger recalibration for an agent
|
|
101
|
+
*
|
|
102
|
+
* @example
|
|
103
|
+
* ```typescript
|
|
104
|
+
* const result = await calibration.retrain('agent_123', {
|
|
105
|
+
* predictionTypes: ['churn_risk', 'escalation_risk'],
|
|
106
|
+
* });
|
|
107
|
+
* console.log(`Retrained: ${result.success}`);
|
|
108
|
+
* ```
|
|
109
|
+
*/
|
|
110
|
+
retrain(agentId: string, options?: {
|
|
111
|
+
predictionTypes?: PredictionType[];
|
|
112
|
+
minSamples?: number;
|
|
113
|
+
}): Promise<{
|
|
114
|
+
success: boolean;
|
|
115
|
+
retrainedTypes: PredictionType[];
|
|
116
|
+
skippedTypes: Array<{
|
|
117
|
+
type: PredictionType;
|
|
118
|
+
reason: string;
|
|
119
|
+
}>;
|
|
120
|
+
newMetrics: CalibrationMetrics[];
|
|
121
|
+
}>;
|
|
122
|
+
/**
|
|
123
|
+
* Get reliability diagram data for visualization
|
|
124
|
+
*
|
|
125
|
+
* @example
|
|
126
|
+
* ```typescript
|
|
127
|
+
* const diagram = await calibration.reliabilityDiagram('agent_123', 'outcome');
|
|
128
|
+
* // Use diagram.buckets to plot predicted vs actual probabilities
|
|
129
|
+
* ```
|
|
130
|
+
*/
|
|
131
|
+
reliabilityDiagram(agentId: string, predictionType: PredictionType): Promise<{
|
|
132
|
+
agentId: string;
|
|
133
|
+
predictionType: PredictionType;
|
|
134
|
+
buckets: CalibrationBucket[];
|
|
135
|
+
perfectCalibrationLine: Array<{
|
|
136
|
+
x: number;
|
|
137
|
+
y: number;
|
|
138
|
+
}>;
|
|
139
|
+
}>;
|
|
140
|
+
};
|
|
141
|
+
/**
|
|
142
|
+
* Calculate Brier score from predictions and outcomes
|
|
143
|
+
* Lower is better, <0.1 is considered good
|
|
144
|
+
*/
|
|
145
|
+
export declare function calculateBrierScore(predictions: Array<{
|
|
146
|
+
predicted: number;
|
|
147
|
+
actual: number;
|
|
148
|
+
}>): number;
|
|
149
|
+
/**
|
|
150
|
+
* Calculate Expected Calibration Error (ECE)
|
|
151
|
+
* Measures how well-calibrated predictions are across confidence buckets
|
|
152
|
+
*/
|
|
153
|
+
export declare function calculateECE(predictions: Array<{
|
|
154
|
+
predicted: number;
|
|
155
|
+
actual: number;
|
|
156
|
+
}>, numBuckets?: number): number;
|
|
157
|
+
/**
|
|
158
|
+
* Check if a model is well-calibrated based on Brier score
|
|
159
|
+
*/
|
|
160
|
+
export declare function isWellCalibrated(brierScore: number): boolean;
|
|
161
|
+
/**
|
|
162
|
+
* Get calibration quality label
|
|
163
|
+
*/
|
|
164
|
+
export declare function getCalibrationQuality(brierScore: number): 'excellent' | 'good' | 'fair' | 'poor';
|
|
165
|
+
/**
|
|
166
|
+
* Format Brier score for display
|
|
167
|
+
*/
|
|
168
|
+
export declare function formatBrierScore(score: number): string;
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ThinkHive SDK v3.0 - Calibration API
|
|
4
|
+
*
|
|
5
|
+
* Prediction accuracy tracking with Brier scores and calibration metrics
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.calibration = void 0;
|
|
9
|
+
exports.calculateBrierScore = calculateBrierScore;
|
|
10
|
+
exports.calculateECE = calculateECE;
|
|
11
|
+
exports.isWellCalibrated = isWellCalibrated;
|
|
12
|
+
exports.getCalibrationQuality = getCalibrationQuality;
|
|
13
|
+
exports.formatBrierScore = formatBrierScore;
|
|
14
|
+
const client_1 = require("../core/client");
|
|
15
|
+
/**
|
|
16
|
+
* Calibration API client for prediction accuracy tracking
|
|
17
|
+
*/
|
|
18
|
+
exports.calibration = {
|
|
19
|
+
/**
|
|
20
|
+
* Get calibration status for an agent
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* ```typescript
|
|
24
|
+
* const status = await calibration.status('agent_123', 'churn_risk');
|
|
25
|
+
* console.log(`Brier score: ${status.brierScore}`);
|
|
26
|
+
* console.log(`Is calibrated: ${status.isCalibrated}`);
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
async status(agentId, predictionType) {
|
|
30
|
+
return (0, client_1.apiRequestWithData)(`/calibration/status/${agentId}?predictionType=${predictionType}`);
|
|
31
|
+
},
|
|
32
|
+
/**
|
|
33
|
+
* Get all calibration metrics for an agent
|
|
34
|
+
*
|
|
35
|
+
* @example
|
|
36
|
+
* ```typescript
|
|
37
|
+
* const metrics = await calibration.allMetrics('agent_123');
|
|
38
|
+
* for (const m of metrics) {
|
|
39
|
+
* console.log(`${m.predictionType}: Brier=${m.brierScore}`);
|
|
40
|
+
* }
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
async allMetrics(agentId) {
|
|
44
|
+
return (0, client_1.apiRequestWithData)(`/calibration/metrics/${agentId}`);
|
|
45
|
+
},
|
|
46
|
+
/**
|
|
47
|
+
* Record a prediction outcome for calibration tracking
|
|
48
|
+
*
|
|
49
|
+
* @example
|
|
50
|
+
* ```typescript
|
|
51
|
+
* // Record a churn prediction outcome
|
|
52
|
+
* await calibration.recordOutcome({
|
|
53
|
+
* runId: 'run_abc123',
|
|
54
|
+
* predictionType: 'churn_risk',
|
|
55
|
+
* predictedValue: 0.7, // We predicted 70% churn risk
|
|
56
|
+
* actualOutcome: 1, // Customer did churn
|
|
57
|
+
* });
|
|
58
|
+
*
|
|
59
|
+
* // Record a resolution time prediction
|
|
60
|
+
* await calibration.recordOutcome({
|
|
61
|
+
* runId: 'run_abc123',
|
|
62
|
+
* predictionType: 'resolution_time',
|
|
63
|
+
* predictedValue: 15, // Predicted 15 minutes
|
|
64
|
+
* actualOutcome: 22, // Actual was 22 minutes
|
|
65
|
+
* });
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
async recordOutcome(input) {
|
|
69
|
+
return (0, client_1.apiRequestWithData)('/calibration/record', {
|
|
70
|
+
method: 'POST',
|
|
71
|
+
body: {
|
|
72
|
+
...input,
|
|
73
|
+
predictedAt: input.predictedAt || new Date().toISOString(),
|
|
74
|
+
observedAt: input.observedAt || new Date().toISOString(),
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
},
|
|
78
|
+
/**
|
|
79
|
+
* Trigger recalibration for an agent
|
|
80
|
+
*
|
|
81
|
+
* @example
|
|
82
|
+
* ```typescript
|
|
83
|
+
* const result = await calibration.retrain('agent_123', {
|
|
84
|
+
* predictionTypes: ['churn_risk', 'escalation_risk'],
|
|
85
|
+
* });
|
|
86
|
+
* console.log(`Retrained: ${result.success}`);
|
|
87
|
+
* ```
|
|
88
|
+
*/
|
|
89
|
+
async retrain(agentId, options = {}) {
|
|
90
|
+
return (0, client_1.apiRequestWithData)(`/calibration/retrain/${agentId}`, {
|
|
91
|
+
method: 'POST',
|
|
92
|
+
body: options,
|
|
93
|
+
});
|
|
94
|
+
},
|
|
95
|
+
/**
|
|
96
|
+
* Get reliability diagram data for visualization
|
|
97
|
+
*
|
|
98
|
+
* @example
|
|
99
|
+
* ```typescript
|
|
100
|
+
* const diagram = await calibration.reliabilityDiagram('agent_123', 'outcome');
|
|
101
|
+
* // Use diagram.buckets to plot predicted vs actual probabilities
|
|
102
|
+
* ```
|
|
103
|
+
*/
|
|
104
|
+
async reliabilityDiagram(agentId, predictionType) {
|
|
105
|
+
return (0, client_1.apiRequestWithData)(`/calibration/diagram/${agentId}?predictionType=${predictionType}`);
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
// ============================================================================
|
|
109
|
+
// HELPER FUNCTIONS
|
|
110
|
+
// ============================================================================
|
|
111
|
+
/**
|
|
112
|
+
* Calculate Brier score from predictions and outcomes
|
|
113
|
+
* Lower is better, <0.1 is considered good
|
|
114
|
+
*/
|
|
115
|
+
function calculateBrierScore(predictions) {
|
|
116
|
+
if (predictions.length === 0)
|
|
117
|
+
return 0;
|
|
118
|
+
const sum = predictions.reduce((acc, { predicted, actual }) => {
|
|
119
|
+
return acc + Math.pow(predicted - actual, 2);
|
|
120
|
+
}, 0);
|
|
121
|
+
return sum / predictions.length;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Calculate Expected Calibration Error (ECE)
|
|
125
|
+
* Measures how well-calibrated predictions are across confidence buckets
|
|
126
|
+
*/
|
|
127
|
+
function calculateECE(predictions, numBuckets = 10) {
|
|
128
|
+
if (predictions.length === 0)
|
|
129
|
+
return 0;
|
|
130
|
+
const buckets = [];
|
|
131
|
+
for (let i = 0; i < numBuckets; i++) {
|
|
132
|
+
buckets.push({ predictions: [], actuals: [] });
|
|
133
|
+
}
|
|
134
|
+
// Assign predictions to buckets
|
|
135
|
+
for (const { predicted, actual } of predictions) {
|
|
136
|
+
const bucketIndex = Math.min(Math.floor(predicted * numBuckets), numBuckets - 1);
|
|
137
|
+
buckets[bucketIndex].predictions.push(predicted);
|
|
138
|
+
buckets[bucketIndex].actuals.push(actual);
|
|
139
|
+
}
|
|
140
|
+
// Calculate ECE
|
|
141
|
+
let ece = 0;
|
|
142
|
+
for (const bucket of buckets) {
|
|
143
|
+
if (bucket.predictions.length === 0)
|
|
144
|
+
continue;
|
|
145
|
+
const avgPredicted = bucket.predictions.reduce((a, b) => a + b, 0) / bucket.predictions.length;
|
|
146
|
+
const avgActual = bucket.actuals.reduce((a, b) => a + b, 0) / bucket.actuals.length;
|
|
147
|
+
const weight = bucket.predictions.length / predictions.length;
|
|
148
|
+
ece += weight * Math.abs(avgPredicted - avgActual);
|
|
149
|
+
}
|
|
150
|
+
return ece;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Check if a model is well-calibrated based on Brier score
|
|
154
|
+
*/
|
|
155
|
+
function isWellCalibrated(brierScore) {
|
|
156
|
+
return brierScore < 0.1;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Get calibration quality label
|
|
160
|
+
*/
|
|
161
|
+
function getCalibrationQuality(brierScore) {
|
|
162
|
+
if (brierScore < 0.05)
|
|
163
|
+
return 'excellent';
|
|
164
|
+
if (brierScore < 0.1)
|
|
165
|
+
return 'good';
|
|
166
|
+
if (brierScore < 0.2)
|
|
167
|
+
return 'fair';
|
|
168
|
+
return 'poor';
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Format Brier score for display
|
|
172
|
+
*/
|
|
173
|
+
function formatBrierScore(score) {
|
|
174
|
+
return score.toFixed(4);
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2FsaWJyYXRpb24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvYXBpL2NhbGlicmF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7OztHQUlHOzs7QUFnTUgsa0RBVUM7QUFNRCxvQ0FvQ0M7QUFLRCw0Q0FFQztBQUtELHNEQU9DO0FBS0QsNENBRUM7QUE1UUQsMkNBQWdFO0FBb0RoRTs7R0FFRztBQUNVLFFBQUEsV0FBVyxHQUFHO0lBQ3pCOzs7Ozs7Ozs7T0FTRztJQUNILEtBQUssQ0FBQyxNQUFNLENBQ1YsT0FBZSxFQUNmLGNBQThCO1FBRTlCLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsdUJBQXVCLE9BQU8sbUJBQW1CLGNBQWMsRUFBRSxDQUNsRSxDQUFDO0lBQ0osQ0FBQztJQUVEOzs7Ozs7Ozs7O09BVUc7SUFDSCxLQUFLLENBQUMsVUFBVSxDQUFDLE9BQWU7UUFDOUIsT0FBTyxJQUFBLDJCQUFrQixFQUN2Qix3QkFBd0IsT0FBTyxFQUFFLENBQ2xDLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztPQXFCRztJQUNILEtBQUssQ0FBQyxhQUFhLENBQUMsS0FBeUI7UUFLM0MsT0FBTyxJQUFBLDJCQUFrQixFQUFDLHFCQUFxQixFQUFFO1lBQy9DLE1BQU0sRUFBRSxNQUFNO1lBQ2QsSUFBSSxFQUFFO2dCQUNKLEdBQUcsS0FBSztnQkFDUixXQUFXLEVBQUUsS0FBSyxDQUFDLFdBQVcsSUFBSSxJQUFJLElBQUksRUFBRSxDQUFDLFdBQVcsRUFBRTtnQkFDMUQsVUFBVSxFQUFFLEtBQUssQ0FBQyxVQUFVLElBQUksSUFBSSxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUU7YUFDekQ7U0FDRixDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQ7Ozs7Ozs7Ozs7T0FVRztJQUNILEtBQUssQ0FBQyxPQUFPLENBQ1gsT0FBZSxFQUNmLFVBR0ksRUFBRTtRQU9OLE9BQU8sSUFBQSwyQkFBa0IsRUFBQyx3QkFBd0IsT0FBTyxFQUFFLEVBQUU7WUFDM0QsTUFBTSxFQUFFLE1BQU07WUFDZCxJQUFJLEVBQUUsT0FBTztTQUNkLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7Ozs7T0FRRztJQUNILEtBQUssQ0FBQyxrQkFBa0IsQ0FDdEIsT0FBZSxFQUNmLGNBQThCO1FBTzlCLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsd0JBQXdCLE9BQU8sbUJBQW1CLGNBQWMsRUFBRSxDQUNuRSxDQUFDO0lBQ0osQ0FBQztDQUNGLENBQUM7QUFFRiwrRUFBK0U7QUFDL0UsbUJBQW1CO0FBQ25CLCtFQUErRTtBQUUvRTs7O0dBR0c7QUFDSCxTQUFnQixtQkFBbUIsQ0FDakMsV0FBeUQ7SUFFekQsSUFBSSxXQUFXLENBQUMsTUFBTSxLQUFLLENBQUM7UUFBRSxPQUFPLENBQUMsQ0FBQztJQUV2QyxNQUFNLEdBQUcsR0FBRyxXQUFXLENBQUMsTUFBTSxDQUFDLENBQUMsR0FBRyxFQUFFLEVBQUUsU0FBUyxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUU7UUFDNUQsT0FBTyxHQUFHLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxTQUFTLEdBQUcsTUFBTSxFQUFFLENBQUMsQ0FBQyxDQUFDO0lBQy9DLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQztJQUVOLE9BQU8sR0FBRyxHQUFHLFdBQVcsQ0FBQyxNQUFNLENBQUM7QUFDbEMsQ0FBQztBQUVEOzs7R0FHRztBQUNILFNBQWdCLFlBQVksQ0FDMUIsV0FBeUQsRUFDekQsYUFBcUIsRUFBRTtJQUV2QixJQUFJLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQztRQUFFLE9BQU8sQ0FBQyxDQUFDO0lBRXZDLE1BQU0sT0FBTyxHQUF3RCxFQUFFLENBQUM7SUFDeEUsS0FBSyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxHQUFHLFVBQVUsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDO1FBQ3BDLE9BQU8sQ0FBQyxJQUFJLENBQUMsRUFBRSxXQUFXLEVBQUUsRUFBRSxFQUFFLE9BQU8sRUFBRSxFQUFFLEVBQUUsQ0FBQyxDQUFDO0lBQ2pELENBQUM7SUFFRCxnQ0FBZ0M7SUFDaEMsS0FBSyxNQUFNLEVBQUUsU0FBUyxFQUFFLE1BQU0sRUFBRSxJQUFJLFdBQVcsRUFBRSxDQUFDO1FBQ2hELE1BQU0sV0FBVyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQzFCLElBQUksQ0FBQyxLQUFLLENBQUMsU0FBUyxHQUFHLFVBQVUsQ0FBQyxFQUNsQyxVQUFVLEdBQUcsQ0FBQyxDQUNmLENBQUM7UUFDRixPQUFPLENBQUMsV0FBVyxDQUFDLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUNqRCxPQUFPLENBQUMsV0FBVyxDQUFDLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUM1QyxDQUFDO0lBRUQsZ0JBQWdCO0lBQ2hCLElBQUksR0FBRyxHQUFHLENBQUMsQ0FBQztJQUNaLEtBQUssTUFBTSxNQUFNLElBQUksT0FBTyxFQUFFLENBQUM7UUFDN0IsSUFBSSxNQUFNLENBQUMsV0FBVyxDQUFDLE1BQU0sS0FBSyxDQUFDO1lBQUUsU0FBUztRQUU5QyxNQUFNLFlBQVksR0FDaEIsTUFBTSxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxHQUFHLE1BQU0sQ0FBQyxXQUFXLENBQUMsTUFBTSxDQUFDO1FBQzVFLE1BQU0sU0FBUyxHQUNiLE1BQU0sQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsR0FBRyxNQUFNLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQztRQUNwRSxNQUFNLE1BQU0sR0FBRyxNQUFNLENBQUMsV0FBVyxDQUFDLE1BQU0sR0FBRyxXQUFXLENBQUMsTUFBTSxDQUFDO1FBRTlELEdBQUcsSUFBSSxNQUFNLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxZQUFZLEdBQUcsU0FBUyxDQUFDLENBQUM7SUFDckQsQ0FBQztJQUVELE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQztBQUVEOztHQUVHO0FBQ0gsU0FBZ0IsZ0JBQWdCLENBQUMsVUFBa0I7SUFDakQsT0FBTyxVQUFVLEdBQUcsR0FBRyxDQUFDO0FBQzFCLENBQUM7QUFFRDs7R0FFRztBQUNILFNBQWdCLHFCQUFxQixDQUNuQyxVQUFrQjtJQUVsQixJQUFJLFVBQVUsR0FBRyxJQUFJO1FBQUUsT0FBTyxXQUFXLENBQUM7SUFDMUMsSUFBSSxVQUFVLEdBQUcsR0FBRztRQUFFLE9BQU8sTUFBTSxDQUFDO0lBQ3BDLElBQUksVUFBVSxHQUFHLEdBQUc7UUFBRSxPQUFPLE1BQU0sQ0FBQztJQUNwQyxPQUFPLE1BQU0sQ0FBQztBQUNoQixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxTQUFnQixnQkFBZ0IsQ0FBQyxLQUFhO0lBQzVDLE9BQU8sS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztBQUMxQixDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLyoqXG4gKiBUaGlua0hpdmUgU0RLIHYzLjAgLSBDYWxpYnJhdGlvbiBBUElcbiAqXG4gKiBQcmVkaWN0aW9uIGFjY3VyYWN5IHRyYWNraW5nIHdpdGggQnJpZXIgc2NvcmVzIGFuZCBjYWxpYnJhdGlvbiBtZXRyaWNzXG4gKi9cblxuaW1wb3J0IHsgYXBpUmVxdWVzdCwgYXBpUmVxdWVzdFdpdGhEYXRhIH0gZnJvbSAnLi4vY29yZS9jbGllbnQnO1xuaW1wb3J0IHR5cGUge1xuICBDYWxpYnJhdGlvblN0YXR1cyxcbiAgQ2FsaWJyYXRpb25CdWNrZXQsXG4gIFByZWRpY3Rpb25UeXBlLFxuICBBcGlSZXNwb25zZSxcbn0gZnJvbSAnLi4vY29yZS90eXBlcyc7XG5cbi8vID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT1cbi8vIENBTElCUkFUSU9OIEFQSSBDTElFTlRcbi8vID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT1cblxuLyoqXG4gKiBSZWNvcmQgb3V0Y29tZSBpbnB1dFxuICovXG5leHBvcnQgaW50ZXJmYWNlIFJlY29yZE91dGNvbWVJbnB1dCB7XG4gIC8qKiBSdW4gSUQgdGhlIHByZWRpY3Rpb24gd2FzIG1hZGUgZm9yICovXG4gIHJ1bklkOiBzdHJpbmc7XG4gIC8qKiBUeXBlIG9mIHByZWRpY3Rpb24gKi9cbiAgcHJlZGljdGlvblR5cGU6IFByZWRpY3Rpb25UeXBlO1xuICAvKiogVGhlIHByZWRpY3RlZCB2YWx1ZSAoMC0xIGZvciBwcm9iYWJpbGl0aWVzKSAqL1xuICBwcmVkaWN0ZWRWYWx1ZTogbnVtYmVyO1xuICAvKiogVGhlIGFjdHVhbCBvdXRjb21lICgwIG9yIDEgZm9yIGJpbmFyeSwgb3IgYWN0dWFsIHZhbHVlKSAqL1xuICBhY3R1YWxPdXRjb21lOiBudW1iZXI7XG4gIC8qKiBXaGVuIHRoZSBwcmVkaWN0aW9uIHdhcyBtYWRlICovXG4gIHByZWRpY3RlZEF0Pzogc3RyaW5nO1xuICAvKiogV2hlbiB0aGUgb3V0Y29tZSB3YXMgb2JzZXJ2ZWQgKi9cbiAgb2JzZXJ2ZWRBdD86IHN0cmluZztcbn1cblxuLyoqXG4gKiBDYWxpYnJhdGlvbiBtZXRyaWNzXG4gKi9cbmV4cG9ydCBpbnRlcmZhY2UgQ2FsaWJyYXRpb25NZXRyaWNzIHtcbiAgYWdlbnRJZDogc3RyaW5nO1xuICBwcmVkaWN0aW9uVHlwZTogUHJlZGljdGlvblR5cGU7XG4gIC8qKiBCcmllciBzY29yZSAobG93ZXIgaXMgYmV0dGVyLCA8MC4xIGlzIGdvb2QpICovXG4gIGJyaWVyU2NvcmU6IG51bWJlcjtcbiAgLyoqIEV4cGVjdGVkIENhbGlicmF0aW9uIEVycm9yICovXG4gIGVjZTogbnVtYmVyO1xuICAvKiogTWF4aW11bSBDYWxpYnJhdGlvbiBFcnJvciAqL1xuICBtY2U6IG51bWJlcjtcbiAgLyoqIFNhbXBsZSBjb3VudCAqL1xuICBzYW1wbGVDb3VudDogbnVtYmVyO1xuICAvKiogSXMgdGhlIG1vZGVsIHdlbGwtY2FsaWJyYXRlZCAqL1xuICBpc0NhbGlicmF0ZWQ6IGJvb2xlYW47XG4gIC8qKiBSZWxpYWJpbGl0eSBkaWFncmFtIGRhdGEgKi9cbiAgcmVsaWFiaWxpdHlEaWFncmFtOiBDYWxpYnJhdGlvbkJ1Y2tldFtdO1xuICAvKiogTGFzdCB1cGRhdGVkICovXG4gIGxhc3RVcGRhdGVkOiBzdHJpbmc7XG59XG5cbi8qKlxuICogQ2FsaWJyYXRpb24gQVBJIGNsaWVudCBmb3IgcHJlZGljdGlvbiBhY2N1cmFjeSB0cmFja2luZ1xuICovXG5leHBvcnQgY29uc3QgY2FsaWJyYXRpb24gPSB7XG4gIC8qKlxuICAgKiBHZXQgY2FsaWJyYXRpb24gc3RhdHVzIGZvciBhbiBhZ2VudFxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IHN0YXR1cyA9IGF3YWl0IGNhbGlicmF0aW9uLnN0YXR1cygnYWdlbnRfMTIzJywgJ2NodXJuX3Jpc2snKTtcbiAgICogY29uc29sZS5sb2coYEJyaWVyIHNjb3JlOiAke3N0YXR1cy5icmllclNjb3JlfWApO1xuICAgKiBjb25zb2xlLmxvZyhgSXMgY2FsaWJyYXRlZDogJHtzdGF0dXMuaXNDYWxpYnJhdGVkfWApO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIHN0YXR1cyhcbiAgICBhZ2VudElkOiBzdHJpbmcsXG4gICAgcHJlZGljdGlvblR5cGU6IFByZWRpY3Rpb25UeXBlXG4gICk6IFByb21pc2U8Q2FsaWJyYXRpb25TdGF0dXM+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPENhbGlicmF0aW9uU3RhdHVzPihcbiAgICAgIGAvY2FsaWJyYXRpb24vc3RhdHVzLyR7YWdlbnRJZH0/cHJlZGljdGlvblR5cGU9JHtwcmVkaWN0aW9uVHlwZX1gXG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IGFsbCBjYWxpYnJhdGlvbiBtZXRyaWNzIGZvciBhbiBhZ2VudFxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IG1ldHJpY3MgPSBhd2FpdCBjYWxpYnJhdGlvbi5hbGxNZXRyaWNzKCdhZ2VudF8xMjMnKTtcbiAgICogZm9yIChjb25zdCBtIG9mIG1ldHJpY3MpIHtcbiAgICogICBjb25zb2xlLmxvZyhgJHttLnByZWRpY3Rpb25UeXBlfTogQnJpZXI9JHttLmJyaWVyU2NvcmV9YCk7XG4gICAqIH1cbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBhbGxNZXRyaWNzKGFnZW50SWQ6IHN0cmluZyk6IFByb21pc2U8Q2FsaWJyYXRpb25NZXRyaWNzW10+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPENhbGlicmF0aW9uTWV0cmljc1tdPihcbiAgICAgIGAvY2FsaWJyYXRpb24vbWV0cmljcy8ke2FnZW50SWR9YFxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIFJlY29yZCBhIHByZWRpY3Rpb24gb3V0Y29tZSBmb3IgY2FsaWJyYXRpb24gdHJhY2tpbmdcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiAvLyBSZWNvcmQgYSBjaHVybiBwcmVkaWN0aW9uIG91dGNvbWVcbiAgICogYXdhaXQgY2FsaWJyYXRpb24ucmVjb3JkT3V0Y29tZSh7XG4gICAqICAgcnVuSWQ6ICdydW5fYWJjMTIzJyxcbiAgICogICBwcmVkaWN0aW9uVHlwZTogJ2NodXJuX3Jpc2snLFxuICAgKiAgIHByZWRpY3RlZFZhbHVlOiAwLjcsICAvLyBXZSBwcmVkaWN0ZWQgNzAlIGNodXJuIHJpc2tcbiAgICogICBhY3R1YWxPdXRjb21lOiAxLCAgICAgLy8gQ3VzdG9tZXIgZGlkIGNodXJuXG4gICAqIH0pO1xuICAgKlxuICAgKiAvLyBSZWNvcmQgYSByZXNvbHV0aW9uIHRpbWUgcHJlZGljdGlvblxuICAgKiBhd2FpdCBjYWxpYnJhdGlvbi5yZWNvcmRPdXRjb21lKHtcbiAgICogICBydW5JZDogJ3J1bl9hYmMxMjMnLFxuICAgKiAgIHByZWRpY3Rpb25UeXBlOiAncmVzb2x1dGlvbl90aW1lJyxcbiAgICogICBwcmVkaWN0ZWRWYWx1ZTogMTUsICAgLy8gUHJlZGljdGVkIDE1IG1pbnV0ZXNcbiAgICogICBhY3R1YWxPdXRjb21lOiAyMiwgICAgLy8gQWN0dWFsIHdhcyAyMiBtaW51dGVzXG4gICAqIH0pO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIHJlY29yZE91dGNvbWUoaW5wdXQ6IFJlY29yZE91dGNvbWVJbnB1dCk6IFByb21pc2U8e1xuICAgIHJlY29yZGVkOiBib29sZWFuO1xuICAgIGJyaWVyQ29udHJpYnV0aW9uOiBudW1iZXI7XG4gICAgbWVzc2FnZTogc3RyaW5nO1xuICB9PiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YSgnL2NhbGlicmF0aW9uL3JlY29yZCcsIHtcbiAgICAgIG1ldGhvZDogJ1BPU1QnLFxuICAgICAgYm9keToge1xuICAgICAgICAuLi5pbnB1dCxcbiAgICAgICAgcHJlZGljdGVkQXQ6IGlucHV0LnByZWRpY3RlZEF0IHx8IG5ldyBEYXRlKCkudG9JU09TdHJpbmcoKSxcbiAgICAgICAgb2JzZXJ2ZWRBdDogaW5wdXQub2JzZXJ2ZWRBdCB8fCBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCksXG4gICAgICB9LFxuICAgIH0pO1xuICB9LFxuXG4gIC8qKlxuICAgKiBUcmlnZ2VyIHJlY2FsaWJyYXRpb24gZm9yIGFuIGFnZW50XG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgcmVzdWx0ID0gYXdhaXQgY2FsaWJyYXRpb24ucmV0cmFpbignYWdlbnRfMTIzJywge1xuICAgKiAgIHByZWRpY3Rpb25UeXBlczogWydjaHVybl9yaXNrJywgJ2VzY2FsYXRpb25fcmlzayddLFxuICAgKiB9KTtcbiAgICogY29uc29sZS5sb2coYFJldHJhaW5lZDogJHtyZXN1bHQuc3VjY2Vzc31gKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyByZXRyYWluKFxuICAgIGFnZW50SWQ6IHN0cmluZyxcbiAgICBvcHRpb25zOiB7XG4gICAgICBwcmVkaWN0aW9uVHlwZXM/OiBQcmVkaWN0aW9uVHlwZVtdO1xuICAgICAgbWluU2FtcGxlcz86IG51bWJlcjtcbiAgICB9ID0ge31cbiAgKTogUHJvbWlzZTx7XG4gICAgc3VjY2VzczogYm9vbGVhbjtcbiAgICByZXRyYWluZWRUeXBlczogUHJlZGljdGlvblR5cGVbXTtcbiAgICBza2lwcGVkVHlwZXM6IEFycmF5PHsgdHlwZTogUHJlZGljdGlvblR5cGU7IHJlYXNvbjogc3RyaW5nIH0+O1xuICAgIG5ld01ldHJpY3M6IENhbGlicmF0aW9uTWV0cmljc1tdO1xuICB9PiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YShgL2NhbGlicmF0aW9uL3JldHJhaW4vJHthZ2VudElkfWAsIHtcbiAgICAgIG1ldGhvZDogJ1BPU1QnLFxuICAgICAgYm9keTogb3B0aW9ucyxcbiAgICB9KTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IHJlbGlhYmlsaXR5IGRpYWdyYW0gZGF0YSBmb3IgdmlzdWFsaXphdGlvblxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IGRpYWdyYW0gPSBhd2FpdCBjYWxpYnJhdGlvbi5yZWxpYWJpbGl0eURpYWdyYW0oJ2FnZW50XzEyMycsICdvdXRjb21lJyk7XG4gICAqIC8vIFVzZSBkaWFncmFtLmJ1Y2tldHMgdG8gcGxvdCBwcmVkaWN0ZWQgdnMgYWN0dWFsIHByb2JhYmlsaXRpZXNcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyByZWxpYWJpbGl0eURpYWdyYW0oXG4gICAgYWdlbnRJZDogc3RyaW5nLFxuICAgIHByZWRpY3Rpb25UeXBlOiBQcmVkaWN0aW9uVHlwZVxuICApOiBQcm9taXNlPHtcbiAgICBhZ2VudElkOiBzdHJpbmc7XG4gICAgcHJlZGljdGlvblR5cGU6IFByZWRpY3Rpb25UeXBlO1xuICAgIGJ1Y2tldHM6IENhbGlicmF0aW9uQnVja2V0W107XG4gICAgcGVyZmVjdENhbGlicmF0aW9uTGluZTogQXJyYXk8eyB4OiBudW1iZXI7IHk6IG51bWJlciB9PjtcbiAgfT4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGEoXG4gICAgICBgL2NhbGlicmF0aW9uL2RpYWdyYW0vJHthZ2VudElkfT9wcmVkaWN0aW9uVHlwZT0ke3ByZWRpY3Rpb25UeXBlfWBcbiAgICApO1xuICB9LFxufTtcblxuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuLy8gSEVMUEVSIEZVTkNUSU9OU1xuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuXG4vKipcbiAqIENhbGN1bGF0ZSBCcmllciBzY29yZSBmcm9tIHByZWRpY3Rpb25zIGFuZCBvdXRjb21lc1xuICogTG93ZXIgaXMgYmV0dGVyLCA8MC4xIGlzIGNvbnNpZGVyZWQgZ29vZFxuICovXG5leHBvcnQgZnVuY3Rpb24gY2FsY3VsYXRlQnJpZXJTY29yZShcbiAgcHJlZGljdGlvbnM6IEFycmF5PHsgcHJlZGljdGVkOiBudW1iZXI7IGFjdHVhbDogbnVtYmVyIH0+XG4pOiBudW1iZXIge1xuICBpZiAocHJlZGljdGlvbnMubGVuZ3RoID09PSAwKSByZXR1cm4gMDtcblxuICBjb25zdCBzdW0gPSBwcmVkaWN0aW9ucy5yZWR1Y2UoKGFjYywgeyBwcmVkaWN0ZWQsIGFjdHVhbCB9KSA9PiB7XG4gICAgcmV0dXJuIGFjYyArIE1hdGgucG93KHByZWRpY3RlZCAtIGFjdHVhbCwgMik7XG4gIH0sIDApO1xuXG4gIHJldHVybiBzdW0gLyBwcmVkaWN0aW9ucy5sZW5ndGg7XG59XG5cbi8qKlxuICogQ2FsY3VsYXRlIEV4cGVjdGVkIENhbGlicmF0aW9uIEVycm9yIChFQ0UpXG4gKiBNZWFzdXJlcyBob3cgd2VsbC1jYWxpYnJhdGVkIHByZWRpY3Rpb25zIGFyZSBhY3Jvc3MgY29uZmlkZW5jZSBidWNrZXRzXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBjYWxjdWxhdGVFQ0UoXG4gIHByZWRpY3Rpb25zOiBBcnJheTx7IHByZWRpY3RlZDogbnVtYmVyOyBhY3R1YWw6IG51bWJlciB9PixcbiAgbnVtQnVja2V0czogbnVtYmVyID0gMTBcbik6IG51bWJlciB7XG4gIGlmIChwcmVkaWN0aW9ucy5sZW5ndGggPT09IDApIHJldHVybiAwO1xuXG4gIGNvbnN0IGJ1Y2tldHM6IEFycmF5PHsgcHJlZGljdGlvbnM6IG51bWJlcltdOyBhY3R1YWxzOiBudW1iZXJbXSB9PiA9IFtdO1xuICBmb3IgKGxldCBpID0gMDsgaSA8IG51bUJ1Y2tldHM7IGkrKykge1xuICAgIGJ1Y2tldHMucHVzaCh7IHByZWRpY3Rpb25zOiBbXSwgYWN0dWFsczogW10gfSk7XG4gIH1cblxuICAvLyBBc3NpZ24gcHJlZGljdGlvbnMgdG8gYnVja2V0c1xuICBmb3IgKGNvbnN0IHsgcHJlZGljdGVkLCBhY3R1YWwgfSBvZiBwcmVkaWN0aW9ucykge1xuICAgIGNvbnN0IGJ1Y2tldEluZGV4ID0gTWF0aC5taW4oXG4gICAgICBNYXRoLmZsb29yKHByZWRpY3RlZCAqIG51bUJ1Y2tldHMpLFxuICAgICAgbnVtQnVja2V0cyAtIDFcbiAgICApO1xuICAgIGJ1Y2tldHNbYnVja2V0SW5kZXhdLnByZWRpY3Rpb25zLnB1c2gocHJlZGljdGVkKTtcbiAgICBidWNrZXRzW2J1Y2tldEluZGV4XS5hY3R1YWxzLnB1c2goYWN0dWFsKTtcbiAgfVxuXG4gIC8vIENhbGN1bGF0ZSBFQ0VcbiAgbGV0IGVjZSA9IDA7XG4gIGZvciAoY29uc3QgYnVja2V0IG9mIGJ1Y2tldHMpIHtcbiAgICBpZiAoYnVja2V0LnByZWRpY3Rpb25zLmxlbmd0aCA9PT0gMCkgY29udGludWU7XG5cbiAgICBjb25zdCBhdmdQcmVkaWN0ZWQgPVxuICAgICAgYnVja2V0LnByZWRpY3Rpb25zLnJlZHVjZSgoYSwgYikgPT4gYSArIGIsIDApIC8gYnVja2V0LnByZWRpY3Rpb25zLmxlbmd0aDtcbiAgICBjb25zdCBhdmdBY3R1YWwgPVxuICAgICAgYnVja2V0LmFjdHVhbHMucmVkdWNlKChhLCBiKSA9PiBhICsgYiwgMCkgLyBidWNrZXQuYWN0dWFscy5sZW5ndGg7XG4gICAgY29uc3Qgd2VpZ2h0ID0gYnVja2V0LnByZWRpY3Rpb25zLmxlbmd0aCAvIHByZWRpY3Rpb25zLmxlbmd0aDtcblxuICAgIGVjZSArPSB3ZWlnaHQgKiBNYXRoLmFicyhhdmdQcmVkaWN0ZWQgLSBhdmdBY3R1YWwpO1xuICB9XG5cbiAgcmV0dXJuIGVjZTtcbn1cblxuLyoqXG4gKiBDaGVjayBpZiBhIG1vZGVsIGlzIHdlbGwtY2FsaWJyYXRlZCBiYXNlZCBvbiBCcmllciBzY29yZVxuICovXG5leHBvcnQgZnVuY3Rpb24gaXNXZWxsQ2FsaWJyYXRlZChicmllclNjb3JlOiBudW1iZXIpOiBib29sZWFuIHtcbiAgcmV0dXJuIGJyaWVyU2NvcmUgPCAwLjE7XG59XG5cbi8qKlxuICogR2V0IGNhbGlicmF0aW9uIHF1YWxpdHkgbGFiZWxcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldENhbGlicmF0aW9uUXVhbGl0eShcbiAgYnJpZXJTY29yZTogbnVtYmVyXG4pOiAnZXhjZWxsZW50JyB8ICdnb29kJyB8ICdmYWlyJyB8ICdwb29yJyB7XG4gIGlmIChicmllclNjb3JlIDwgMC4wNSkgcmV0dXJuICdleGNlbGxlbnQnO1xuICBpZiAoYnJpZXJTY29yZSA8IDAuMSkgcmV0dXJuICdnb29kJztcbiAgaWYgKGJyaWVyU2NvcmUgPCAwLjIpIHJldHVybiAnZmFpcic7XG4gIHJldHVybiAncG9vcic7XG59XG5cbi8qKlxuICogRm9ybWF0IEJyaWVyIHNjb3JlIGZvciBkaXNwbGF5XG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBmb3JtYXRCcmllclNjb3JlKHNjb3JlOiBudW1iZXIpOiBzdHJpbmcge1xuICByZXR1cm4gc2NvcmUudG9GaXhlZCg0KTtcbn1cbiJdfQ==
|