@thinkhive/sdk 3.1.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +83 -12
- package/README.md +279 -128
- package/dist/api/agents.d.ts +169 -0
- package/dist/api/agents.js +185 -0
- package/dist/api/apiKeys.d.ts +252 -0
- package/dist/api/apiKeys.js +298 -0
- package/dist/api/business-metrics.d.ts +188 -0
- package/dist/api/business-metrics.js +213 -0
- package/dist/api/calibration.d.ts +0 -62
- package/dist/api/calibration.js +5 -48
- package/dist/api/claims.js +10 -7
- package/dist/api/conversation-eval.d.ts +200 -0
- package/dist/api/conversation-eval.js +235 -0
- package/dist/api/deterministic-graders.d.ts +205 -0
- package/dist/api/deterministic-graders.js +191 -0
- package/dist/api/eval-health.d.ts +250 -0
- package/dist/api/eval-health.js +224 -0
- package/dist/api/human-review.d.ts +275 -0
- package/dist/api/human-review.js +236 -0
- package/dist/api/nondeterminism.d.ts +300 -0
- package/dist/api/nondeterminism.js +250 -0
- package/dist/api/quality-metrics.d.ts +303 -0
- package/dist/api/quality-metrics.js +198 -0
- package/dist/api/roi-analytics.d.ts +263 -0
- package/dist/api/roi-analytics.js +204 -0
- package/dist/api/runs.js +12 -6
- package/dist/api/transcript-patterns.d.ts +204 -0
- package/dist/api/transcript-patterns.js +227 -0
- package/dist/core/client.d.ts +83 -9
- package/dist/core/client.js +229 -34
- package/dist/core/config.d.ts +2 -3
- package/dist/core/config.js +3 -4
- package/dist/core/types.d.ts +57 -4
- package/dist/core/types.js +1 -1
- package/dist/index.d.ts +429 -76
- package/dist/index.js +262 -42
- package/package.json +2 -2
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ThinkHive SDK v3.0 - Human Review API
|
|
4
|
+
*
|
|
5
|
+
* API for managing human review queue, calibration sets, and reviewer management
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.humanReview = void 0;
|
|
9
|
+
const client_1 = require("../core/client");
|
|
10
|
+
// ============================================================================
|
|
11
|
+
// HUMAN REVIEW API CLIENT
|
|
12
|
+
// ============================================================================
|
|
13
|
+
/**
|
|
14
|
+
* Human Review API client for managing human review queue and calibration
|
|
15
|
+
*/
|
|
16
|
+
exports.humanReview = {
|
|
17
|
+
/**
|
|
18
|
+
* Get pending review queue items
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```typescript
|
|
22
|
+
* const items = await humanReview.getQueue({
|
|
23
|
+
* agentId: 'agent_123',
|
|
24
|
+
* status: 'pending',
|
|
25
|
+
* limit: 20,
|
|
26
|
+
* });
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
async getQueue(options = {}) {
|
|
30
|
+
const params = new URLSearchParams();
|
|
31
|
+
if (options.agentId)
|
|
32
|
+
params.set('agentId', options.agentId);
|
|
33
|
+
if (options.status)
|
|
34
|
+
params.set('status', options.status);
|
|
35
|
+
if (options.reviewType)
|
|
36
|
+
params.set('reviewType', options.reviewType);
|
|
37
|
+
if (options.isCalibration !== undefined)
|
|
38
|
+
params.set('isCalibration', String(options.isCalibration));
|
|
39
|
+
if (options.minPriority !== undefined)
|
|
40
|
+
params.set('minPriority', String(options.minPriority));
|
|
41
|
+
if (options.limit)
|
|
42
|
+
params.set('limit', String(options.limit));
|
|
43
|
+
if (options.offset)
|
|
44
|
+
params.set('offset', String(options.offset));
|
|
45
|
+
return (0, client_1.apiRequestWithData)(`/human-review/queue?${params.toString()}`, { apiVersion: 'none' });
|
|
46
|
+
},
|
|
47
|
+
/**
|
|
48
|
+
* Add an item to the review queue
|
|
49
|
+
*
|
|
50
|
+
* @example
|
|
51
|
+
* ```typescript
|
|
52
|
+
* const item = await humanReview.addToQueue({
|
|
53
|
+
* traceId: 'trace_123',
|
|
54
|
+
* agentId: 'agent_123',
|
|
55
|
+
* reviewType: 'disagreement',
|
|
56
|
+
* priority: 80,
|
|
57
|
+
* llmScore: 65,
|
|
58
|
+
* llmPassed: true,
|
|
59
|
+
* });
|
|
60
|
+
* ```
|
|
61
|
+
*/
|
|
62
|
+
async addToQueue(options) {
|
|
63
|
+
return (0, client_1.apiRequestWithData)('/human-review/queue', {
|
|
64
|
+
method: 'POST',
|
|
65
|
+
body: options,
|
|
66
|
+
apiVersion: 'none',
|
|
67
|
+
});
|
|
68
|
+
},
|
|
69
|
+
/**
|
|
70
|
+
* Get a specific review item
|
|
71
|
+
*
|
|
72
|
+
* @example
|
|
73
|
+
* ```typescript
|
|
74
|
+
* const item = await humanReview.getItem('item_123');
|
|
75
|
+
* ```
|
|
76
|
+
*/
|
|
77
|
+
async getItem(itemId) {
|
|
78
|
+
return (0, client_1.apiRequestWithData)(`/human-review/queue/${itemId}`, { apiVersion: 'none' });
|
|
79
|
+
},
|
|
80
|
+
/**
|
|
81
|
+
* Claim a review item for processing
|
|
82
|
+
*
|
|
83
|
+
* @example
|
|
84
|
+
* ```typescript
|
|
85
|
+
* const item = await humanReview.claim('item_123');
|
|
86
|
+
* ```
|
|
87
|
+
*/
|
|
88
|
+
async claim(itemId) {
|
|
89
|
+
return (0, client_1.apiRequestWithData)(`/human-review/queue/${itemId}/claim`, { method: 'POST', apiVersion: 'none' });
|
|
90
|
+
},
|
|
91
|
+
/**
|
|
92
|
+
* Release a claimed review item
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```typescript
|
|
96
|
+
* await humanReview.release('item_123');
|
|
97
|
+
* ```
|
|
98
|
+
*/
|
|
99
|
+
async release(itemId) {
|
|
100
|
+
return (0, client_1.apiRequestWithData)(`/human-review/queue/${itemId}/release`, { method: 'POST', apiVersion: 'none' });
|
|
101
|
+
},
|
|
102
|
+
/**
|
|
103
|
+
* Skip a review item
|
|
104
|
+
*
|
|
105
|
+
* @example
|
|
106
|
+
* ```typescript
|
|
107
|
+
* await humanReview.skip('item_123');
|
|
108
|
+
* ```
|
|
109
|
+
*/
|
|
110
|
+
async skip(itemId) {
|
|
111
|
+
return (0, client_1.apiRequestWithData)(`/human-review/queue/${itemId}/skip`, { method: 'POST', apiVersion: 'none' });
|
|
112
|
+
},
|
|
113
|
+
/**
|
|
114
|
+
* Submit a review
|
|
115
|
+
*
|
|
116
|
+
* @example
|
|
117
|
+
* ```typescript
|
|
118
|
+
* const result = await humanReview.submit('item_123', {
|
|
119
|
+
* passed: true,
|
|
120
|
+
* score: 85,
|
|
121
|
+
* reasoning: 'Response accurately addressed the query',
|
|
122
|
+
* durationMs: 45000,
|
|
123
|
+
* });
|
|
124
|
+
* ```
|
|
125
|
+
*/
|
|
126
|
+
async submit(itemId, review) {
|
|
127
|
+
return (0, client_1.apiRequestWithData)(`/human-review/queue/${itemId}/submit`, { method: 'POST', body: review, apiVersion: 'none' });
|
|
128
|
+
},
|
|
129
|
+
/**
|
|
130
|
+
* Get queue statistics
|
|
131
|
+
*
|
|
132
|
+
* @example
|
|
133
|
+
* ```typescript
|
|
134
|
+
* const stats = await humanReview.getStats('agent_123');
|
|
135
|
+
* ```
|
|
136
|
+
*/
|
|
137
|
+
async getStats(agentId) {
|
|
138
|
+
const params = agentId ? `?agentId=${agentId}` : '';
|
|
139
|
+
return (0, client_1.apiRequestWithData)(`/human-review/stats${params}`, { apiVersion: 'none' });
|
|
140
|
+
},
|
|
141
|
+
/**
|
|
142
|
+
* Get next item for a reviewer
|
|
143
|
+
*
|
|
144
|
+
* @example
|
|
145
|
+
* ```typescript
|
|
146
|
+
* const nextItem = await humanReview.getNextItem('agent_123');
|
|
147
|
+
* ```
|
|
148
|
+
*/
|
|
149
|
+
async getNextItem(agentId) {
|
|
150
|
+
const params = agentId ? `?agentId=${agentId}` : '';
|
|
151
|
+
return (0, client_1.apiRequestWithData)(`/human-review/next-item${params}`, { apiVersion: 'none' });
|
|
152
|
+
},
|
|
153
|
+
/**
|
|
154
|
+
* Get available review types
|
|
155
|
+
*
|
|
156
|
+
* @example
|
|
157
|
+
* ```typescript
|
|
158
|
+
* const types = await humanReview.getReviewTypes();
|
|
159
|
+
* ```
|
|
160
|
+
*/
|
|
161
|
+
async getReviewTypes() {
|
|
162
|
+
return (0, client_1.apiRequestWithData)('/human-review/review-types', { apiVersion: 'none' });
|
|
163
|
+
},
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
165
|
+
// CALIBRATION SETS
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
/**
|
|
168
|
+
* Get calibration sets
|
|
169
|
+
*
|
|
170
|
+
* @example
|
|
171
|
+
* ```typescript
|
|
172
|
+
* const sets = await humanReview.getCalibrationSets('agent_123');
|
|
173
|
+
* ```
|
|
174
|
+
*/
|
|
175
|
+
async getCalibrationSets(agentId) {
|
|
176
|
+
const params = agentId ? `?agentId=${agentId}` : '';
|
|
177
|
+
return (0, client_1.apiRequestWithData)(`/human-review/calibration-sets${params}`, { apiVersion: 'none' });
|
|
178
|
+
},
|
|
179
|
+
/**
|
|
180
|
+
* Create a calibration set
|
|
181
|
+
*
|
|
182
|
+
* @example
|
|
183
|
+
* ```typescript
|
|
184
|
+
* const set = await humanReview.createCalibrationSet({
|
|
185
|
+
* name: 'Quality Calibration Q1',
|
|
186
|
+
* agentId: 'agent_123',
|
|
187
|
+
* targetAgreement: 0.85,
|
|
188
|
+
* minSamples: 50,
|
|
189
|
+
* });
|
|
190
|
+
* ```
|
|
191
|
+
*/
|
|
192
|
+
async createCalibrationSet(options) {
|
|
193
|
+
return (0, client_1.apiRequestWithData)('/human-review/calibration-sets', {
|
|
194
|
+
method: 'POST',
|
|
195
|
+
body: options,
|
|
196
|
+
apiVersion: 'none',
|
|
197
|
+
});
|
|
198
|
+
},
|
|
199
|
+
/**
|
|
200
|
+
* Get a calibration set by ID
|
|
201
|
+
*
|
|
202
|
+
* @example
|
|
203
|
+
* ```typescript
|
|
204
|
+
* const set = await humanReview.getCalibrationSet('set_123');
|
|
205
|
+
* ```
|
|
206
|
+
*/
|
|
207
|
+
async getCalibrationSet(setId) {
|
|
208
|
+
return (0, client_1.apiRequestWithData)(`/human-review/calibration-sets/${setId}`, { apiVersion: 'none' });
|
|
209
|
+
},
|
|
210
|
+
// ---------------------------------------------------------------------------
|
|
211
|
+
// REVIEWER CALIBRATION
|
|
212
|
+
// ---------------------------------------------------------------------------
|
|
213
|
+
/**
|
|
214
|
+
* Get certified reviewers for a calibration set
|
|
215
|
+
*
|
|
216
|
+
* @example
|
|
217
|
+
* ```typescript
|
|
218
|
+
* const reviewers = await humanReview.getCertifiedReviewers('set_123');
|
|
219
|
+
* ```
|
|
220
|
+
*/
|
|
221
|
+
async getCertifiedReviewers(calibrationSetId) {
|
|
222
|
+
return (0, client_1.apiRequestWithData)(`/human-review/reviewers?calibrationSetId=${calibrationSetId}`, { apiVersion: 'none' });
|
|
223
|
+
},
|
|
224
|
+
/**
|
|
225
|
+
* Get reviewer calibration status
|
|
226
|
+
*
|
|
227
|
+
* @example
|
|
228
|
+
* ```typescript
|
|
229
|
+
* const calibrations = await humanReview.getReviewerCalibrations('user_123');
|
|
230
|
+
* ```
|
|
231
|
+
*/
|
|
232
|
+
async getReviewerCalibrations(userId) {
|
|
233
|
+
return (0, client_1.apiRequestWithData)(`/human-review/calibration/${userId}`, { apiVersion: 'none' });
|
|
234
|
+
},
|
|
235
|
+
};
|
|
236
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHVtYW4tcmV2aWV3LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2FwaS9odW1hbi1yZXZpZXcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7O0dBSUc7OztBQUVILDJDQUFnRTtBQXVIaEUsK0VBQStFO0FBQy9FLDBCQUEwQjtBQUMxQiwrRUFBK0U7QUFFL0U7O0dBRUc7QUFDVSxRQUFBLFdBQVcsR0FBRztJQUN6Qjs7Ozs7Ozs7Ozs7T0FXRztJQUNILEtBQUssQ0FBQyxRQUFRLENBQUMsVUFBNEIsRUFBRTtRQUMzQyxNQUFNLE1BQU0sR0FBRyxJQUFJLGVBQWUsRUFBRSxDQUFDO1FBQ3JDLElBQUksT0FBTyxDQUFDLE9BQU87WUFBRSxNQUFNLENBQUMsR0FBRyxDQUFDLFNBQVMsRUFBRSxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDNUQsSUFBSSxPQUFPLENBQUMsTUFBTTtZQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUN6RCxJQUFJLE9BQU8sQ0FBQyxVQUFVO1lBQUUsTUFBTSxDQUFDLEdBQUcsQ0FBQyxZQUFZLEVBQUUsT0FBTyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3JFLElBQUksT0FBTyxDQUFDLGFBQWEsS0FBSyxTQUFTO1lBQUUsTUFBTSxDQUFDLEdBQUcsQ0FBQyxlQUFlLEVBQUUsTUFBTSxDQUFDLE9BQU8sQ0FBQyxhQUFhLENBQUMsQ0FBQyxDQUFDO1FBQ3BHLElBQUksT0FBTyxDQUFDLFdBQVcsS0FBSyxTQUFTO1lBQUUsTUFBTSxDQUFDLEdBQUcsQ0FBQyxhQUFhLEVBQUUsTUFBTSxDQUFDLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDO1FBQzlGLElBQUksT0FBTyxDQUFDLEtBQUs7WUFBRSxNQUFNLENBQUMsR0FBRyxDQUFDLE9BQU8sRUFBRSxNQUFNLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUM7UUFDOUQsSUFBSSxPQUFPLENBQUMsTUFBTTtZQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsUUFBUSxFQUFFLE1BQU0sQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQztRQUVqRSxPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLHVCQUF1QixNQUFNLENBQUMsUUFBUSxFQUFFLEVBQUUsRUFDMUMsRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZCLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7Ozs7Ozs7O09BY0c7SUFDSCxLQUFLLENBQUMsVUFBVSxDQUFDLE9BQTBCO1FBQ3pDLE9BQU8sSUFBQSwyQkFBa0IsRUFBdUIscUJBQXFCLEVBQUU7WUFDckUsTUFBTSxFQUFFLE1BQU07WUFDZCxJQUFJLEVBQUUsT0FBTztZQUNiLFVBQVUsRUFBRSxNQUFNO1NBQ25CLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLE9BQU8sQ0FBQyxNQUFjO1FBQzFCLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsdUJBQXVCLE1BQU0sRUFBRSxFQUMvQixFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLEtBQUssQ0FBQyxNQUFjO1FBQ3hCLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsdUJBQXVCLE1BQU0sUUFBUSxFQUNyQyxFQUFFLE1BQU0sRUFBRSxNQUFNLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxDQUN2QyxDQUFDO0lBQ0osQ0FBQztJQUVEOzs7Ozs7O09BT0c7SUFDSCxLQUFLLENBQUMsT0FBTyxDQUFDLE1BQWM7UUFDMUIsT0FBTyxJQUFBLDJCQUFrQixFQUN2Qix1QkFBdUIsTUFBTSxVQUFVLEVBQ3ZDLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZDLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILEtBQUssQ0FBQyxJQUFJLENBQUMsTUFBYztRQUN2QixPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLHVCQUF1QixNQUFNLE9BQU8sRUFDcEMsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkMsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7Ozs7Ozs7O09BWUc7SUFDSCxLQUFLLENBQUMsTUFBTSxDQUFDLE1BQWMsRUFBRSxNQUEyQjtRQUN0RCxPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLHVCQUF1QixNQUFNLFNBQVMsRUFDdEMsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxDQUNyRCxDQUFDO0lBQ0osQ0FBQztJQUVEOzs7Ozs7O09BT0c7SUFDSCxLQUFLLENBQUMsUUFBUSxDQUFDLE9BQWdCO1FBQzdCLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxDQUFDLENBQUMsWUFBWSxPQUFPLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQ3BELE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsc0JBQXNCLE1BQU0sRUFBRSxFQUM5QixFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLFdBQVcsQ0FBQyxPQUFnQjtRQUNoQyxNQUFNLE1BQU0sR0FBRyxPQUFPLENBQUMsQ0FBQyxDQUFDLFlBQVksT0FBTyxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUNwRCxPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLDBCQUEwQixNQUFNLEVBQUUsRUFDbEMsRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZCLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILEtBQUssQ0FBQyxjQUFjO1FBTWxCLE9BQU8sSUFBQSwyQkFBa0IsRUFBQyw0QkFBNEIsRUFBRSxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FBQyxDQUFDO0lBQ2xGLENBQUM7SUFFRCw4RUFBOEU7SUFDOUUsbUJBQW1CO0lBQ25CLDhFQUE4RTtJQUU5RTs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLGtCQUFrQixDQUFDLE9BQWdCO1FBQ3ZDLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxDQUFDLENBQUMsWUFBWSxPQUFPLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQ3BELE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsaUNBQWlDLE1BQU0sRUFBRSxFQUN6QyxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7Ozs7Ozs7O09BWUc7SUFDSCxLQUFLLENBQUMsb0JBQW9CLENBQUMsT0FBb0M7UUFDN0QsT0FBTyxJQUFBLDJCQUFrQixFQUFpQixnQ0FBZ0MsRUFBRTtZQUMxRSxNQUFNLEVBQUUsTUFBTTtZQUNkLElBQUksRUFBRSxPQUFPO1lBQ2IsVUFBVSxFQUFFLE1BQU07U0FDbkIsQ0FBQyxDQUFDO0lBQ0wsQ0FBQztJQUVEOzs7Ozs7O09BT0c7SUFDSCxLQUFLLENBQUMsaUJBQWlCLENBQUMsS0FBYTtRQUNuQyxPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLGtDQUFrQyxLQUFLLEVBQUUsRUFDekMsRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZCLENBQUM7SUFDSixDQUFDO0lBRUQsOEVBQThFO0lBQzlFLHVCQUF1QjtJQUN2Qiw4RUFBOEU7SUFFOUU7Ozs7Ozs7T0FPRztJQUNILEtBQUssQ0FBQyxxQkFBcUIsQ0FBQyxnQkFBd0I7UUFDbEQsT0FBTyxJQUFBLDJCQUFrQixFQUN2Qiw0Q0FBNEMsZ0JBQWdCLEVBQUUsRUFDOUQsRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZCLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILEtBQUssQ0FBQyx1QkFBdUIsQ0FBQyxNQUFjO1FBQzFDLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsNkJBQTZCLE1BQU0sRUFBRSxFQUNyQyxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkIsQ0FBQztJQUNKLENBQUM7Q0FDRixDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLyoqXG4gKiBUaGlua0hpdmUgU0RLIHYzLjAgLSBIdW1hbiBSZXZpZXcgQVBJXG4gKlxuICogQVBJIGZvciBtYW5hZ2luZyBodW1hbiByZXZpZXcgcXVldWUsIGNhbGlicmF0aW9uIHNldHMsIGFuZCByZXZpZXdlciBtYW5hZ2VtZW50XG4gKi9cblxuaW1wb3J0IHsgYXBpUmVxdWVzdCwgYXBpUmVxdWVzdFdpdGhEYXRhIH0gZnJvbSAnLi4vY29yZS9jbGllbnQnO1xuXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG4vLyBUWVBFU1xuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuXG5leHBvcnQgdHlwZSBIdW1hblJldmlld1N0YXR1cyA9ICdwZW5kaW5nJyB8ICdpbl9wcm9ncmVzcycgfCAnY29tcGxldGVkJyB8ICdza2lwcGVkJyB8ICdleHBpcmVkJztcbmV4cG9ydCB0eXBlIEh1bWFuUmV2aWV3VHlwZSA9ICdkaXNhZ3JlZW1lbnQnIHwgJ2xvd19jb25maWRlbmNlJyB8ICdjYWxpYnJhdGlvbicgfCAncmFuZG9tX3NhbXBsZScgfCAnZmxhZ2dlZCc7XG5cbmV4cG9ydCBpbnRlcmZhY2UgSHVtYW5SZXZpZXdRdWV1ZUl0ZW0ge1xuICBpZDogc3RyaW5nO1xuICBjb21wYW55SWQ6IHN0cmluZztcbiAgYWdlbnRJZDogc3RyaW5nO1xuICB0cmFjZUlkOiBzdHJpbmc7XG4gIGNyaXRlcmlvbklkPzogc3RyaW5nO1xuICByZXZpZXdUeXBlOiBIdW1hblJldmlld1R5cGU7XG4gIHByaW9yaXR5OiBudW1iZXI7XG4gIHN0YXR1czogSHVtYW5SZXZpZXdTdGF0dXM7XG4gIGxsbVNjb3JlPzogbnVtYmVyO1xuICBsbG1QYXNzZWQ/OiBib29sZWFuO1xuICBsbG1SZWFzb25pbmc/OiBzdHJpbmc7XG4gIGxsbUNvbmZpZGVuY2U/OiBudW1iZXI7XG4gIHJldmlld2VySWQ/OiBzdHJpbmc7XG4gIGh1bWFuU2NvcmU/OiBudW1iZXI7XG4gIGh1bWFuUGFzc2VkPzogYm9vbGVhbjtcbiAgaHVtYW5SZWFzb25pbmc/OiBzdHJpbmc7XG4gIHJldmlld0R1cmF0aW9uTXM/OiBudW1iZXI7XG4gIGlzQ2FsaWJyYXRpb25TYW1wbGU/OiBib29sZWFuO1xuICBjYWxpYnJhdGlvblNldElkPzogc3RyaW5nO1xuICBleHBlY3RlZFNjb3JlPzogbnVtYmVyO1xuICBleHBlY3RlZFBhc3NlZD86IGJvb2xlYW47XG4gIGV4cGlyZXNBdD86IHN0cmluZztcbiAgY2xhaW1lZEF0Pzogc3RyaW5nO1xuICBjb21wbGV0ZWRBdD86IHN0cmluZztcbiAgY3JlYXRlZEF0OiBzdHJpbmc7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgQWRkVG9RdWV1ZU9wdGlvbnMge1xuICB0cmFjZUlkOiBzdHJpbmc7XG4gIGNyaXRlcmlvbklkPzogc3RyaW5nO1xuICBhZ2VudElkOiBzdHJpbmc7XG4gIHJldmlld1R5cGU6IEh1bWFuUmV2aWV3VHlwZTtcbiAgcHJpb3JpdHk/OiBudW1iZXI7XG4gIGxsbVNjb3JlPzogbnVtYmVyO1xuICBsbG1QYXNzZWQ/OiBib29sZWFuO1xuICBsbG1SZWFzb25pbmc/OiBzdHJpbmc7XG4gIGxsbUNvbmZpZGVuY2U/OiBudW1iZXI7XG4gIGlzQ2FsaWJyYXRpb25TYW1wbGU/OiBib29sZWFuO1xuICBjYWxpYnJhdGlvblNldElkPzogc3RyaW5nO1xuICBleHBlY3RlZFNjb3JlPzogbnVtYmVyO1xuICBleHBlY3RlZFBhc3NlZD86IGJvb2xlYW47XG4gIGV4cGlyZXNJbk1zPzogbnVtYmVyO1xuICBtZXRhZGF0YT86IFJlY29yZDxzdHJpbmcsIHVua25vd24+O1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIFN1Ym1pdFJldmlld09wdGlvbnMge1xuICBwYXNzZWQ6IGJvb2xlYW47XG4gIHNjb3JlOiBudW1iZXI7XG4gIHJlYXNvbmluZzogc3RyaW5nO1xuICBkdXJhdGlvbk1zOiBudW1iZXI7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgQ2FsaWJyYXRpb25TZXQge1xuICBpZDogc3RyaW5nO1xuICBjb21wYW55SWQ6IHN0cmluZztcbiAgYWdlbnRJZD86IHN0cmluZztcbiAgbmFtZTogc3RyaW5nO1xuICBkZXNjcmlwdGlvbj86IHN0cmluZztcbiAgdGFyZ2V0U2FtcGxlQ291bnQ6IG51bWJlcjtcbiAgY3VycmVudFNhbXBsZUNvdW50OiBudW1iZXI7XG4gIG1pbkFncmVlbWVudFJhdGU6IG51bWJlcjtcbiAgcGFzc2luZ1Njb3JlVGhyZXNob2xkOiBudW1iZXI7XG4gIGNyaXRlcmlhSWRzOiBzdHJpbmdbXTtcbiAgaXNBY3RpdmU6IGJvb2xlYW47XG4gIGNyZWF0ZWRCeT86IHN0cmluZztcbiAgY3JlYXRlZEF0OiBzdHJpbmc7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgQ3JlYXRlQ2FsaWJyYXRpb25TZXRPcHRpb25zIHtcbiAgbmFtZTogc3RyaW5nO1xuICBkZXNjcmlwdGlvbj86IHN0cmluZztcbiAgYWdlbnRJZDogc3RyaW5nO1xuICBjcml0ZXJpb25JZD86IHN0cmluZztcbiAgdGFyZ2V0QWdyZWVtZW50PzogbnVtYmVyO1xuICBtaW5TYW1wbGVzPzogbnVtYmVyO1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIFJldmlld2VyQ2FsaWJyYXRpb24ge1xuICBpZDogc3RyaW5nO1xuICB1c2VySWQ6IHN0cmluZztcbiAgY2FsaWJyYXRpb25TZXRJZDogc3RyaW5nO1xuICB0b3RhbFJldmlld3M6IG51bWJlcjtcbiAgYWdyZWVtZW50UmF0ZTogbnVtYmVyO1xuICBtZWFuQWJzb2x1dGVFcnJvcjogbnVtYmVyO1xuICBpc0NlcnRpZmllZDogYm9vbGVhbjtcbiAgY2VydGlmaWVkQXQ/OiBzdHJpbmc7XG4gIGxhc3RSZXZpZXdBdD86IHN0cmluZztcbn1cblxuZXhwb3J0IGludGVyZmFjZSBRdWV1ZVN0YXRzIHtcbiAgcGVuZGluZzogbnVtYmVyO1xuICBpblByb2dyZXNzOiBudW1iZXI7XG4gIGNvbXBsZXRlZDogbnVtYmVyO1xuICBza2lwcGVkOiBudW1iZXI7XG4gIGV4cGlyZWQ6IG51bWJlcjtcbiAgYXZnUmV2aWV3RHVyYXRpb25Ncz86IG51bWJlcjtcbiAgYXZnQWdyZWVtZW50UmF0ZT86IG51bWJlcjtcbn1cblxuZXhwb3J0IGludGVyZmFjZSBMaXN0UXVldWVPcHRpb25zIHtcbiAgYWdlbnRJZD86IHN0cmluZztcbiAgc3RhdHVzPzogSHVtYW5SZXZpZXdTdGF0dXM7XG4gIHJldmlld1R5cGU/OiBIdW1hblJldmlld1R5cGU7XG4gIGlzQ2FsaWJyYXRpb24/OiBib29sZWFuO1xuICBtaW5Qcmlvcml0eT86IG51bWJlcjtcbiAgbGltaXQ/OiBudW1iZXI7XG4gIG9mZnNldD86IG51bWJlcjtcbn1cblxuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuLy8gSFVNQU4gUkVWSUVXIEFQSSBDTElFTlRcbi8vID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT1cblxuLyoqXG4gKiBIdW1hbiBSZXZpZXcgQVBJIGNsaWVudCBmb3IgbWFuYWdpbmcgaHVtYW4gcmV2aWV3IHF1ZXVlIGFuZCBjYWxpYnJhdGlvblxuICovXG5leHBvcnQgY29uc3QgaHVtYW5SZXZpZXcgPSB7XG4gIC8qKlxuICAgKiBHZXQgcGVuZGluZyByZXZpZXcgcXVldWUgaXRlbXNcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBpdGVtcyA9IGF3YWl0IGh1bWFuUmV2aWV3LmdldFF1ZXVlKHtcbiAgICogICBhZ2VudElkOiAnYWdlbnRfMTIzJyxcbiAgICogICBzdGF0dXM6ICdwZW5kaW5nJyxcbiAgICogICBsaW1pdDogMjAsXG4gICAqIH0pO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldFF1ZXVlKG9wdGlvbnM6IExpc3RRdWV1ZU9wdGlvbnMgPSB7fSk6IFByb21pc2U8SHVtYW5SZXZpZXdRdWV1ZUl0ZW1bXT4ge1xuICAgIGNvbnN0IHBhcmFtcyA9IG5ldyBVUkxTZWFyY2hQYXJhbXMoKTtcbiAgICBpZiAob3B0aW9ucy5hZ2VudElkKSBwYXJhbXMuc2V0KCdhZ2VudElkJywgb3B0aW9ucy5hZ2VudElkKTtcbiAgICBpZiAob3B0aW9ucy5zdGF0dXMpIHBhcmFtcy5zZXQoJ3N0YXR1cycsIG9wdGlvbnMuc3RhdHVzKTtcbiAgICBpZiAob3B0aW9ucy5yZXZpZXdUeXBlKSBwYXJhbXMuc2V0KCdyZXZpZXdUeXBlJywgb3B0aW9ucy5yZXZpZXdUeXBlKTtcbiAgICBpZiAob3B0aW9ucy5pc0NhbGlicmF0aW9uICE9PSB1bmRlZmluZWQpIHBhcmFtcy5zZXQoJ2lzQ2FsaWJyYXRpb24nLCBTdHJpbmcob3B0aW9ucy5pc0NhbGlicmF0aW9uKSk7XG4gICAgaWYgKG9wdGlvbnMubWluUHJpb3JpdHkgIT09IHVuZGVmaW5lZCkgcGFyYW1zLnNldCgnbWluUHJpb3JpdHknLCBTdHJpbmcob3B0aW9ucy5taW5Qcmlvcml0eSkpO1xuICAgIGlmIChvcHRpb25zLmxpbWl0KSBwYXJhbXMuc2V0KCdsaW1pdCcsIFN0cmluZyhvcHRpb25zLmxpbWl0KSk7XG4gICAgaWYgKG9wdGlvbnMub2Zmc2V0KSBwYXJhbXMuc2V0KCdvZmZzZXQnLCBTdHJpbmcob3B0aW9ucy5vZmZzZXQpKTtcblxuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8SHVtYW5SZXZpZXdRdWV1ZUl0ZW1bXT4oXG4gICAgICBgL2h1bWFuLXJldmlldy9xdWV1ZT8ke3BhcmFtcy50b1N0cmluZygpfWAsXG4gICAgICB7IGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogQWRkIGFuIGl0ZW0gdG8gdGhlIHJldmlldyBxdWV1ZVxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IGl0ZW0gPSBhd2FpdCBodW1hblJldmlldy5hZGRUb1F1ZXVlKHtcbiAgICogICB0cmFjZUlkOiAndHJhY2VfMTIzJyxcbiAgICogICBhZ2VudElkOiAnYWdlbnRfMTIzJyxcbiAgICogICByZXZpZXdUeXBlOiAnZGlzYWdyZWVtZW50JyxcbiAgICogICBwcmlvcml0eTogODAsXG4gICAqICAgbGxtU2NvcmU6IDY1LFxuICAgKiAgIGxsbVBhc3NlZDogdHJ1ZSxcbiAgICogfSk7XG4gICAqIGBgYFxuICAgKi9cbiAgYXN5bmMgYWRkVG9RdWV1ZShvcHRpb25zOiBBZGRUb1F1ZXVlT3B0aW9ucyk6IFByb21pc2U8SHVtYW5SZXZpZXdRdWV1ZUl0ZW0+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPEh1bWFuUmV2aWV3UXVldWVJdGVtPignL2h1bWFuLXJldmlldy9xdWV1ZScsIHtcbiAgICAgIG1ldGhvZDogJ1BPU1QnLFxuICAgICAgYm9keTogb3B0aW9ucyxcbiAgICAgIGFwaVZlcnNpb246ICdub25lJyxcbiAgICB9KTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IGEgc3BlY2lmaWMgcmV2aWV3IGl0ZW1cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBpdGVtID0gYXdhaXQgaHVtYW5SZXZpZXcuZ2V0SXRlbSgnaXRlbV8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRJdGVtKGl0ZW1JZDogc3RyaW5nKTogUHJvbWlzZTxIdW1hblJldmlld1F1ZXVlSXRlbT4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8SHVtYW5SZXZpZXdRdWV1ZUl0ZW0+KFxuICAgICAgYC9odW1hbi1yZXZpZXcvcXVldWUvJHtpdGVtSWR9YCxcbiAgICAgIHsgYXBpVmVyc2lvbjogJ25vbmUnIH1cbiAgICApO1xuICB9LFxuXG4gIC8qKlxuICAgKiBDbGFpbSBhIHJldmlldyBpdGVtIGZvciBwcm9jZXNzaW5nXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgaXRlbSA9IGF3YWl0IGh1bWFuUmV2aWV3LmNsYWltKCdpdGVtXzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGNsYWltKGl0ZW1JZDogc3RyaW5nKTogUHJvbWlzZTxIdW1hblJldmlld1F1ZXVlSXRlbT4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8SHVtYW5SZXZpZXdRdWV1ZUl0ZW0+KFxuICAgICAgYC9odW1hbi1yZXZpZXcvcXVldWUvJHtpdGVtSWR9L2NsYWltYCxcbiAgICAgIHsgbWV0aG9kOiAnUE9TVCcsIGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogUmVsZWFzZSBhIGNsYWltZWQgcmV2aWV3IGl0ZW1cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBhd2FpdCBodW1hblJldmlldy5yZWxlYXNlKCdpdGVtXzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIHJlbGVhc2UoaXRlbUlkOiBzdHJpbmcpOiBQcm9taXNlPEh1bWFuUmV2aWV3UXVldWVJdGVtPiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxIdW1hblJldmlld1F1ZXVlSXRlbT4oXG4gICAgICBgL2h1bWFuLXJldmlldy9xdWV1ZS8ke2l0ZW1JZH0vcmVsZWFzZWAsXG4gICAgICB7IG1ldGhvZDogJ1BPU1QnLCBhcGlWZXJzaW9uOiAnbm9uZScgfVxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIFNraXAgYSByZXZpZXcgaXRlbVxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGF3YWl0IGh1bWFuUmV2aWV3LnNraXAoJ2l0ZW1fMTIzJyk7XG4gICAqIGBgYFxuICAgKi9cbiAgYXN5bmMgc2tpcChpdGVtSWQ6IHN0cmluZyk6IFByb21pc2U8SHVtYW5SZXZpZXdRdWV1ZUl0ZW0+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPEh1bWFuUmV2aWV3UXVldWVJdGVtPihcbiAgICAgIGAvaHVtYW4tcmV2aWV3L3F1ZXVlLyR7aXRlbUlkfS9za2lwYCxcbiAgICAgIHsgbWV0aG9kOiAnUE9TVCcsIGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogU3VibWl0IGEgcmV2aWV3XG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgcmVzdWx0ID0gYXdhaXQgaHVtYW5SZXZpZXcuc3VibWl0KCdpdGVtXzEyMycsIHtcbiAgICogICBwYXNzZWQ6IHRydWUsXG4gICAqICAgc2NvcmU6IDg1LFxuICAgKiAgIHJlYXNvbmluZzogJ1Jlc3BvbnNlIGFjY3VyYXRlbHkgYWRkcmVzc2VkIHRoZSBxdWVyeScsXG4gICAqICAgZHVyYXRpb25NczogNDUwMDAsXG4gICAqIH0pO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIHN1Ym1pdChpdGVtSWQ6IHN0cmluZywgcmV2aWV3OiBTdWJtaXRSZXZpZXdPcHRpb25zKTogUHJvbWlzZTxIdW1hblJldmlld1F1ZXVlSXRlbT4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8SHVtYW5SZXZpZXdRdWV1ZUl0ZW0+KFxuICAgICAgYC9odW1hbi1yZXZpZXcvcXVldWUvJHtpdGVtSWR9L3N1Ym1pdGAsXG4gICAgICB7IG1ldGhvZDogJ1BPU1QnLCBib2R5OiByZXZpZXcsIGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IHF1ZXVlIHN0YXRpc3RpY3NcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzdGF0cyA9IGF3YWl0IGh1bWFuUmV2aWV3LmdldFN0YXRzKCdhZ2VudF8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRTdGF0cyhhZ2VudElkPzogc3RyaW5nKTogUHJvbWlzZTxRdWV1ZVN0YXRzPiB7XG4gICAgY29uc3QgcGFyYW1zID0gYWdlbnRJZCA/IGA/YWdlbnRJZD0ke2FnZW50SWR9YCA6ICcnO1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8UXVldWVTdGF0cz4oXG4gICAgICBgL2h1bWFuLXJldmlldy9zdGF0cyR7cGFyYW1zfWAsXG4gICAgICB7IGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IG5leHQgaXRlbSBmb3IgYSByZXZpZXdlclxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IG5leHRJdGVtID0gYXdhaXQgaHVtYW5SZXZpZXcuZ2V0TmV4dEl0ZW0oJ2FnZW50XzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldE5leHRJdGVtKGFnZW50SWQ/OiBzdHJpbmcpOiBQcm9taXNlPEh1bWFuUmV2aWV3UXVldWVJdGVtIHwgbnVsbD4ge1xuICAgIGNvbnN0IHBhcmFtcyA9IGFnZW50SWQgPyBgP2FnZW50SWQ9JHthZ2VudElkfWAgOiAnJztcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPEh1bWFuUmV2aWV3UXVldWVJdGVtIHwgbnVsbD4oXG4gICAgICBgL2h1bWFuLXJldmlldy9uZXh0LWl0ZW0ke3BhcmFtc31gLFxuICAgICAgeyBhcGlWZXJzaW9uOiAnbm9uZScgfVxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIEdldCBhdmFpbGFibGUgcmV2aWV3IHR5cGVzXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgdHlwZXMgPSBhd2FpdCBodW1hblJldmlldy5nZXRSZXZpZXdUeXBlcygpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldFJldmlld1R5cGVzKCk6IFByb21pc2U8QXJyYXk8e1xuICAgIGlkOiBzdHJpbmc7XG4gICAgbmFtZTogc3RyaW5nO1xuICAgIGRlc2NyaXB0aW9uOiBzdHJpbmc7XG4gICAgYXV0b1RyaWdnZXI6IGJvb2xlYW47XG4gIH0+PiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YSgnL2h1bWFuLXJldmlldy9yZXZpZXctdHlwZXMnLCB7IGFwaVZlcnNpb246ICdub25lJyB9KTtcbiAgfSxcblxuICAvLyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS1cbiAgLy8gQ0FMSUJSQVRJT04gU0VUU1xuICAvLyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS1cblxuICAvKipcbiAgICogR2V0IGNhbGlicmF0aW9uIHNldHNcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzZXRzID0gYXdhaXQgaHVtYW5SZXZpZXcuZ2V0Q2FsaWJyYXRpb25TZXRzKCdhZ2VudF8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRDYWxpYnJhdGlvblNldHMoYWdlbnRJZD86IHN0cmluZyk6IFByb21pc2U8Q2FsaWJyYXRpb25TZXRbXT4ge1xuICAgIGNvbnN0IHBhcmFtcyA9IGFnZW50SWQgPyBgP2FnZW50SWQ9JHthZ2VudElkfWAgOiAnJztcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPENhbGlicmF0aW9uU2V0W10+KFxuICAgICAgYC9odW1hbi1yZXZpZXcvY2FsaWJyYXRpb24tc2V0cyR7cGFyYW1zfWAsXG4gICAgICB7IGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogQ3JlYXRlIGEgY2FsaWJyYXRpb24gc2V0XG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3Qgc2V0ID0gYXdhaXQgaHVtYW5SZXZpZXcuY3JlYXRlQ2FsaWJyYXRpb25TZXQoe1xuICAgKiAgIG5hbWU6ICdRdWFsaXR5IENhbGlicmF0aW9uIFExJyxcbiAgICogICBhZ2VudElkOiAnYWdlbnRfMTIzJyxcbiAgICogICB0YXJnZXRBZ3JlZW1lbnQ6IDAuODUsXG4gICAqICAgbWluU2FtcGxlczogNTAsXG4gICAqIH0pO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGNyZWF0ZUNhbGlicmF0aW9uU2V0KG9wdGlvbnM6IENyZWF0ZUNhbGlicmF0aW9uU2V0T3B0aW9ucyk6IFByb21pc2U8Q2FsaWJyYXRpb25TZXQ+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPENhbGlicmF0aW9uU2V0PignL2h1bWFuLXJldmlldy9jYWxpYnJhdGlvbi1zZXRzJywge1xuICAgICAgbWV0aG9kOiAnUE9TVCcsXG4gICAgICBib2R5OiBvcHRpb25zLFxuICAgICAgYXBpVmVyc2lvbjogJ25vbmUnLFxuICAgIH0pO1xuICB9LFxuXG4gIC8qKlxuICAgKiBHZXQgYSBjYWxpYnJhdGlvbiBzZXQgYnkgSURcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzZXQgPSBhd2FpdCBodW1hblJldmlldy5nZXRDYWxpYnJhdGlvblNldCgnc2V0XzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldENhbGlicmF0aW9uU2V0KHNldElkOiBzdHJpbmcpOiBQcm9taXNlPENhbGlicmF0aW9uU2V0PiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxDYWxpYnJhdGlvblNldD4oXG4gICAgICBgL2h1bWFuLXJldmlldy9jYWxpYnJhdGlvbi1zZXRzLyR7c2V0SWR9YCxcbiAgICAgIHsgYXBpVmVyc2lvbjogJ25vbmUnIH1cbiAgICApO1xuICB9LFxuXG4gIC8vIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLVxuICAvLyBSRVZJRVdFUiBDQUxJQlJBVElPTlxuICAvLyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS1cblxuICAvKipcbiAgICogR2V0IGNlcnRpZmllZCByZXZpZXdlcnMgZm9yIGEgY2FsaWJyYXRpb24gc2V0XG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgcmV2aWV3ZXJzID0gYXdhaXQgaHVtYW5SZXZpZXcuZ2V0Q2VydGlmaWVkUmV2aWV3ZXJzKCdzZXRfMTIzJyk7XG4gICAqIGBgYFxuICAgKi9cbiAgYXN5bmMgZ2V0Q2VydGlmaWVkUmV2aWV3ZXJzKGNhbGlicmF0aW9uU2V0SWQ6IHN0cmluZyk6IFByb21pc2U8UmV2aWV3ZXJDYWxpYnJhdGlvbltdPiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxSZXZpZXdlckNhbGlicmF0aW9uW10+KFxuICAgICAgYC9odW1hbi1yZXZpZXcvcmV2aWV3ZXJzP2NhbGlicmF0aW9uU2V0SWQ9JHtjYWxpYnJhdGlvblNldElkfWAsXG4gICAgICB7IGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IHJldmlld2VyIGNhbGlicmF0aW9uIHN0YXR1c1xuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IGNhbGlicmF0aW9ucyA9IGF3YWl0IGh1bWFuUmV2aWV3LmdldFJldmlld2VyQ2FsaWJyYXRpb25zKCd1c2VyXzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldFJldmlld2VyQ2FsaWJyYXRpb25zKHVzZXJJZDogc3RyaW5nKTogUHJvbWlzZTxSZXZpZXdlckNhbGlicmF0aW9uW10+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPFJldmlld2VyQ2FsaWJyYXRpb25bXT4oXG4gICAgICBgL2h1bWFuLXJldmlldy9jYWxpYnJhdGlvbi8ke3VzZXJJZH1gLFxuICAgICAgeyBhcGlWZXJzaW9uOiAnbm9uZScgfVxuICAgICk7XG4gIH0sXG59O1xuIl19
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ThinkHive SDK v3.0 - Non-Determinism API
|
|
3
|
+
*
|
|
4
|
+
* API for pass@k / pass^k analysis to measure LLM evaluation reliability
|
|
5
|
+
*/
|
|
6
|
+
export type NondeterminismRunType = 'pass_at_k' | 'pass_to_k' | 'variance' | 'reliability';
|
|
7
|
+
export type NondeterminismRunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
|
8
|
+
export interface NondeterminismRun {
|
|
9
|
+
id: string;
|
|
10
|
+
companyId: string;
|
|
11
|
+
agentId: string;
|
|
12
|
+
runType: NondeterminismRunType;
|
|
13
|
+
kValue: number;
|
|
14
|
+
status: NondeterminismRunStatus;
|
|
15
|
+
traceCount: number;
|
|
16
|
+
criterionId?: string;
|
|
17
|
+
criteriaIds: string[];
|
|
18
|
+
temperature?: string;
|
|
19
|
+
model?: string;
|
|
20
|
+
progressPercent: number;
|
|
21
|
+
passAtKRate?: string;
|
|
22
|
+
passToKRate?: string;
|
|
23
|
+
avgVariance?: string;
|
|
24
|
+
reliabilityScore?: string;
|
|
25
|
+
startedAt?: string;
|
|
26
|
+
completedAt?: string;
|
|
27
|
+
createdBy?: string;
|
|
28
|
+
createdAt: string;
|
|
29
|
+
}
|
|
30
|
+
export interface NondeterminismSample {
|
|
31
|
+
id: string;
|
|
32
|
+
runId: string;
|
|
33
|
+
traceId: string;
|
|
34
|
+
criterionId: string;
|
|
35
|
+
sampleIndex: number;
|
|
36
|
+
score: string;
|
|
37
|
+
passed: boolean;
|
|
38
|
+
reasoning?: string;
|
|
39
|
+
confidence?: string;
|
|
40
|
+
tokensUsed?: number;
|
|
41
|
+
costUsd?: string;
|
|
42
|
+
model?: string;
|
|
43
|
+
temperature?: string;
|
|
44
|
+
latencyMs?: number;
|
|
45
|
+
error?: string;
|
|
46
|
+
createdAt: string;
|
|
47
|
+
}
|
|
48
|
+
export interface CreateRunOptions {
|
|
49
|
+
agentId: string;
|
|
50
|
+
criterionId?: string;
|
|
51
|
+
criteriaIds?: string[];
|
|
52
|
+
kValue: number;
|
|
53
|
+
traceIds: string[];
|
|
54
|
+
runType?: NondeterminismRunType;
|
|
55
|
+
temperature?: number;
|
|
56
|
+
model?: string;
|
|
57
|
+
}
|
|
58
|
+
export interface RecordSampleOptions {
|
|
59
|
+
runId: string;
|
|
60
|
+
traceId: string;
|
|
61
|
+
criterionId: string;
|
|
62
|
+
sampleIndex: number;
|
|
63
|
+
score: number;
|
|
64
|
+
passed: boolean;
|
|
65
|
+
reasoning?: string;
|
|
66
|
+
confidence?: number;
|
|
67
|
+
tokensUsed?: number;
|
|
68
|
+
costUsd?: number;
|
|
69
|
+
model?: string;
|
|
70
|
+
temperature?: number;
|
|
71
|
+
latencyMs?: number;
|
|
72
|
+
error?: string;
|
|
73
|
+
}
|
|
74
|
+
export interface TraceAnalysis {
|
|
75
|
+
traceId: string;
|
|
76
|
+
samples: NondeterminismSample[];
|
|
77
|
+
passCount: number;
|
|
78
|
+
totalCount: number;
|
|
79
|
+
passRate: number;
|
|
80
|
+
scoreVariance: number;
|
|
81
|
+
meanScore: number;
|
|
82
|
+
isConsistent: boolean;
|
|
83
|
+
}
|
|
84
|
+
export interface CriterionAnalysis {
|
|
85
|
+
criterionId: string;
|
|
86
|
+
traceAnalyses: TraceAnalysis[];
|
|
87
|
+
passAtKRate: number;
|
|
88
|
+
passToKRate: number;
|
|
89
|
+
reliabilityScore: number;
|
|
90
|
+
isReliable: boolean;
|
|
91
|
+
recommendation: string;
|
|
92
|
+
}
|
|
93
|
+
export interface RunSummary {
|
|
94
|
+
run: NondeterminismRun;
|
|
95
|
+
traceAnalyses: TraceAnalysis[];
|
|
96
|
+
criterionAnalyses: CriterionAnalysis[];
|
|
97
|
+
}
|
|
98
|
+
export interface ListRunsOptions {
|
|
99
|
+
agentId?: string;
|
|
100
|
+
status?: NondeterminismRunStatus;
|
|
101
|
+
limit?: number;
|
|
102
|
+
offset?: number;
|
|
103
|
+
}
|
|
104
|
+
export interface PassAtKInfo {
|
|
105
|
+
concepts: {
|
|
106
|
+
passAtK: {
|
|
107
|
+
name: string;
|
|
108
|
+
description: string;
|
|
109
|
+
formula: string;
|
|
110
|
+
useCase: string;
|
|
111
|
+
};
|
|
112
|
+
passToK: {
|
|
113
|
+
name: string;
|
|
114
|
+
description: string;
|
|
115
|
+
formula: string;
|
|
116
|
+
useCase: string;
|
|
117
|
+
};
|
|
118
|
+
variance: {
|
|
119
|
+
name: string;
|
|
120
|
+
description: string;
|
|
121
|
+
useCase: string;
|
|
122
|
+
};
|
|
123
|
+
reliability: {
|
|
124
|
+
name: string;
|
|
125
|
+
description: string;
|
|
126
|
+
useCase: string;
|
|
127
|
+
};
|
|
128
|
+
};
|
|
129
|
+
recommendations: Record<string, string>;
|
|
130
|
+
defaults: {
|
|
131
|
+
kValue: number;
|
|
132
|
+
reliabilityThreshold: number;
|
|
133
|
+
varianceThreshold: number;
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Non-Determinism API client for pass@k analysis and reliability measurement
|
|
138
|
+
*/
|
|
139
|
+
export declare const nondeterminism: {
|
|
140
|
+
/**
|
|
141
|
+
* Create a new non-determinism analysis run
|
|
142
|
+
*
|
|
143
|
+
* @example
|
|
144
|
+
* ```typescript
|
|
145
|
+
* const run = await nondeterminism.createRun({
|
|
146
|
+
* agentId: 'agent_123',
|
|
147
|
+
* criterionId: 'criterion_456',
|
|
148
|
+
* kValue: 5,
|
|
149
|
+
* traceIds: ['trace_1', 'trace_2', 'trace_3'],
|
|
150
|
+
* runType: 'pass_at_k',
|
|
151
|
+
* });
|
|
152
|
+
* ```
|
|
153
|
+
*/
|
|
154
|
+
createRun(options: CreateRunOptions): Promise<NondeterminismRun>;
|
|
155
|
+
/**
|
|
156
|
+
* Get non-determinism runs
|
|
157
|
+
*
|
|
158
|
+
* @example
|
|
159
|
+
* ```typescript
|
|
160
|
+
* const runs = await nondeterminism.getRuns({ agentId: 'agent_123' });
|
|
161
|
+
* ```
|
|
162
|
+
*/
|
|
163
|
+
getRuns(options?: ListRunsOptions): Promise<NondeterminismRun[]>;
|
|
164
|
+
/**
|
|
165
|
+
* Get a specific run
|
|
166
|
+
*
|
|
167
|
+
* @example
|
|
168
|
+
* ```typescript
|
|
169
|
+
* const run = await nondeterminism.getRun('run_123');
|
|
170
|
+
* ```
|
|
171
|
+
*/
|
|
172
|
+
getRun(runId: string): Promise<NondeterminismRun>;
|
|
173
|
+
/**
|
|
174
|
+
* Start a run
|
|
175
|
+
*
|
|
176
|
+
* @example
|
|
177
|
+
* ```typescript
|
|
178
|
+
* await nondeterminism.startRun('run_123');
|
|
179
|
+
* ```
|
|
180
|
+
*/
|
|
181
|
+
startRun(runId: string): Promise<void>;
|
|
182
|
+
/**
|
|
183
|
+
* Complete a run
|
|
184
|
+
*
|
|
185
|
+
* @example
|
|
186
|
+
* ```typescript
|
|
187
|
+
* await nondeterminism.completeRun('run_123');
|
|
188
|
+
* ```
|
|
189
|
+
*/
|
|
190
|
+
completeRun(runId: string): Promise<void>;
|
|
191
|
+
/**
|
|
192
|
+
* Record a sample result
|
|
193
|
+
*
|
|
194
|
+
* @example
|
|
195
|
+
* ```typescript
|
|
196
|
+
* const sample = await nondeterminism.recordSample({
|
|
197
|
+
* runId: 'run_123',
|
|
198
|
+
* traceId: 'trace_456',
|
|
199
|
+
* criterionId: 'criterion_789',
|
|
200
|
+
* sampleIndex: 0,
|
|
201
|
+
* score: 85,
|
|
202
|
+
* passed: true,
|
|
203
|
+
* reasoning: 'Response meets quality criteria',
|
|
204
|
+
* });
|
|
205
|
+
* ```
|
|
206
|
+
*/
|
|
207
|
+
recordSample(options: RecordSampleOptions): Promise<NondeterminismSample>;
|
|
208
|
+
/**
|
|
209
|
+
* Get samples for a run
|
|
210
|
+
*
|
|
211
|
+
* @example
|
|
212
|
+
* ```typescript
|
|
213
|
+
* const samples = await nondeterminism.getSamples('run_123');
|
|
214
|
+
* ```
|
|
215
|
+
*/
|
|
216
|
+
getSamples(runId: string): Promise<NondeterminismSample[]>;
|
|
217
|
+
/**
|
|
218
|
+
* Get run summary with analysis
|
|
219
|
+
*
|
|
220
|
+
* @example
|
|
221
|
+
* ```typescript
|
|
222
|
+
* const summary = await nondeterminism.getRunSummary('run_123');
|
|
223
|
+
* console.log(`Pass@k rate: ${summary.criterionAnalyses[0].passAtKRate}`);
|
|
224
|
+
* ```
|
|
225
|
+
*/
|
|
226
|
+
getRunSummary(runId: string): Promise<RunSummary>;
|
|
227
|
+
/**
|
|
228
|
+
* Trigger analysis of a completed run
|
|
229
|
+
*
|
|
230
|
+
* @example
|
|
231
|
+
* ```typescript
|
|
232
|
+
* const summary = await nondeterminism.analyzeRun('run_123');
|
|
233
|
+
* ```
|
|
234
|
+
*/
|
|
235
|
+
analyzeRun(runId: string): Promise<RunSummary>;
|
|
236
|
+
/**
|
|
237
|
+
* Get information about pass@k analysis
|
|
238
|
+
*
|
|
239
|
+
* @example
|
|
240
|
+
* ```typescript
|
|
241
|
+
* const info = await nondeterminism.getInfo();
|
|
242
|
+
* console.log(info.concepts.passAtK.description);
|
|
243
|
+
* ```
|
|
244
|
+
*/
|
|
245
|
+
getInfo(): Promise<PassAtKInfo>;
|
|
246
|
+
};
|
|
247
|
+
/**
|
|
248
|
+
* Calculate pass@k probability from pass rate
|
|
249
|
+
*
|
|
250
|
+
* @param passRate - Single-run pass rate (0-1)
|
|
251
|
+
* @param k - Number of runs
|
|
252
|
+
* @returns Probability that at least 1 of k runs passes
|
|
253
|
+
*
|
|
254
|
+
* @example
|
|
255
|
+
* ```typescript
|
|
256
|
+
* const passAtK = calculatePassAtK(0.7, 3); // ~0.973
|
|
257
|
+
* ```
|
|
258
|
+
*/
|
|
259
|
+
export declare function calculatePassAtK(passRate: number, k: number): number;
|
|
260
|
+
/**
|
|
261
|
+
* Calculate pass^k probability from pass rate
|
|
262
|
+
*
|
|
263
|
+
* @param passRate - Single-run pass rate (0-1)
|
|
264
|
+
* @param k - Number of runs
|
|
265
|
+
* @returns Probability that all k runs pass
|
|
266
|
+
*
|
|
267
|
+
* @example
|
|
268
|
+
* ```typescript
|
|
269
|
+
* const passToK = calculatePassToK(0.7, 3); // ~0.343
|
|
270
|
+
* ```
|
|
271
|
+
*/
|
|
272
|
+
export declare function calculatePassToK(passRate: number, k: number): number;
|
|
273
|
+
/**
|
|
274
|
+
* Calculate required pass rate to achieve target pass@k
|
|
275
|
+
*
|
|
276
|
+
* @param targetPassAtK - Desired pass@k probability
|
|
277
|
+
* @param k - Number of runs
|
|
278
|
+
* @returns Required single-run pass rate
|
|
279
|
+
*
|
|
280
|
+
* @example
|
|
281
|
+
* ```typescript
|
|
282
|
+
* const requiredRate = requiredPassRateForPassAtK(0.95, 3); // ~0.632
|
|
283
|
+
* ```
|
|
284
|
+
*/
|
|
285
|
+
export declare function requiredPassRateForPassAtK(targetPassAtK: number, k: number): number;
|
|
286
|
+
/**
|
|
287
|
+
* Determine if evaluation is reliable based on analysis
|
|
288
|
+
*
|
|
289
|
+
* @param analysis - Criterion analysis result
|
|
290
|
+
* @param reliabilityThreshold - Minimum reliability score (default 0.8)
|
|
291
|
+
* @returns Whether the evaluation is considered reliable
|
|
292
|
+
*/
|
|
293
|
+
export declare function isReliableEvaluation(analysis: CriterionAnalysis, reliabilityThreshold?: number): boolean;
|
|
294
|
+
/**
|
|
295
|
+
* Get recommendation based on reliability analysis
|
|
296
|
+
*
|
|
297
|
+
* @param analysis - Criterion analysis result
|
|
298
|
+
* @returns Actionable recommendation string
|
|
299
|
+
*/
|
|
300
|
+
export declare function getReliabilityRecommendation(analysis: CriterionAnalysis): string;
|