@thinkhive/sdk 3.1.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +83 -12
- package/README.md +279 -128
- package/dist/api/agents.d.ts +169 -0
- package/dist/api/agents.js +185 -0
- package/dist/api/apiKeys.d.ts +252 -0
- package/dist/api/apiKeys.js +298 -0
- package/dist/api/business-metrics.d.ts +188 -0
- package/dist/api/business-metrics.js +213 -0
- package/dist/api/calibration.d.ts +0 -62
- package/dist/api/calibration.js +5 -48
- package/dist/api/claims.js +10 -7
- package/dist/api/conversation-eval.d.ts +200 -0
- package/dist/api/conversation-eval.js +235 -0
- package/dist/api/deterministic-graders.d.ts +205 -0
- package/dist/api/deterministic-graders.js +191 -0
- package/dist/api/eval-health.d.ts +250 -0
- package/dist/api/eval-health.js +224 -0
- package/dist/api/human-review.d.ts +275 -0
- package/dist/api/human-review.js +236 -0
- package/dist/api/nondeterminism.d.ts +300 -0
- package/dist/api/nondeterminism.js +250 -0
- package/dist/api/quality-metrics.d.ts +303 -0
- package/dist/api/quality-metrics.js +198 -0
- package/dist/api/roi-analytics.d.ts +263 -0
- package/dist/api/roi-analytics.js +204 -0
- package/dist/api/runs.js +12 -6
- package/dist/api/transcript-patterns.d.ts +204 -0
- package/dist/api/transcript-patterns.js +227 -0
- package/dist/core/client.d.ts +83 -9
- package/dist/core/client.js +229 -34
- package/dist/core/config.d.ts +2 -3
- package/dist/core/config.js +3 -4
- package/dist/core/types.d.ts +57 -4
- package/dist/core/types.js +1 -1
- package/dist/index.d.ts +429 -76
- package/dist/index.js +262 -42
- package/package.json +2 -2
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ThinkHive SDK v3.0 - Evaluation Health API
|
|
4
|
+
*
|
|
5
|
+
* API for eval saturation monitoring, regression detection, and health reports
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.evalHealth = void 0;
|
|
9
|
+
exports.hasHealthIssue = hasHealthIssue;
|
|
10
|
+
exports.getSeverityLevel = getSeverityLevel;
|
|
11
|
+
exports.isSaturated = isSaturated;
|
|
12
|
+
exports.getSaturationRecommendation = getSaturationRecommendation;
|
|
13
|
+
const client_1 = require("../core/client");
|
|
14
|
+
// ============================================================================
|
|
15
|
+
// EVAL HEALTH API CLIENT
|
|
16
|
+
// ============================================================================
|
|
17
|
+
/**
|
|
18
|
+
* Evaluation Health API client for monitoring eval quality and detecting regressions
|
|
19
|
+
*/
|
|
20
|
+
exports.evalHealth = {
|
|
21
|
+
/**
|
|
22
|
+
* Get comprehensive health report for an agent
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```typescript
|
|
26
|
+
* const report = await evalHealth.getReport('agent_123');
|
|
27
|
+
* console.log(`Overall health: ${report.overallHealth}`);
|
|
28
|
+
* console.log(`Active regressions: ${report.regressionCount}`);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
async getReport(agentId) {
|
|
32
|
+
return (0, client_1.apiRequestWithData)(`/eval-health/report?agentId=${agentId}`, { apiVersion: 'none' });
|
|
33
|
+
},
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
// SNAPSHOTS
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
/**
|
|
38
|
+
* Get historical health snapshots
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const snapshots = await evalHealth.getSnapshots({
|
|
43
|
+
* agentId: 'agent_123',
|
|
44
|
+
* startDate: '2024-01-01T00:00:00Z',
|
|
45
|
+
* endDate: '2024-01-31T23:59:59Z',
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
async getSnapshots(options) {
|
|
50
|
+
const params = new URLSearchParams();
|
|
51
|
+
params.set('agentId', options.agentId);
|
|
52
|
+
if (options.criterionId)
|
|
53
|
+
params.set('criterionId', options.criterionId);
|
|
54
|
+
if (options.startDate)
|
|
55
|
+
params.set('startDate', options.startDate);
|
|
56
|
+
if (options.endDate)
|
|
57
|
+
params.set('endDate', options.endDate);
|
|
58
|
+
return (0, client_1.apiRequestWithData)(`/eval-health/snapshots?${params.toString()}`, { apiVersion: 'none' });
|
|
59
|
+
},
|
|
60
|
+
/**
|
|
61
|
+
* Get latest health snapshot
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* ```typescript
|
|
65
|
+
* const snapshot = await evalHealth.getLatestSnapshot('agent_123');
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
async getLatestSnapshot(agentId, criterionId) {
|
|
69
|
+
const params = new URLSearchParams();
|
|
70
|
+
params.set('agentId', agentId);
|
|
71
|
+
if (criterionId)
|
|
72
|
+
params.set('criterionId', criterionId);
|
|
73
|
+
return (0, client_1.apiRequestWithData)(`/eval-health/snapshots/latest?${params.toString()}`, { apiVersion: 'none' });
|
|
74
|
+
},
|
|
75
|
+
/**
|
|
76
|
+
* Record a health snapshot
|
|
77
|
+
*
|
|
78
|
+
* @example
|
|
79
|
+
* ```typescript
|
|
80
|
+
* const snapshot = await evalHealth.recordSnapshot({
|
|
81
|
+
* agentId: 'agent_123',
|
|
82
|
+
* snapshotDate: new Date().toISOString(),
|
|
83
|
+
* passRate: '0.85',
|
|
84
|
+
* evalCount: 150,
|
|
85
|
+
* healthStatus: 'healthy',
|
|
86
|
+
* });
|
|
87
|
+
* ```
|
|
88
|
+
*/
|
|
89
|
+
async recordSnapshot(options) {
|
|
90
|
+
return (0, client_1.apiRequestWithData)('/eval-health/snapshots', {
|
|
91
|
+
method: 'POST',
|
|
92
|
+
body: options,
|
|
93
|
+
apiVersion: 'none',
|
|
94
|
+
});
|
|
95
|
+
},
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
// REGRESSIONS
|
|
98
|
+
// ---------------------------------------------------------------------------
|
|
99
|
+
/**
|
|
100
|
+
* Get unresolved regressions for an agent
|
|
101
|
+
*
|
|
102
|
+
* @example
|
|
103
|
+
* ```typescript
|
|
104
|
+
* const regressions = await evalHealth.getRegressions('agent_123');
|
|
105
|
+
* for (const regression of regressions) {
|
|
106
|
+
* console.log(`${regression.severity}: ${regression.delta}% drop`);
|
|
107
|
+
* }
|
|
108
|
+
* ```
|
|
109
|
+
*/
|
|
110
|
+
async getRegressions(agentId) {
|
|
111
|
+
return (0, client_1.apiRequestWithData)(`/eval-health/regressions?agentId=${agentId}`, { apiVersion: 'none' });
|
|
112
|
+
},
|
|
113
|
+
/**
|
|
114
|
+
* Record a new regression
|
|
115
|
+
*
|
|
116
|
+
* @example
|
|
117
|
+
* ```typescript
|
|
118
|
+
* const regression = await evalHealth.recordRegression({
|
|
119
|
+
* agentId: 'agent_123',
|
|
120
|
+
* severity: 'moderate',
|
|
121
|
+
* baselinePassRate: '0.92',
|
|
122
|
+
* currentPassRate: '0.78',
|
|
123
|
+
* delta: '-0.14',
|
|
124
|
+
* baselinePeriodStart: '2024-01-01T00:00:00Z',
|
|
125
|
+
* baselinePeriodEnd: '2024-01-15T23:59:59Z',
|
|
126
|
+
* currentPeriodStart: '2024-01-16T00:00:00Z',
|
|
127
|
+
* currentPeriodEnd: '2024-01-31T23:59:59Z',
|
|
128
|
+
* });
|
|
129
|
+
* ```
|
|
130
|
+
*/
|
|
131
|
+
async recordRegression(options) {
|
|
132
|
+
return (0, client_1.apiRequestWithData)('/eval-health/regressions', {
|
|
133
|
+
method: 'POST',
|
|
134
|
+
body: options,
|
|
135
|
+
apiVersion: 'none',
|
|
136
|
+
});
|
|
137
|
+
},
|
|
138
|
+
/**
|
|
139
|
+
* Resolve a regression
|
|
140
|
+
*
|
|
141
|
+
* @example
|
|
142
|
+
* ```typescript
|
|
143
|
+
* await evalHealth.resolveRegression('regression_123', {
|
|
144
|
+
* resolutionType: 'fixed',
|
|
145
|
+
* notes: 'Updated prompt template to address quality issues',
|
|
146
|
+
* });
|
|
147
|
+
* ```
|
|
148
|
+
*/
|
|
149
|
+
async resolveRegression(regressionId, options) {
|
|
150
|
+
await (0, client_1.apiRequest)(`/eval-health/regressions/${regressionId}/resolve`, {
|
|
151
|
+
method: 'POST',
|
|
152
|
+
body: options,
|
|
153
|
+
apiVersion: 'none',
|
|
154
|
+
});
|
|
155
|
+
},
|
|
156
|
+
/**
|
|
157
|
+
* Acknowledge a regression
|
|
158
|
+
*
|
|
159
|
+
* @example
|
|
160
|
+
* ```typescript
|
|
161
|
+
* await evalHealth.acknowledgeRegression('regression_123');
|
|
162
|
+
* ```
|
|
163
|
+
*/
|
|
164
|
+
async acknowledgeRegression(regressionId) {
|
|
165
|
+
await (0, client_1.apiRequest)(`/eval-health/regressions/${regressionId}/acknowledge`, {
|
|
166
|
+
method: 'POST',
|
|
167
|
+
apiVersion: 'none',
|
|
168
|
+
});
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
// ============================================================================
|
|
172
|
+
// HELPER FUNCTIONS
|
|
173
|
+
// ============================================================================
|
|
174
|
+
/**
|
|
175
|
+
* Check if health status indicates an issue
|
|
176
|
+
*
|
|
177
|
+
* @param status - Health status to check
|
|
178
|
+
* @returns Whether the status indicates a problem
|
|
179
|
+
*/
|
|
180
|
+
function hasHealthIssue(status) {
|
|
181
|
+
return status === 'warning' || status === 'critical';
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Get severity level as numeric value for sorting
|
|
185
|
+
*
|
|
186
|
+
* @param severity - Regression severity
|
|
187
|
+
* @returns Numeric severity (1-3, higher is worse)
|
|
188
|
+
*/
|
|
189
|
+
function getSeverityLevel(severity) {
|
|
190
|
+
switch (severity) {
|
|
191
|
+
case 'minor': return 1;
|
|
192
|
+
case 'moderate': return 2;
|
|
193
|
+
case 'severe': return 3;
|
|
194
|
+
default: return 0;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Check if evaluation is saturated
|
|
199
|
+
*
|
|
200
|
+
* @param snapshot - Health snapshot to check
|
|
201
|
+
* @returns Whether evaluation is at ceiling or floor
|
|
202
|
+
*/
|
|
203
|
+
function isSaturated(snapshot) {
|
|
204
|
+
return snapshot.saturationType === 'ceiling' || snapshot.saturationType === 'floor';
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Get recommendation for saturation type
|
|
208
|
+
*
|
|
209
|
+
* @param saturationType - Type of saturation
|
|
210
|
+
* @returns Recommendation string
|
|
211
|
+
*/
|
|
212
|
+
function getSaturationRecommendation(saturationType) {
|
|
213
|
+
switch (saturationType) {
|
|
214
|
+
case 'ceiling':
|
|
215
|
+
return 'Evaluation criteria may be too lenient. Consider adding stricter checks or more challenging test cases.';
|
|
216
|
+
case 'floor':
|
|
217
|
+
return 'Evaluation criteria may be too strict. Consider relaxing thresholds or reviewing criteria for accuracy.';
|
|
218
|
+
case 'healthy':
|
|
219
|
+
return 'Evaluation is operating within healthy parameters.';
|
|
220
|
+
default:
|
|
221
|
+
return 'Unable to determine saturation status.';
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZXZhbC1oZWFsdGguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvYXBpL2V2YWwtaGVhbHRoLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7OztHQUlHOzs7QUE0VEgsd0NBRUM7QUFRRCw0Q0FPQztBQVFELGtDQUVDO0FBUUQsa0VBV0M7QUF4V0QsMkNBQWdFO0FBOEhoRSwrRUFBK0U7QUFDL0UseUJBQXlCO0FBQ3pCLCtFQUErRTtBQUUvRTs7R0FFRztBQUNVLFFBQUEsVUFBVSxHQUFHO0lBQ3hCOzs7Ozs7Ozs7T0FTRztJQUNILEtBQUssQ0FBQyxTQUFTLENBQUMsT0FBZTtRQUM3QixPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLCtCQUErQixPQUFPLEVBQUUsRUFDeEMsRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZCLENBQUM7SUFDSixDQUFDO0lBRUQsOEVBQThFO0lBQzlFLFlBQVk7SUFDWiw4RUFBOEU7SUFFOUU7Ozs7Ozs7Ozs7O09BV0c7SUFDSCxLQUFLLENBQUMsWUFBWSxDQUFDLE9BQTRCO1FBQzdDLE1BQU0sTUFBTSxHQUFHLElBQUksZUFBZSxFQUFFLENBQUM7UUFDckMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxTQUFTLEVBQUUsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQ3ZDLElBQUksT0FBTyxDQUFDLFdBQVc7WUFBRSxNQUFNLENBQUMsR0FBRyxDQUFDLGFBQWEsRUFBRSxPQUFPLENBQUMsV0FBVyxDQUFDLENBQUM7UUFDeEUsSUFBSSxPQUFPLENBQUMsU0FBUztZQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsV0FBVyxFQUFFLE9BQU8sQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUNsRSxJQUFJLE9BQU8sQ0FBQyxPQUFPO1lBQUUsTUFBTSxDQUFDLEdBQUcsQ0FBQyxTQUFTLEVBQUUsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBRTVELE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsMEJBQTBCLE1BQU0sQ0FBQyxRQUFRLEVBQUUsRUFBRSxFQUM3QyxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLGlCQUFpQixDQUFDLE9BQWUsRUFBRSxXQUFvQjtRQUMzRCxNQUFNLE1BQU0sR0FBRyxJQUFJLGVBQWUsRUFBRSxDQUFDO1FBQ3JDLE1BQU0sQ0FBQyxHQUFHLENBQUMsU0FBUyxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQy9CLElBQUksV0FBVztZQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsYUFBYSxFQUFFLFdBQVcsQ0FBQyxDQUFDO1FBRXhELE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsaUNBQWlDLE1BQU0sQ0FBQyxRQUFRLEVBQUUsRUFBRSxFQUNwRCxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsQ0FDdkIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7Ozs7Ozs7OztPQWFHO0lBQ0gsS0FBSyxDQUFDLGNBQWMsQ0FBQyxPQUE4QjtRQUNqRCxPQUFPLElBQUEsMkJBQWtCLEVBQXFCLHdCQUF3QixFQUFFO1lBQ3RFLE1BQU0sRUFBRSxNQUFNO1lBQ2QsSUFBSSxFQUFFLE9BQU87WUFDYixVQUFVLEVBQUUsTUFBTTtTQUNuQixDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQsOEVBQThFO0lBQzlFLGNBQWM7SUFDZCw4RUFBOEU7SUFFOUU7Ozs7Ozs7Ozs7T0FVRztJQUNILEtBQUssQ0FBQyxjQUFjLENBQUMsT0FBZTtRQUNsQyxPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLG9DQUFvQyxPQUFPLEVBQUUsRUFDN0MsRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLENBQ3ZCLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7Ozs7Ozs7Ozs7O09BaUJHO0lBQ0gsS0FBSyxDQUFDLGdCQUFnQixDQUFDLE9BQWdDO1FBQ3JELE9BQU8sSUFBQSwyQkFBa0IsRUFBaUIsMEJBQTBCLEVBQUU7WUFDcEUsTUFBTSxFQUFFLE1BQU07WUFDZCxJQUFJLEVBQUUsT0FBTztZQUNiLFVBQVUsRUFBRSxNQUFNO1NBQ25CLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7Ozs7OztPQVVHO0lBQ0gsS0FBSyxDQUFDLGlCQUFpQixDQUFDLFlBQW9CLEVBQUUsT0FBaUM7UUFDN0UsTUFBTSxJQUFBLG1CQUFVLEVBQUMsNEJBQTRCLFlBQVksVUFBVSxFQUFFO1lBQ25FLE1BQU0sRUFBRSxNQUFNO1lBQ2QsSUFBSSxFQUFFLE9BQU87WUFDYixVQUFVLEVBQUUsTUFBTTtTQUNuQixDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILEtBQUssQ0FBQyxxQkFBcUIsQ0FBQyxZQUFvQjtRQUM5QyxNQUFNLElBQUEsbUJBQVUsRUFBQyw0QkFBNEIsWUFBWSxjQUFjLEVBQUU7WUFDdkUsTUFBTSxFQUFFLE1BQU07WUFDZCxVQUFVLEVBQUUsTUFBTTtTQUNuQixDQUFDLENBQUM7SUFDTCxDQUFDO0NBQ0YsQ0FBQztBQUVGLCtFQUErRTtBQUMvRSxtQkFBbUI7QUFDbkIsK0VBQStFO0FBRS9FOzs7OztHQUtHO0FBQ0gsU0FBZ0IsY0FBYyxDQUFDLE1BQW9CO0lBQ2pELE9BQU8sTUFBTSxLQUFLLFNBQVMsSUFBSSxNQUFNLEtBQUssVUFBVSxDQUFDO0FBQ3ZELENBQUM7QUFFRDs7Ozs7R0FLRztBQUNILFNBQWdCLGdCQUFnQixDQUFDLFFBQTRCO0lBQzNELFFBQVEsUUFBUSxFQUFFLENBQUM7UUFDakIsS0FBSyxPQUFPLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUN2QixLQUFLLFVBQVUsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQzFCLEtBQUssUUFBUSxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDeEIsT0FBTyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUM7SUFDcEIsQ0FBQztBQUNILENBQUM7QUFFRDs7Ozs7R0FLRztBQUNILFNBQWdCLFdBQVcsQ0FBQyxRQUE0QjtJQUN0RCxPQUFPLFFBQVEsQ0FBQyxjQUFjLEtBQUssU0FBUyxJQUFJLFFBQVEsQ0FBQyxjQUFjLEtBQUssT0FBTyxDQUFDO0FBQ3RGLENBQUM7QUFFRDs7Ozs7R0FLRztBQUNILFNBQWdCLDJCQUEyQixDQUFDLGNBQThCO0lBQ3hFLFFBQVEsY0FBYyxFQUFFLENBQUM7UUFDdkIsS0FBSyxTQUFTO1lBQ1osT0FBTyx5R0FBeUcsQ0FBQztRQUNuSCxLQUFLLE9BQU87WUFDVixPQUFPLHlHQUF5RyxDQUFDO1FBQ25ILEtBQUssU0FBUztZQUNaLE9BQU8sb0RBQW9ELENBQUM7UUFDOUQ7WUFDRSxPQUFPLHdDQUF3QyxDQUFDO0lBQ3BELENBQUM7QUFDSCxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLyoqXG4gKiBUaGlua0hpdmUgU0RLIHYzLjAgLSBFdmFsdWF0aW9uIEhlYWx0aCBBUElcbiAqXG4gKiBBUEkgZm9yIGV2YWwgc2F0dXJhdGlvbiBtb25pdG9yaW5nLCByZWdyZXNzaW9uIGRldGVjdGlvbiwgYW5kIGhlYWx0aCByZXBvcnRzXG4gKi9cblxuaW1wb3J0IHsgYXBpUmVxdWVzdCwgYXBpUmVxdWVzdFdpdGhEYXRhIH0gZnJvbSAnLi4vY29yZS9jbGllbnQnO1xuXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG4vLyBUWVBFU1xuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuXG5leHBvcnQgdHlwZSBTYXR1cmF0aW9uVHlwZSA9ICdjZWlsaW5nJyB8ICdmbG9vcicgfCAnaGVhbHRoeSc7XG5leHBvcnQgdHlwZSBIZWFsdGhTdGF0dXMgPSAnaGVhbHRoeScgfCAnd2FybmluZycgfCAnY3JpdGljYWwnO1xuZXhwb3J0IHR5cGUgUmVncmVzc2lvblNldmVyaXR5ID0gJ21pbm9yJyB8ICdtb2RlcmF0ZScgfCAnc2V2ZXJlJztcblxuZXhwb3J0IGludGVyZmFjZSBFdmFsSGVhbHRoU25hcHNob3Qge1xuICBpZDogc3RyaW5nO1xuICBjb21wYW55SWQ6IHN0cmluZztcbiAgYWdlbnRJZDogc3RyaW5nO1xuICBjcml0ZXJpb25JZD86IHN0cmluZztcbiAgc25hcHNob3REYXRlOiBzdHJpbmc7XG4gIHBhc3NSYXRlPzogc3RyaW5nO1xuICBldmFsQ291bnQ/OiBudW1iZXI7XG4gIG1lYW5TY29yZT86IHN0cmluZztcbiAgc2F0dXJhdGlvblR5cGU/OiBTYXR1cmF0aW9uVHlwZTtcbiAgZGF5c0F0U2F0dXJhdGlvbj86IG51bWJlcjtcbiAgdHJlbmREaXJlY3Rpb24/OiBzdHJpbmc7XG4gIHRyZW5kU3RyZW5ndGg/OiBzdHJpbmc7XG4gIGhlYWx0aFN0YXR1cz86IEhlYWx0aFN0YXR1cztcbiAgaGVhbHRoU2NvcmU/OiBzdHJpbmc7XG4gIGNyZWF0ZWRBdDogc3RyaW5nO1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIEV2YWxSZWdyZXNzaW9uIHtcbiAgaWQ6IHN0cmluZztcbiAgY29tcGFueUlkOiBzdHJpbmc7XG4gIGFnZW50SWQ6IHN0cmluZztcbiAgY3JpdGVyaW9uSWQ/OiBzdHJpbmc7XG4gIHNldmVyaXR5OiBSZWdyZXNzaW9uU2V2ZXJpdHk7XG4gIGJhc2VsaW5lUGFzc1JhdGU6IHN0cmluZztcbiAgY3VycmVudFBhc3NSYXRlOiBzdHJpbmc7XG4gIGRlbHRhOiBzdHJpbmc7XG4gIGRlbHRhUGVyY2VudD86IHN0cmluZztcbiAgYmFzZWxpbmVQZXJpb2RTdGFydDogc3RyaW5nO1xuICBiYXNlbGluZVBlcmlvZEVuZDogc3RyaW5nO1xuICBjdXJyZW50UGVyaW9kU3RhcnQ6IHN0cmluZztcbiAgY3VycmVudFBlcmlvZEVuZDogc3RyaW5nO1xuICBiYXNlbGluZUV2YWxDb3VudD86IG51bWJlcjtcbiAgY3VycmVudEV2YWxDb3VudD86IG51bWJlcjtcbiAgc3VzcGVjdGVkQ2F1c2VzPzogdW5rbm93bltdO1xuICBpc1NpZ25pZmljYW50PzogYm9vbGVhbjtcbiAgaXNSZXNvbHZlZDogYm9vbGVhbjtcbiAgaXNBY2tub3dsZWRnZWQ6IGJvb2xlYW47XG4gIHJlc29sdmVkQXQ/OiBzdHJpbmc7XG4gIHJlc29sdmVkQnk/OiBzdHJpbmc7XG4gIHJlc29sdXRpb25UeXBlPzogc3RyaW5nO1xuICByZXNvbHV0aW9uTm90ZXM/OiBzdHJpbmc7XG4gIGFja25vd2xlZGdlZEF0Pzogc3RyaW5nO1xuICBhY2tub3dsZWRnZWRCeT86IHN0cmluZztcbiAgZGV0ZWN0ZWRBdDogc3RyaW5nO1xuICBjcmVhdGVkQXQ6IHN0cmluZztcbn1cblxuZXhwb3J0IGludGVyZmFjZSBIZWFsdGhSZXBvcnQge1xuICBhZ2VudElkOiBzdHJpbmc7XG4gIGdlbmVyYXRlZEF0OiBzdHJpbmc7XG4gIG92ZXJhbGxIZWFsdGg6IEhlYWx0aFN0YXR1cztcbiAgb3ZlcmFsbFNjb3JlOiBudW1iZXI7XG4gIHBhc3NSYXRlOiBudW1iZXI7XG4gIGV2YWxDb3VudDogbnVtYmVyO1xuICBzYXR1cmF0aW9uU3RhdHVzOiB7XG4gICAgdHlwZTogU2F0dXJhdGlvblR5cGU7XG4gICAgZGF5c0F0U2F0dXJhdGlvbjogbnVtYmVyO1xuICAgIHJlY29tbWVuZGF0aW9uOiBzdHJpbmc7XG4gIH07XG4gIHJlZ3Jlc3Npb25Db3VudDogbnVtYmVyO1xuICBhY3RpdmVSZWdyZXNzaW9uczogRXZhbFJlZ3Jlc3Npb25bXTtcbiAgdHJlbmQ6IHtcbiAgICBkaXJlY3Rpb246ICdpbXByb3ZpbmcnIHwgJ3N0YWJsZScgfCAnZGVjbGluaW5nJztcbiAgICBzdHJlbmd0aDogbnVtYmVyO1xuICAgIGRlc2NyaXB0aW9uOiBzdHJpbmc7XG4gIH07XG4gIHJlY29tbWVuZGF0aW9uczogc3RyaW5nW107XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgQ3JlYXRlU25hcHNob3RPcHRpb25zIHtcbiAgYWdlbnRJZDogc3RyaW5nO1xuICBjcml0ZXJpb25JZD86IHN0cmluZztcbiAgc25hcHNob3REYXRlOiBzdHJpbmc7XG4gIHBhc3NSYXRlPzogc3RyaW5nO1xuICBldmFsQ291bnQ/OiBudW1iZXI7XG4gIG1lYW5TY29yZT86IHN0cmluZztcbiAgc2F0dXJhdGlvblR5cGU/OiBTYXR1cmF0aW9uVHlwZTtcbiAgZGF5c0F0U2F0dXJhdGlvbj86IG51bWJlcjtcbiAgdHJlbmREaXJlY3Rpb24/OiBzdHJpbmc7XG4gIHRyZW5kU3RyZW5ndGg/OiBzdHJpbmc7XG4gIGhlYWx0aFN0YXR1cz86IEhlYWx0aFN0YXR1cztcbiAgaGVhbHRoU2NvcmU/OiBzdHJpbmc7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgQ3JlYXRlUmVncmVzc2lvbk9wdGlvbnMge1xuICBhZ2VudElkOiBzdHJpbmc7XG4gIGNyaXRlcmlvbklkPzogc3RyaW5nO1xuICBzZXZlcml0eTogUmVncmVzc2lvblNldmVyaXR5O1xuICBiYXNlbGluZVBhc3NSYXRlOiBzdHJpbmc7XG4gIGN1cnJlbnRQYXNzUmF0ZTogc3RyaW5nO1xuICBkZWx0YTogc3RyaW5nO1xuICBkZWx0YVBlcmNlbnQ/OiBzdHJpbmc7XG4gIGJhc2VsaW5lUGVyaW9kU3RhcnQ6IHN0cmluZztcbiAgYmFzZWxpbmVQZXJpb2RFbmQ6IHN0cmluZztcbiAgY3VycmVudFBlcmlvZFN0YXJ0OiBzdHJpbmc7XG4gIGN1cnJlbnRQZXJpb2RFbmQ6IHN0cmluZztcbiAgYmFzZWxpbmVFdmFsQ291bnQ/OiBudW1iZXI7XG4gIGN1cnJlbnRFdmFsQ291bnQ/OiBudW1iZXI7XG4gIHN1c3BlY3RlZENhdXNlcz86IHVua25vd25bXTtcbiAgaXNTaWduaWZpY2FudD86IGJvb2xlYW47XG4gIGRldGVjdGVkQXQ/OiBzdHJpbmc7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgUmVzb2x2ZVJlZ3Jlc3Npb25PcHRpb25zIHtcbiAgcmVzb2x1dGlvblR5cGU6IHN0cmluZztcbiAgbm90ZXM/OiBzdHJpbmc7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgR2V0U25hcHNob3RzT3B0aW9ucyB7XG4gIGFnZW50SWQ6IHN0cmluZztcbiAgY3JpdGVyaW9uSWQ/OiBzdHJpbmc7XG4gIHN0YXJ0RGF0ZT86IHN0cmluZztcbiAgZW5kRGF0ZT86IHN0cmluZztcbn1cblxuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuLy8gRVZBTCBIRUFMVEggQVBJIENMSUVOVFxuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuXG4vKipcbiAqIEV2YWx1YXRpb24gSGVhbHRoIEFQSSBjbGllbnQgZm9yIG1vbml0b3JpbmcgZXZhbCBxdWFsaXR5IGFuZCBkZXRlY3RpbmcgcmVncmVzc2lvbnNcbiAqL1xuZXhwb3J0IGNvbnN0IGV2YWxIZWFsdGggPSB7XG4gIC8qKlxuICAgKiBHZXQgY29tcHJlaGVuc2l2ZSBoZWFsdGggcmVwb3J0IGZvciBhbiBhZ2VudFxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IHJlcG9ydCA9IGF3YWl0IGV2YWxIZWFsdGguZ2V0UmVwb3J0KCdhZ2VudF8xMjMnKTtcbiAgICogY29uc29sZS5sb2coYE92ZXJhbGwgaGVhbHRoOiAke3JlcG9ydC5vdmVyYWxsSGVhbHRofWApO1xuICAgKiBjb25zb2xlLmxvZyhgQWN0aXZlIHJlZ3Jlc3Npb25zOiAke3JlcG9ydC5yZWdyZXNzaW9uQ291bnR9YCk7XG4gICAqIGBgYFxuICAgKi9cbiAgYXN5bmMgZ2V0UmVwb3J0KGFnZW50SWQ6IHN0cmluZyk6IFByb21pc2U8SGVhbHRoUmVwb3J0PiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxIZWFsdGhSZXBvcnQ+KFxuICAgICAgYC9ldmFsLWhlYWx0aC9yZXBvcnQ/YWdlbnRJZD0ke2FnZW50SWR9YCxcbiAgICAgIHsgYXBpVmVyc2lvbjogJ25vbmUnIH1cbiAgICApO1xuICB9LFxuXG4gIC8vIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLVxuICAvLyBTTkFQU0hPVFNcbiAgLy8gLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tXG5cbiAgLyoqXG4gICAqIEdldCBoaXN0b3JpY2FsIGhlYWx0aCBzbmFwc2hvdHNcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzbmFwc2hvdHMgPSBhd2FpdCBldmFsSGVhbHRoLmdldFNuYXBzaG90cyh7XG4gICAqICAgYWdlbnRJZDogJ2FnZW50XzEyMycsXG4gICAqICAgc3RhcnREYXRlOiAnMjAyNC0wMS0wMVQwMDowMDowMFonLFxuICAgKiAgIGVuZERhdGU6ICcyMDI0LTAxLTMxVDIzOjU5OjU5WicsXG4gICAqIH0pO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldFNuYXBzaG90cyhvcHRpb25zOiBHZXRTbmFwc2hvdHNPcHRpb25zKTogUHJvbWlzZTxFdmFsSGVhbHRoU25hcHNob3RbXT4ge1xuICAgIGNvbnN0IHBhcmFtcyA9IG5ldyBVUkxTZWFyY2hQYXJhbXMoKTtcbiAgICBwYXJhbXMuc2V0KCdhZ2VudElkJywgb3B0aW9ucy5hZ2VudElkKTtcbiAgICBpZiAob3B0aW9ucy5jcml0ZXJpb25JZCkgcGFyYW1zLnNldCgnY3JpdGVyaW9uSWQnLCBvcHRpb25zLmNyaXRlcmlvbklkKTtcbiAgICBpZiAob3B0aW9ucy5zdGFydERhdGUpIHBhcmFtcy5zZXQoJ3N0YXJ0RGF0ZScsIG9wdGlvbnMuc3RhcnREYXRlKTtcbiAgICBpZiAob3B0aW9ucy5lbmREYXRlKSBwYXJhbXMuc2V0KCdlbmREYXRlJywgb3B0aW9ucy5lbmREYXRlKTtcblxuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8RXZhbEhlYWx0aFNuYXBzaG90W10+KFxuICAgICAgYC9ldmFsLWhlYWx0aC9zbmFwc2hvdHM/JHtwYXJhbXMudG9TdHJpbmcoKX1gLFxuICAgICAgeyBhcGlWZXJzaW9uOiAnbm9uZScgfVxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIEdldCBsYXRlc3QgaGVhbHRoIHNuYXBzaG90XG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3Qgc25hcHNob3QgPSBhd2FpdCBldmFsSGVhbHRoLmdldExhdGVzdFNuYXBzaG90KCdhZ2VudF8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRMYXRlc3RTbmFwc2hvdChhZ2VudElkOiBzdHJpbmcsIGNyaXRlcmlvbklkPzogc3RyaW5nKTogUHJvbWlzZTxFdmFsSGVhbHRoU25hcHNob3QgfCBudWxsPiB7XG4gICAgY29uc3QgcGFyYW1zID0gbmV3IFVSTFNlYXJjaFBhcmFtcygpO1xuICAgIHBhcmFtcy5zZXQoJ2FnZW50SWQnLCBhZ2VudElkKTtcbiAgICBpZiAoY3JpdGVyaW9uSWQpIHBhcmFtcy5zZXQoJ2NyaXRlcmlvbklkJywgY3JpdGVyaW9uSWQpO1xuXG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxFdmFsSGVhbHRoU25hcHNob3QgfCBudWxsPihcbiAgICAgIGAvZXZhbC1oZWFsdGgvc25hcHNob3RzL2xhdGVzdD8ke3BhcmFtcy50b1N0cmluZygpfWAsXG4gICAgICB7IGFwaVZlcnNpb246ICdub25lJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogUmVjb3JkIGEgaGVhbHRoIHNuYXBzaG90XG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3Qgc25hcHNob3QgPSBhd2FpdCBldmFsSGVhbHRoLnJlY29yZFNuYXBzaG90KHtcbiAgICogICBhZ2VudElkOiAnYWdlbnRfMTIzJyxcbiAgICogICBzbmFwc2hvdERhdGU6IG5ldyBEYXRlKCkudG9JU09TdHJpbmcoKSxcbiAgICogICBwYXNzUmF0ZTogJzAuODUnLFxuICAgKiAgIGV2YWxDb3VudDogMTUwLFxuICAgKiAgIGhlYWx0aFN0YXR1czogJ2hlYWx0aHknLFxuICAgKiB9KTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyByZWNvcmRTbmFwc2hvdChvcHRpb25zOiBDcmVhdGVTbmFwc2hvdE9wdGlvbnMpOiBQcm9taXNlPEV2YWxIZWFsdGhTbmFwc2hvdD4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8RXZhbEhlYWx0aFNuYXBzaG90PignL2V2YWwtaGVhbHRoL3NuYXBzaG90cycsIHtcbiAgICAgIG1ldGhvZDogJ1BPU1QnLFxuICAgICAgYm9keTogb3B0aW9ucyxcbiAgICAgIGFwaVZlcnNpb246ICdub25lJyxcbiAgICB9KTtcbiAgfSxcblxuICAvLyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS1cbiAgLy8gUkVHUkVTU0lPTlNcbiAgLy8gLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tXG5cbiAgLyoqXG4gICAqIEdldCB1bnJlc29sdmVkIHJlZ3Jlc3Npb25zIGZvciBhbiBhZ2VudFxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IHJlZ3Jlc3Npb25zID0gYXdhaXQgZXZhbEhlYWx0aC5nZXRSZWdyZXNzaW9ucygnYWdlbnRfMTIzJyk7XG4gICAqIGZvciAoY29uc3QgcmVncmVzc2lvbiBvZiByZWdyZXNzaW9ucykge1xuICAgKiAgIGNvbnNvbGUubG9nKGAke3JlZ3Jlc3Npb24uc2V2ZXJpdHl9OiAke3JlZ3Jlc3Npb24uZGVsdGF9JSBkcm9wYCk7XG4gICAqIH1cbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRSZWdyZXNzaW9ucyhhZ2VudElkOiBzdHJpbmcpOiBQcm9taXNlPEV2YWxSZWdyZXNzaW9uW10+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPEV2YWxSZWdyZXNzaW9uW10+KFxuICAgICAgYC9ldmFsLWhlYWx0aC9yZWdyZXNzaW9ucz9hZ2VudElkPSR7YWdlbnRJZH1gLFxuICAgICAgeyBhcGlWZXJzaW9uOiAnbm9uZScgfVxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIFJlY29yZCBhIG5ldyByZWdyZXNzaW9uXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgcmVncmVzc2lvbiA9IGF3YWl0IGV2YWxIZWFsdGgucmVjb3JkUmVncmVzc2lvbih7XG4gICAqICAgYWdlbnRJZDogJ2FnZW50XzEyMycsXG4gICAqICAgc2V2ZXJpdHk6ICdtb2RlcmF0ZScsXG4gICAqICAgYmFzZWxpbmVQYXNzUmF0ZTogJzAuOTInLFxuICAgKiAgIGN1cnJlbnRQYXNzUmF0ZTogJzAuNzgnLFxuICAgKiAgIGRlbHRhOiAnLTAuMTQnLFxuICAgKiAgIGJhc2VsaW5lUGVyaW9kU3RhcnQ6ICcyMDI0LTAxLTAxVDAwOjAwOjAwWicsXG4gICAqICAgYmFzZWxpbmVQZXJpb2RFbmQ6ICcyMDI0LTAxLTE1VDIzOjU5OjU5WicsXG4gICAqICAgY3VycmVudFBlcmlvZFN0YXJ0OiAnMjAyNC0wMS0xNlQwMDowMDowMFonLFxuICAgKiAgIGN1cnJlbnRQZXJpb2RFbmQ6ICcyMDI0LTAxLTMxVDIzOjU5OjU5WicsXG4gICAqIH0pO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIHJlY29yZFJlZ3Jlc3Npb24ob3B0aW9uczogQ3JlYXRlUmVncmVzc2lvbk9wdGlvbnMpOiBQcm9taXNlPEV2YWxSZWdyZXNzaW9uPiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxFdmFsUmVncmVzc2lvbj4oJy9ldmFsLWhlYWx0aC9yZWdyZXNzaW9ucycsIHtcbiAgICAgIG1ldGhvZDogJ1BPU1QnLFxuICAgICAgYm9keTogb3B0aW9ucyxcbiAgICAgIGFwaVZlcnNpb246ICdub25lJyxcbiAgICB9KTtcbiAgfSxcblxuICAvKipcbiAgICogUmVzb2x2ZSBhIHJlZ3Jlc3Npb25cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBhd2FpdCBldmFsSGVhbHRoLnJlc29sdmVSZWdyZXNzaW9uKCdyZWdyZXNzaW9uXzEyMycsIHtcbiAgICogICByZXNvbHV0aW9uVHlwZTogJ2ZpeGVkJyxcbiAgICogICBub3RlczogJ1VwZGF0ZWQgcHJvbXB0IHRlbXBsYXRlIHRvIGFkZHJlc3MgcXVhbGl0eSBpc3N1ZXMnLFxuICAgKiB9KTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyByZXNvbHZlUmVncmVzc2lvbihyZWdyZXNzaW9uSWQ6IHN0cmluZywgb3B0aW9uczogUmVzb2x2ZVJlZ3Jlc3Npb25PcHRpb25zKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgYXdhaXQgYXBpUmVxdWVzdChgL2V2YWwtaGVhbHRoL3JlZ3Jlc3Npb25zLyR7cmVncmVzc2lvbklkfS9yZXNvbHZlYCwge1xuICAgICAgbWV0aG9kOiAnUE9TVCcsXG4gICAgICBib2R5OiBvcHRpb25zLFxuICAgICAgYXBpVmVyc2lvbjogJ25vbmUnLFxuICAgIH0pO1xuICB9LFxuXG4gIC8qKlxuICAgKiBBY2tub3dsZWRnZSBhIHJlZ3Jlc3Npb25cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBhd2FpdCBldmFsSGVhbHRoLmFja25vd2xlZGdlUmVncmVzc2lvbigncmVncmVzc2lvbl8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBhY2tub3dsZWRnZVJlZ3Jlc3Npb24ocmVncmVzc2lvbklkOiBzdHJpbmcpOiBQcm9taXNlPHZvaWQ+IHtcbiAgICBhd2FpdCBhcGlSZXF1ZXN0KGAvZXZhbC1oZWFsdGgvcmVncmVzc2lvbnMvJHtyZWdyZXNzaW9uSWR9L2Fja25vd2xlZGdlYCwge1xuICAgICAgbWV0aG9kOiAnUE9TVCcsXG4gICAgICBhcGlWZXJzaW9uOiAnbm9uZScsXG4gICAgfSk7XG4gIH0sXG59O1xuXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG4vLyBIRUxQRVIgRlVOQ1RJT05TXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG5cbi8qKlxuICogQ2hlY2sgaWYgaGVhbHRoIHN0YXR1cyBpbmRpY2F0ZXMgYW4gaXNzdWVcbiAqXG4gKiBAcGFyYW0gc3RhdHVzIC0gSGVhbHRoIHN0YXR1cyB0byBjaGVja1xuICogQHJldHVybnMgV2hldGhlciB0aGUgc3RhdHVzIGluZGljYXRlcyBhIHByb2JsZW1cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGhhc0hlYWx0aElzc3VlKHN0YXR1czogSGVhbHRoU3RhdHVzKTogYm9vbGVhbiB7XG4gIHJldHVybiBzdGF0dXMgPT09ICd3YXJuaW5nJyB8fCBzdGF0dXMgPT09ICdjcml0aWNhbCc7XG59XG5cbi8qKlxuICogR2V0IHNldmVyaXR5IGxldmVsIGFzIG51bWVyaWMgdmFsdWUgZm9yIHNvcnRpbmdcbiAqXG4gKiBAcGFyYW0gc2V2ZXJpdHkgLSBSZWdyZXNzaW9uIHNldmVyaXR5XG4gKiBAcmV0dXJucyBOdW1lcmljIHNldmVyaXR5ICgxLTMsIGhpZ2hlciBpcyB3b3JzZSlcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGdldFNldmVyaXR5TGV2ZWwoc2V2ZXJpdHk6IFJlZ3Jlc3Npb25TZXZlcml0eSk6IG51bWJlciB7XG4gIHN3aXRjaCAoc2V2ZXJpdHkpIHtcbiAgICBjYXNlICdtaW5vcic6IHJldHVybiAxO1xuICAgIGNhc2UgJ21vZGVyYXRlJzogcmV0dXJuIDI7XG4gICAgY2FzZSAnc2V2ZXJlJzogcmV0dXJuIDM7XG4gICAgZGVmYXVsdDogcmV0dXJuIDA7XG4gIH1cbn1cblxuLyoqXG4gKiBDaGVjayBpZiBldmFsdWF0aW9uIGlzIHNhdHVyYXRlZFxuICpcbiAqIEBwYXJhbSBzbmFwc2hvdCAtIEhlYWx0aCBzbmFwc2hvdCB0byBjaGVja1xuICogQHJldHVybnMgV2hldGhlciBldmFsdWF0aW9uIGlzIGF0IGNlaWxpbmcgb3IgZmxvb3JcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGlzU2F0dXJhdGVkKHNuYXBzaG90OiBFdmFsSGVhbHRoU25hcHNob3QpOiBib29sZWFuIHtcbiAgcmV0dXJuIHNuYXBzaG90LnNhdHVyYXRpb25UeXBlID09PSAnY2VpbGluZycgfHwgc25hcHNob3Quc2F0dXJhdGlvblR5cGUgPT09ICdmbG9vcic7XG59XG5cbi8qKlxuICogR2V0IHJlY29tbWVuZGF0aW9uIGZvciBzYXR1cmF0aW9uIHR5cGVcbiAqXG4gKiBAcGFyYW0gc2F0dXJhdGlvblR5cGUgLSBUeXBlIG9mIHNhdHVyYXRpb25cbiAqIEByZXR1cm5zIFJlY29tbWVuZGF0aW9uIHN0cmluZ1xuICovXG5leHBvcnQgZnVuY3Rpb24gZ2V0U2F0dXJhdGlvblJlY29tbWVuZGF0aW9uKHNhdHVyYXRpb25UeXBlOiBTYXR1cmF0aW9uVHlwZSk6IHN0cmluZyB7XG4gIHN3aXRjaCAoc2F0dXJhdGlvblR5cGUpIHtcbiAgICBjYXNlICdjZWlsaW5nJzpcbiAgICAgIHJldHVybiAnRXZhbHVhdGlvbiBjcml0ZXJpYSBtYXkgYmUgdG9vIGxlbmllbnQuIENvbnNpZGVyIGFkZGluZyBzdHJpY3RlciBjaGVja3Mgb3IgbW9yZSBjaGFsbGVuZ2luZyB0ZXN0IGNhc2VzLic7XG4gICAgY2FzZSAnZmxvb3InOlxuICAgICAgcmV0dXJuICdFdmFsdWF0aW9uIGNyaXRlcmlhIG1heSBiZSB0b28gc3RyaWN0LiBDb25zaWRlciByZWxheGluZyB0aHJlc2hvbGRzIG9yIHJldmlld2luZyBjcml0ZXJpYSBmb3IgYWNjdXJhY3kuJztcbiAgICBjYXNlICdoZWFsdGh5JzpcbiAgICAgIHJldHVybiAnRXZhbHVhdGlvbiBpcyBvcGVyYXRpbmcgd2l0aGluIGhlYWx0aHkgcGFyYW1ldGVycy4nO1xuICAgIGRlZmF1bHQ6XG4gICAgICByZXR1cm4gJ1VuYWJsZSB0byBkZXRlcm1pbmUgc2F0dXJhdGlvbiBzdGF0dXMuJztcbiAgfVxufVxuIl19
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ThinkHive SDK v3.0 - Human Review API
|
|
3
|
+
*
|
|
4
|
+
* API for managing human review queue, calibration sets, and reviewer management
|
|
5
|
+
*/
|
|
6
|
+
export type HumanReviewStatus = 'pending' | 'in_progress' | 'completed' | 'skipped' | 'expired';
|
|
7
|
+
export type HumanReviewType = 'disagreement' | 'low_confidence' | 'calibration' | 'random_sample' | 'flagged';
|
|
8
|
+
export interface HumanReviewQueueItem {
|
|
9
|
+
id: string;
|
|
10
|
+
companyId: string;
|
|
11
|
+
agentId: string;
|
|
12
|
+
traceId: string;
|
|
13
|
+
criterionId?: string;
|
|
14
|
+
reviewType: HumanReviewType;
|
|
15
|
+
priority: number;
|
|
16
|
+
status: HumanReviewStatus;
|
|
17
|
+
llmScore?: number;
|
|
18
|
+
llmPassed?: boolean;
|
|
19
|
+
llmReasoning?: string;
|
|
20
|
+
llmConfidence?: number;
|
|
21
|
+
reviewerId?: string;
|
|
22
|
+
humanScore?: number;
|
|
23
|
+
humanPassed?: boolean;
|
|
24
|
+
humanReasoning?: string;
|
|
25
|
+
reviewDurationMs?: number;
|
|
26
|
+
isCalibrationSample?: boolean;
|
|
27
|
+
calibrationSetId?: string;
|
|
28
|
+
expectedScore?: number;
|
|
29
|
+
expectedPassed?: boolean;
|
|
30
|
+
expiresAt?: string;
|
|
31
|
+
claimedAt?: string;
|
|
32
|
+
completedAt?: string;
|
|
33
|
+
createdAt: string;
|
|
34
|
+
}
|
|
35
|
+
export interface AddToQueueOptions {
|
|
36
|
+
traceId: string;
|
|
37
|
+
criterionId?: string;
|
|
38
|
+
agentId: string;
|
|
39
|
+
reviewType: HumanReviewType;
|
|
40
|
+
priority?: number;
|
|
41
|
+
llmScore?: number;
|
|
42
|
+
llmPassed?: boolean;
|
|
43
|
+
llmReasoning?: string;
|
|
44
|
+
llmConfidence?: number;
|
|
45
|
+
isCalibrationSample?: boolean;
|
|
46
|
+
calibrationSetId?: string;
|
|
47
|
+
expectedScore?: number;
|
|
48
|
+
expectedPassed?: boolean;
|
|
49
|
+
expiresInMs?: number;
|
|
50
|
+
metadata?: Record<string, unknown>;
|
|
51
|
+
}
|
|
52
|
+
export interface SubmitReviewOptions {
|
|
53
|
+
passed: boolean;
|
|
54
|
+
score: number;
|
|
55
|
+
reasoning: string;
|
|
56
|
+
durationMs: number;
|
|
57
|
+
}
|
|
58
|
+
export interface CalibrationSet {
|
|
59
|
+
id: string;
|
|
60
|
+
companyId: string;
|
|
61
|
+
agentId?: string;
|
|
62
|
+
name: string;
|
|
63
|
+
description?: string;
|
|
64
|
+
targetSampleCount: number;
|
|
65
|
+
currentSampleCount: number;
|
|
66
|
+
minAgreementRate: number;
|
|
67
|
+
passingScoreThreshold: number;
|
|
68
|
+
criteriaIds: string[];
|
|
69
|
+
isActive: boolean;
|
|
70
|
+
createdBy?: string;
|
|
71
|
+
createdAt: string;
|
|
72
|
+
}
|
|
73
|
+
export interface CreateCalibrationSetOptions {
|
|
74
|
+
name: string;
|
|
75
|
+
description?: string;
|
|
76
|
+
agentId: string;
|
|
77
|
+
criterionId?: string;
|
|
78
|
+
targetAgreement?: number;
|
|
79
|
+
minSamples?: number;
|
|
80
|
+
}
|
|
81
|
+
export interface ReviewerCalibration {
|
|
82
|
+
id: string;
|
|
83
|
+
userId: string;
|
|
84
|
+
calibrationSetId: string;
|
|
85
|
+
totalReviews: number;
|
|
86
|
+
agreementRate: number;
|
|
87
|
+
meanAbsoluteError: number;
|
|
88
|
+
isCertified: boolean;
|
|
89
|
+
certifiedAt?: string;
|
|
90
|
+
lastReviewAt?: string;
|
|
91
|
+
}
|
|
92
|
+
export interface QueueStats {
|
|
93
|
+
pending: number;
|
|
94
|
+
inProgress: number;
|
|
95
|
+
completed: number;
|
|
96
|
+
skipped: number;
|
|
97
|
+
expired: number;
|
|
98
|
+
avgReviewDurationMs?: number;
|
|
99
|
+
avgAgreementRate?: number;
|
|
100
|
+
}
|
|
101
|
+
export interface ListQueueOptions {
|
|
102
|
+
agentId?: string;
|
|
103
|
+
status?: HumanReviewStatus;
|
|
104
|
+
reviewType?: HumanReviewType;
|
|
105
|
+
isCalibration?: boolean;
|
|
106
|
+
minPriority?: number;
|
|
107
|
+
limit?: number;
|
|
108
|
+
offset?: number;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Human Review API client for managing human review queue and calibration
|
|
112
|
+
*/
|
|
113
|
+
export declare const humanReview: {
|
|
114
|
+
/**
|
|
115
|
+
* Get pending review queue items
|
|
116
|
+
*
|
|
117
|
+
* @example
|
|
118
|
+
* ```typescript
|
|
119
|
+
* const items = await humanReview.getQueue({
|
|
120
|
+
* agentId: 'agent_123',
|
|
121
|
+
* status: 'pending',
|
|
122
|
+
* limit: 20,
|
|
123
|
+
* });
|
|
124
|
+
* ```
|
|
125
|
+
*/
|
|
126
|
+
getQueue(options?: ListQueueOptions): Promise<HumanReviewQueueItem[]>;
|
|
127
|
+
/**
|
|
128
|
+
* Add an item to the review queue
|
|
129
|
+
*
|
|
130
|
+
* @example
|
|
131
|
+
* ```typescript
|
|
132
|
+
* const item = await humanReview.addToQueue({
|
|
133
|
+
* traceId: 'trace_123',
|
|
134
|
+
* agentId: 'agent_123',
|
|
135
|
+
* reviewType: 'disagreement',
|
|
136
|
+
* priority: 80,
|
|
137
|
+
* llmScore: 65,
|
|
138
|
+
* llmPassed: true,
|
|
139
|
+
* });
|
|
140
|
+
* ```
|
|
141
|
+
*/
|
|
142
|
+
addToQueue(options: AddToQueueOptions): Promise<HumanReviewQueueItem>;
|
|
143
|
+
/**
|
|
144
|
+
* Get a specific review item
|
|
145
|
+
*
|
|
146
|
+
* @example
|
|
147
|
+
* ```typescript
|
|
148
|
+
* const item = await humanReview.getItem('item_123');
|
|
149
|
+
* ```
|
|
150
|
+
*/
|
|
151
|
+
getItem(itemId: string): Promise<HumanReviewQueueItem>;
|
|
152
|
+
/**
|
|
153
|
+
* Claim a review item for processing
|
|
154
|
+
*
|
|
155
|
+
* @example
|
|
156
|
+
* ```typescript
|
|
157
|
+
* const item = await humanReview.claim('item_123');
|
|
158
|
+
* ```
|
|
159
|
+
*/
|
|
160
|
+
claim(itemId: string): Promise<HumanReviewQueueItem>;
|
|
161
|
+
/**
|
|
162
|
+
* Release a claimed review item
|
|
163
|
+
*
|
|
164
|
+
* @example
|
|
165
|
+
* ```typescript
|
|
166
|
+
* await humanReview.release('item_123');
|
|
167
|
+
* ```
|
|
168
|
+
*/
|
|
169
|
+
release(itemId: string): Promise<HumanReviewQueueItem>;
|
|
170
|
+
/**
|
|
171
|
+
* Skip a review item
|
|
172
|
+
*
|
|
173
|
+
* @example
|
|
174
|
+
* ```typescript
|
|
175
|
+
* await humanReview.skip('item_123');
|
|
176
|
+
* ```
|
|
177
|
+
*/
|
|
178
|
+
skip(itemId: string): Promise<HumanReviewQueueItem>;
|
|
179
|
+
/**
|
|
180
|
+
* Submit a review
|
|
181
|
+
*
|
|
182
|
+
* @example
|
|
183
|
+
* ```typescript
|
|
184
|
+
* const result = await humanReview.submit('item_123', {
|
|
185
|
+
* passed: true,
|
|
186
|
+
* score: 85,
|
|
187
|
+
* reasoning: 'Response accurately addressed the query',
|
|
188
|
+
* durationMs: 45000,
|
|
189
|
+
* });
|
|
190
|
+
* ```
|
|
191
|
+
*/
|
|
192
|
+
submit(itemId: string, review: SubmitReviewOptions): Promise<HumanReviewQueueItem>;
|
|
193
|
+
/**
|
|
194
|
+
* Get queue statistics
|
|
195
|
+
*
|
|
196
|
+
* @example
|
|
197
|
+
* ```typescript
|
|
198
|
+
* const stats = await humanReview.getStats('agent_123');
|
|
199
|
+
* ```
|
|
200
|
+
*/
|
|
201
|
+
getStats(agentId?: string): Promise<QueueStats>;
|
|
202
|
+
/**
|
|
203
|
+
* Get next item for a reviewer
|
|
204
|
+
*
|
|
205
|
+
* @example
|
|
206
|
+
* ```typescript
|
|
207
|
+
* const nextItem = await humanReview.getNextItem('agent_123');
|
|
208
|
+
* ```
|
|
209
|
+
*/
|
|
210
|
+
getNextItem(agentId?: string): Promise<HumanReviewQueueItem | null>;
|
|
211
|
+
/**
|
|
212
|
+
* Get available review types
|
|
213
|
+
*
|
|
214
|
+
* @example
|
|
215
|
+
* ```typescript
|
|
216
|
+
* const types = await humanReview.getReviewTypes();
|
|
217
|
+
* ```
|
|
218
|
+
*/
|
|
219
|
+
getReviewTypes(): Promise<Array<{
|
|
220
|
+
id: string;
|
|
221
|
+
name: string;
|
|
222
|
+
description: string;
|
|
223
|
+
autoTrigger: boolean;
|
|
224
|
+
}>>;
|
|
225
|
+
/**
|
|
226
|
+
* Get calibration sets
|
|
227
|
+
*
|
|
228
|
+
* @example
|
|
229
|
+
* ```typescript
|
|
230
|
+
* const sets = await humanReview.getCalibrationSets('agent_123');
|
|
231
|
+
* ```
|
|
232
|
+
*/
|
|
233
|
+
getCalibrationSets(agentId?: string): Promise<CalibrationSet[]>;
|
|
234
|
+
/**
|
|
235
|
+
* Create a calibration set
|
|
236
|
+
*
|
|
237
|
+
* @example
|
|
238
|
+
* ```typescript
|
|
239
|
+
* const set = await humanReview.createCalibrationSet({
|
|
240
|
+
* name: 'Quality Calibration Q1',
|
|
241
|
+
* agentId: 'agent_123',
|
|
242
|
+
* targetAgreement: 0.85,
|
|
243
|
+
* minSamples: 50,
|
|
244
|
+
* });
|
|
245
|
+
* ```
|
|
246
|
+
*/
|
|
247
|
+
createCalibrationSet(options: CreateCalibrationSetOptions): Promise<CalibrationSet>;
|
|
248
|
+
/**
|
|
249
|
+
* Get a calibration set by ID
|
|
250
|
+
*
|
|
251
|
+
* @example
|
|
252
|
+
* ```typescript
|
|
253
|
+
* const set = await humanReview.getCalibrationSet('set_123');
|
|
254
|
+
* ```
|
|
255
|
+
*/
|
|
256
|
+
getCalibrationSet(setId: string): Promise<CalibrationSet>;
|
|
257
|
+
/**
|
|
258
|
+
* Get certified reviewers for a calibration set
|
|
259
|
+
*
|
|
260
|
+
* @example
|
|
261
|
+
* ```typescript
|
|
262
|
+
* const reviewers = await humanReview.getCertifiedReviewers('set_123');
|
|
263
|
+
* ```
|
|
264
|
+
*/
|
|
265
|
+
getCertifiedReviewers(calibrationSetId: string): Promise<ReviewerCalibration[]>;
|
|
266
|
+
/**
|
|
267
|
+
* Get reviewer calibration status
|
|
268
|
+
*
|
|
269
|
+
* @example
|
|
270
|
+
* ```typescript
|
|
271
|
+
* const calibrations = await humanReview.getReviewerCalibrations('user_123');
|
|
272
|
+
* ```
|
|
273
|
+
*/
|
|
274
|
+
getReviewerCalibrations(userId: string): Promise<ReviewerCalibration[]>;
|
|
275
|
+
};
|