@thinkhive/sdk 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +279 -128
- package/dist/api/apiKeys.d.ts +252 -0
- package/dist/api/apiKeys.js +298 -0
- package/dist/api/business-metrics.d.ts +188 -0
- package/dist/api/business-metrics.js +213 -0
- package/dist/api/conversation-eval.d.ts +200 -0
- package/dist/api/conversation-eval.js +235 -0
- package/dist/api/deterministic-graders.d.ts +205 -0
- package/dist/api/deterministic-graders.js +191 -0
- package/dist/api/eval-health.d.ts +250 -0
- package/dist/api/eval-health.js +224 -0
- package/dist/api/human-review.d.ts +275 -0
- package/dist/api/human-review.js +236 -0
- package/dist/api/nondeterminism.d.ts +300 -0
- package/dist/api/nondeterminism.js +250 -0
- package/dist/api/quality-metrics.d.ts +303 -0
- package/dist/api/quality-metrics.js +198 -0
- package/dist/api/roi-analytics.d.ts +263 -0
- package/dist/api/roi-analytics.js +204 -0
- package/dist/api/transcript-patterns.d.ts +204 -0
- package/dist/api/transcript-patterns.js +227 -0
- package/dist/core/client.d.ts +82 -8
- package/dist/core/client.js +223 -32
- package/dist/core/config.d.ts +1 -1
- package/dist/core/config.js +2 -2
- package/dist/core/types.d.ts +27 -2
- package/dist/core/types.js +1 -1
- package/dist/index.d.ts +415 -62
- package/dist/index.js +253 -37
- package/package.json +8 -4
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ThinkHive SDK v3.0 - Non-Determinism API
|
|
3
|
+
*
|
|
4
|
+
* API for pass@k / pass^k analysis to measure LLM evaluation reliability
|
|
5
|
+
*/
|
|
6
|
+
export type NondeterminismRunType = 'pass_at_k' | 'pass_to_k' | 'variance' | 'reliability';
|
|
7
|
+
export type NondeterminismRunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
|
8
|
+
export interface NondeterminismRun {
|
|
9
|
+
id: string;
|
|
10
|
+
companyId: string;
|
|
11
|
+
agentId: string;
|
|
12
|
+
runType: NondeterminismRunType;
|
|
13
|
+
kValue: number;
|
|
14
|
+
status: NondeterminismRunStatus;
|
|
15
|
+
traceCount: number;
|
|
16
|
+
criterionId?: string;
|
|
17
|
+
criteriaIds: string[];
|
|
18
|
+
temperature?: string;
|
|
19
|
+
model?: string;
|
|
20
|
+
progressPercent: number;
|
|
21
|
+
passAtKRate?: string;
|
|
22
|
+
passToKRate?: string;
|
|
23
|
+
avgVariance?: string;
|
|
24
|
+
reliabilityScore?: string;
|
|
25
|
+
startedAt?: string;
|
|
26
|
+
completedAt?: string;
|
|
27
|
+
createdBy?: string;
|
|
28
|
+
createdAt: string;
|
|
29
|
+
}
|
|
30
|
+
export interface NondeterminismSample {
|
|
31
|
+
id: string;
|
|
32
|
+
runId: string;
|
|
33
|
+
traceId: string;
|
|
34
|
+
criterionId: string;
|
|
35
|
+
sampleIndex: number;
|
|
36
|
+
score: string;
|
|
37
|
+
passed: boolean;
|
|
38
|
+
reasoning?: string;
|
|
39
|
+
confidence?: string;
|
|
40
|
+
tokensUsed?: number;
|
|
41
|
+
costUsd?: string;
|
|
42
|
+
model?: string;
|
|
43
|
+
temperature?: string;
|
|
44
|
+
latencyMs?: number;
|
|
45
|
+
error?: string;
|
|
46
|
+
createdAt: string;
|
|
47
|
+
}
|
|
48
|
+
export interface CreateRunOptions {
|
|
49
|
+
agentId: string;
|
|
50
|
+
criterionId?: string;
|
|
51
|
+
criteriaIds?: string[];
|
|
52
|
+
kValue: number;
|
|
53
|
+
traceIds: string[];
|
|
54
|
+
runType?: NondeterminismRunType;
|
|
55
|
+
temperature?: number;
|
|
56
|
+
model?: string;
|
|
57
|
+
}
|
|
58
|
+
export interface RecordSampleOptions {
|
|
59
|
+
runId: string;
|
|
60
|
+
traceId: string;
|
|
61
|
+
criterionId: string;
|
|
62
|
+
sampleIndex: number;
|
|
63
|
+
score: number;
|
|
64
|
+
passed: boolean;
|
|
65
|
+
reasoning?: string;
|
|
66
|
+
confidence?: number;
|
|
67
|
+
tokensUsed?: number;
|
|
68
|
+
costUsd?: number;
|
|
69
|
+
model?: string;
|
|
70
|
+
temperature?: number;
|
|
71
|
+
latencyMs?: number;
|
|
72
|
+
error?: string;
|
|
73
|
+
}
|
|
74
|
+
export interface TraceAnalysis {
|
|
75
|
+
traceId: string;
|
|
76
|
+
samples: NondeterminismSample[];
|
|
77
|
+
passCount: number;
|
|
78
|
+
totalCount: number;
|
|
79
|
+
passRate: number;
|
|
80
|
+
scoreVariance: number;
|
|
81
|
+
meanScore: number;
|
|
82
|
+
isConsistent: boolean;
|
|
83
|
+
}
|
|
84
|
+
export interface CriterionAnalysis {
|
|
85
|
+
criterionId: string;
|
|
86
|
+
traceAnalyses: TraceAnalysis[];
|
|
87
|
+
passAtKRate: number;
|
|
88
|
+
passToKRate: number;
|
|
89
|
+
reliabilityScore: number;
|
|
90
|
+
isReliable: boolean;
|
|
91
|
+
recommendation: string;
|
|
92
|
+
}
|
|
93
|
+
export interface RunSummary {
|
|
94
|
+
run: NondeterminismRun;
|
|
95
|
+
traceAnalyses: TraceAnalysis[];
|
|
96
|
+
criterionAnalyses: CriterionAnalysis[];
|
|
97
|
+
}
|
|
98
|
+
export interface ListRunsOptions {
|
|
99
|
+
agentId?: string;
|
|
100
|
+
status?: NondeterminismRunStatus;
|
|
101
|
+
limit?: number;
|
|
102
|
+
offset?: number;
|
|
103
|
+
}
|
|
104
|
+
export interface PassAtKInfo {
|
|
105
|
+
concepts: {
|
|
106
|
+
passAtK: {
|
|
107
|
+
name: string;
|
|
108
|
+
description: string;
|
|
109
|
+
formula: string;
|
|
110
|
+
useCase: string;
|
|
111
|
+
};
|
|
112
|
+
passToK: {
|
|
113
|
+
name: string;
|
|
114
|
+
description: string;
|
|
115
|
+
formula: string;
|
|
116
|
+
useCase: string;
|
|
117
|
+
};
|
|
118
|
+
variance: {
|
|
119
|
+
name: string;
|
|
120
|
+
description: string;
|
|
121
|
+
useCase: string;
|
|
122
|
+
};
|
|
123
|
+
reliability: {
|
|
124
|
+
name: string;
|
|
125
|
+
description: string;
|
|
126
|
+
useCase: string;
|
|
127
|
+
};
|
|
128
|
+
};
|
|
129
|
+
recommendations: Record<string, string>;
|
|
130
|
+
defaults: {
|
|
131
|
+
kValue: number;
|
|
132
|
+
reliabilityThreshold: number;
|
|
133
|
+
varianceThreshold: number;
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Non-Determinism API client for pass@k analysis and reliability measurement
|
|
138
|
+
*/
|
|
139
|
+
export declare const nondeterminism: {
|
|
140
|
+
/**
|
|
141
|
+
* Create a new non-determinism analysis run
|
|
142
|
+
*
|
|
143
|
+
* @example
|
|
144
|
+
* ```typescript
|
|
145
|
+
* const run = await nondeterminism.createRun({
|
|
146
|
+
* agentId: 'agent_123',
|
|
147
|
+
* criterionId: 'criterion_456',
|
|
148
|
+
* kValue: 5,
|
|
149
|
+
* traceIds: ['trace_1', 'trace_2', 'trace_3'],
|
|
150
|
+
* runType: 'pass_at_k',
|
|
151
|
+
* });
|
|
152
|
+
* ```
|
|
153
|
+
*/
|
|
154
|
+
createRun(options: CreateRunOptions): Promise<NondeterminismRun>;
|
|
155
|
+
/**
|
|
156
|
+
* Get non-determinism runs
|
|
157
|
+
*
|
|
158
|
+
* @example
|
|
159
|
+
* ```typescript
|
|
160
|
+
* const runs = await nondeterminism.getRuns({ agentId: 'agent_123' });
|
|
161
|
+
* ```
|
|
162
|
+
*/
|
|
163
|
+
getRuns(options?: ListRunsOptions): Promise<NondeterminismRun[]>;
|
|
164
|
+
/**
|
|
165
|
+
* Get a specific run
|
|
166
|
+
*
|
|
167
|
+
* @example
|
|
168
|
+
* ```typescript
|
|
169
|
+
* const run = await nondeterminism.getRun('run_123');
|
|
170
|
+
* ```
|
|
171
|
+
*/
|
|
172
|
+
getRun(runId: string): Promise<NondeterminismRun>;
|
|
173
|
+
/**
|
|
174
|
+
* Start a run
|
|
175
|
+
*
|
|
176
|
+
* @example
|
|
177
|
+
* ```typescript
|
|
178
|
+
* await nondeterminism.startRun('run_123');
|
|
179
|
+
* ```
|
|
180
|
+
*/
|
|
181
|
+
startRun(runId: string): Promise<void>;
|
|
182
|
+
/**
|
|
183
|
+
* Complete a run
|
|
184
|
+
*
|
|
185
|
+
* @example
|
|
186
|
+
* ```typescript
|
|
187
|
+
* await nondeterminism.completeRun('run_123');
|
|
188
|
+
* ```
|
|
189
|
+
*/
|
|
190
|
+
completeRun(runId: string): Promise<void>;
|
|
191
|
+
/**
|
|
192
|
+
* Record a sample result
|
|
193
|
+
*
|
|
194
|
+
* @example
|
|
195
|
+
* ```typescript
|
|
196
|
+
* const sample = await nondeterminism.recordSample({
|
|
197
|
+
* runId: 'run_123',
|
|
198
|
+
* traceId: 'trace_456',
|
|
199
|
+
* criterionId: 'criterion_789',
|
|
200
|
+
* sampleIndex: 0,
|
|
201
|
+
* score: 85,
|
|
202
|
+
* passed: true,
|
|
203
|
+
* reasoning: 'Response meets quality criteria',
|
|
204
|
+
* });
|
|
205
|
+
* ```
|
|
206
|
+
*/
|
|
207
|
+
recordSample(options: RecordSampleOptions): Promise<NondeterminismSample>;
|
|
208
|
+
/**
|
|
209
|
+
* Get samples for a run
|
|
210
|
+
*
|
|
211
|
+
* @example
|
|
212
|
+
* ```typescript
|
|
213
|
+
* const samples = await nondeterminism.getSamples('run_123');
|
|
214
|
+
* ```
|
|
215
|
+
*/
|
|
216
|
+
getSamples(runId: string): Promise<NondeterminismSample[]>;
|
|
217
|
+
/**
|
|
218
|
+
* Get run summary with analysis
|
|
219
|
+
*
|
|
220
|
+
* @example
|
|
221
|
+
* ```typescript
|
|
222
|
+
* const summary = await nondeterminism.getRunSummary('run_123');
|
|
223
|
+
* console.log(`Pass@k rate: ${summary.criterionAnalyses[0].passAtKRate}`);
|
|
224
|
+
* ```
|
|
225
|
+
*/
|
|
226
|
+
getRunSummary(runId: string): Promise<RunSummary>;
|
|
227
|
+
/**
|
|
228
|
+
* Trigger analysis of a completed run
|
|
229
|
+
*
|
|
230
|
+
* @example
|
|
231
|
+
* ```typescript
|
|
232
|
+
* const summary = await nondeterminism.analyzeRun('run_123');
|
|
233
|
+
* ```
|
|
234
|
+
*/
|
|
235
|
+
analyzeRun(runId: string): Promise<RunSummary>;
|
|
236
|
+
/**
|
|
237
|
+
* Get information about pass@k analysis
|
|
238
|
+
*
|
|
239
|
+
* @example
|
|
240
|
+
* ```typescript
|
|
241
|
+
* const info = await nondeterminism.getInfo();
|
|
242
|
+
* console.log(info.concepts.passAtK.description);
|
|
243
|
+
* ```
|
|
244
|
+
*/
|
|
245
|
+
getInfo(): Promise<PassAtKInfo>;
|
|
246
|
+
};
|
|
247
|
+
/**
|
|
248
|
+
* Calculate pass@k probability from pass rate
|
|
249
|
+
*
|
|
250
|
+
* @param passRate - Single-run pass rate (0-1)
|
|
251
|
+
* @param k - Number of runs
|
|
252
|
+
* @returns Probability that at least 1 of k runs passes
|
|
253
|
+
*
|
|
254
|
+
* @example
|
|
255
|
+
* ```typescript
|
|
256
|
+
* const passAtK = calculatePassAtK(0.7, 3); // ~0.973
|
|
257
|
+
* ```
|
|
258
|
+
*/
|
|
259
|
+
export declare function calculatePassAtK(passRate: number, k: number): number;
|
|
260
|
+
/**
|
|
261
|
+
* Calculate pass^k probability from pass rate
|
|
262
|
+
*
|
|
263
|
+
* @param passRate - Single-run pass rate (0-1)
|
|
264
|
+
* @param k - Number of runs
|
|
265
|
+
* @returns Probability that all k runs pass
|
|
266
|
+
*
|
|
267
|
+
* @example
|
|
268
|
+
* ```typescript
|
|
269
|
+
* const passToK = calculatePassToK(0.7, 3); // ~0.343
|
|
270
|
+
* ```
|
|
271
|
+
*/
|
|
272
|
+
export declare function calculatePassToK(passRate: number, k: number): number;
|
|
273
|
+
/**
|
|
274
|
+
* Calculate required pass rate to achieve target pass@k
|
|
275
|
+
*
|
|
276
|
+
* @param targetPassAtK - Desired pass@k probability
|
|
277
|
+
* @param k - Number of runs
|
|
278
|
+
* @returns Required single-run pass rate
|
|
279
|
+
*
|
|
280
|
+
* @example
|
|
281
|
+
* ```typescript
|
|
282
|
+
* const requiredRate = requiredPassRateForPassAtK(0.95, 3); // ~0.632
|
|
283
|
+
* ```
|
|
284
|
+
*/
|
|
285
|
+
export declare function requiredPassRateForPassAtK(targetPassAtK: number, k: number): number;
|
|
286
|
+
/**
|
|
287
|
+
* Determine if evaluation is reliable based on analysis
|
|
288
|
+
*
|
|
289
|
+
* @param analysis - Criterion analysis result
|
|
290
|
+
* @param reliabilityThreshold - Minimum reliability score (default 0.8)
|
|
291
|
+
* @returns Whether the evaluation is considered reliable
|
|
292
|
+
*/
|
|
293
|
+
export declare function isReliableEvaluation(analysis: CriterionAnalysis, reliabilityThreshold?: number): boolean;
|
|
294
|
+
/**
|
|
295
|
+
* Get recommendation based on reliability analysis
|
|
296
|
+
*
|
|
297
|
+
* @param analysis - Criterion analysis result
|
|
298
|
+
* @returns Actionable recommendation string
|
|
299
|
+
*/
|
|
300
|
+
export declare function getReliabilityRecommendation(analysis: CriterionAnalysis): string;
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ThinkHive SDK v3.0 - Non-Determinism API
|
|
4
|
+
*
|
|
5
|
+
* API for pass@k / pass^k analysis to measure LLM evaluation reliability
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.nondeterminism = void 0;
|
|
9
|
+
exports.calculatePassAtK = calculatePassAtK;
|
|
10
|
+
exports.calculatePassToK = calculatePassToK;
|
|
11
|
+
exports.requiredPassRateForPassAtK = requiredPassRateForPassAtK;
|
|
12
|
+
exports.isReliableEvaluation = isReliableEvaluation;
|
|
13
|
+
exports.getReliabilityRecommendation = getReliabilityRecommendation;
|
|
14
|
+
const client_1 = require("../core/client");
|
|
15
|
+
// ============================================================================
|
|
16
|
+
// NON-DETERMINISM API CLIENT
|
|
17
|
+
// ============================================================================
|
|
18
|
+
/**
|
|
19
|
+
* Non-Determinism API client for pass@k analysis and reliability measurement
|
|
20
|
+
*/
|
|
21
|
+
exports.nondeterminism = {
|
|
22
|
+
/**
|
|
23
|
+
* Create a new non-determinism analysis run
|
|
24
|
+
*
|
|
25
|
+
* @example
|
|
26
|
+
* ```typescript
|
|
27
|
+
* const run = await nondeterminism.createRun({
|
|
28
|
+
* agentId: 'agent_123',
|
|
29
|
+
* criterionId: 'criterion_456',
|
|
30
|
+
* kValue: 5,
|
|
31
|
+
* traceIds: ['trace_1', 'trace_2', 'trace_3'],
|
|
32
|
+
* runType: 'pass_at_k',
|
|
33
|
+
* });
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
async createRun(options) {
|
|
37
|
+
return (0, client_1.apiRequestWithData)('/nondeterminism/runs', {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
body: options,
|
|
40
|
+
apiVersion: 'v1',
|
|
41
|
+
});
|
|
42
|
+
},
|
|
43
|
+
/**
|
|
44
|
+
* Get non-determinism runs
|
|
45
|
+
*
|
|
46
|
+
* @example
|
|
47
|
+
* ```typescript
|
|
48
|
+
* const runs = await nondeterminism.getRuns({ agentId: 'agent_123' });
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
async getRuns(options = {}) {
|
|
52
|
+
const params = new URLSearchParams();
|
|
53
|
+
if (options.agentId)
|
|
54
|
+
params.set('agentId', options.agentId);
|
|
55
|
+
if (options.status)
|
|
56
|
+
params.set('status', options.status);
|
|
57
|
+
if (options.limit)
|
|
58
|
+
params.set('limit', String(options.limit));
|
|
59
|
+
if (options.offset)
|
|
60
|
+
params.set('offset', String(options.offset));
|
|
61
|
+
return (0, client_1.apiRequestWithData)(`/nondeterminism/runs?${params.toString()}`, { apiVersion: 'v1' });
|
|
62
|
+
},
|
|
63
|
+
/**
|
|
64
|
+
* Get a specific run
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```typescript
|
|
68
|
+
* const run = await nondeterminism.getRun('run_123');
|
|
69
|
+
* ```
|
|
70
|
+
*/
|
|
71
|
+
async getRun(runId) {
|
|
72
|
+
return (0, client_1.apiRequestWithData)(`/nondeterminism/runs/${runId}`, { apiVersion: 'v1' });
|
|
73
|
+
},
|
|
74
|
+
/**
|
|
75
|
+
* Start a run
|
|
76
|
+
*
|
|
77
|
+
* @example
|
|
78
|
+
* ```typescript
|
|
79
|
+
* await nondeterminism.startRun('run_123');
|
|
80
|
+
* ```
|
|
81
|
+
*/
|
|
82
|
+
async startRun(runId) {
|
|
83
|
+
await (0, client_1.apiRequest)(`/nondeterminism/runs/${runId}/start`, {
|
|
84
|
+
method: 'POST',
|
|
85
|
+
apiVersion: 'v1',
|
|
86
|
+
});
|
|
87
|
+
},
|
|
88
|
+
/**
|
|
89
|
+
* Complete a run
|
|
90
|
+
*
|
|
91
|
+
* @example
|
|
92
|
+
* ```typescript
|
|
93
|
+
* await nondeterminism.completeRun('run_123');
|
|
94
|
+
* ```
|
|
95
|
+
*/
|
|
96
|
+
async completeRun(runId) {
|
|
97
|
+
await (0, client_1.apiRequest)(`/nondeterminism/runs/${runId}/complete`, {
|
|
98
|
+
method: 'POST',
|
|
99
|
+
apiVersion: 'v1',
|
|
100
|
+
});
|
|
101
|
+
},
|
|
102
|
+
/**
|
|
103
|
+
* Record a sample result
|
|
104
|
+
*
|
|
105
|
+
* @example
|
|
106
|
+
* ```typescript
|
|
107
|
+
* const sample = await nondeterminism.recordSample({
|
|
108
|
+
* runId: 'run_123',
|
|
109
|
+
* traceId: 'trace_456',
|
|
110
|
+
* criterionId: 'criterion_789',
|
|
111
|
+
* sampleIndex: 0,
|
|
112
|
+
* score: 85,
|
|
113
|
+
* passed: true,
|
|
114
|
+
* reasoning: 'Response meets quality criteria',
|
|
115
|
+
* });
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
async recordSample(options) {
|
|
119
|
+
return (0, client_1.apiRequestWithData)('/nondeterminism/samples', {
|
|
120
|
+
method: 'POST',
|
|
121
|
+
body: options,
|
|
122
|
+
apiVersion: 'v1',
|
|
123
|
+
});
|
|
124
|
+
},
|
|
125
|
+
/**
|
|
126
|
+
* Get samples for a run
|
|
127
|
+
*
|
|
128
|
+
* @example
|
|
129
|
+
* ```typescript
|
|
130
|
+
* const samples = await nondeterminism.getSamples('run_123');
|
|
131
|
+
* ```
|
|
132
|
+
*/
|
|
133
|
+
async getSamples(runId) {
|
|
134
|
+
return (0, client_1.apiRequestWithData)(`/nondeterminism/runs/${runId}/samples`, { apiVersion: 'v1' });
|
|
135
|
+
},
|
|
136
|
+
/**
|
|
137
|
+
* Get run summary with analysis
|
|
138
|
+
*
|
|
139
|
+
* @example
|
|
140
|
+
* ```typescript
|
|
141
|
+
* const summary = await nondeterminism.getRunSummary('run_123');
|
|
142
|
+
* console.log(`Pass@k rate: ${summary.criterionAnalyses[0].passAtKRate}`);
|
|
143
|
+
* ```
|
|
144
|
+
*/
|
|
145
|
+
async getRunSummary(runId) {
|
|
146
|
+
return (0, client_1.apiRequestWithData)(`/nondeterminism/runs/${runId}/summary`, { apiVersion: 'v1' });
|
|
147
|
+
},
|
|
148
|
+
/**
|
|
149
|
+
* Trigger analysis of a completed run
|
|
150
|
+
*
|
|
151
|
+
* @example
|
|
152
|
+
* ```typescript
|
|
153
|
+
* const summary = await nondeterminism.analyzeRun('run_123');
|
|
154
|
+
* ```
|
|
155
|
+
*/
|
|
156
|
+
async analyzeRun(runId) {
|
|
157
|
+
return (0, client_1.apiRequestWithData)(`/nondeterminism/runs/${runId}/analyze`, { method: 'POST', apiVersion: 'v1' });
|
|
158
|
+
},
|
|
159
|
+
/**
|
|
160
|
+
* Get information about pass@k analysis
|
|
161
|
+
*
|
|
162
|
+
* @example
|
|
163
|
+
* ```typescript
|
|
164
|
+
* const info = await nondeterminism.getInfo();
|
|
165
|
+
* console.log(info.concepts.passAtK.description);
|
|
166
|
+
* ```
|
|
167
|
+
*/
|
|
168
|
+
async getInfo() {
|
|
169
|
+
return (0, client_1.apiRequestWithData)('/nondeterminism/info', { apiVersion: 'v1' });
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
// ============================================================================
|
|
173
|
+
// HELPER FUNCTIONS
|
|
174
|
+
// ============================================================================
|
|
175
|
+
/**
|
|
176
|
+
* Calculate pass@k probability from pass rate
|
|
177
|
+
*
|
|
178
|
+
* @param passRate - Single-run pass rate (0-1)
|
|
179
|
+
* @param k - Number of runs
|
|
180
|
+
* @returns Probability that at least 1 of k runs passes
|
|
181
|
+
*
|
|
182
|
+
* @example
|
|
183
|
+
* ```typescript
|
|
184
|
+
* const passAtK = calculatePassAtK(0.7, 3); // ~0.973
|
|
185
|
+
* ```
|
|
186
|
+
*/
|
|
187
|
+
function calculatePassAtK(passRate, k) {
|
|
188
|
+
return 1 - Math.pow(1 - passRate, k);
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Calculate pass^k probability from pass rate
|
|
192
|
+
*
|
|
193
|
+
* @param passRate - Single-run pass rate (0-1)
|
|
194
|
+
* @param k - Number of runs
|
|
195
|
+
* @returns Probability that all k runs pass
|
|
196
|
+
*
|
|
197
|
+
* @example
|
|
198
|
+
* ```typescript
|
|
199
|
+
* const passToK = calculatePassToK(0.7, 3); // ~0.343
|
|
200
|
+
* ```
|
|
201
|
+
*/
|
|
202
|
+
function calculatePassToK(passRate, k) {
|
|
203
|
+
return Math.pow(passRate, k);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Calculate required pass rate to achieve target pass@k
|
|
207
|
+
*
|
|
208
|
+
* @param targetPassAtK - Desired pass@k probability
|
|
209
|
+
* @param k - Number of runs
|
|
210
|
+
* @returns Required single-run pass rate
|
|
211
|
+
*
|
|
212
|
+
* @example
|
|
213
|
+
* ```typescript
|
|
214
|
+
* const requiredRate = requiredPassRateForPassAtK(0.95, 3); // ~0.632
|
|
215
|
+
* ```
|
|
216
|
+
*/
|
|
217
|
+
function requiredPassRateForPassAtK(targetPassAtK, k) {
|
|
218
|
+
return 1 - Math.pow(1 - targetPassAtK, 1 / k);
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Determine if evaluation is reliable based on analysis
|
|
222
|
+
*
|
|
223
|
+
* @param analysis - Criterion analysis result
|
|
224
|
+
* @param reliabilityThreshold - Minimum reliability score (default 0.8)
|
|
225
|
+
* @returns Whether the evaluation is considered reliable
|
|
226
|
+
*/
|
|
227
|
+
function isReliableEvaluation(analysis, reliabilityThreshold = 0.8) {
|
|
228
|
+
return analysis.reliabilityScore >= reliabilityThreshold;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Get recommendation based on reliability analysis
|
|
232
|
+
*
|
|
233
|
+
* @param analysis - Criterion analysis result
|
|
234
|
+
* @returns Actionable recommendation string
|
|
235
|
+
*/
|
|
236
|
+
function getReliabilityRecommendation(analysis) {
|
|
237
|
+
if (analysis.reliabilityScore >= 0.9) {
|
|
238
|
+
return 'Evaluation is highly reliable. No changes needed.';
|
|
239
|
+
}
|
|
240
|
+
else if (analysis.reliabilityScore >= 0.8) {
|
|
241
|
+
return 'Evaluation is reliable. Consider minor criteria refinements.';
|
|
242
|
+
}
|
|
243
|
+
else if (analysis.reliabilityScore >= 0.6) {
|
|
244
|
+
return 'Evaluation has moderate reliability. Add more specific criteria or examples.';
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
return 'Evaluation is unreliable. Consider using deterministic checks or restructuring criteria.';
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9uZGV0ZXJtaW5pc20uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvYXBpL25vbmRldGVybWluaXNtLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7OztHQUlHOzs7QUFxVUgsNENBRUM7QUFjRCw0Q0FFQztBQWNELGdFQUVDO0FBU0Qsb0RBS0M7QUFRRCxvRUFVQztBQXJZRCwyQ0FBZ0U7QUE0SGhFLCtFQUErRTtBQUMvRSw2QkFBNkI7QUFDN0IsK0VBQStFO0FBRS9FOztHQUVHO0FBQ1UsUUFBQSxjQUFjLEdBQUc7SUFDNUI7Ozs7Ozs7Ozs7Ozs7T0FhRztJQUNILEtBQUssQ0FBQyxTQUFTLENBQUMsT0FBeUI7UUFDdkMsT0FBTyxJQUFBLDJCQUFrQixFQUFvQixzQkFBc0IsRUFBRTtZQUNuRSxNQUFNLEVBQUUsTUFBTTtZQUNkLElBQUksRUFBRSxPQUFPO1lBQ2IsVUFBVSxFQUFFLElBQUk7U0FDakIsQ0FBQyxDQUFDO0lBQ0wsQ0FBQztJQUVEOzs7Ozs7O09BT0c7SUFDSCxLQUFLLENBQUMsT0FBTyxDQUFDLFVBQTJCLEVBQUU7UUFDekMsTUFBTSxNQUFNLEdBQUcsSUFBSSxlQUFlLEVBQUUsQ0FBQztRQUNyQyxJQUFJLE9BQU8sQ0FBQyxPQUFPO1lBQUUsTUFBTSxDQUFDLEdBQUcsQ0FBQyxTQUFTLEVBQUUsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQzVELElBQUksT0FBTyxDQUFDLE1BQU07WUFBRSxNQUFNLENBQUMsR0FBRyxDQUFDLFFBQVEsRUFBRSxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDekQsSUFBSSxPQUFPLENBQUMsS0FBSztZQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQztRQUM5RCxJQUFJLE9BQU8sQ0FBQyxNQUFNO1lBQUUsTUFBTSxDQUFDLEdBQUcsQ0FBQyxRQUFRLEVBQUUsTUFBTSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDO1FBRWpFLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsd0JBQXdCLE1BQU0sQ0FBQyxRQUFRLEVBQUUsRUFBRSxFQUMzQyxFQUFFLFVBQVUsRUFBRSxJQUFJLEVBQUUsQ0FDckIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLE1BQU0sQ0FBQyxLQUFhO1FBQ3hCLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsd0JBQXdCLEtBQUssRUFBRSxFQUMvQixFQUFFLFVBQVUsRUFBRSxJQUFJLEVBQUUsQ0FDckIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLFFBQVEsQ0FBQyxLQUFhO1FBQzFCLE1BQU0sSUFBQSxtQkFBVSxFQUFDLHdCQUF3QixLQUFLLFFBQVEsRUFBRTtZQUN0RCxNQUFNLEVBQUUsTUFBTTtZQUNkLFVBQVUsRUFBRSxJQUFJO1NBQ2pCLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLFdBQVcsQ0FBQyxLQUFhO1FBQzdCLE1BQU0sSUFBQSxtQkFBVSxFQUFDLHdCQUF3QixLQUFLLFdBQVcsRUFBRTtZQUN6RCxNQUFNLEVBQUUsTUFBTTtZQUNkLFVBQVUsRUFBRSxJQUFJO1NBQ2pCLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7Ozs7Ozs7Ozs7O09BZUc7SUFDSCxLQUFLLENBQUMsWUFBWSxDQUFDLE9BQTRCO1FBQzdDLE9BQU8sSUFBQSwyQkFBa0IsRUFBdUIseUJBQXlCLEVBQUU7WUFDekUsTUFBTSxFQUFFLE1BQU07WUFDZCxJQUFJLEVBQUUsT0FBTztZQUNiLFVBQVUsRUFBRSxJQUFJO1NBQ2pCLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0gsS0FBSyxDQUFDLFVBQVUsQ0FBQyxLQUFhO1FBQzVCLE9BQU8sSUFBQSwyQkFBa0IsRUFDdkIsd0JBQXdCLEtBQUssVUFBVSxFQUN2QyxFQUFFLFVBQVUsRUFBRSxJQUFJLEVBQUUsQ0FDckIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7Ozs7T0FRRztJQUNILEtBQUssQ0FBQyxhQUFhLENBQUMsS0FBYTtRQUMvQixPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLHdCQUF3QixLQUFLLFVBQVUsRUFDdkMsRUFBRSxVQUFVLEVBQUUsSUFBSSxFQUFFLENBQ3JCLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILEtBQUssQ0FBQyxVQUFVLENBQUMsS0FBYTtRQUM1QixPQUFPLElBQUEsMkJBQWtCLEVBQ3ZCLHdCQUF3QixLQUFLLFVBQVUsRUFDdkMsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLFVBQVUsRUFBRSxJQUFJLEVBQUUsQ0FDckMsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7Ozs7T0FRRztJQUNILEtBQUssQ0FBQyxPQUFPO1FBQ1gsT0FBTyxJQUFBLDJCQUFrQixFQUN2QixzQkFBc0IsRUFDdEIsRUFBRSxVQUFVLEVBQUUsSUFBSSxFQUFFLENBQ3JCLENBQUM7SUFDSixDQUFDO0NBQ0YsQ0FBQztBQUVGLCtFQUErRTtBQUMvRSxtQkFBbUI7QUFDbkIsK0VBQStFO0FBRS9FOzs7Ozs7Ozs7OztHQVdHO0FBQ0gsU0FBZ0IsZ0JBQWdCLENBQUMsUUFBZ0IsRUFBRSxDQUFTO0lBQzFELE9BQU8sQ0FBQyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQyxHQUFHLFFBQVEsRUFBRSxDQUFDLENBQUMsQ0FBQztBQUN2QyxDQUFDO0FBRUQ7Ozs7Ozs7Ozs7O0dBV0c7QUFDSCxTQUFnQixnQkFBZ0IsQ0FBQyxRQUFnQixFQUFFLENBQVM7SUFDMUQsT0FBTyxJQUFJLENBQUMsR0FBRyxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUMsQ0FBQztBQUMvQixDQUFDO0FBRUQ7Ozs7Ozs7Ozs7O0dBV0c7QUFDSCxTQUFnQiwwQkFBMEIsQ0FBQyxhQUFxQixFQUFFLENBQVM7SUFDekUsT0FBTyxDQUFDLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLEdBQUcsYUFBYSxFQUFFLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztBQUNoRCxDQUFDO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsU0FBZ0Isb0JBQW9CLENBQ2xDLFFBQTJCLEVBQzNCLG9CQUFvQixHQUFHLEdBQUc7SUFFMUIsT0FBTyxRQUFRLENBQUMsZ0JBQWdCLElBQUksb0JBQW9CLENBQUM7QUFDM0QsQ0FBQztBQUVEOzs7OztHQUtHO0FBQ0gsU0FBZ0IsNEJBQTRCLENBQUMsUUFBMkI7SUFDdEUsSUFBSSxRQUFRLENBQUMsZ0JBQWdCLElBQUksR0FBRyxFQUFFLENBQUM7UUFDckMsT0FBTyxtREFBbUQsQ0FBQztJQUM3RCxDQUFDO1NBQU0sSUFBSSxRQUFRLENBQUMsZ0JBQWdCLElBQUksR0FBRyxFQUFFLENBQUM7UUFDNUMsT0FBTyw4REFBOEQsQ0FBQztJQUN4RSxDQUFDO1NBQU0sSUFBSSxRQUFRLENBQUMsZ0JBQWdCLElBQUksR0FBRyxFQUFFLENBQUM7UUFDNUMsT0FBTyw4RUFBOEUsQ0FBQztJQUN4RixDQUFDO1NBQU0sQ0FBQztRQUNOLE9BQU8sMEZBQTBGLENBQUM7SUFDcEcsQ0FBQztBQUNILENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFRoaW5rSGl2ZSBTREsgdjMuMCAtIE5vbi1EZXRlcm1pbmlzbSBBUElcbiAqXG4gKiBBUEkgZm9yIHBhc3NAayAvIHBhc3NeayBhbmFseXNpcyB0byBtZWFzdXJlIExMTSBldmFsdWF0aW9uIHJlbGlhYmlsaXR5XG4gKi9cblxuaW1wb3J0IHsgYXBpUmVxdWVzdCwgYXBpUmVxdWVzdFdpdGhEYXRhIH0gZnJvbSAnLi4vY29yZS9jbGllbnQnO1xuXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG4vLyBUWVBFU1xuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuXG5leHBvcnQgdHlwZSBOb25kZXRlcm1pbmlzbVJ1blR5cGUgPSAncGFzc19hdF9rJyB8ICdwYXNzX3RvX2snIHwgJ3ZhcmlhbmNlJyB8ICdyZWxpYWJpbGl0eSc7XG5leHBvcnQgdHlwZSBOb25kZXRlcm1pbmlzbVJ1blN0YXR1cyA9ICdwZW5kaW5nJyB8ICdydW5uaW5nJyB8ICdjb21wbGV0ZWQnIHwgJ2ZhaWxlZCcgfCAnY2FuY2VsbGVkJztcblxuZXhwb3J0IGludGVyZmFjZSBOb25kZXRlcm1pbmlzbVJ1biB7XG4gIGlkOiBzdHJpbmc7XG4gIGNvbXBhbnlJZDogc3RyaW5nO1xuICBhZ2VudElkOiBzdHJpbmc7XG4gIHJ1blR5cGU6IE5vbmRldGVybWluaXNtUnVuVHlwZTtcbiAga1ZhbHVlOiBudW1iZXI7XG4gIHN0YXR1czogTm9uZGV0ZXJtaW5pc21SdW5TdGF0dXM7XG4gIHRyYWNlQ291bnQ6IG51bWJlcjtcbiAgY3JpdGVyaW9uSWQ/OiBzdHJpbmc7XG4gIGNyaXRlcmlhSWRzOiBzdHJpbmdbXTtcbiAgdGVtcGVyYXR1cmU/OiBzdHJpbmc7XG4gIG1vZGVsPzogc3RyaW5nO1xuICBwcm9ncmVzc1BlcmNlbnQ6IG51bWJlcjtcbiAgcGFzc0F0S1JhdGU/OiBzdHJpbmc7XG4gIHBhc3NUb0tSYXRlPzogc3RyaW5nO1xuICBhdmdWYXJpYW5jZT86IHN0cmluZztcbiAgcmVsaWFiaWxpdHlTY29yZT86IHN0cmluZztcbiAgc3RhcnRlZEF0Pzogc3RyaW5nO1xuICBjb21wbGV0ZWRBdD86IHN0cmluZztcbiAgY3JlYXRlZEJ5Pzogc3RyaW5nO1xuICBjcmVhdGVkQXQ6IHN0cmluZztcbn1cblxuZXhwb3J0IGludGVyZmFjZSBOb25kZXRlcm1pbmlzbVNhbXBsZSB7XG4gIGlkOiBzdHJpbmc7XG4gIHJ1bklkOiBzdHJpbmc7XG4gIHRyYWNlSWQ6IHN0cmluZztcbiAgY3JpdGVyaW9uSWQ6IHN0cmluZztcbiAgc2FtcGxlSW5kZXg6IG51bWJlcjtcbiAgc2NvcmU6IHN0cmluZztcbiAgcGFzc2VkOiBib29sZWFuO1xuICByZWFzb25pbmc/OiBzdHJpbmc7XG4gIGNvbmZpZGVuY2U/OiBzdHJpbmc7XG4gIHRva2Vuc1VzZWQ/OiBudW1iZXI7XG4gIGNvc3RVc2Q/OiBzdHJpbmc7XG4gIG1vZGVsPzogc3RyaW5nO1xuICB0ZW1wZXJhdHVyZT86IHN0cmluZztcbiAgbGF0ZW5jeU1zPzogbnVtYmVyO1xuICBlcnJvcj86IHN0cmluZztcbiAgY3JlYXRlZEF0OiBzdHJpbmc7XG59XG5cbmV4cG9ydCBpbnRlcmZhY2UgQ3JlYXRlUnVuT3B0aW9ucyB7XG4gIGFnZW50SWQ6IHN0cmluZztcbiAgY3JpdGVyaW9uSWQ/OiBzdHJpbmc7XG4gIGNyaXRlcmlhSWRzPzogc3RyaW5nW107XG4gIGtWYWx1ZTogbnVtYmVyO1xuICB0cmFjZUlkczogc3RyaW5nW107XG4gIHJ1blR5cGU/OiBOb25kZXRlcm1pbmlzbVJ1blR5cGU7XG4gIHRlbXBlcmF0dXJlPzogbnVtYmVyO1xuICBtb2RlbD86IHN0cmluZztcbn1cblxuZXhwb3J0IGludGVyZmFjZSBSZWNvcmRTYW1wbGVPcHRpb25zIHtcbiAgcnVuSWQ6IHN0cmluZztcbiAgdHJhY2VJZDogc3RyaW5nO1xuICBjcml0ZXJpb25JZDogc3RyaW5nO1xuICBzYW1wbGVJbmRleDogbnVtYmVyO1xuICBzY29yZTogbnVtYmVyO1xuICBwYXNzZWQ6IGJvb2xlYW47XG4gIHJlYXNvbmluZz86IHN0cmluZztcbiAgY29uZmlkZW5jZT86IG51bWJlcjtcbiAgdG9rZW5zVXNlZD86IG51bWJlcjtcbiAgY29zdFVzZD86IG51bWJlcjtcbiAgbW9kZWw/OiBzdHJpbmc7XG4gIHRlbXBlcmF0dXJlPzogbnVtYmVyO1xuICBsYXRlbmN5TXM/OiBudW1iZXI7XG4gIGVycm9yPzogc3RyaW5nO1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIFRyYWNlQW5hbHlzaXMge1xuICB0cmFjZUlkOiBzdHJpbmc7XG4gIHNhbXBsZXM6IE5vbmRldGVybWluaXNtU2FtcGxlW107XG4gIHBhc3NDb3VudDogbnVtYmVyO1xuICB0b3RhbENvdW50OiBudW1iZXI7XG4gIHBhc3NSYXRlOiBudW1iZXI7XG4gIHNjb3JlVmFyaWFuY2U6IG51bWJlcjtcbiAgbWVhblNjb3JlOiBudW1iZXI7XG4gIGlzQ29uc2lzdGVudDogYm9vbGVhbjtcbn1cblxuZXhwb3J0IGludGVyZmFjZSBDcml0ZXJpb25BbmFseXNpcyB7XG4gIGNyaXRlcmlvbklkOiBzdHJpbmc7XG4gIHRyYWNlQW5hbHlzZXM6IFRyYWNlQW5hbHlzaXNbXTtcbiAgcGFzc0F0S1JhdGU6IG51bWJlcjtcbiAgcGFzc1RvS1JhdGU6IG51bWJlcjtcbiAgcmVsaWFiaWxpdHlTY29yZTogbnVtYmVyO1xuICBpc1JlbGlhYmxlOiBib29sZWFuO1xuICByZWNvbW1lbmRhdGlvbjogc3RyaW5nO1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIFJ1blN1bW1hcnkge1xuICBydW46IE5vbmRldGVybWluaXNtUnVuO1xuICB0cmFjZUFuYWx5c2VzOiBUcmFjZUFuYWx5c2lzW107XG4gIGNyaXRlcmlvbkFuYWx5c2VzOiBDcml0ZXJpb25BbmFseXNpc1tdO1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIExpc3RSdW5zT3B0aW9ucyB7XG4gIGFnZW50SWQ/OiBzdHJpbmc7XG4gIHN0YXR1cz86IE5vbmRldGVybWluaXNtUnVuU3RhdHVzO1xuICBsaW1pdD86IG51bWJlcjtcbiAgb2Zmc2V0PzogbnVtYmVyO1xufVxuXG5leHBvcnQgaW50ZXJmYWNlIFBhc3NBdEtJbmZvIHtcbiAgY29uY2VwdHM6IHtcbiAgICBwYXNzQXRLOiB7IG5hbWU6IHN0cmluZzsgZGVzY3JpcHRpb246IHN0cmluZzsgZm9ybXVsYTogc3RyaW5nOyB1c2VDYXNlOiBzdHJpbmcgfTtcbiAgICBwYXNzVG9LOiB7IG5hbWU6IHN0cmluZzsgZGVzY3JpcHRpb246IHN0cmluZzsgZm9ybXVsYTogc3RyaW5nOyB1c2VDYXNlOiBzdHJpbmcgfTtcbiAgICB2YXJpYW5jZTogeyBuYW1lOiBzdHJpbmc7IGRlc2NyaXB0aW9uOiBzdHJpbmc7IHVzZUNhc2U6IHN0cmluZyB9O1xuICAgIHJlbGlhYmlsaXR5OiB7IG5hbWU6IHN0cmluZzsgZGVzY3JpcHRpb246IHN0cmluZzsgdXNlQ2FzZTogc3RyaW5nIH07XG4gIH07XG4gIHJlY29tbWVuZGF0aW9uczogUmVjb3JkPHN0cmluZywgc3RyaW5nPjtcbiAgZGVmYXVsdHM6IHsga1ZhbHVlOiBudW1iZXI7IHJlbGlhYmlsaXR5VGhyZXNob2xkOiBudW1iZXI7IHZhcmlhbmNlVGhyZXNob2xkOiBudW1iZXIgfTtcbn1cblxuLy8gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuLy8gTk9OLURFVEVSTUlOSVNNIEFQSSBDTElFTlRcbi8vID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT1cblxuLyoqXG4gKiBOb24tRGV0ZXJtaW5pc20gQVBJIGNsaWVudCBmb3IgcGFzc0BrIGFuYWx5c2lzIGFuZCByZWxpYWJpbGl0eSBtZWFzdXJlbWVudFxuICovXG5leHBvcnQgY29uc3Qgbm9uZGV0ZXJtaW5pc20gPSB7XG4gIC8qKlxuICAgKiBDcmVhdGUgYSBuZXcgbm9uLWRldGVybWluaXNtIGFuYWx5c2lzIHJ1blxuICAgKlxuICAgKiBAZXhhbXBsZVxuICAgKiBgYGB0eXBlc2NyaXB0XG4gICAqIGNvbnN0IHJ1biA9IGF3YWl0IG5vbmRldGVybWluaXNtLmNyZWF0ZVJ1bih7XG4gICAqICAgYWdlbnRJZDogJ2FnZW50XzEyMycsXG4gICAqICAgY3JpdGVyaW9uSWQ6ICdjcml0ZXJpb25fNDU2JyxcbiAgICogICBrVmFsdWU6IDUsXG4gICAqICAgdHJhY2VJZHM6IFsndHJhY2VfMScsICd0cmFjZV8yJywgJ3RyYWNlXzMnXSxcbiAgICogICBydW5UeXBlOiAncGFzc19hdF9rJyxcbiAgICogfSk7XG4gICAqIGBgYFxuICAgKi9cbiAgYXN5bmMgY3JlYXRlUnVuKG9wdGlvbnM6IENyZWF0ZVJ1bk9wdGlvbnMpOiBQcm9taXNlPE5vbmRldGVybWluaXNtUnVuPiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxOb25kZXRlcm1pbmlzbVJ1bj4oJy9ub25kZXRlcm1pbmlzbS9ydW5zJywge1xuICAgICAgbWV0aG9kOiAnUE9TVCcsXG4gICAgICBib2R5OiBvcHRpb25zLFxuICAgICAgYXBpVmVyc2lvbjogJ3YxJyxcbiAgICB9KTtcbiAgfSxcblxuICAvKipcbiAgICogR2V0IG5vbi1kZXRlcm1pbmlzbSBydW5zXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgcnVucyA9IGF3YWl0IG5vbmRldGVybWluaXNtLmdldFJ1bnMoeyBhZ2VudElkOiAnYWdlbnRfMTIzJyB9KTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRSdW5zKG9wdGlvbnM6IExpc3RSdW5zT3B0aW9ucyA9IHt9KTogUHJvbWlzZTxOb25kZXRlcm1pbmlzbVJ1bltdPiB7XG4gICAgY29uc3QgcGFyYW1zID0gbmV3IFVSTFNlYXJjaFBhcmFtcygpO1xuICAgIGlmIChvcHRpb25zLmFnZW50SWQpIHBhcmFtcy5zZXQoJ2FnZW50SWQnLCBvcHRpb25zLmFnZW50SWQpO1xuICAgIGlmIChvcHRpb25zLnN0YXR1cykgcGFyYW1zLnNldCgnc3RhdHVzJywgb3B0aW9ucy5zdGF0dXMpO1xuICAgIGlmIChvcHRpb25zLmxpbWl0KSBwYXJhbXMuc2V0KCdsaW1pdCcsIFN0cmluZyhvcHRpb25zLmxpbWl0KSk7XG4gICAgaWYgKG9wdGlvbnMub2Zmc2V0KSBwYXJhbXMuc2V0KCdvZmZzZXQnLCBTdHJpbmcob3B0aW9ucy5vZmZzZXQpKTtcblxuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8Tm9uZGV0ZXJtaW5pc21SdW5bXT4oXG4gICAgICBgL25vbmRldGVybWluaXNtL3J1bnM/JHtwYXJhbXMudG9TdHJpbmcoKX1gLFxuICAgICAgeyBhcGlWZXJzaW9uOiAndjEnIH1cbiAgICApO1xuICB9LFxuXG4gIC8qKlxuICAgKiBHZXQgYSBzcGVjaWZpYyBydW5cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBydW4gPSBhd2FpdCBub25kZXRlcm1pbmlzbS5nZXRSdW4oJ3J1bl8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRSdW4ocnVuSWQ6IHN0cmluZyk6IFByb21pc2U8Tm9uZGV0ZXJtaW5pc21SdW4+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPE5vbmRldGVybWluaXNtUnVuPihcbiAgICAgIGAvbm9uZGV0ZXJtaW5pc20vcnVucy8ke3J1bklkfWAsXG4gICAgICB7IGFwaVZlcnNpb246ICd2MScgfVxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIFN0YXJ0IGEgcnVuXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogYXdhaXQgbm9uZGV0ZXJtaW5pc20uc3RhcnRSdW4oJ3J1bl8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBzdGFydFJ1bihydW5JZDogc3RyaW5nKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgYXdhaXQgYXBpUmVxdWVzdChgL25vbmRldGVybWluaXNtL3J1bnMvJHtydW5JZH0vc3RhcnRgLCB7XG4gICAgICBtZXRob2Q6ICdQT1NUJyxcbiAgICAgIGFwaVZlcnNpb246ICd2MScsXG4gICAgfSk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIENvbXBsZXRlIGEgcnVuXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogYXdhaXQgbm9uZGV0ZXJtaW5pc20uY29tcGxldGVSdW4oJ3J1bl8xMjMnKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBjb21wbGV0ZVJ1bihydW5JZDogc3RyaW5nKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgYXdhaXQgYXBpUmVxdWVzdChgL25vbmRldGVybWluaXNtL3J1bnMvJHtydW5JZH0vY29tcGxldGVgLCB7XG4gICAgICBtZXRob2Q6ICdQT1NUJyxcbiAgICAgIGFwaVZlcnNpb246ICd2MScsXG4gICAgfSk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIFJlY29yZCBhIHNhbXBsZSByZXN1bHRcbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzYW1wbGUgPSBhd2FpdCBub25kZXRlcm1pbmlzbS5yZWNvcmRTYW1wbGUoe1xuICAgKiAgIHJ1bklkOiAncnVuXzEyMycsXG4gICAqICAgdHJhY2VJZDogJ3RyYWNlXzQ1NicsXG4gICAqICAgY3JpdGVyaW9uSWQ6ICdjcml0ZXJpb25fNzg5JyxcbiAgICogICBzYW1wbGVJbmRleDogMCxcbiAgICogICBzY29yZTogODUsXG4gICAqICAgcGFzc2VkOiB0cnVlLFxuICAgKiAgIHJlYXNvbmluZzogJ1Jlc3BvbnNlIG1lZXRzIHF1YWxpdHkgY3JpdGVyaWEnLFxuICAgKiB9KTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyByZWNvcmRTYW1wbGUob3B0aW9uczogUmVjb3JkU2FtcGxlT3B0aW9ucyk6IFByb21pc2U8Tm9uZGV0ZXJtaW5pc21TYW1wbGU+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPE5vbmRldGVybWluaXNtU2FtcGxlPignL25vbmRldGVybWluaXNtL3NhbXBsZXMnLCB7XG4gICAgICBtZXRob2Q6ICdQT1NUJyxcbiAgICAgIGJvZHk6IG9wdGlvbnMsXG4gICAgICBhcGlWZXJzaW9uOiAndjEnLFxuICAgIH0pO1xuICB9LFxuXG4gIC8qKlxuICAgKiBHZXQgc2FtcGxlcyBmb3IgYSBydW5cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzYW1wbGVzID0gYXdhaXQgbm9uZGV0ZXJtaW5pc20uZ2V0U2FtcGxlcygncnVuXzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGdldFNhbXBsZXMocnVuSWQ6IHN0cmluZyk6IFByb21pc2U8Tm9uZGV0ZXJtaW5pc21TYW1wbGVbXT4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8Tm9uZGV0ZXJtaW5pc21TYW1wbGVbXT4oXG4gICAgICBgL25vbmRldGVybWluaXNtL3J1bnMvJHtydW5JZH0vc2FtcGxlc2AsXG4gICAgICB7IGFwaVZlcnNpb246ICd2MScgfVxuICAgICk7XG4gIH0sXG5cbiAgLyoqXG4gICAqIEdldCBydW4gc3VtbWFyeSB3aXRoIGFuYWx5c2lzXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3Qgc3VtbWFyeSA9IGF3YWl0IG5vbmRldGVybWluaXNtLmdldFJ1blN1bW1hcnkoJ3J1bl8xMjMnKTtcbiAgICogY29uc29sZS5sb2coYFBhc3NAayByYXRlOiAke3N1bW1hcnkuY3JpdGVyaW9uQW5hbHlzZXNbMF0ucGFzc0F0S1JhdGV9YCk7XG4gICAqIGBgYFxuICAgKi9cbiAgYXN5bmMgZ2V0UnVuU3VtbWFyeShydW5JZDogc3RyaW5nKTogUHJvbWlzZTxSdW5TdW1tYXJ5PiB7XG4gICAgcmV0dXJuIGFwaVJlcXVlc3RXaXRoRGF0YTxSdW5TdW1tYXJ5PihcbiAgICAgIGAvbm9uZGV0ZXJtaW5pc20vcnVucy8ke3J1bklkfS9zdW1tYXJ5YCxcbiAgICAgIHsgYXBpVmVyc2lvbjogJ3YxJyB9XG4gICAgKTtcbiAgfSxcblxuICAvKipcbiAgICogVHJpZ2dlciBhbmFseXNpcyBvZiBhIGNvbXBsZXRlZCBydW5cbiAgICpcbiAgICogQGV4YW1wbGVcbiAgICogYGBgdHlwZXNjcmlwdFxuICAgKiBjb25zdCBzdW1tYXJ5ID0gYXdhaXQgbm9uZGV0ZXJtaW5pc20uYW5hbHl6ZVJ1bigncnVuXzEyMycpO1xuICAgKiBgYGBcbiAgICovXG4gIGFzeW5jIGFuYWx5emVSdW4ocnVuSWQ6IHN0cmluZyk6IFByb21pc2U8UnVuU3VtbWFyeT4ge1xuICAgIHJldHVybiBhcGlSZXF1ZXN0V2l0aERhdGE8UnVuU3VtbWFyeT4oXG4gICAgICBgL25vbmRldGVybWluaXNtL3J1bnMvJHtydW5JZH0vYW5hbHl6ZWAsXG4gICAgICB7IG1ldGhvZDogJ1BPU1QnLCBhcGlWZXJzaW9uOiAndjEnIH1cbiAgICApO1xuICB9LFxuXG4gIC8qKlxuICAgKiBHZXQgaW5mb3JtYXRpb24gYWJvdXQgcGFzc0BrIGFuYWx5c2lzXG4gICAqXG4gICAqIEBleGFtcGxlXG4gICAqIGBgYHR5cGVzY3JpcHRcbiAgICogY29uc3QgaW5mbyA9IGF3YWl0IG5vbmRldGVybWluaXNtLmdldEluZm8oKTtcbiAgICogY29uc29sZS5sb2coaW5mby5jb25jZXB0cy5wYXNzQXRLLmRlc2NyaXB0aW9uKTtcbiAgICogYGBgXG4gICAqL1xuICBhc3luYyBnZXRJbmZvKCk6IFByb21pc2U8UGFzc0F0S0luZm8+IHtcbiAgICByZXR1cm4gYXBpUmVxdWVzdFdpdGhEYXRhPFBhc3NBdEtJbmZvPihcbiAgICAgICcvbm9uZGV0ZXJtaW5pc20vaW5mbycsXG4gICAgICB7IGFwaVZlcnNpb246ICd2MScgfVxuICAgICk7XG4gIH0sXG59O1xuXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG4vLyBIRUxQRVIgRlVOQ1RJT05TXG4vLyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09XG5cbi8qKlxuICogQ2FsY3VsYXRlIHBhc3NAayBwcm9iYWJpbGl0eSBmcm9tIHBhc3MgcmF0ZVxuICpcbiAqIEBwYXJhbSBwYXNzUmF0ZSAtIFNpbmdsZS1ydW4gcGFzcyByYXRlICgwLTEpXG4gKiBAcGFyYW0gayAtIE51bWJlciBvZiBydW5zXG4gKiBAcmV0dXJucyBQcm9iYWJpbGl0eSB0aGF0IGF0IGxlYXN0IDEgb2YgayBydW5zIHBhc3Nlc1xuICpcbiAqIEBleGFtcGxlXG4gKiBgYGB0eXBlc2NyaXB0XG4gKiBjb25zdCBwYXNzQXRLID0gY2FsY3VsYXRlUGFzc0F0SygwLjcsIDMpOyAvLyB+MC45NzNcbiAqIGBgYFxuICovXG5leHBvcnQgZnVuY3Rpb24gY2FsY3VsYXRlUGFzc0F0SyhwYXNzUmF0ZTogbnVtYmVyLCBrOiBudW1iZXIpOiBudW1iZXIge1xuICByZXR1cm4gMSAtIE1hdGgucG93KDEgLSBwYXNzUmF0ZSwgayk7XG59XG5cbi8qKlxuICogQ2FsY3VsYXRlIHBhc3NeayBwcm9iYWJpbGl0eSBmcm9tIHBhc3MgcmF0ZVxuICpcbiAqIEBwYXJhbSBwYXNzUmF0ZSAtIFNpbmdsZS1ydW4gcGFzcyByYXRlICgwLTEpXG4gKiBAcGFyYW0gayAtIE51bWJlciBvZiBydW5zXG4gKiBAcmV0dXJucyBQcm9iYWJpbGl0eSB0aGF0IGFsbCBrIHJ1bnMgcGFzc1xuICpcbiAqIEBleGFtcGxlXG4gKiBgYGB0eXBlc2NyaXB0XG4gKiBjb25zdCBwYXNzVG9LID0gY2FsY3VsYXRlUGFzc1RvSygwLjcsIDMpOyAvLyB+MC4zNDNcbiAqIGBgYFxuICovXG5leHBvcnQgZnVuY3Rpb24gY2FsY3VsYXRlUGFzc1RvSyhwYXNzUmF0ZTogbnVtYmVyLCBrOiBudW1iZXIpOiBudW1iZXIge1xuICByZXR1cm4gTWF0aC5wb3cocGFzc1JhdGUsIGspO1xufVxuXG4vKipcbiAqIENhbGN1bGF0ZSByZXF1aXJlZCBwYXNzIHJhdGUgdG8gYWNoaWV2ZSB0YXJnZXQgcGFzc0BrXG4gKlxuICogQHBhcmFtIHRhcmdldFBhc3NBdEsgLSBEZXNpcmVkIHBhc3NAayBwcm9iYWJpbGl0eVxuICogQHBhcmFtIGsgLSBOdW1iZXIgb2YgcnVuc1xuICogQHJldHVybnMgUmVxdWlyZWQgc2luZ2xlLXJ1biBwYXNzIHJhdGVcbiAqXG4gKiBAZXhhbXBsZVxuICogYGBgdHlwZXNjcmlwdFxuICogY29uc3QgcmVxdWlyZWRSYXRlID0gcmVxdWlyZWRQYXNzUmF0ZUZvclBhc3NBdEsoMC45NSwgMyk7IC8vIH4wLjYzMlxuICogYGBgXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiByZXF1aXJlZFBhc3NSYXRlRm9yUGFzc0F0Syh0YXJnZXRQYXNzQXRLOiBudW1iZXIsIGs6IG51bWJlcik6IG51bWJlciB7XG4gIHJldHVybiAxIC0gTWF0aC5wb3coMSAtIHRhcmdldFBhc3NBdEssIDEgLyBrKTtcbn1cblxuLyoqXG4gKiBEZXRlcm1pbmUgaWYgZXZhbHVhdGlvbiBpcyByZWxpYWJsZSBiYXNlZCBvbiBhbmFseXNpc1xuICpcbiAqIEBwYXJhbSBhbmFseXNpcyAtIENyaXRlcmlvbiBhbmFseXNpcyByZXN1bHRcbiAqIEBwYXJhbSByZWxpYWJpbGl0eVRocmVzaG9sZCAtIE1pbmltdW0gcmVsaWFiaWxpdHkgc2NvcmUgKGRlZmF1bHQgMC44KVxuICogQHJldHVybnMgV2hldGhlciB0aGUgZXZhbHVhdGlvbiBpcyBjb25zaWRlcmVkIHJlbGlhYmxlXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBpc1JlbGlhYmxlRXZhbHVhdGlvbihcbiAgYW5hbHlzaXM6IENyaXRlcmlvbkFuYWx5c2lzLFxuICByZWxpYWJpbGl0eVRocmVzaG9sZCA9IDAuOFxuKTogYm9vbGVhbiB7XG4gIHJldHVybiBhbmFseXNpcy5yZWxpYWJpbGl0eVNjb3JlID49IHJlbGlhYmlsaXR5VGhyZXNob2xkO1xufVxuXG4vKipcbiAqIEdldCByZWNvbW1lbmRhdGlvbiBiYXNlZCBvbiByZWxpYWJpbGl0eSBhbmFseXNpc1xuICpcbiAqIEBwYXJhbSBhbmFseXNpcyAtIENyaXRlcmlvbiBhbmFseXNpcyByZXN1bHRcbiAqIEByZXR1cm5zIEFjdGlvbmFibGUgcmVjb21tZW5kYXRpb24gc3RyaW5nXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBnZXRSZWxpYWJpbGl0eVJlY29tbWVuZGF0aW9uKGFuYWx5c2lzOiBDcml0ZXJpb25BbmFseXNpcyk6IHN0cmluZyB7XG4gIGlmIChhbmFseXNpcy5yZWxpYWJpbGl0eVNjb3JlID49IDAuOSkge1xuICAgIHJldHVybiAnRXZhbHVhdGlvbiBpcyBoaWdobHkgcmVsaWFibGUuIE5vIGNoYW5nZXMgbmVlZGVkLic7XG4gIH0gZWxzZSBpZiAoYW5hbHlzaXMucmVsaWFiaWxpdHlTY29yZSA+PSAwLjgpIHtcbiAgICByZXR1cm4gJ0V2YWx1YXRpb24gaXMgcmVsaWFibGUuIENvbnNpZGVyIG1pbm9yIGNyaXRlcmlhIHJlZmluZW1lbnRzLic7XG4gIH0gZWxzZSBpZiAoYW5hbHlzaXMucmVsaWFiaWxpdHlTY29yZSA+PSAwLjYpIHtcbiAgICByZXR1cm4gJ0V2YWx1YXRpb24gaGFzIG1vZGVyYXRlIHJlbGlhYmlsaXR5LiBBZGQgbW9yZSBzcGVjaWZpYyBjcml0ZXJpYSBvciBleGFtcGxlcy4nO1xuICB9IGVsc2Uge1xuICAgIHJldHVybiAnRXZhbHVhdGlvbiBpcyB1bnJlbGlhYmxlLiBDb25zaWRlciB1c2luZyBkZXRlcm1pbmlzdGljIGNoZWNrcyBvciByZXN0cnVjdHVyaW5nIGNyaXRlcmlhLic7XG4gIH1cbn1cbiJdfQ==
|