@artemiskit/sdk 0.4.5 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/index.js +93 -2
- package/dist/types.d.ts +8 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/artemiskit.ts +32 -0
- package/src/types.ts +8 -3
package/CHANGELOG.md
CHANGED
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyCH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAuBzC;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/index.js
CHANGED
|
@@ -28128,6 +28128,62 @@ function chunkArray(array, size) {
|
|
|
28128
28128
|
}
|
|
28129
28129
|
return chunks;
|
|
28130
28130
|
}
|
|
28131
|
+
function assessComparisonEligibility(baseline, current) {
|
|
28132
|
+
const reasons = [];
|
|
28133
|
+
if (baseline.config.scenario !== current.config.scenario) {
|
|
28134
|
+
reasons.push({ code: "scenario_mismatch" });
|
|
28135
|
+
}
|
|
28136
|
+
const baselineIdentity = baseline.workload_identity;
|
|
28137
|
+
const currentIdentity = current.workload_identity;
|
|
28138
|
+
if (!baselineIdentity || !currentIdentity) {
|
|
28139
|
+
if (!baselineIdentity || !currentIdentity)
|
|
28140
|
+
reasons.push({ code: "workload_identity_missing" });
|
|
28141
|
+
if (!baselineIdentity || !currentIdentity)
|
|
28142
|
+
reasons.push({ code: "rubric_identity_missing" });
|
|
28143
|
+
} else {
|
|
28144
|
+
if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
|
|
28145
|
+
reasons.push({ code: "workload_mismatch" });
|
|
28146
|
+
}
|
|
28147
|
+
if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
|
|
28148
|
+
reasons.push({ code: "rubric_mismatch" });
|
|
28149
|
+
}
|
|
28150
|
+
}
|
|
28151
|
+
if (reasons.some((reason) => isIncomparableReason(reason.code))) {
|
|
28152
|
+
return { schema_version: "1", status: "incomparable", reasons };
|
|
28153
|
+
}
|
|
28154
|
+
const baselineExecution = baseline.execution_provenance;
|
|
28155
|
+
const currentExecution = current.execution_provenance;
|
|
28156
|
+
if (!baselineExecution || !currentExecution) {
|
|
28157
|
+
reasons.push({ code: "execution_provenance_missing" });
|
|
28158
|
+
} else {
|
|
28159
|
+
if (baselineExecution.target.provider !== currentExecution.target.provider) {
|
|
28160
|
+
reasons.push({ code: "target_provider_changed" });
|
|
28161
|
+
}
|
|
28162
|
+
if (!sameStringSet(baselineExecution.target.requested_models, currentExecution.target.requested_models)) {
|
|
28163
|
+
reasons.push({ code: "target_model_changed" });
|
|
28164
|
+
}
|
|
28165
|
+
if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
|
|
28166
|
+
reasons.push({ code: "generation_settings_changed" });
|
|
28167
|
+
}
|
|
28168
|
+
}
|
|
28169
|
+
return {
|
|
28170
|
+
schema_version: "1",
|
|
28171
|
+
status: reasons.length === 0 ? "compatible" : "qualified",
|
|
28172
|
+
reasons
|
|
28173
|
+
};
|
|
28174
|
+
}
|
|
28175
|
+
function isComparisonAvailable(eligibility) {
|
|
28176
|
+
return eligibility.status !== "incomparable";
|
|
28177
|
+
}
|
|
28178
|
+
function isIncomparableReason(code) {
|
|
28179
|
+
return code === "scenario_mismatch" || code === "workload_mismatch" || code === "rubric_mismatch";
|
|
28180
|
+
}
|
|
28181
|
+
function sameStringSet(left, right) {
|
|
28182
|
+
return JSON.stringify([...left ?? []].sort()) === JSON.stringify([...right ?? []].sort());
|
|
28183
|
+
}
|
|
28184
|
+
function sameGeneration(left, right) {
|
|
28185
|
+
return left?.temperature === right?.temperature && left?.max_tokens === right?.max_tokens && left?.seed === right?.seed;
|
|
28186
|
+
}
|
|
28131
28187
|
function getManifestType(manifest) {
|
|
28132
28188
|
if ("type" in manifest) {
|
|
28133
28189
|
if (manifest.type === "redteam")
|
|
@@ -28267,9 +28323,14 @@ class LocalStorageAdapter {
|
|
|
28267
28323
|
this.loadRun(baselineId),
|
|
28268
28324
|
this.loadRun(currentId)
|
|
28269
28325
|
]);
|
|
28326
|
+
const eligibility = assessComparisonEligibility(baseline, current);
|
|
28327
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
28328
|
+
return { baseline, current, eligibility };
|
|
28329
|
+
}
|
|
28270
28330
|
return {
|
|
28271
28331
|
baseline,
|
|
28272
28332
|
current,
|
|
28333
|
+
eligibility,
|
|
28273
28334
|
delta: {
|
|
28274
28335
|
successRate: current.metrics.success_rate - baseline.metrics.success_rate,
|
|
28275
28336
|
latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
|
|
@@ -28368,7 +28429,7 @@ class LocalStorageAdapter {
|
|
|
28368
28429
|
return null;
|
|
28369
28430
|
}
|
|
28370
28431
|
const comparison = await this.compare(baseline.runId, runId);
|
|
28371
|
-
const hasRegression = comparison.delta.successRate < -regressionThreshold;
|
|
28432
|
+
const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
|
|
28372
28433
|
return {
|
|
28373
28434
|
baseline,
|
|
28374
28435
|
comparison,
|
|
@@ -36959,9 +37020,14 @@ class SupabaseStorageAdapter {
|
|
|
36959
37020
|
}
|
|
36960
37021
|
async compare(baselineId, currentId) {
|
|
36961
37022
|
const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
|
|
37023
|
+
const eligibility = assessComparisonEligibility(baseline, current);
|
|
37024
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
37025
|
+
return { baseline, current, eligibility };
|
|
37026
|
+
}
|
|
36962
37027
|
return {
|
|
36963
37028
|
baseline,
|
|
36964
37029
|
current,
|
|
37030
|
+
eligibility,
|
|
36965
37031
|
delta: {
|
|
36966
37032
|
successRate: current.metrics.success_rate - baseline.metrics.success_rate,
|
|
36967
37033
|
latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
|
|
@@ -37092,7 +37158,7 @@ class SupabaseStorageAdapter {
|
|
|
37092
37158
|
return null;
|
|
37093
37159
|
}
|
|
37094
37160
|
const comparison = await this.compare(baseline.runId, runId);
|
|
37095
|
-
const hasRegression = comparison.delta.successRate < -regressionThreshold;
|
|
37161
|
+
const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
|
|
37096
37162
|
return {
|
|
37097
37163
|
baseline,
|
|
37098
37164
|
comparison,
|
|
@@ -51475,6 +51541,29 @@ class ArtemisKit {
|
|
|
51475
51541
|
});
|
|
51476
51542
|
const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
|
|
51477
51543
|
const currentMetrics = this.extractRunMetrics(currentManifest, current);
|
|
51544
|
+
const eligibility = assessComparisonEligibility(baselineManifest, currentManifest);
|
|
51545
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
51546
|
+
this.emit("progress", {
|
|
51547
|
+
message: `Comparison unavailable: ${eligibility.reasons.map((reason) => reason.code).join(", ")}`,
|
|
51548
|
+
phase: "teardown",
|
|
51549
|
+
progress: 100
|
|
51550
|
+
});
|
|
51551
|
+
return {
|
|
51552
|
+
baseline: baselineMetrics,
|
|
51553
|
+
current: currentMetrics,
|
|
51554
|
+
comparison: {
|
|
51555
|
+
comparisonAvailable: false,
|
|
51556
|
+
newFailures: [],
|
|
51557
|
+
newPasses: [],
|
|
51558
|
+
unchanged: [],
|
|
51559
|
+
addedCases: [],
|
|
51560
|
+
removedCases: []
|
|
51561
|
+
},
|
|
51562
|
+
eligibility,
|
|
51563
|
+
hasRegression: false,
|
|
51564
|
+
threshold
|
|
51565
|
+
};
|
|
51566
|
+
}
|
|
51478
51567
|
const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
|
|
51479
51568
|
this.emit("progress", {
|
|
51480
51569
|
message: "Comparing cases...",
|
|
@@ -51492,9 +51581,11 @@ class ArtemisKit {
|
|
|
51492
51581
|
baseline: baselineMetrics,
|
|
51493
51582
|
current: currentMetrics,
|
|
51494
51583
|
comparison: {
|
|
51584
|
+
comparisonAvailable: true,
|
|
51495
51585
|
successRateDelta,
|
|
51496
51586
|
...caseComparison
|
|
51497
51587
|
},
|
|
51588
|
+
eligibility,
|
|
51498
51589
|
hasRegression,
|
|
51499
51590
|
threshold
|
|
51500
51591
|
};
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Type definitions for the programmatic SDK
|
|
4
4
|
*/
|
|
5
|
-
import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
5
|
+
import type { AdapterConfig, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
6
6
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
7
|
import type { Scenario } from '@artemiskit/core';
|
|
8
8
|
/**
|
|
@@ -327,8 +327,10 @@ export interface RunSummary {
|
|
|
327
327
|
* Comparison details between two runs
|
|
328
328
|
*/
|
|
329
329
|
export interface ComparisonDetails {
|
|
330
|
-
/**
|
|
331
|
-
|
|
330
|
+
/** True when the runs are eligible for a metric/case delta. */
|
|
331
|
+
comparisonAvailable: boolean;
|
|
332
|
+
/** Change in success rate (current - baseline), absent for incomparable runs. */
|
|
333
|
+
successRateDelta?: number;
|
|
332
334
|
/** Cases that passed in baseline but failed in current */
|
|
333
335
|
newFailures: Array<{
|
|
334
336
|
caseId: string;
|
|
@@ -371,7 +373,9 @@ export interface CompareResult {
|
|
|
371
373
|
current: RunSummary;
|
|
372
374
|
/** Detailed comparison */
|
|
373
375
|
comparison: ComparisonDetails;
|
|
374
|
-
/**
|
|
376
|
+
/** Versioned compatibility decision that governs the comparison. */
|
|
377
|
+
eligibility: ComparisonEligibility;
|
|
378
|
+
/** Whether a regression was detected; always false for incomparable runs. */
|
|
375
379
|
hasRegression: boolean;
|
|
376
380
|
/** Threshold used for regression detection */
|
|
377
381
|
threshold: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.6",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.5.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.5.3",
|
|
75
|
+
"@artemiskit/redteam": "0.3.7"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
package/src/artemiskit.ts
CHANGED
|
@@ -15,10 +15,12 @@ import {
|
|
|
15
15
|
type StressManifest,
|
|
16
16
|
type StressMetrics,
|
|
17
17
|
type StressRequestResult,
|
|
18
|
+
assessComparisonEligibility,
|
|
18
19
|
runScenario as coreRunScenario,
|
|
19
20
|
createAdapter,
|
|
20
21
|
createStorageAdapter,
|
|
21
22
|
getGitInfo,
|
|
23
|
+
isComparisonAvailable,
|
|
22
24
|
parseScenarioFile,
|
|
23
25
|
resolveScenarioPaths,
|
|
24
26
|
} from '@artemiskit/core';
|
|
@@ -1025,6 +1027,34 @@ export class ArtemisKit {
|
|
|
1025
1027
|
// Extract metrics from manifests
|
|
1026
1028
|
const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
|
|
1027
1029
|
const currentMetrics = this.extractRunMetrics(currentManifest, current);
|
|
1030
|
+
const eligibility = assessComparisonEligibility(
|
|
1031
|
+
baselineManifest as RunManifest,
|
|
1032
|
+
currentManifest as RunManifest
|
|
1033
|
+
);
|
|
1034
|
+
|
|
1035
|
+
if (!isComparisonAvailable(eligibility)) {
|
|
1036
|
+
this.emit('progress', {
|
|
1037
|
+
message: `Comparison unavailable: ${eligibility.reasons.map((reason) => reason.code).join(', ')}`,
|
|
1038
|
+
phase: 'teardown',
|
|
1039
|
+
progress: 100,
|
|
1040
|
+
});
|
|
1041
|
+
|
|
1042
|
+
return {
|
|
1043
|
+
baseline: baselineMetrics,
|
|
1044
|
+
current: currentMetrics,
|
|
1045
|
+
comparison: {
|
|
1046
|
+
comparisonAvailable: false,
|
|
1047
|
+
newFailures: [],
|
|
1048
|
+
newPasses: [],
|
|
1049
|
+
unchanged: [],
|
|
1050
|
+
addedCases: [],
|
|
1051
|
+
removedCases: [],
|
|
1052
|
+
},
|
|
1053
|
+
eligibility,
|
|
1054
|
+
hasRegression: false,
|
|
1055
|
+
threshold,
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1028
1058
|
|
|
1029
1059
|
// Calculate success rate delta
|
|
1030
1060
|
const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
|
|
@@ -1054,9 +1084,11 @@ export class ArtemisKit {
|
|
|
1054
1084
|
baseline: baselineMetrics,
|
|
1055
1085
|
current: currentMetrics,
|
|
1056
1086
|
comparison: {
|
|
1087
|
+
comparisonAvailable: true,
|
|
1057
1088
|
successRateDelta,
|
|
1058
1089
|
...caseComparison,
|
|
1059
1090
|
},
|
|
1091
|
+
eligibility,
|
|
1060
1092
|
hasRegression,
|
|
1061
1093
|
threshold,
|
|
1062
1094
|
};
|
package/src/types.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import type {
|
|
7
7
|
AdapterConfig,
|
|
8
8
|
CaseResult,
|
|
9
|
+
ComparisonEligibility,
|
|
9
10
|
ModelClient,
|
|
10
11
|
RedTeamManifest,
|
|
11
12
|
RunManifest,
|
|
@@ -393,8 +394,10 @@ export interface RunSummary {
|
|
|
393
394
|
* Comparison details between two runs
|
|
394
395
|
*/
|
|
395
396
|
export interface ComparisonDetails {
|
|
396
|
-
/**
|
|
397
|
-
|
|
397
|
+
/** True when the runs are eligible for a metric/case delta. */
|
|
398
|
+
comparisonAvailable: boolean;
|
|
399
|
+
/** Change in success rate (current - baseline), absent for incomparable runs. */
|
|
400
|
+
successRateDelta?: number;
|
|
398
401
|
/** Cases that passed in baseline but failed in current */
|
|
399
402
|
newFailures: Array<{
|
|
400
403
|
caseId: string;
|
|
@@ -438,7 +441,9 @@ export interface CompareResult {
|
|
|
438
441
|
current: RunSummary;
|
|
439
442
|
/** Detailed comparison */
|
|
440
443
|
comparison: ComparisonDetails;
|
|
441
|
-
/**
|
|
444
|
+
/** Versioned compatibility decision that governs the comparison. */
|
|
445
|
+
eligibility: ComparisonEligibility;
|
|
446
|
+
/** Whether a regression was detected; always false for incomparable runs. */
|
|
442
447
|
hasRegression: boolean;
|
|
443
448
|
/** Threshold used for regression detection */
|
|
444
449
|
threshold: number;
|